diff --git a/.env.example b/.env.example index bf06357e1..f11a17f8f 100644 --- a/.env.example +++ b/.env.example @@ -158,6 +158,15 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS="600" ALIYUN_OSS_POST_MAX_SIZE_BYTES="20971520" ALIYUN_OSS_SUCCESS_ACTION_STATUS="200" +# AGC 项目定时快照上传目标。对象只落在服务端私有前缀 +# `agc/project-snapshots/v1/{user}/{project}/` 下，客户端直传票据不覆盖该前缀。 +# bucket 与凭据可以与资源 bucket 分离；凭据未设置时回退使用 ALIYUN_OSS_ACCESS_KEY_*， +# 但 bucket / endpoint 默认指向 AGC 发行 bucket，需要该凭据具备目标 bucket 的 PutObject 权限。 +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET="agc-dev" +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT="oss-rg-china-mainland.aliyuncs.com" +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID="" +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET="" + # BgFilter 受限资源 worker。父 api-server / external-generation-worker 与唯一的 # `GENARRATIVE_PROCESS_ROLE=bgfilter-worker` 进程必须使用同一个内部 Token。 # `npm run dev` 与 `npm run dev:api-server` 都会自动带起并验活唯一 worker，不要再开第二个终端重复启动。 diff --git a/apps/ai-game-creator-shell/scripts/check-config.mjs b/apps/ai-game-creator-shell/scripts/check-config.mjs index e32dfb4b7..ab7a0270c 100644 --- a/apps/ai-game-creator-shell/scripts/check-config.mjs +++ b/apps/ai-game-creator-shell/scripts/check-config.mjs @@ -121,6 +121,10 @@ const allowedUncalledTauriCommands = [ 'open_game_creator_launcher_window', 'open_game_creator_workspace_window', 'read_direct_project_conversation', + // 项目定时快照上传只在 Rust 侧触发（周期定时器 / 工作区窗口关闭）与排障调用； + // 按产品口径不做客户端可见界面，因此同 `open_game_creator_*_window` 一样按 native-only 登记。 + 'read_local_project_snapshot_state', + 'sync_local_project_snapshot', 'reset_design_agent_session', 'stop_local_game_preview_if_matches', 'start_game_creator_external_mcp', diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index 7aafd8b45..60d3113a3 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -106,6 +106,7 @@ mod preview; mod process_session; mod process_session_bridge; mod project; +mod project_snapshot; mod provider_handoff; mod provider_retry; mod repository_context; @@ -148,6 +149,7 @@ use plugin_host::{ use preview::*; use process_session::*; use project::*; +use project_snapshot::*; use repository_context::*; use resource_inspect::*; use resource_preview_scheduler::*; @@ -2125,6 +2127,9 @@ where fn handle_game_creator_gui_run_event(event: &tauri::RunEvent) { if matches!(event, tauri::RunEvent::Exit) { + // 窗口关闭时已经按项目触发过一次快照同步；退出路径只负责在有界预算内 + // 等在途同步收尾，不重复发起（此刻窗口已销毁，重新枚举项目只会是空集）。 + wait_for_project_snapshot_syncs_on_exit(); // 退出时统一收尾本地预览：进程内监听线程随进程消失，但 `.agent/manifest.json` // 里的 preview 记录会留在 running 上，下次进项目就照着它渲染打不开的运行界面。 if let Err(error) = @@ -2390,6 +2395,7 @@ fn main() { .plugin(tauri_plugin_clipboard_manager::init()) .plugin(tauri_plugin_updater::Builder::new().build()) .plugin(context_menu::init()) + .on_window_event(|window, event| handle_project_snapshot_window_event(window, event)) .manage(game_creator_preview_registry()) .manage(ProjectResourcePreviewReadManager::default()) .manage(PluginHost::default()) @@ -2418,6 +2424,7 @@ fn main() { setup_log.fail("startup.appdata.resolve.failed details=config-dir-uninitialized"); error })?; + spawn_project_snapshot_scheduler(app.handle().clone()); if let Err(error) = builtin_plugins::initialize(&config_dir) { app_log!("startup.builtin-plugins.initialize.failed: {error}"); } @@ -2680,6 +2687,8 @@ fn main() { report_client_error, get_pending_error_reports, ack_error_reports, + sync_local_project_snapshot, + read_local_project_snapshot_state, ]) .build(tauri_context); let app = match app { diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs new file mode 100644 index 000000000..c6de3f6f1 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs @@ -0,0 +1,151 @@ +use super::*; +use std::collections::BTreeSet; + +/// 需要上传的一个文件。摘要与字节数来自本次差异对比，上传阶段不再重新计算。 +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotUploadCandidate { + pub(crate) relative_path: String, + pub(crate) size_bytes: u64, + pub(crate) modified_ms: u64, + pub(crate) checksum: String, + pub(crate) absolute_path: PathBuf, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotDiff { + pub(crate) uploads: Vec, + /// 本地已删除、只在远端清单中需要消失的路径。 + pub(crate) deleted: Vec, + /// 内容与上次同步一致、只有元数据变化的路径；只需刷新本地索引。 + pub(crate) metadata_only: Vec, + pub(crate) skipped: Vec, + /// 本轮因为累计上限没有上传、留给下一次同步的路径。 + pub(crate) deferred: Vec, + /// 同步期间被改写、本轮不参与上传与索引推进的路径。 + pub(crate) pending: Vec, + pub(crate) upload_bytes: u64, + /// 扫描后的完整清单（尚未扣除上传失败与延后项），成功同步后就是新的索引。 + pub(crate) current: BTreeMap, +} + +impl ProjectSnapshotDiff { + pub(crate) fn has_changes(&self) -> bool { + !self.uploads.is_empty() || !self.deleted.is_empty() || !self.metadata_only.is_empty() + } +} + +/// 增量差异对比：先用 `(字节数, 修改时间)` 判定是否需要读盘，只有元数据变化时 +/// 才重新计算摘要，再用摘要判断内容是否真的变了。 +pub(crate) fn compute_project_snapshot_diff( + scan: &ProjectSnapshotScanResult, + previous: &BTreeMap, + max_sync_bytes: u64, +) -> Result { + let mut diff = ProjectSnapshotDiff { + skipped: scan.skipped.clone(), + ..ProjectSnapshotDiff::default() + }; + let mut metadata_only = Vec::new(); + + for file in &scan.files { + let previous_entry = previous.get(&file.relative_path); + if let Some(entry) = previous_entry { + if entry.size_bytes == file.size_bytes + && entry.modified_ms == file.modified_ms + && !entry.checksum.is_empty() + { + diff.current + .insert(file.relative_path.clone(), entry.clone()); + continue; + } + } + + let bytes = read_project_snapshot_file_bytes(&file.absolute_path)?; + // 同步不持项目写锁：读完立刻复核一次，文件在读取期间被改写就留给下一轮， + // 既不把这份内容写进索引，也不上传它。 + if !project_snapshot_file_matches(&file.absolute_path, file.size_bytes, file.modified_ms) { + diff.pending.push(ProjectSnapshotSkippedPath { + relative_path: file.relative_path.clone(), + reason: "同步期间文件发生变化，留到下一次".to_string(), + }); + // 上一轮已同步过这条路径时沿用旧记录：文件仍然存在，只是这一轮读不到 + // 一致版本，所以既不能算删除（会被远端 GC 掉），也不能记成本轮已同步。 + if let Some(entry) = previous_entry { + diff.current + .insert(file.relative_path.clone(), entry.clone()); + } + continue; + } + let checksum = project_snapshot_checksum(&bytes); + match previous_entry { + Some(entry) if entry.checksum == checksum => { + metadata_only.push(file.relative_path.clone()) + } + _ => diff.uploads.push(ProjectSnapshotUploadCandidate { + relative_path: file.relative_path.clone(), + size_bytes: file.size_bytes, + modified_ms: file.modified_ms, + checksum: checksum.clone(), + absolute_path: file.absolute_path.clone(), + }), + } + diff.current.insert( + file.relative_path.clone(), + ProjectSnapshotIndexedFile { + size_bytes: file.size_bytes, + modified_ms: file.modified_ms, + checksum, + }, + ); + } + + diff.deleted = previous + .keys() + .filter(|relative_path| !diff.current.contains_key(*relative_path)) + .cloned() + .collect(); + diff.deleted.sort(); + diff.metadata_only = metadata_only; + + apply_project_snapshot_sync_budget(&mut diff, max_sync_bytes); + Ok(diff) +} + +/// 按路径顺序累计上传体积；超出上限的文件进入延后清单，不静默截断也不上传残缺内容。 +fn apply_project_snapshot_sync_budget(diff: &mut ProjectSnapshotDiff, max_sync_bytes: u64) { + let mut accepted = Vec::new(); + let mut deferred = Vec::new(); + let mut bytes = 0_u64; + for candidate in diff.uploads.drain(..) { + if bytes.saturating_add(candidate.size_bytes) > max_sync_bytes { + deferred.push(ProjectSnapshotSkippedPath { + relative_path: candidate.relative_path, + reason: format!("单次同步超过 {max_sync_bytes} 字节上限，留到下一次"), + }); + continue; + } + bytes = bytes.saturating_add(candidate.size_bytes); + accepted.push(candidate); + } + diff.upload_bytes = bytes; + diff.uploads = accepted; + diff.deferred = deferred; +} + +/// 成功同步后的索引清单：延后项与上传失败项都要剔除，否则下次同步会把它们 +/// 当成"已经同步过"而漏传。 +pub(crate) fn build_project_snapshot_synced_files( + diff: &ProjectSnapshotDiff, + uploaded_paths: &BTreeSet, +) -> BTreeMap { + let mut retained = diff.current.clone(); + for candidate in &diff.uploads { + if !uploaded_paths.contains(&candidate.relative_path) { + retained.remove(&candidate.relative_path); + } + } + for deferred in &diff.deferred { + retained.remove(&deferred.relative_path); + } + retained +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/index.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/index.rs new file mode 100644 index 000000000..f81f9e747 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/index.rs @@ -0,0 +1,131 @@ +use super::*; + +/// 本地增量索引里的一条文件记录。 +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotIndexedFile { + pub(crate) size_bytes: u64, + pub(crate) modified_ms: u64, + pub(crate) checksum: String, +} + +/// 上次成功同步的快照。索引只在本机 AppData 中，不进入用户项目目录。 +/// +/// `user_id` 是远端前缀的一部分：换号后旧索引不再代表同一个远端命名空间， +/// 因此读取时按用户身份判等，不一致就当作冷启动重新全量对比。 +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotIndex { + pub(crate) schema_version: u32, + pub(crate) project_id: String, + pub(crate) user_id: String, + pub(crate) sync_revision: u64, + pub(crate) synced_at_ms: u64, + #[serde(default)] + pub(crate) files: BTreeMap, +} + +pub(crate) fn empty_project_snapshot_index( + project_id: &str, + user_id: &str, +) -> ProjectSnapshotIndex { + ProjectSnapshotIndex { + schema_version: PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION, + project_id: project_id.to_string(), + user_id: user_id.to_string(), + sync_revision: 0, + synced_at_ms: 0, + files: BTreeMap::new(), + } +} + +/// 项目 ID 同时用作 AppData 目录名与远端键段：这里额外拒绝 `.` 与 `..`，避免 +/// 把索引写到目录之外。 +pub(crate) fn validate_project_snapshot_project_id(project_id: &str) -> Result { + shared_contracts::agc_project_snapshots::validate_agc_project_snapshot_project_id(project_id)?; + if project_id == "." || project_id == ".." { + return Err("项目 ID 不能是相对目录".to_string()); + } + Ok(project_id.to_string()) +} + +pub(crate) fn project_snapshot_index_directory(project_id: &str) -> Result { + let config_dir = game_creator_runtime_config_dir() + .ok_or_else(|| "客户端 AppData 配置目录未初始化".to_string())?; + project_snapshot_index_directory_at(&config_dir, project_id) +} + +pub(crate) fn project_snapshot_index_directory_at( + config_dir: &Path, + project_id: &str, +) -> Result { + let project_id = validate_project_snapshot_project_id(project_id)?; + Ok(config_dir.join(PROJECT_SNAPSHOT_DIRECTORY).join(project_id)) +} + +pub(crate) fn project_snapshot_index_path(project_id: &str) -> Result { + Ok(project_snapshot_index_path_at( + &project_snapshot_index_directory(project_id)?, + )) +} + +pub(crate) fn project_snapshot_index_path_at(directory: &Path) -> PathBuf { + directory.join(PROJECT_SNAPSHOT_INDEX_FILE_NAME) +} + +/// 读取本地索引。索引不存在、版本不符或内容损坏时返回空索引：这种情况下 +/// 全量对比会重新上传所有文件，不会漏传，也不会因为坏索引中断同步。 +pub(crate) fn read_project_snapshot_index( + project_id: &str, +) -> Result { + read_project_snapshot_index_at(&project_snapshot_index_directory(project_id)?, project_id) +} + +pub(crate) fn read_project_snapshot_index_at( + directory: &Path, + project_id: &str, +) -> Result { + let path = project_snapshot_index_path_at(directory); + if !path.exists() { + return Ok(empty_project_snapshot_index(project_id, "")); + } + let content = match read_game_creator_private_file_to_string( + &path, + "项目快照索引", + PROJECT_SNAPSHOT_INDEX_MAX_BYTES, + ) { + Ok(content) => content, + Err(error) => { + app_log!("project_snapshot.index.read.failed: {error}"); + return Ok(empty_project_snapshot_index(project_id, "")); + } + }; + match serde_json::from_str::(&content) { + Ok(index) + if index.schema_version == PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION + && index.project_id == project_id => + { + Ok(index) + } + Ok(_) => Ok(empty_project_snapshot_index(project_id, "")), + Err(error) => { + app_log!("project_snapshot.index.parse.failed: {error}"); + Ok(empty_project_snapshot_index(project_id, "")) + } + } +} + +pub(crate) fn write_project_snapshot_index(index: &ProjectSnapshotIndex) -> Result<(), String> { + write_project_snapshot_index_at(&project_snapshot_index_directory(&index.project_id)?, index) +} + +pub(crate) fn write_project_snapshot_index_at( + directory: &Path, + index: &ProjectSnapshotIndex, +) -> Result<(), String> { + let path = project_snapshot_index_path_at(directory); + let mut encoded = serde_json::to_vec_pretty(index) + .map_err(|error| format!("序列化项目快照索引失败：{error}"))?; + encoded.push(b'\n'); + write_game_creator_private_file(&path, &encoded, "项目快照索引") +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs new file mode 100644 index 000000000..bcdf2ce21 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs @@ -0,0 +1,547 @@ +//! AGC 项目定时快照上传。 +//! +//! 客户端在这里负责三件事：按项目目录扫描与增量差异对比、按触发时机编排上传、 +//! 把已成功同步的清单写回本机索引。远端对象键与存储凭据由 api-server 决定， +//! 客户端不持有 OSS 凭据，也不直连 OSS。 + +use super::*; +use std::collections::BTreeSet; +use std::sync::{Condvar, MutexGuard}; +use std::time::Instant; + +mod diff; +mod index; +mod scan; +mod transport; + +#[cfg(test)] +mod tests; + +pub(crate) use diff::*; +pub(crate) use index::*; +pub(crate) use scan::*; +pub(crate) use transport::*; + +/// 本机项目快照索引在 AppData 配置目录下的位置。 +pub(crate) const PROJECT_SNAPSHOT_DIRECTORY: &str = "project-snapshots"; +const PROJECT_SNAPSHOT_INDEX_FILE_NAME: &str = "index.json"; +const PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION: u32 = 1; +const PROJECT_SNAPSHOT_INDEX_MAX_BYTES: u64 = 32 * 1024 * 1024; + +pub(crate) const PROJECT_SNAPSHOT_MAX_FILE_BYTES: u64 = + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES; +pub(crate) const PROJECT_SNAPSHOT_MAX_SYNC_BYTES: u64 = + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_SYNC_BYTES; +/// 单项目常驻占用上限。超过时不再尝试上传，直接给出明确失败而不是反复被服务端拒绝。 +pub(crate) const PROJECT_SNAPSHOT_MAX_PROJECT_BYTES: u64 = + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES; + +const PROJECT_SNAPSHOT_FILES_ENDPOINT: &str = "/api/agc/project-snapshots/files"; +const PROJECT_SNAPSHOT_MANIFEST_ENDPOINT: &str = "/api/agc/project-snapshots/manifest"; + +/// 单个请求的完成上限；关闭与退出路径依赖它保持有界。 +const PROJECT_SNAPSHOT_REQUEST_TIMEOUT_SECONDS: u64 = 60; +/// 应用退出前等待在途同步完成的上限。 +const PROJECT_SNAPSHOT_EXIT_BUDGET_SECONDS: u64 = 15; +const PROJECT_SNAPSHOT_DEFAULT_SYNC_INTERVAL_SECONDS: u64 = 300; +const PROJECT_SNAPSHOT_SYNC_INTERVAL_ENV: &str = + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_SYNC_INTERVAL_SECONDS"; +const PROJECT_SNAPSHOT_DISABLED_ENV: &str = "GENARRATIVE_AGC_PROJECT_SNAPSHOT_DISABLED"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProjectSnapshotSyncTrigger { + Periodic, + ProjectClose, + Manual, +} + +impl ProjectSnapshotSyncTrigger { + fn as_str(self) -> &'static str { + match self { + Self::Periodic => "periodic", + Self::ProjectClose => "project-close", + Self::Manual => "manual", + } + } +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotFailureView { + pub(crate) relative_path: String, + pub(crate) code: String, + pub(crate) detail: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotSyncReport { + pub(crate) project_id: String, + pub(crate) trigger: String, + /// `synced` 全部成功，`partial` 部分成功，`failed` 全部失败，`no-op` 无改动。 + pub(crate) status: String, + pub(crate) sync_revision: u64, + pub(crate) uploaded_files: usize, + pub(crate) uploaded_bytes: u64, + pub(crate) remote_skipped_files: usize, + pub(crate) deleted_files: usize, + pub(crate) deferred_files: usize, + pub(crate) metadata_only_files: usize, + pub(crate) skipped_files: Vec, + /// 同步期间被改写、本轮未参与上传与索引推进的路径。 + pub(crate) pending_files: Vec, + pub(crate) failed_files: Vec, + pub(crate) synced_at_ms: u64, +} + +/// 同步开关。停用时不发起任何请求，也不推进索引。 +pub(crate) fn project_snapshot_sync_enabled() -> bool { + match std::env::var_os(PROJECT_SNAPSHOT_DISABLED_ENV) { + Some(value) => { + let value = value.to_string_lossy().trim().to_ascii_lowercase(); + !matches!(value.as_str(), "1" | "true" | "yes" | "on") + } + None => true, + } +} + +pub(crate) fn project_snapshot_sync_interval() -> Duration { + let seconds = std::env::var(PROJECT_SNAPSHOT_SYNC_INTERVAL_ENV) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|seconds| *seconds > 0) + .unwrap_or(PROJECT_SNAPSHOT_DEFAULT_SYNC_INTERVAL_SECONDS); + Duration::from_secs(seconds) +} + +/// 同一项目的同步串行执行；周期触发遇到在途同步直接让位，不排队堆积。 +static PROJECT_SNAPSHOT_PROJECT_LOCKS: OnceLock>>>> = + OnceLock::new(); + +struct ProjectSnapshotInFlight { + active: Mutex, + idle: Condvar, +} + +static PROJECT_SNAPSHOT_IN_FLIGHT: OnceLock = OnceLock::new(); + +fn project_snapshot_project_locks() -> &'static Mutex>>> { + PROJECT_SNAPSHOT_PROJECT_LOCKS.get_or_init(|| Mutex::new(BTreeMap::new())) +} + +fn project_snapshot_in_flight() -> &'static ProjectSnapshotInFlight { + PROJECT_SNAPSHOT_IN_FLIGHT.get_or_init(|| ProjectSnapshotInFlight { + active: Mutex::new(0), + idle: Condvar::new(), + }) +} + +fn project_snapshot_project_lock(key: &str) -> Arc> { + let mut locks = project_snapshot_project_locks() + .lock() + .expect("project snapshot lock registry"); + locks + .entry(key.to_string()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone() +} + +struct ProjectSnapshotInFlightGuard; + +impl ProjectSnapshotInFlightGuard { + fn begin() -> Self { + let state = project_snapshot_in_flight(); + let mut active = state + .active + .lock() + .expect("project snapshot in-flight lock"); + *active += 1; + Self + } +} + +impl Drop for ProjectSnapshotInFlightGuard { + fn drop(&mut self) { + let state = project_snapshot_in_flight(); + if let Ok(mut active) = state.active.lock() { + *active = active.saturating_sub(1); + if *active == 0 { + state.idle.notify_all(); + } + } + } +} + +/// 周期触发：已有同项目同步在途时返回 `None`。 +pub(crate) fn try_run_project_snapshot_sync(key: &str, run: impl FnOnce() -> T) -> Option { + let lock = project_snapshot_project_lock(key); + let guard = lock.try_lock().ok()?; + let _in_flight = ProjectSnapshotInFlightGuard::begin(); + let result = run(); + drop(guard); + Some(result) +} + +/// 关闭与手动触发：等待在途同步结束后再执行一次，保证最后一次状态被记录。 +pub(crate) fn run_project_snapshot_sync(key: &str, run: impl FnOnce() -> T) -> T { + let lock = project_snapshot_project_lock(key); + let guard: MutexGuard<'_, ()> = lock.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let _in_flight = ProjectSnapshotInFlightGuard::begin(); + let result = run(); + drop(guard); + result +} + +/// 退出前等待在途同步收尾。返回是否在预算内全部结束。 +pub(crate) fn wait_for_project_snapshot_syncs(timeout: Duration) -> bool { + let state = project_snapshot_in_flight(); + let deadline = Instant::now() + timeout; + let mut active = state + .active + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + while *active > 0 { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return false; + } + let (guard, _) = state + .idle + .wait_timeout(active, remaining) + .unwrap_or_else(|poisoned| poisoned.into_inner()); + active = guard; + } + true +} + +/// 把项目同步请求交给后台线程：窗口关闭与应用退出路径都不能被网络等待阻塞。 +pub(crate) fn request_project_snapshot_sync( + project_root: PathBuf, + trigger: ProjectSnapshotSyncTrigger, +) { + if !project_snapshot_sync_enabled() { + return; + } + let key = project_snapshot_sync_key(&project_root); + let spawn = std::thread::Builder::new() + .name("agc-project-snapshot".to_string()) + .spawn(move || { + let run = || sync_project_snapshot_blocking(&project_root, trigger); + let outcome = if matches!(trigger, ProjectSnapshotSyncTrigger::Periodic) { + try_run_project_snapshot_sync(&key, run) + } else { + Some(run_project_snapshot_sync(&key, run)) + }; + if let Some(Err(error)) = outcome { + app_log!( + "project_snapshot.sync.failed trigger={}: {error}", + trigger.as_str() + ); + } + }); + if let Err(error) = spawn { + app_log!("project_snapshot.sync.spawn.failed: {error}"); + } +} + +pub(crate) fn project_snapshot_sync_key(project_root: &Path) -> String { + let value = project_root.to_string_lossy().replace('\\', "/"); + #[cfg(windows)] + { + return value.trim_end_matches('/').to_ascii_lowercase(); + } + #[cfg(not(windows))] + { + value.trim_end_matches('/').to_string() + } +} + +fn sync_project_snapshot_blocking( + project_root: &Path, + trigger: ProjectSnapshotSyncTrigger, +) -> Result { + tauri::async_runtime::block_on(sync_project_snapshot_async(project_root, trigger)) +} + +async fn sync_project_snapshot_async( + project_root: &Path, + trigger: ProjectSnapshotSyncTrigger, +) -> Result { + if !project_snapshot_sync_enabled() { + return Err("项目快照同步已停用".to_string()); + } + validate_project_root(project_root)?; + let session = current_platform_session() + .ok_or_else(|| "authentication-required: 请先登录陶泥儿账号".to_string())?; + let manifest = read_existing_manifest_for_project(project_root)?; + let project_id = validate_project_snapshot_project_id(manifest.project_id.trim())?; + + let previous = read_project_snapshot_index(&project_id)?; + let previous = if previous.user_id == session.user_id { + previous + } else { + empty_project_snapshot_index(&project_id, &session.user_id) + }; + + let scan = scan_project_snapshot_files(project_root, PROJECT_SNAPSHOT_MAX_FILE_BYTES)?; + // 项目总量上限在本地先判：超限时明确失败，不再逐个文件上传后被服务端整体拒绝。 + let scan_total_bytes = scan + .files + .iter() + .fold(0_u64, |total, file| total.saturating_add(file.size_bytes)); + if scan_total_bytes > PROJECT_SNAPSHOT_MAX_PROJECT_BYTES { + return Err(format!( + "项目体积 {scan_total_bytes} 字节超过项目快照单项目上限 {PROJECT_SNAPSHOT_MAX_PROJECT_BYTES} 字节" + )); + } + let diff = + compute_project_snapshot_diff(&scan, &previous.files, PROJECT_SNAPSHOT_MAX_SYNC_BYTES)?; + if !diff.has_changes() { + return Ok(ProjectSnapshotSyncReport { + project_id, + trigger: trigger.as_str().to_string(), + status: "no-op".to_string(), + sync_revision: previous.sync_revision, + uploaded_files: 0, + uploaded_bytes: 0, + remote_skipped_files: 0, + deleted_files: 0, + deferred_files: 0, + metadata_only_files: diff.metadata_only.len(), + skipped_files: failure_views(&diff.skipped), + pending_files: failure_views(&diff.pending), + failed_files: Vec::new(), + synced_at_ms: previous.synced_at_ms, + }); + } + + let upload = upload_project_snapshot_diff(&session, &project_id, &diff).await; + let synced_files = build_project_snapshot_synced_files(&diff, &upload.uploaded_paths); + let next_revision = previous.sync_revision.saturating_add(1); + let synced_at_ms = u64::try_from(unix_millis()).unwrap_or(u64::MAX); + let payload = shared_contracts::agc_project_snapshots::AgcProjectSnapshotManifestRequest { + schema_version: + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION, + project_id: project_id.clone(), + sync_revision: next_revision, + synced_at_ms, + files: synced_files + .iter() + .map(|(relative_path, file)| { + shared_contracts::agc_project_snapshots::AgcProjectSnapshotManifestFile { + relative_path: relative_path.clone(), + size_bytes: file.size_bytes, + checksum: file.checksum.clone(), + } + }) + .collect(), + }; + // 清单失败就不推进索引：宁可下一次重算，也不能把本地记成"已同步"。 + upload_project_snapshot_manifest(&session, &payload) + .await + .map_err(|error| error.message())?; + write_project_snapshot_index(&ProjectSnapshotIndex { + schema_version: PROJECT_SNAPSHOT_INDEX_SCHEMA_VERSION, + project_id: project_id.clone(), + user_id: session.user_id.clone(), + sync_revision: next_revision, + synced_at_ms, + files: synced_files, + })?; + + let status = if upload.failures.is_empty() { + "synced" + } else if upload.uploaded_paths.is_empty() { + "failed" + } else { + "partial" + }; + let report = ProjectSnapshotSyncReport { + project_id, + trigger: trigger.as_str().to_string(), + status: status.to_string(), + sync_revision: next_revision, + uploaded_files: upload.uploaded_paths.len(), + uploaded_bytes: upload.uploaded_bytes, + remote_skipped_files: upload.remote_skipped_files, + deleted_files: diff.deleted.len(), + deferred_files: diff.deferred.len(), + metadata_only_files: diff.metadata_only.len(), + skipped_files: failure_views(&diff.skipped), + pending_files: failure_views(&diff.pending), + failed_files: upload + .failures + .iter() + .map(|failure| ProjectSnapshotFailureView { + relative_path: failure.relative_path.clone(), + code: failure.code.clone(), + detail: failure.detail.clone(), + }) + .collect(), + synced_at_ms, + }; + app_log!( + "project_snapshot.sync.completed trigger={} status={} revision={} uploaded={} skippedRemote={} deleted={} deferred={} failed={}", + report.trigger, + report.status, + report.sync_revision, + report.uploaded_files, + report.remote_skipped_files, + report.deleted_files, + report.deferred_files, + report.failed_files.len() + ); + Ok(report) +} + +fn failure_views(skipped: &[ProjectSnapshotSkippedPath]) -> Vec { + skipped + .iter() + .map(|entry| ProjectSnapshotFailureView { + relative_path: entry.relative_path.clone(), + code: "skipped".to_string(), + detail: entry.reason.clone(), + }) + .collect() +} + +/// 从窗口 URL 读取项目路径。只有带 `projectPath` 的窗口才代表打开的项目。 +pub(crate) fn project_snapshot_project_path_from_url(url: &url::Url) -> Option { + url.query_pairs() + .find_map(|(key, value)| (key == "projectPath").then(|| value.into_owned())) + .filter(|value| !value.trim().is_empty()) +} + +fn open_project_snapshot_workspaces(app: &tauri::AppHandle) -> Vec { + let mut paths = BTreeSet::new(); + for window in app.webview_windows().into_values() { + let Ok(url) = window.url() else { + continue; + }; + if let Some(project_path) = project_snapshot_project_path_from_url(&url) { + paths.insert(project_path); + } + } + paths.into_iter().collect() +} + +/// 工作区窗口关闭即视为项目关闭：立刻补一次同步。 +pub(crate) fn handle_project_snapshot_window_event( + window: &tauri::Window, + event: &tauri::WindowEvent, +) { + if !project_snapshot_sync_enabled() { + return; + } + if !matches!(event, tauri::WindowEvent::CloseRequested { .. }) { + return; + } + // `tauri::Window` 不暴露 WebView 地址，按标签取回对应的 WebView 窗口再读 URL。 + let Some(webview) = window.app_handle().get_webview_window(window.label()) else { + return; + }; + let Ok(url) = webview.url() else { + return; + }; + let Some(project_path) = project_snapshot_project_path_from_url(&url) else { + return; + }; + request_project_snapshot_sync( + PathBuf::from(project_path), + ProjectSnapshotSyncTrigger::ProjectClose, + ); +} + +/// 应用退出前等待在途同步收尾。 +/// +/// 退出时刻窗口已销毁，按窗口重新枚举项目只会得到空集，因此这里不重复发起同步： +/// 关窗触发的同步已经在 `CloseRequested` 时启动，退出路径只需要给它一个有界窗口。 +pub(crate) fn wait_for_project_snapshot_syncs_on_exit() { + if !project_snapshot_sync_enabled() { + return; + } + if !wait_for_project_snapshot_syncs(Duration::from_secs(PROJECT_SNAPSHOT_EXIT_BUDGET_SECONDS)) { + app_log!("project_snapshot.sync.exit.budget-exhausted"); + } +} + +/// 周期定时器：只为当前仍打开的项目触发，进程内项目集合由窗口 URL 决定。 +pub(crate) fn spawn_project_snapshot_scheduler(app: tauri::AppHandle) { + if !project_snapshot_sync_enabled() { + app_log!("project_snapshot.scheduler.disabled"); + return; + } + let interval = project_snapshot_sync_interval(); + let spawned = std::thread::Builder::new() + .name("agc-project-snapshot-scheduler".to_string()) + .spawn(move || loop { + std::thread::sleep(interval); + for project_path in open_project_snapshot_workspaces(&app) { + request_project_snapshot_sync( + PathBuf::from(project_path), + ProjectSnapshotSyncTrigger::Periodic, + ); + } + }); + if let Err(error) = spawned { + app_log!("project_snapshot.scheduler.spawn.failed: {error}"); + } +} + +fn resolve_project_snapshot_root(project_path: &str) -> Result { + let trimmed = project_path.trim(); + if trimmed.is_empty() { + return Err("请提供项目绝对路径".to_string()); + } + let root = PathBuf::from(trimmed); + validate_project_root(&root)?; + Ok(root) +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotStateView { + project_id: String, + index_path: String, + index_present: bool, + file_count: usize, + sync_revision: u64, + synced_at_ms: u64, + enabled: bool, +} + +/// 手动触发一次项目快照同步。同步本身跑在阻塞工作线程上，不占用窗口线程。 +#[tauri::command] +pub(crate) async fn sync_local_project_snapshot( + project_path: String, +) -> Result { + let root = resolve_project_snapshot_root(&project_path)?; + let key = project_snapshot_sync_key(&root); + tauri::async_runtime::spawn_blocking(move || { + run_project_snapshot_sync(&key, || { + sync_project_snapshot_blocking(&root, ProjectSnapshotSyncTrigger::Manual) + }) + }) + .await + .map_err(|error| format!("项目快照同步任务失败：{error}"))? +} + +#[tauri::command] +pub(crate) fn read_local_project_snapshot_state( + project_path: String, +) -> Result { + let root = resolve_project_snapshot_root(&project_path)?; + let manifest = read_existing_manifest_for_project(&root)?; + let project_id = validate_project_snapshot_project_id(manifest.project_id.trim())?; + let index_path = project_snapshot_index_path(&project_id)?; + let index = read_project_snapshot_index(&project_id)?; + Ok(ProjectSnapshotStateView { + project_id, + index_path: index_path.to_string_lossy().into_owned(), + index_present: index_path.exists(), + file_count: index.files.len(), + sync_revision: index.sync_revision, + synced_at_ms: index.synced_at_ms, + enabled: project_snapshot_sync_enabled(), + }) +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs new file mode 100644 index 000000000..2163c4b24 --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs @@ -0,0 +1,157 @@ +use super::*; + +/// 扫描阶段的候选文件：只看元数据，不读内容，差异对比阶段才决定是否读盘。 +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotScannedFile { + pub(crate) relative_path: String, + pub(crate) size_bytes: u64, + pub(crate) modified_ms: u64, + pub(crate) absolute_path: PathBuf, +} + +/// 被排除或不参与本次同步的路径。原因只用于本机日志与同步报告，不进入远端。 +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ProjectSnapshotSkippedPath { + pub(crate) relative_path: String, + pub(crate) reason: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotScanResult { + pub(crate) files: Vec, + pub(crate) skipped: Vec, +} + +/// 扫描项目目录，复用 checkpoint / 项目索引同一份排除口径： +/// `.agent`、版本控制目录、依赖与构建产物目录、凭据目录、符号链接与重解析点 +/// 都不参与同步，超出单文件上限的文件进入跳过清单而不是静默丢弃。 +pub(crate) fn scan_project_snapshot_files( + root: &Path, + max_file_bytes: u64, +) -> Result { + let mut result = ProjectSnapshotScanResult::default(); + if !root.exists() { + return Ok(result); + } + let root_metadata = + fs::symlink_metadata(root).map_err(|error| format!("读取项目目录元数据失败：{error}"))?; + if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() { + return Err("项目快照只支持普通项目目录".to_string()); + } + + let mut directories = vec![root.to_path_buf()]; + while let Some(directory) = directories.pop() { + let entries = fs::read_dir(&directory) + .map_err(|error| format!("读取项目目录失败：{}: {error}", directory.display()))?; + for entry in entries { + let entry = entry + .map_err(|error| format!("读取项目文件失败：{}: {error}", directory.display()))?; + let path = entry.path(); + // 路径不在项目根内（含符号链接跳转）时直接跳过，不猜测归属。 + let Ok(relative_path) = relative_project_path(root, &path) else { + continue; + }; + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) => { + result.skipped.push(ProjectSnapshotSkippedPath { + relative_path, + reason: format!("读取元数据失败：{error}"), + }); + continue; + } + }; + if should_skip_project_snapshot_path(&relative_path) { + continue; + } + if metadata.file_type().is_symlink() || windows_metadata_is_reparse_point(&metadata) { + result.skipped.push(ProjectSnapshotSkippedPath { + relative_path, + reason: "符号链接或重解析点不参与项目快照".to_string(), + }); + continue; + } + if metadata.is_dir() { + directories.push(path); + continue; + } + if !metadata.is_file() { + result.skipped.push(ProjectSnapshotSkippedPath { + relative_path, + reason: "不是普通文件".to_string(), + }); + continue; + } + if metadata.len() > max_file_bytes { + result.skipped.push(ProjectSnapshotSkippedPath { + relative_path, + reason: format!("单个文件超过 {max_file_bytes} 字节上限"), + }); + continue; + } + result.files.push(ProjectSnapshotScannedFile { + relative_path, + size_bytes: metadata.len(), + modified_ms: project_snapshot_modified_ms(&metadata), + absolute_path: path, + }); + } + } + + result + .files + .sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + result + .skipped + .sort_by(|left, right| left.relative_path.cmp(&right.relative_path)); + Ok(result) +} + +/// 修改时间缺失或早于 Unix 纪元时返回 0：这样的文件每轮都会重算摘要，不会 +/// 被误判成"未改动"。 +pub(crate) fn project_snapshot_modified_ms(metadata: &fs::Metadata) -> u64 { + metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +/// 安全打开并读取项目文件内容。拒绝符号链接、重解析点与硬链接，读取后不再 +/// 按路径名二次访问。 +pub(crate) fn read_project_snapshot_file_bytes(path: &Path) -> Result, String> { + let (mut file, metadata) = open_project_snapshot_regular_file(path, "项目快照文件")?; + let mut bytes = Vec::with_capacity(usize::try_from(metadata.len()).unwrap_or_default()); + file.read_to_end(&mut bytes) + .map_err(|error| format!("读取项目快照文件失败：{}: {error}", path.display()))?; + Ok(bytes) +} + +/// 与项目索引同口径的校验和，便于同一份内容在不同入口之间比较。 +pub(crate) fn project_snapshot_checksum(bytes: &[u8]) -> String { + format!("fnv1a64:{:016x}", fnv1a64(bytes)) +} + +/// 复核文件是否仍是扫描时刻的那一份内容。 +/// +/// 同步不持有项目写锁：Agent 或编辑器可能在扫描之后、读取或上传之前改写文件。 +/// 读取前后都按 `(字节数, 修改时间)` 比对，任一处不一致就判定为"同步期间发生变化"， +/// 该文件不计入索引、留给下一次同步，而不是上传一份自相矛盾的快照。 +pub(crate) fn project_snapshot_file_matches( + path: &Path, + size_bytes: u64, + modified_ms: u64, +) -> bool { + let Ok(metadata) = fs::symlink_metadata(path) else { + return false; + }; + if metadata.file_type().is_symlink() || windows_metadata_is_reparse_point(&metadata) { + return false; + } + if !metadata.is_file() { + return false; + } + metadata.len() == size_bytes && project_snapshot_modified_ms(&metadata) == modified_ms +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs new file mode 100644 index 000000000..ff57bdfae --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs @@ -0,0 +1,590 @@ +use super::*; +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn fixture_root() -> tempfile::TempDir { + tempfile::tempdir().expect("create project snapshot fixture root") +} + +fn write_fixture_file(root: &Path, relative_path: &str, bytes: &[u8]) { + let path = root.join(relative_path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).expect("create fixture parent directory"); + } + fs::write(&path, bytes).expect("write fixture file"); +} + +fn scan_fixture(root: &Path) -> ProjectSnapshotScanResult { + scan_project_snapshot_files(root, PROJECT_SNAPSHOT_MAX_FILE_BYTES) + .expect("scan fixture project") +} + +fn paths_of(files: &[ProjectSnapshotUploadCandidate]) -> Vec { + files + .iter() + .map(|candidate| candidate.relative_path.clone()) + .collect() +} + +#[test] +fn project_snapshot_project_id_validation_rejects_paths_and_unsafe_values() { + for value in [ + "", + " ", + ".", + "..", + "../escape", + "a/b", + "a\\b", + "a b", + "-leading", + ".hidden", + ] { + assert!( + validate_project_snapshot_project_id(value).is_err(), + "unsafe project id must be rejected: {value}" + ); + } + assert_eq!( + validate_project_snapshot_project_id("gameagent-1a2b3c4d").expect("stable project id"), + "gameagent-1a2b3c4d" + ); +} + +#[test] +fn project_snapshot_index_round_trips_and_recovers_from_corruption() { + let root = fixture_root(); + let directory = + project_snapshot_index_directory_at(root.path(), "project-1").expect("index directory"); + assert!(read_project_snapshot_index_at(&directory, "project-1") + .expect("missing index reads as empty") + .files + .is_empty()); + + let mut index = empty_project_snapshot_index("project-1", "user-1"); + index.sync_revision = 3; + index.synced_at_ms = 1_700_000_000_000; + index.files.insert( + "game/index.html".to_string(), + ProjectSnapshotIndexedFile { + size_bytes: 13, + modified_ms: 34, + checksum: "fnv1a64:0000000000000001".to_string(), + }, + ); + write_project_snapshot_index_at(&directory, &index).expect("write index"); + assert_eq!( + read_project_snapshot_index_at(&directory, "project-1").expect("read index"), + index + ); + + let path = project_snapshot_index_path_at(&directory); + fs::write(&path, b"{ not json").expect("corrupt index"); + assert!( + read_project_snapshot_index_at(&directory, "project-1") + .expect("corrupt index reads as empty") + .files + .is_empty(), + "损坏索引必须退化为空索引，让下一次同步全量重算" + ); +} + +#[test] +fn project_snapshot_scan_skips_excluded_paths_and_oversized_files() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/index.html", b""); + write_fixture_file(root.path(), "assets/manifest.json", b"{}"); + write_fixture_file(root.path(), ".agent/runtime/state.json", b"{}"); + write_fixture_file(root.path(), ".agent/manifest.json", b"{}"); + write_fixture_file(root.path(), "node_modules/pkg/index.js", b"export {};"); + write_fixture_file(root.path(), "game/dist/bundle.js", b"bundle"); + write_fixture_file(root.path(), "secrets/key.pem", b"private-key"); + write_fixture_file(root.path(), "assets/big.bin", b"0123456789"); + + let scan = scan_project_snapshot_files(root.path(), 4).expect("scan fixture project"); + let scanned = scan + .files + .iter() + .map(|file| file.relative_path.clone()) + .collect::>(); + assert_eq!( + scanned, + vec!["assets/manifest.json".to_string()], + ".agent、node_modules、dist 与凭据目录里的文件不能进入候选集合" + ); + let skipped = scan + .skipped + .iter() + .map(|entry| entry.relative_path.clone()) + .collect::>(); + assert_eq!( + skipped, + vec!["assets/big.bin".to_string(), "game/index.html".to_string()], + "超限文件进入跳过清单，不静默丢弃" + ); +} + +#[test] +fn project_snapshot_diff_reuses_metadata_and_reports_a_single_modification() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + write_fixture_file(root.path(), "game/b.txt", b"beta!"); + + let first_scan = scan_fixture(root.path()); + let first = compute_project_snapshot_diff( + &first_scan, + &BTreeMap::new(), + PROJECT_SNAPSHOT_MAX_SYNC_BYTES, + ) + .expect("first diff"); + assert_eq!(paths_of(&first.uploads), vec!["game/a.txt", "game/b.txt"]); + assert!(first.deleted.is_empty()); + + // 第二次：元数据未变，必须复用摘要并且不产生上传项。 + let second = + compute_project_snapshot_diff(&first_scan, &first.current, PROJECT_SNAPSHOT_MAX_SYNC_BYTES) + .expect("second diff"); + assert!(second.uploads.is_empty(), "{:?}", second.uploads); + assert!(second.deleted.is_empty()); + assert!(second.metadata_only.is_empty()); + assert!(!second.has_changes()); + assert_eq!(second.current, first.current); + + // 只改一个文件：差异集合恰好包含它。 + write_fixture_file(root.path(), "game/a.txt", b"alpha-changed"); + let third = compute_project_snapshot_diff( + &scan_fixture(root.path()), + &first.current, + PROJECT_SNAPSHOT_MAX_SYNC_BYTES, + ) + .expect("third diff"); + assert_eq!(paths_of(&third.uploads), vec!["game/a.txt"]); + + // 删除文件只体现在清单：没有上传项，但 deleted 里能读到。 + fs::remove_file(root.path().join("game/b.txt")).expect("delete fixture file"); + let fourth = compute_project_snapshot_diff( + &scan_fixture(root.path()), + &third.current, + PROJECT_SNAPSHOT_MAX_SYNC_BYTES, + ) + .expect("fourth diff"); + assert!(fourth.uploads.is_empty()); + assert_eq!(fourth.deleted, vec!["game/b.txt".to_string()]); +} + +#[test] +fn project_snapshot_diff_treats_touched_but_identical_content_as_metadata_only() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + let scan = scan_fixture(root.path()); + let checksum = project_snapshot_checksum(b"alpha"); + let mut previous = BTreeMap::new(); + previous.insert( + "game/a.txt".to_string(), + ProjectSnapshotIndexedFile { + size_bytes: 5, + // 修改时间不同但内容一致：只能靠摘要判定，不能重复上传。 + modified_ms: scan.files[0].modified_ms.saturating_sub(1_000), + checksum, + }, + ); + let diff = compute_project_snapshot_diff(&scan, &previous, PROJECT_SNAPSHOT_MAX_SYNC_BYTES) + .expect("metadata-only diff"); + assert!(diff.uploads.is_empty()); + assert_eq!(diff.metadata_only, vec!["game/a.txt".to_string()]); + assert!(diff.has_changes()); +} + +#[test] +fn project_snapshot_diff_defers_files_over_the_sync_budget() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"0123456789"); + write_fixture_file(root.path(), "game/b.txt", b"0123456789"); + let scan = scan_fixture(root.path()); + let diff = compute_project_snapshot_diff(&scan, &BTreeMap::new(), 15).expect("budget diff"); + assert_eq!(paths_of(&diff.uploads), vec!["game/a.txt"]); + assert_eq!( + diff.deferred + .iter() + .map(|entry| entry.relative_path.clone()) + .collect::>(), + vec!["game/b.txt".to_string()] + ); + assert_eq!(diff.upload_bytes, 10); +} + +#[test] +fn project_snapshot_synced_files_exclude_failed_and_deferred_paths() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"a"); + write_fixture_file(root.path(), "game/b.txt", b"b"); + write_fixture_file(root.path(), "game/c.txt", b"c"); + let scan = scan_fixture(root.path()); + let mut diff = + compute_project_snapshot_diff(&scan, &BTreeMap::new(), PROJECT_SNAPSHOT_MAX_SYNC_BYTES) + .expect("diff"); + diff.deferred.push(ProjectSnapshotSkippedPath { + relative_path: "game/c.txt".to_string(), + reason: "fixture".to_string(), + }); + let uploaded = BTreeSet::from(["game/a.txt".to_string()]); + let synced = build_project_snapshot_synced_files(&diff, &uploaded); + assert_eq!( + synced.keys().cloned().collect::>(), + vec!["game/a.txt".to_string()], + "未成功上传与延后的路径不能进入新索引，否则下一次同步会漏传" + ); +} + +#[test] +fn project_snapshot_sync_is_serialized_per_project() { + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let key = "c:/fixture/serialized-project"; + let mut handles = Vec::new(); + for _ in 0..3 { + let active = Arc::clone(&active); + let peak = Arc::clone(&peak); + handles.push(std::thread::spawn(move || { + run_project_snapshot_sync(key, || { + let current = active.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(current, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis(30)); + active.fetch_sub(1, Ordering::SeqCst); + }); + })); + } + for handle in handles { + handle.join().expect("join serialized sync thread"); + } + assert_eq!(peak.load(Ordering::SeqCst), 1); +} + +#[test] +fn project_snapshot_periodic_trigger_yields_while_a_sync_is_in_flight() { + let key = "c:/fixture/periodic-project"; + let (started_sender, started_receiver) = mpsc::channel(); + let handle = std::thread::spawn(move || { + run_project_snapshot_sync(key, || { + started_sender.send(()).expect("signal started sync"); + std::thread::sleep(Duration::from_millis(200)); + }); + }); + started_receiver.recv().expect("wait for in-flight sync"); + assert!( + try_run_project_snapshot_sync(key, || ()).is_none(), + "在途同步期间周期触发必须直接让位" + ); + handle.join().expect("join in-flight sync"); + assert!(try_run_project_snapshot_sync(key, || ()).is_some()); +} + +fn read_http_request_with_body(stream: &mut TcpStream) -> String { + stream + .set_read_timeout(Some(Duration::from_secs(5))) + .expect("set fixture read timeout"); + let mut bytes = Vec::new(); + let mut buffer = [0_u8; 4096]; + let header_end = loop { + let read = stream.read(&mut buffer).expect("read fixture request"); + assert!(read > 0, "fixture request closed before headers"); + bytes.extend_from_slice(&buffer[..read]); + if let Some(index) = bytes.windows(4).position(|window| window == b"\r\n\r\n") { + break index + 4; + } + }; + let headers = String::from_utf8_lossy(&bytes[..header_end]).into_owned(); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok())? + }) + .unwrap_or(0); + while bytes.len() < header_end + content_length { + let read = stream.read(&mut buffer).expect("read fixture request body"); + if read == 0 { + break; + } + bytes.extend_from_slice(&buffer[..read]); + } + String::from_utf8_lossy(&bytes).into_owned() +} + +fn spawn_snapshot_fixture( + response_status: &'static str, + response_body: String, +) -> (String, std::thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind fixture listener"); + let address = listener.local_addr().expect("read fixture address"); + let handle = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept fixture request"); + let request = read_http_request_with_body(&mut stream); + let response = format!( + "HTTP/1.1 {response_status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{response_body}", + response_body.len() + ); + stream + .write_all(response.as_bytes()) + .expect("write fixture response"); + request + }); + (format!("http://{address}"), handle) +} + +fn fixture_session(api_base_url: String) -> PlatformSessionSnapshot { + PlatformSessionSnapshot { + user_id: "user-fixture".to_string(), + access_token: "fixture-token".to_string(), + api_base_url, + identity_generation: 1, + revision: 1, + } +} + +fn fixture_candidate( + root: &Path, + relative_path: &str, + bytes: &[u8], +) -> ProjectSnapshotUploadCandidate { + write_fixture_file(root, relative_path, bytes); + let path = root.join(relative_path); + let metadata = fs::symlink_metadata(&path).expect("read fixture metadata"); + ProjectSnapshotUploadCandidate { + relative_path: relative_path.to_string(), + size_bytes: bytes.len() as u64, + // 用真实修改时间：上传前的一致性复核会拿它比对，写死常量会把请求提前挡掉。 + modified_ms: project_snapshot_modified_ms(&metadata), + checksum: project_snapshot_checksum(bytes), + absolute_path: path, + } +} + +#[test] +fn project_snapshot_upload_marks_remote_duplicates_and_sends_marker_headers() { + let root = fixture_root(); + let candidate = fixture_candidate(root.path(), "game/a.txt", b"alpha"); + let body = serde_json::json!({ + "projectId": "project-1", + "relativePath": "game/a.txt", + "objectKey": "agc/project-snapshots/v1/user-fixture/project-1/game/a.txt", + "skipped": true, + "checksum": candidate.checksum, + "sizeBytes": candidate.size_bytes, + }) + .to_string(); + let (base_url, fixture) = spawn_snapshot_fixture("200 OK", body); + let session = fixture_session(base_url); + let diff = ProjectSnapshotDiff { + uploads: vec![candidate], + ..ProjectSnapshotDiff::default() + }; + let report = + tauri::async_runtime::block_on(upload_project_snapshot_diff(&session, "project-1", &diff)); + let request = fixture.join().expect("join fixture"); + + assert!(report.failures.is_empty(), "{:?}", report.failures); + assert_eq!(report.remote_skipped_files, 1); + assert_eq!(report.uploaded_bytes, 5); + assert_eq!( + report.uploaded_paths, + BTreeSet::from(["game/a.txt".to_string()]) + ); + assert!(request.starts_with("POST /api/agc/project-snapshots/files?")); + assert!(request.contains("projectId=project-1")); + assert!(request.contains("relativePath=game%2Fa.txt")); + assert!(request + .to_ascii_lowercase() + .contains("authorization: bearer fixture-token")); + assert!(request + .to_ascii_lowercase() + .contains("x-genarrative-client: agc")); + assert!(request.ends_with("alpha")); +} + +#[test] +fn project_snapshot_file_matches_detects_metadata_drift() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + let path = root.path().join("game/a.txt"); + let metadata = fs::symlink_metadata(&path).expect("fixture metadata"); + let size = metadata.len(); + let modified = project_snapshot_modified_ms(&metadata); + + assert!(project_snapshot_file_matches(&path, size, modified)); + assert!( + !project_snapshot_file_matches(&path, size + 1, modified), + "字节数变化必须被判为已改写" + ); + assert!( + !project_snapshot_file_matches(&path, size, modified + 1), + "修改时间变化必须被判为已改写" + ); + assert!( + !project_snapshot_file_matches(&root.path().join("game/missing.txt"), size, modified), + "文件消失必须被判为已改写" + ); +} + +#[test] +fn project_snapshot_diff_defers_files_that_changed_while_being_read() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + let mut previous = BTreeMap::new(); + previous.insert( + "game/a.txt".to_string(), + ProjectSnapshotIndexedFile { + size_bytes: 5, + modified_ms: 42, + checksum: "fnv1a64:00000000000000aa".to_string(), + }, + ); + // 扫描时刻的元数据与磁盘现状不一致：等价于"扫描之后文件被改写"。 + let scan = ProjectSnapshotScanResult { + files: vec![ProjectSnapshotScannedFile { + relative_path: "game/a.txt".to_string(), + size_bytes: 5, + modified_ms: 41, + absolute_path: root.path().join("game/a.txt"), + }], + skipped: Vec::new(), + }; + + let diff = compute_project_snapshot_diff(&scan, &previous, PROJECT_SNAPSHOT_MAX_SYNC_BYTES) + .expect("diff"); + assert!(diff.uploads.is_empty(), "{:?}", diff.uploads); + assert_eq!(diff.pending.len(), 1); + assert_eq!(diff.pending[0].relative_path, "game/a.txt"); + assert!( + diff.deleted.is_empty(), + "同步期间被改写的文件不能被当成删除，否则远端 GC 会误删" + ); + assert!( + diff.current.contains_key("game/a.txt"), + "应沿用上一轮记录，保持清单与远端对象一致" + ); +} + +#[test] +fn project_snapshot_upload_skips_files_that_changed_after_the_diff() { + let root = fixture_root(); + let mut candidate = fixture_candidate(root.path(), "game/a.txt", b"alpha"); + candidate.modified_ms = candidate.modified_ms.saturating_sub(1_000); + // 服务地址故意指向未监听端口：只要请求真的发出去就会出现 transport 失败， + // 因此这里同时证明"没有发出请求"和"分类是 file-changed"。 + let session = fixture_session("http://127.0.0.1:9".to_string()); + let diff = ProjectSnapshotDiff { + uploads: vec![candidate], + ..ProjectSnapshotDiff::default() + }; + + let report = + tauri::async_runtime::block_on(upload_project_snapshot_diff(&session, "project-1", &diff)); + assert!(report.uploaded_paths.is_empty()); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].code, "file-changed"); + assert!( + ProjectSnapshotUploadErrorKind::FileChanged.is_retryable(), + "文件被改写不是终态失败，下一次触发应重试" + ); +} + +#[test] +fn project_snapshot_upload_stops_after_a_deterministic_authentication_failure() { + let root = fixture_root(); + let first = fixture_candidate(root.path(), "game/a.txt", b"alpha"); + let second = fixture_candidate(root.path(), "game/b.txt", b"beta!"); + let (base_url, fixture) = spawn_snapshot_fixture("401 Unauthorized", "{}".to_string()); + let session = fixture_session(base_url); + let diff = ProjectSnapshotDiff { + uploads: vec![first, second], + ..ProjectSnapshotDiff::default() + }; + let report = + tauri::async_runtime::block_on(upload_project_snapshot_diff(&session, "project-1", &diff)); + fixture.join().expect("join fixture"); + + assert!(report.uploaded_paths.is_empty()); + assert_eq!(report.failures.len(), 2); + assert_eq!(report.failures[0].code, "authentication-required"); + assert_eq!(report.failures[0].relative_path, "game/a.txt"); + assert_eq!(report.failures[1].relative_path, "game/b.txt"); + assert!( + !ProjectSnapshotUploadErrorKind::Authentication.is_retryable(), + "鉴权失败不能进入自动重试" + ); +} + +#[test] +fn project_snapshot_project_path_is_read_from_window_urls_only() { + let url = url::Url::parse("tauri://localhost/index.html?main&projectPath=C%3A%5Cgames%5Cdemo") + .expect("parse fixture url"); + assert_eq!( + project_snapshot_project_path_from_url(&url), + Some("C:\\games\\demo".to_string()) + ); + let launcher = url::Url::parse("tauri://localhost/index.html?launcher").expect("parse url"); + assert_eq!(project_snapshot_project_path_from_url(&launcher), None); +} + +/// 真实链路冒烟：客户端差异引擎 → 本地 api-server → 真实 OSS。 +/// +/// 默认忽略；需要显式提供目标项目与登录态： +/// +/// ```text +/// $env:GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_PROJECT="<项目绝对路径>" +/// $env:GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_TOKEN="" +/// $env:GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_API_BASE_URL="http://127.0.0.1:8082" +/// cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml \ +/// project_snapshot_live_sync -- --ignored --nocapture +/// ``` +/// +/// 判定口径：第一次必须 `synced` 且上传数大于 0；紧接着的第二次必须 `no-op` 且上传 0 个文件， +/// 用来证明差异对比在真实服务端与真实 OSS 上确实避免了重复上传。 +/// +/// 另需 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_CONFIG_DIR` 指定 AppData 配置目录， +/// 否则测试进程里没有窗口 setup 初始化过该目录，索引无处可写。 +#[test] +#[ignore = "live smoke：需要真实 api-server 与 OSS 凭据"] +fn project_snapshot_live_sync_uploads_once_then_reports_no_op() { + let Ok(project_path) = std::env::var("GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_PROJECT") else { + println!("[skip] 未设置 GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_PROJECT"); + return; + }; + let Ok(access_token) = std::env::var("GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_TOKEN") else { + println!("[skip] 未设置 GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_TOKEN"); + return; + }; + let api_base_url = std::env::var("GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_API_BASE_URL") + .unwrap_or_else(|_| "http://127.0.0.1:8082".to_string()); + let user_id = std::env::var("GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_USER_ID") + .unwrap_or_else(|_| "snapshot-live-smoke-user".to_string()); + let Ok(config_dir) = std::env::var("GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_CONFIG_DIR") else { + println!("[skip] 未设置 GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_CONFIG_DIR"); + return; + }; + set_game_creator_runtime_config_dir(PathBuf::from(config_dir.trim())); + + let _session = install_test_platform_session(&user_id, &access_token, &api_base_url); + let root = PathBuf::from(project_path.trim()); + let first = tauri::async_runtime::block_on(sync_project_snapshot_async( + &root, + ProjectSnapshotSyncTrigger::Manual, + )) + .expect("第一次项目快照同步必须成功"); + println!("第一次同步：{first:#?}"); + assert_eq!(first.status, "synced", "{first:#?}"); + assert!(first.uploaded_files > 0, "{first:#?}"); + assert!(first.failed_files.is_empty(), "{first:#?}"); + + let second = tauri::async_runtime::block_on(sync_project_snapshot_async( + &root, + ProjectSnapshotSyncTrigger::Manual, + )) + .expect("第二次项目快照同步必须成功"); + println!("第二次同步：{second:#?}"); + assert_eq!(second.status, "no-op", "{second:#?}"); + assert_eq!(second.uploaded_files, 0, "{second:#?}"); +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs new file mode 100644 index 000000000..6ff4c226a --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs @@ -0,0 +1,337 @@ +use super::*; +use std::collections::BTreeSet; + +/// 上传失败的分类。鉴权与权限类失败不重试，其余交给下一次周期或关闭触发。 +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProjectSnapshotUploadErrorKind { + Authentication, + Permission, + /// 同步期间文件被改写：本轮跳过该文件，下一次重算即可，不是终态失败。 + FileChanged, + /// 服务端按配额或频率拒绝：等下一次触发再试。 + Throttled, + Rejected, + Upstream, + Transport, +} + +impl ProjectSnapshotUploadErrorKind { + pub(crate) fn code(self) -> &'static str { + match self { + Self::Authentication => "authentication-required", + Self::Permission => "permission-denied", + Self::FileChanged => "file-changed", + Self::Throttled => "throttled", + Self::Rejected => "request-rejected", + Self::Upstream => "upstream-failed", + Self::Transport => "transport-failed", + } + } + + /// 鉴权、权限与请求被拒都是确定性失败；重复提交同样的内容不会变好。 + pub(crate) fn is_retryable(self) -> bool { + matches!( + self, + Self::Upstream | Self::Transport | Self::FileChanged | Self::Throttled + ) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct ProjectSnapshotUploadError { + kind: ProjectSnapshotUploadErrorKind, + detail: String, +} + +impl ProjectSnapshotUploadError { + fn new(kind: ProjectSnapshotUploadErrorKind, detail: impl Into) -> Self { + Self { + kind, + detail: detail.into(), + } + } + + pub(crate) fn kind(&self) -> ProjectSnapshotUploadErrorKind { + self.kind + } + + pub(crate) fn code(&self) -> &'static str { + self.kind.code() + } + + pub(crate) fn message(&self) -> String { + format!("{}: {}", self.kind.code(), self.detail) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotUploadFailure { + pub(crate) relative_path: String, + pub(crate) code: String, + pub(crate) detail: String, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct ProjectSnapshotUploadReport { + pub(crate) uploaded_paths: BTreeSet, + pub(crate) uploaded_bytes: u64, + pub(crate) remote_skipped_files: usize, + pub(crate) failures: Vec, +} + +impl ProjectSnapshotUploadReport { + fn from_hard_failure(paths: &[String], error: &ProjectSnapshotUploadError) -> Self { + Self { + uploaded_paths: BTreeSet::new(), + uploaded_bytes: 0, + remote_skipped_files: 0, + failures: paths + .iter() + .map(|relative_path| ProjectSnapshotUploadFailure { + relative_path: relative_path.clone(), + code: error.code().to_string(), + detail: error.detail.clone(), + }) + .collect(), + } + } +} + +/// 逐文件上传本次差异集合。单个文件失败只影响该文件；鉴权或权限失败会中止 +/// 后续请求，因为同样的凭据不会在这一次同步里变好。 +pub(crate) async fn upload_project_snapshot_diff( + session: &PlatformSessionSnapshot, + project_id: &str, + diff: &ProjectSnapshotDiff, +) -> ProjectSnapshotUploadReport { + let mut report = ProjectSnapshotUploadReport::default(); + if diff.uploads.is_empty() { + return report; + } + let client = match crate::http_client::agc_main_site_client_builder() + .timeout(Duration::from_secs( + PROJECT_SNAPSHOT_REQUEST_TIMEOUT_SECONDS, + )) + .build() + { + Ok(client) => client, + Err(error) => { + let failure = ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Transport, + format!("创建项目快照上传客户端失败：{error}"), + ); + let paths = diff + .uploads + .iter() + .map(|candidate| candidate.relative_path.clone()) + .collect::>(); + return ProjectSnapshotUploadReport::from_hard_failure(&paths, &failure); + } + }; + + for (index, candidate) in diff.uploads.iter().enumerate() { + match upload_project_snapshot_file(&client, session, project_id, candidate).await { + Ok(skipped_remote) => { + report + .uploaded_paths + .insert(candidate.relative_path.clone()); + report.uploaded_bytes = report.uploaded_bytes.saturating_add(candidate.size_bytes); + if skipped_remote { + report.remote_skipped_files += 1; + } + } + Err(error) => { + let fatal = !error.kind().is_retryable(); + report.failures.push(ProjectSnapshotUploadFailure { + relative_path: candidate.relative_path.clone(), + code: error.code().to_string(), + detail: error.detail.clone(), + }); + if fatal { + for remaining in diff.uploads.iter().skip(index + 1) { + report.failures.push(ProjectSnapshotUploadFailure { + relative_path: remaining.relative_path.clone(), + code: error.code().to_string(), + detail: "上一次请求已确定性失败，本轮不再继续".to_string(), + }); + } + break; + } + } + } + } + report +} + +/// 返回是否为"远端已存在同内容对象"的跳过。 +async fn upload_project_snapshot_file( + client: &reqwest::Client, + session: &PlatformSessionSnapshot, + project_id: &str, + candidate: &ProjectSnapshotUploadCandidate, +) -> Result { + let bytes = read_project_snapshot_file_bytes(&candidate.absolute_path).map_err(|error| { + ProjectSnapshotUploadError::new(ProjectSnapshotUploadErrorKind::Rejected, error) + })?; + if bytes.len() as u64 != candidate.size_bytes { + return Err(ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::FileChanged, + format!( + "项目快照文件在同步期间发生变化：{}", + candidate.relative_path + ), + )); + } + // 上传前再复核一次元数据：只比长度会漏掉"改完又改回同样大小"的情况， + // 这种文件本轮不传，也不进入索引。 + if !project_snapshot_file_matches( + &candidate.absolute_path, + candidate.size_bytes, + candidate.modified_ms, + ) { + return Err(ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::FileChanged, + format!( + "项目快照文件在同步期间发生变化：{}", + candidate.relative_path + ), + )); + } + let url = project_snapshot_endpoint( + session, + PROJECT_SNAPSHOT_FILES_ENDPOINT, + [ + ("projectId", project_id.to_string()), + ("relativePath", candidate.relative_path.clone()), + ("checksum", candidate.checksum.clone()), + ("sizeBytes", candidate.size_bytes.to_string()), + ], + )?; + let request = client + .post(url) + .bearer_auth(&session.access_token) + .header(reqwest::header::CONTENT_TYPE, "application/octet-stream") + .body(bytes); + let response = crate::http_client::with_agc_main_site_marker(request) + .send() + .await + .map_err(|error| { + ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Transport, + format!("项目快照上传请求失败：{error}"), + ) + })?; + let status = response.status(); + if !status.is_success() { + return Err(project_snapshot_response_error(status, response).await); + } + let parsed = response + .json::() + .await + .map_err(|error| { + ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Upstream, + format!("项目快照上传响应无法解析：{error}"), + ) + })?; + if parsed.relative_path != candidate.relative_path || parsed.checksum != candidate.checksum { + return Err(ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Upstream, + format!("项目快照上传响应与请求不一致：{}", candidate.relative_path), + )); + } + Ok(parsed.skipped) +} + +/// 提交本次同步的远端清单；删除只在这里表达。 +pub(crate) async fn upload_project_snapshot_manifest( + session: &PlatformSessionSnapshot, + payload: &shared_contracts::agc_project_snapshots::AgcProjectSnapshotManifestRequest, +) -> Result<(), ProjectSnapshotUploadError> { + let client = crate::http_client::agc_main_site_client_builder() + .timeout(Duration::from_secs( + PROJECT_SNAPSHOT_REQUEST_TIMEOUT_SECONDS, + )) + .build() + .map_err(|error| { + ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Transport, + format!("创建项目快照上传客户端失败：{error}"), + ) + })?; + let url = project_snapshot_endpoint(session, PROJECT_SNAPSHOT_MANIFEST_ENDPOINT, [])?; + let request = client + .post(url) + .bearer_auth(&session.access_token) + .json(payload); + let response = crate::http_client::with_agc_main_site_marker(request) + .send() + .await + .map_err(|error| { + ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Transport, + format!("项目快照清单上传请求失败：{error}"), + ) + })?; + let status = response.status(); + if !status.is_success() { + return Err(project_snapshot_response_error(status, response).await); + } + Ok(()) +} + +fn project_snapshot_endpoint( + session: &PlatformSessionSnapshot, + path: &str, + query: impl IntoIterator, +) -> Result { + let base_url = session.api_base_url.trim_end_matches('/'); + if base_url.is_empty() { + return Err(ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Authentication, + "登录态缺少 API Server 地址", + )); + } + let mut url = reqwest::Url::parse(&format!("{base_url}{path}")).map_err(|error| { + ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::Transport, + format!("项目快照接口地址无效：{error}"), + ) + })?; + { + let mut pairs = url.query_pairs_mut(); + for (key, value) in query { + pairs.append_pair(key, &value); + } + } + Ok(url) +} + +async fn project_snapshot_response_error( + status: reqwest::StatusCode, + response: reqwest::Response, +) -> ProjectSnapshotUploadError { + let kind = match status.as_u16() { + 401 => ProjectSnapshotUploadErrorKind::Authentication, + 403 => ProjectSnapshotUploadErrorKind::Permission, + // 服务端配额与频率闸门：等下一次触发，不做本轮内重试。 + 429 => ProjectSnapshotUploadErrorKind::Throttled, + code if (400..500).contains(&code) => ProjectSnapshotUploadErrorKind::Rejected, + _ => ProjectSnapshotUploadErrorKind::Upstream, + }; + // 上游正文可能带内部信息，只保留状态码与有界摘要。 + let detail = response + .text() + .await + .map(|body| body.chars().take(200).collect::()) + .unwrap_or_default(); + let detail = detail.trim().to_string(); + if detail.is_empty() { + ProjectSnapshotUploadError::new(kind, format!("项目快照接口返回 HTTP {}", status.as_u16())) + } else { + ProjectSnapshotUploadError::new( + kind, + format!("项目快照接口返回 HTTP {}：{detail}", status.as_u16()), + ) + } +} diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 9b508dac1..eefb1ae55 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -162,6 +162,16 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS=600 ALIYUN_OSS_POST_MAX_SIZE_BYTES=20971520 ALIYUN_OSS_SUCCESS_ACTION_STATUS=200 +# AGC 项目定时快照上传目标。对象只落在服务端私有前缀 +# agc/project-snapshots/v1/{user}/{project}/ 下；AccessKey 为空时回退 ALIYUN_OSS_ACCESS_KEY_*， +# 因此回退凭据必须对目标 bucket 具备该前缀的 PutObject/GetObject/DeleteObject 权限。 +# bucket 未单独配置时默认 agc-dev，未配置凭据时 api-server 跳过该客户端， +# 接口返回 503 且客户端失败关闭（不写空对象、不推进本地索引）。 +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET=agc-dev +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT=oss-rg-china-mainland.aliyuncs.com +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID= +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET= + # SpacetimeDB 数据目录 OSS 冷备份配置。可由 cron / Jenkins 调用发布包内 scripts/database-backup-to-oss.mjs。 GENARRATIVE_DATABASE_BACKUP_DATA_DIR=/stdb GENARRATIVE_DATABASE_BACKUP_WORK_DIR=/var/lib/genarrative/database-backups diff --git a/docs/project-memory/plans/【实施计划】AGC项目定时快照上传-2026-09-17.md b/docs/project-memory/plans/【实施计划】AGC项目定时快照上传-2026-09-17.md new file mode 100644 index 000000000..3fe5a8630 --- /dev/null +++ b/docs/project-memory/plans/【实施计划】AGC项目定时快照上传-2026-09-17.md @@ -0,0 +1,43 @@ +# AGC 项目定时快照上传实施计划 + +Version: 1.0 +Status: active +Date: 2026-09-17 +Parent Milestone: `【里程碑】AGC项目定时快照上传-2026-09-17.md` + +## 修改边界 + +1. `server-rs/crates/shared-contracts/src/`：新增 `agc_project_snapshots` DTO（单文件上传请求/响应、同步清单信封），只放共享字段，不放 OSS 细节。 +2. `apps/ai-game-creator-shell/src-tauri/src/project_snapshot/`：新增客户端模块，包含扫描与排除规则、索引读写、差异对比、上传编排、状态与日志；不修改 `project/` 下既有 manifest 与写锁语义。 +3. `apps/ai-game-creator-shell/src-tauri/src/main.rs`：注册新模块、命令与生命周期钩子；`windows.rs` 的窗口关闭与应用退出路径接入触发调用，不改变现有窗口创建/关闭顺序。 +4. `server-rs/crates/platform-oss/src/lib.rs`：新增项目快照私有前缀常量与（必要时）独立 bucket 配置入口；不改动既有前缀枚举语义与资源写路径。 +5. `server-rs/crates/api-server/src/project_snapshots.rs`：新增路由、鉴权、校验与 OSS 写入；不改动 `error_reports` 与 `assets` 既有路由。 +6. `.env.example`：补充 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_*` 说明与默认值。 +7. `docs/`：主规范已更新；完成后把持久结论合并回主规范并删除本计划。 + +## 实现顺序 + +1. 先写 `shared-contracts` DTO 与客户端差异引擎（扫描、排除、索引、diff）及单测，此时无网络依赖，可独立验证。 +2. 接上传编排：按差异集合逐文件提交，成功后再提交清单，最后推进索引；用本地 TCP stub server 覆盖成功、幂等跳过、鉴权失败与部分失败路径。 +3. 接触发接线：周期定时器、工作区窗口关闭与应用退出；确认关闭路径的有界超时和串行化。 +4. 最后接服务端路由与 OSS 写入，补参数校验与幂等跳过测试；服务端完成前客户端按"未配置即失败关闭、不写入索引"处理。 + +每一步都保留既有失败关闭行为；新模块默认不改变其它同步路径（Runner、项目写锁、Resource Editor）。 + +## 验证命令 + +- `cargo fmt --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml -- --check` +- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml project_snapshot -- --nocapture` +- `cargo test -p api-server --bin api-server project_snapshots -- --nocapture` +- `cargo fmt -p api-server -p shared-contracts -p platform-oss -- --check` +- `npm run --prefix apps/ai-game-creator-shell typecheck`（若触及前端） +- `npm run check:encoding` +- `git diff --check` +- 运行时按需：`npm run agc` 打开项目观察索引写入与同步日志，关闭窗口确认关闭触发。 + +## 风险与回滚 + +- 上传体积与带宽：首轮全量可能很大，先设单文件与单次同步总量上限并把超限项记入跳过清单；不静默截断。 +- 数据出境边界：只上传项目目录内普通文件，排除 `.agent/runtime`、`.agent/logs`、`.git`、构建产物与临时文件；凭据类文件不在白名单内。 +- 服务端未配置 bucket 时客户端必须失败关闭，不能把本地索引推进成"已同步"，否则后续同步会漏传。 +- 回滚：客户端可停用触发接线（保留模块与测试）即可回到无上传行为；服务端路由与配置项可单独移除，不影响既有 OSS 前缀与错误报告链路。 diff --git a/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md b/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md new file mode 100644 index 000000000..25f0c802e --- /dev/null +++ b/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md @@ -0,0 +1,49 @@ +# AGC 项目定时快照上传 + +Version: 1.0 +Status: active +Date: 2026-09-17 +Parent Spec: `docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md` 的“2026-09-17 AGC 项目定时快照上传（agc-dev）” + +## 目标 + +AGC 在项目打开期间按周期把用户项目增量上传到 OSS `agc-dev`，并在项目关闭时立即补一次同步；只上传内容发生变化的文件，重复内容不重复上传，远端缺少对应对象时才新建。 + +## 范围 + +- 客户端 `src-tauri/src/project_snapshot/`（`scan.rs` / `diff.rs` / `index.rs` / `transport.rs`）：项目扫描、排除规则、增量索引与差异对比、上传编排、状态查询。 +- 触发接线：工作区窗口存活周期定时器、工作区窗口关闭（`CloseRequested`）；应用退出只做有界等待，不重复发起同步。 +- 服务端 `POST /api/agc/project-snapshots/files` 与 `POST /api/agc/project-snapshots/manifest`：登录态鉴权、参数校验、私有前缀 OSS 写入、HEAD 幂等跳过。 +- 目标 bucket 配置：`GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_*`，默认 `agc-dev`。 +- 契约：`shared-contracts::agc_project_snapshots` 新增请求/响应 DTO 与项目 ID、相对路径、摘要校验函数。 +- 客户端增量索引：`/project-snapshots//index.json`，按用户身份判等，换号后按冷启动全量重算。 +- 排除口径：复用 `should_skip_project_snapshot_path`（整个 `.agent`、`.git`、构建与依赖目录、凭据目录、敏感后缀、符号链接与重解析点）。 + +## 不做 + +- 不做云端下载/恢复、跨设备合并、版本回滚。 +- 不保留多版本历史：清单写入成功后回收不再被引用的旧对象，同一路径只保留当前内容。 +- 不下发 bucket 生命周期策略；不做跨节点的用户级总量配额与计费口径。 +- 不新增 SpacetimeDB 表或 procedure，不修改 `/api/external/v1` 与 External OpenAPI。 +- 不在客户端暴露上传状态、时间线或入口按钮；状态只落本机诊断日志，排障走 native-only 命令。 + +## 验收标准 + +1. 首次同步上传项目内全部符合条件的普通文件；再次同步在无改动时上传 0 个文件。 +2. 只修改一个文件时，差异集合恰好包含一个修改项；删除一个文件时上传集合为空且清单中不再包含该文件。 +3. `(字节数, 修改时间)` 未变的文件复用已存摘要，不重复读取内容计算摘要。 +4. 排除规则命中项（`.agent/runtime`、`.agent/logs`、`.git`、`node_modules`、构建产物、临时文件、符号链接）与超限文件进入跳过清单，不进入上传集合。 +5. 任一次同步失败（非鉴权类）不推进本地索引，下一次触发重算并重试；鉴权/权限类失败不自动重试。 +6. 同一项目的并发触发串行执行，不产生两路重复上传。 +7. 工作区窗口关闭与应用退出都会触发一次同步，且关闭路径不因同步失败而阻塞退出超过超时上限。 +8. 服务端拒绝越界 `projectId`、相对路径与摘要；相同摘要重复提交走跳过分支且不写入新对象。 +9. 新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。 +10. 清单写入成功后，上一版清单里不再被引用的对象被回收；上一版清单不可读时整轮不删除任何对象。 +11. 单项目超过 2 GiB 时客户端明确失败、服务端按 413 拒绝；超过服务端小时配额或 5 秒最小间隔时返回 429 且带 `Retry-After`。 +12. 同步期间被改写的文件既不上传也不推进索引，沿用上一轮记录，且不会被误判成删除。 + +## 依赖 + +- 现有 `platform_session`（用户身份与 Access Token）、项目 manifest（稳定 `project_id`）。 +- 现有 `platform-oss`（PUT/HEAD、私有访问）、`api-server` 登录态中间件与 `shared-contracts`。 +- 现有 AppData 私有文件写入与目录解析工具。 diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 0828b8fda..550e75f0c 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -1530,3 +1530,56 @@ Direct 回合的所有权属于进程内项目身份锁，不属于当前页面 - 两个窗口同时对同一项目发起 Runtime 写请求时，用户体验仍由项目级写锁串行决定；本次不引入跨窗口排队提示。 - 平台会话在窗口间传播依赖共享 localStorage 与 Runner 权威；渲染层不做跨窗口事件推送，另一个窗口在下一次会话校验或刷新时收敛。 + +## 2026-09-17 AGC 项目定时快照上传（agc-dev） + +### 目标与非目标 + +- 目标：AGC 在项目工作区打开期间按固定周期把用户项目增量上传到 OSS `agc-dev`，并在项目关闭时立即补一次同步；重复内容不重复上传，远端占用跟随当前清单收敛。 +- 非目标：不做云端下载/恢复、不做跨设备合并、不保留多版本历史、不新增面向用户的上传界面、不修改 `/api/external/v1` 与 OpenAPI、不新增 SpacetimeDB 表。 +- 非目标：不把 OSS AccessKey 放进客户端；客户端不直连 OSS。 + +### 参与入口、状态与跨模块边界 + +- 触发入口有两个：工作区窗口 `main` 存活期间的周期定时器、工作区窗口关闭事件（`CloseRequested`）。两者共用同一个进程内同步器，同一项目的同步串行执行，周期触发在已有同步进行时直接让位，不排队堆积。 +- 应用退出（`RunEvent::Exit`）不重复发起同步：该时刻窗口已销毁，按窗口重新枚举项目只会得到空集；退出路径只负责在有界预算（15 秒）内等待在途同步收尾，让关窗触发的那一次同步能写完索引再退出。 +- 客户端扫描、差异对比、索引持久化与上传编排都在 Tauri Rust 进程（`src-tauri/src/project_snapshot/`）；WebView 只读状态，不参与差异计算。 +- 本地索引是增量对比的唯一依据：`/project-snapshots//index.json` 保存上次成功同步的相对路径、校验和、字节数和修改时间。项目根使用现有 manifest 的稳定 `project_id` 作为远端身份，路径不再作为身份。 +- 可观测性按产品口径收敛到本机日志：同步结果、失败分类、延后与跳过计数只写入 AppData 诊断日志（`project_snapshot.sync.*` 前缀），客户端界面不暴露上传状态、时间线或入口按钮。`read_local_project_snapshot_state` 与 `sync_local_project_snapshot` 两条命令仅作为 native-only 的排障与联调入口登记，不在渲染层调用。 +- 远端写入经 `api-server`，客户端只持平台登录态 Access Token。两条登录态路由：`POST /api/agc/project-snapshots/files`（单文件，正文为原始字节，元数据走查询串）与 `POST /api/agc/project-snapshots/manifest`（本次同步后的完整清单）。 +- 对象键与清单由服务端决定：文件键为 `agc/project-snapshots/v1/{userId}/{projectId}/files/{sizeBytes}-{checksumDigest}/{relPath}`，清单键为 `agc/project-snapshots/v1/{userId}/{projectId}/manifest.json`。键里带字节数与摘要，因此"对象已存在且长度一致"可以作为内容一致的判据；路径按原始大小写保留，不走 `put_object` 的低位规范化。`agc` 前缀继续是服务端专用私有前缀，通用对象键解析与客户端直传票据都不覆盖它。 +- 目标 bucket 使用独立配置 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET` / `_ENDPOINT` / `_ACCESS_KEY_ID` / `_ACCESS_KEY_SECRET`，默认 `agc-dev` + `oss-rg-china-mainland.aliyuncs.com`，未配置时回退 `ALIYUN_OSS_*`；与"资源 bucket 与备份 bucket 分离"的既有口径一致。 + +### 正常、失败、重试与幂等行为 + +- 差异对比口径：先按 `相对路径 + 字节数 + 修改时间` 判定是否候选变更，命中旧记录则复用已存 `sha256`，只有 `(size, mtime)` 变化才重算摘要。产出新增、修改、删除三类集合，只上传新增与修改的文件。 +- 每次成功同步的最后一步上传该项目的 `manifest.json`（当前全量文件清单：相对路径、摘要、字节数、同步序号）。清单描述的是项目当前全量内容，因此清单体积就是该项目在 OSS 上的常驻占用。 +- 远端回收：清单写入成功后，服务端读取上一版清单，按 `(路径, 字节数, 摘要)` 反推出不再被当前清单引用的对象键并删除。只处理上一版清单登记过的键，不做 LIST，因此不可能误删其它项目或其它功能的对象；单次最多回收 2000 个对象，剩余部分留到下一次清单写入继续；上一版清单读不到或解析失败时整轮跳过回收（fail-closed）。单个删除失败只记日志，不影响本次同步语义。 +- 因此本功能是"当前状态镜像 + 清单"，不保留历史版本：同一路径的内容变化会覆盖式替换远端对象，回滚能力不在本轮范围内。 +- 幂等：同一摘要与字节数的对象重复提交由服务端 HEAD 校验后跳过；探测失败按"未存在"处理并照常 PUT，宁可多传一次也不漏传。索引只在清单写入成功后推进，失败时保留旧索引以便下次重算。 +- 与项目写锁解耦：同步不持有项目写锁，也不阻塞 Agent 写入。读取摘要后与上传前各按 `(字节数, 修改时间)` 复核一次，任一处不一致就判定该文件"同步期间发生变化"：本轮不上传、不写入索引；若该路径上一轮已同步过则沿用旧记录，避免被误判成删除而触发远端回收。这类文件在下一个周期或下次关窗时重算重试。 +- 失败关闭：单个文件失败不推进该文件的索引项，失败文件与剩余文件在下一次周期或下次关闭时重试。鉴权失败（401/403）与格式类拒绝（400/413）是确定性失败，停止本轮剩余请求并等待用户处理后重试；服务端配额或频率拒绝（429）与传输类失败按可重试处理。 +- 配额与限流：单文件 64 MiB、单次同步上传预算 512 MiB（超出部分延后到下一次）、单项目常驻上限 2 GiB（客户端在扫描后先判，超限直接给出明确失败；服务端按清单累计体积复核并返回 413）；服务端按用户做进程内小时配额（文件 3000 次、清单 120 次）并对同一项目强制 5 秒最小清单间隔，超限返回 429 且带 `Retry-After`。进程内配额只用于抑制异常客户端与失控重试，跨节点配额由"单项目上限 + 清单引用回收"保证。 +- 生命周期：同步有界超时（单文件与整次同步分别设上限），项目关闭与应用退出路径不因同步失败而阻塞或延迟退出超过超时上限。 +- 上传内容边界：复用项目索引与 checkpoint 同一份 `should_skip_project_snapshot_path` 口径——整个 `.agent`（含 runtime、logs、checkpoint、manifest、project.lock）、版本控制目录、`node_modules`/`target`/`dist`/`build`/`coverage`/`.cache`、凭据目录与 `.pem`/`.key` 等敏感后缀都不参与同步；符号链接与重解析点同样跳过。单文件（64 MiB）与单次同步总量（512 MiB）各有上限，超限文件进入跳过或延后清单而不是静默丢弃。 + +### 契约与兼容 + +- 新增登录态内部路由 `POST /api/agc/project-snapshots/files`，请求 DTO 放在 `shared-contracts`；不属于 `/api/external/v1`，因此不更新 External OpenAPI，与 `/api/error-reports` 同类。 +- 服务端校验 `projectId` 形态（拒绝路径分隔符、`..`、控制字符与超长值）、相对路径规范（正斜杠、拒绝绝对路径与穿越）、摘要形态（`fnv1a64:` + 16 位十六进制）和字节数上限，任何越界返回 4xx 而不是写入 OSS。 +- 不改变客户端与 Runner 的本机协议、平台会话语义、项目写锁与 manifest 结构；新增索引文件位于 AppData，不进入用户项目目录。 + +### 验收标准与证据来源 + +- 定向 Rust 测试：首次同步全量、仅改一个文件时只产生一个修改项、删除文件只体现在清单、`(size,mtime)` 未变时复用旧摘要、排除规则与上限跳过、同步失败不推进索引、同一项目并发触发串行化。 +- 服务端测试：越界 `projectId`/相对路径/摘要被拒；相同摘要重复提交走跳过分支；超过单项目上限返回 413；超过用户小时配额返回 429；鉴权缺失返回 401；OSS 未配置返回明确的 5xx 而不是写入空对象。 +- 运行时 smoke：AGC 开发态打开项目、观察索引写入与同步日志、关闭工作区窗口后确认关闭触发的那次同步执行；报告为"客户端 diff 已验证 / 服务端已配置环境联调"两层，不合并成一句"已通"。 +- 边界：新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。 + +### 未决问题 + +- 用户侧看不到同步状态与失败原因（界面按产品口径不暴露），排障只能读 AppData 诊断日志或调用 native-only 命令；如果后续要支持用户自助排查，需要先确认是否允许在客户端出现上传相关 UI。 +- 历史版本：本轮只保留"当前状态镜像 + 清单"，旧内容对象在清单写入成功后即被回收，没有回滚能力；要保留历史版本需要先定"保留几个 revision + 由谁回收"的策略。 +- 用户级配额：跨节点的用户总量配额与计费口径未定；当前用单项目 2 GiB 上限 + 清单引用回收保证常驻占用有界，用户级总量只能靠项目数间接约束。 +- 目标 bucket 的生命周期规则（例如转低频/过期删除）需要在部署环境确认后单独收口；功能本身已不再依赖它来控制增长。 +- 大项目（素材数量多、单文件大）的首轮全量上传耗时与带宽占用未实测；单次预算 512 MiB 会把超出部分留到下一次同步，但并发上限与断点续传仍未引入。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 38285c41a..d13920f03 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -540,6 +540,50 @@ curl -fsS --max-time 5 http://127.0.0.1/api/editor/showcase/resources >/dev/null 角色动画源帧 PUT、透明帧 PUT 和最终帧 HEAD 使用 `AppState` 内同一个 OSS HTTP Client/连接池，并受进程级 8 路 OSS permit 保护；BgFilter、阿里云抠图和本地处理不占用该 permit。每个 OSS attempt 最多 3 次（首次 + 2 次重试），退避为 250ms、500ms；只重试 timeout、无 HTTP 响应传输错误、OSS PutObject 的 `400 + RequestTimeout`、PUT 400 错误体读取失败（未解析出 `Code`，按 timeout/transport 归类）、408、429 和 500–599。动作帧 PUT 收到 400 时只读取最多 16 KiB OSS 错误 XML，提取 `Code` 和 `RequestId`；`oss_request_id` 优先使用响应头 `x-oss-request-id`，XML 字段只作回退。错误体读取超时/断流不再按确定性 400 处理：已解析出的 `Code` 优先生效；未解析出 `Code` 时按读取失败原因置 `timeout`/`transport` 并重试，message 追加「错误响应体读取失败」。日志字段包括 `frame_index`、`object_key`、`operation=source_put|final_put|final_head`、`attempt`、`max_attempts`、`retryable`、`will_retry`、`retry_delay_ms`、`permit_wait_ms`、`timeout`、`connect`、`transport`、`oss_code`、`oss_request_id`、`status` 和 `elapsed_ms`。`请求 OSS 失败` 时，`timeout/connect/transport=true` 表示传输类失败；`status=400, oss_code=RequestTimeout, timeout=true`、`status=429` 或 `500–599` 表示暂时性失败，PUT 的 `status=400`、`oss_code` 为空且 `timeout=true` 或 `transport=true`（message 含「错误响应体读取失败」）同样是暂时性失败。除 `RequestTimeout` 和该错误体读取失败两类例外外，其他 400、401/403/404、配置、URL 和签名错误是确定性失败，不会重试。最终帧 HEAD 失败只会重试 HEAD，不会重复 PUT；如果任一帧最终失败，确认整段动作已排空已启动 Future，并检查任务按现有契约退款且没有发布缺帧动画。 +### AGC 项目快照上传目标 + +AGC 客户端按周期与项目关闭时机把用户项目增量上传到 `agc-dev`。客户端只持有平台登录态 Access +Token，经 `POST /api/agc/project-snapshots/files`（单文件原始字节）与 +`POST /api/agc/project-snapshots/manifest`（本次同步清单）交给 `api-server`，由服务端写入私有前缀 +`agc/project-snapshots/v1/{user}/{project}/`；客户端不直连 OSS，也不持有 OSS 凭据。 + +```env +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET=agc-dev +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT=oss-rg-china-mainland.aliyuncs.com +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID= +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET= +``` + +专用凭据为空时回退 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，bucket 与 endpoint +仍默认指向 AGC 发行 bucket，因此回退凭据必须具备目标 bucket 该前缀的 `PutObject` / `GetObject` / +`DeleteObject` 权限；`api-server` 启动时会打印一行 `AGC 项目快照 OSS 客户端已启用`（含 bucket、 +endpoint 与凭据来源，不含密钥），凭据缺失或只配一半则跳过该客户端、接口返回 `503`，客户端按失败关闭 +处理：不写空对象，也不推进本地增量索引，下一次触发重算重试。 + +远端占用按当前清单收敛：每次清单写入成功后，服务端用上一版清单反推不再被引用的对象键并删除，单次最多 +回收 2000 个，上一版清单不可读时整轮跳过（不会误删）。因此不需要额外配置 bucket 生命周期来防止无界 +增长；`agc/project-snapshots/v1/` 下同一路径只保留当前内容，历史版本不保留。配额口径：单文件 64 MiB、 +单次同步上传预算 512 MiB、单项目常驻 2 GiB；超限分别表现为跳过/延后/413。服务端另有进程内小时配额 +（文件 3000 次、清单 120 次）与同一项目 5 秒最小清单间隔，超限返回 `429` 并带 `Retry-After`。 + +两层可重复的现场验证： + +```bash +# 1. 存储层：直接对真实 bucket 做内部前缀写入、读回与清理，并在结束时删除探针对象。 +cargo run -p platform-oss --example agc_project_snapshot_live_smoke --manifest-path server-rs/Cargo.toml + +# 2. 客户端链路：真实差异引擎 → 本地 api-server → 真实 OSS。第一次必须 synced 且上传 > 0， +# 紧接着的第二次必须 no-op 且上传 0 个文件；需要先取得登录态并指定项目与索引目录。 +cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml \ + project_snapshot_live_sync -- --ignored --nocapture +``` + +客户端冒烟读取 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_LIVE_PROJECT`（项目绝对路径，建议用可丢弃的副本）、 +`..._LIVE_TOKEN`、`..._LIVE_API_BASE_URL`、`..._LIVE_USER_ID` 与 `..._LIVE_CONFIG_DIR`（索引目录， +`cargo test` 进程没有窗口 setup 初始化 AppData 配置目录，必须显式指定）；未设置时用例自我跳过。 +本地联调可用 `npm run dev:api-server` 起 api-server，并用密码登录（开发态默认允许未知手机号自动注册） +取得 Access Token。 + ## 生产运维 生产部署当前口径： diff --git a/server-rs/crates/api-server/src/app.rs b/server-rs/crates/api-server/src/app.rs index d4d081884..4c34bda86 100644 --- a/server-rs/crates/api-server/src/app.rs +++ b/server-rs/crates/api-server/src/app.rs @@ -51,6 +51,7 @@ pub fn build_router(state: AppState) -> Router { .merge(modules::external_generation::router(state.clone())) .merge(modules::platform_support::router(state.clone())) .merge(modules::raw::router(state.clone())) + .merge(modules::project_snapshots::router(state.clone())) .merge(crate::error_reports::router(state.clone())) .route( "/api/profile/recharge/wechat/notify", diff --git a/server-rs/crates/api-server/src/config.rs b/server-rs/crates/api-server/src/config.rs index 94ea4d4e4..6a50319a2 100644 --- a/server-rs/crates/api-server/src/config.rs +++ b/server-rs/crates/api-server/src/config.rs @@ -27,6 +27,9 @@ const DEFAULT_EDITOR_BGFILTER_CIRCUIT_FAILURE_THRESHOLD: u32 = 3; const DEFAULT_EDITOR_BGFILTER_CIRCUIT_COOLDOWN_SECONDS: u64 = 120; const DEFAULT_ALIYUN_MATTING_ENDPOINT: &str = "imageseg.cn-shanghai.aliyuncs.com"; const DEFAULT_ALIYUN_MATTING_REQUEST_TIMEOUT_MS: u64 = 30_000; +/// AGC 项目快照默认落到 AGC 发行 bucket；私有前缀由 platform-oss 单独限制。 +const DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET: &str = "agc-dev"; +const DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT: &str = "oss-rg-china-mainland.aliyuncs.com"; pub(crate) const OFFICIAL_LLM_ROUTER_BASE_URL: &str = "https://router.genarrative.world/v1"; pub(crate) const OFFICIAL_LLM_ROUTER_MODEL: &str = "gpt-6-astra"; const LLM_ROUTER_KEY_ENCRYPTION_DOMAIN: &[u8] = b"genarrative:llm-router-api-key-encryption:v1\0"; @@ -169,6 +172,12 @@ pub struct AppConfig { pub oss_post_expire_seconds: u64, pub oss_post_max_size_bytes: u64, pub oss_success_action_status: u16, + /// AGC 项目快照上传目标。默认指向 AGC 发行用的公开 bucket，可用独立凭据覆盖； + /// 未单独配置时回退 `ALIYUN_OSS_*`，与数据库备份 bucket 的分离开关同口径。 + pub project_snapshot_oss_bucket: String, + pub project_snapshot_oss_endpoint: String, + pub project_snapshot_oss_access_key_id: Option, + pub project_snapshot_oss_access_key_secret: Option, pub spacetime_server_url: String, pub spacetime_database: String, pub spacetime_token: Option, @@ -476,6 +485,10 @@ impl Default for AppConfig { oss_post_expire_seconds: 10 * 60, oss_post_max_size_bytes: 20 * 1024 * 1024, oss_success_action_status: 200, + project_snapshot_oss_bucket: DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET.to_string(), + project_snapshot_oss_endpoint: DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT.to_string(), + project_snapshot_oss_access_key_id: None, + project_snapshot_oss_access_key_secret: None, spacetime_server_url: "http://127.0.0.1:3000".to_string(), spacetime_database: "genarrative-dev".to_string(), spacetime_token: None, @@ -1121,6 +1134,24 @@ impl AppConfig { { config.oss_success_action_status = oss_success_action_status; } + config.project_snapshot_oss_bucket = read_first_non_empty_env(&[ + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET", + "ALIYUN_OSS_BUCKET", + ]) + .unwrap_or_else(|| DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_BUCKET.to_string()); + config.project_snapshot_oss_endpoint = read_first_non_empty_env(&[ + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT", + "ALIYUN_OSS_ENDPOINT", + ]) + .unwrap_or_else(|| DEFAULT_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT.to_string()); + config.project_snapshot_oss_access_key_id = read_first_non_empty_env(&[ + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID", + "ALIYUN_OSS_ACCESS_KEY_ID", + ]); + config.project_snapshot_oss_access_key_secret = read_first_non_empty_env(&[ + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET", + "ALIYUN_OSS_ACCESS_KEY_SECRET", + ]); if let Some(spacetime_server_url) = read_first_non_empty_env(&["GENARRATIVE_SPACETIME_SERVER_URL"]) diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 25e42cd28..c0980823f 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -66,6 +66,7 @@ mod process_metrics; mod profile_identity; mod profile_recharge_expiration_listener; mod profile_recharge_refund_reconciliation; +mod project_snapshots; mod prompt; mod raw_image; mod refresh_session; diff --git a/server-rs/crates/api-server/src/modules.rs b/server-rs/crates/api-server/src/modules.rs index 76f5958b7..0b833ab21 100644 --- a/server-rs/crates/api-server/src/modules.rs +++ b/server-rs/crates/api-server/src/modules.rs @@ -10,4 +10,5 @@ pub mod internal; pub mod platform; pub mod platform_support; pub mod profile; +pub mod project_snapshots; pub mod raw; diff --git a/server-rs/crates/api-server/src/modules/project_snapshots.rs b/server-rs/crates/api-server/src/modules/project_snapshots.rs new file mode 100644 index 000000000..65683b00f --- /dev/null +++ b/server-rs/crates/api-server/src/modules/project_snapshots.rs @@ -0,0 +1,31 @@ +use axum::{Router, extract::DefaultBodyLimit, middleware, routing::post}; + +use crate::{ + auth::require_bearer_auth, + project_snapshots::{ + MAX_FILE_REQUEST_BODY_BYTES, MAX_MANIFEST_REQUEST_BODY_BYTES, upload_project_snapshot_file, + upload_project_snapshot_manifest, + }, + state::AppState, +}; + +/// AGC 项目快照只接受登录态客户端；两条路由都带体积门禁，超限请求在进入业务 +/// 处理前就被拒绝。 +pub fn router(state: AppState) -> Router { + Router::new() + .route( + "/api/agc/project-snapshots/files", + post(upload_project_snapshot_file) + .route_layer(middleware::from_fn_with_state( + state.clone(), + require_bearer_auth, + )) + .layer(DefaultBodyLimit::max(MAX_FILE_REQUEST_BODY_BYTES)), + ) + .route( + "/api/agc/project-snapshots/manifest", + post(upload_project_snapshot_manifest) + .route_layer(middleware::from_fn_with_state(state, require_bearer_auth)) + .layer(DefaultBodyLimit::max(MAX_MANIFEST_REQUEST_BODY_BYTES)), + ) +} diff --git a/server-rs/crates/api-server/src/project_snapshots.rs b/server-rs/crates/api-server/src/project_snapshots.rs new file mode 100644 index 000000000..d745d7493 --- /dev/null +++ b/server-rs/crates/api-server/src/project_snapshots.rs @@ -0,0 +1,517 @@ +//! AGC 项目定时快照上传的服务端入口。 +//! +//! 客户端只提交"某个项目里发生变化的文件内容"和"当前清单"，对象键、bucket 与 +//! 存储凭据都由服务端决定。写入固定在内部前缀下，客户端直传票据不覆盖该前缀。 + +use crate::{ + api_response::json_success_body, auth::AuthenticatedAccessToken, http_error::AppError, + request_context::RequestContext, state::AppState, +}; +use axum::{ + Json, + body::Bytes, + extract::{Extension, Query, State}, + http::StatusCode, +}; +use platform_oss::{ + OssDeleteObjectRequest, OssGetObjectRequest, OssInternalPutObjectRequest, OssObjectAccess, + agc_project_snapshot_file_object_key, agc_project_snapshot_manifest_object_key, +}; +use serde_json::Value; +use shared_contracts::agc_project_snapshots::{ + AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES, AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES, + AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES, AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION, + AgcProjectSnapshotFileUploadQuery, AgcProjectSnapshotFileUploadResponse, + AgcProjectSnapshotManifestRequest, AgcProjectSnapshotManifestResponse, + validate_agc_project_snapshot_checksum, validate_agc_project_snapshot_project_id, + validate_agc_project_snapshot_relative_path, +}; +use std::{ + collections::{HashMap, HashSet}, + sync::{Mutex, OnceLock}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tracing::{debug, info, warn}; + +/// 单文件请求体上限比文件上限留一点余量，超限请求由 body limit 直接拒绝。 +pub(crate) const MAX_FILE_REQUEST_BODY_BYTES: usize = + AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES as usize + 1024; +/// 清单请求体上限：条目数本身有上限，这里再给一个字节级兜底。 +pub(crate) const MAX_MANIFEST_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024; +/// 清单里单条路径的字段长度上限（与契约校验口径一致）。 +const MAX_MANIFEST_PATH_CHARS: usize = 1024; +/// 单个用户每小时允许的文件上传次数。进程内计数，用于抑制异常客户端，不是计费级配额。 +const MAX_FILE_UPLOADS_PER_USER_PER_HOUR: usize = 3_000; +/// 单个用户每小时允许的清单写入次数。 +const MAX_MANIFEST_UPLOADS_PER_USER_PER_HOUR: usize = 120; +/// 同一项目两次清单写入的最小间隔，避免异常客户端高频覆盖清单。 +const MIN_MANIFEST_INTERVAL_MS: u64 = 5_000; +/// 单次清单写入最多回收多少个不再被引用的对象，避免一次请求做过量删除。 +const MAX_OBJECTS_RECLAIMED_PER_MANIFEST: usize = 2_000; + +/// 写入一次增量同步里的单个文件。 +/// +/// 对象键由字节数和内容摘要共同决定，因此"对象已存在且长度一致"就是内容已存在的 +/// 充分判据；探测失败按未存在处理，PUT 本身是幂等的。 +pub async fn upload_project_snapshot_file( + State(state): State, + Extension(ctx): Extension, + Extension(auth): Extension, + Query(query): Query, + body: Bytes, +) -> Result, AppError> { + consume_user_upload_quota(auth.claims().user_id(), ProjectSnapshotUploadKind::File)?; + validate_agc_project_snapshot_project_id(&query.project_id).map_err(bad_request)?; + validate_agc_project_snapshot_relative_path(&query.relative_path).map_err(bad_request)?; + validate_agc_project_snapshot_checksum(&query.checksum).map_err(bad_request)?; + if body.is_empty() { + return Err(bad_request("项目快照文件内容不能为空")); + } + let size_bytes = + u64::try_from(body.len()).map_err(|_| bad_request("项目快照文件长度超出可支持范围"))?; + if size_bytes > AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES { + return Err(bad_request("项目快照文件超过单文件上限")); + } + if size_bytes != query.size_bytes { + return Err(bad_request("项目快照文件长度与声明不一致")); + } + let digest = query + .checksum + .strip_prefix("fnv1a64:") + .unwrap_or_default() + .to_string(); + let object_key = agc_project_snapshot_file_object_key( + auth.claims().user_id(), + &query.project_id, + size_bytes, + &digest, + &query.relative_path, + ) + .map_err(|error| bad_request(error.to_string()))?; + + let oss = project_snapshot_oss(&state)?; + let skipped = match oss + .head_internal_object(state.editor_oss_http_client(), &object_key) + .await + { + Ok(Some(existing)) => existing.content_length == size_bytes, + // 确定不存在、或探测失败时都继续写入：PUT 幂等，宁可多传一次也不漏传。 + Ok(None) | Err(_) => false, + }; + if !skipped { + oss.put_internal_object( + state.editor_oss_http_client(), + OssInternalPutObjectRequest { + object_key: object_key.clone(), + content_type: Some("application/octet-stream".to_string()), + access: OssObjectAccess::Private, + metadata: Default::default(), + body: body.to_vec(), + }, + ) + .await + .map_err(|_| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_message("项目快照文件上传失败") + })?; + } + + Ok(json_success_body( + Some(&ctx), + AgcProjectSnapshotFileUploadResponse { + project_id: query.project_id, + relative_path: query.relative_path, + object_key, + skipped, + checksum: query.checksum, + size_bytes, + }, + )) +} + +/// 覆盖写入该项目的远端清单。删除文件只在这里消失，本期不删除远端对象。 +pub async fn upload_project_snapshot_manifest( + State(state): State, + Extension(ctx): Extension, + Extension(auth): Extension, + Json(payload): Json, +) -> Result, AppError> { + consume_user_upload_quota(auth.claims().user_id(), ProjectSnapshotUploadKind::Manifest)?; + validate_manifest(&payload)?; + let user_id = auth.claims().user_id().to_string(); + let object_key = agc_project_snapshot_manifest_object_key(&user_id, &payload.project_id) + .map_err(|error| bad_request(error.to_string()))?; + // 上一版清单同时承担两个职责：项目级写入频率闸门，以及本轮远端对象回收的引用基线。 + // 读不到或解析失败时只跳过回收，绝不据此删除任何对象。 + let previous = read_project_snapshot_manifest(&state, &object_key).await; + if let Some(previous) = previous.as_ref() + && unix_millis_now().saturating_sub(previous.synced_at_ms) < MIN_MANIFEST_INTERVAL_MS + { + return Err(AppError::from_status(StatusCode::TOO_MANY_REQUESTS) + .with_message("同一项目的项目快照写入过于频繁，请稍后重试") + .with_header("retry-after", axum::http::HeaderValue::from_static("5"))); + } + let body = serde_json::to_vec(&payload) + .map_err(|error| internal(format!("序列化项目快照清单失败：{error}")))?; + let total_bytes = payload + .files + .iter() + .fold(0_u64, |total, file| total.saturating_add(file.size_bytes)); + let oss = project_snapshot_oss(&state)?; + oss.put_internal_object( + state.editor_oss_http_client(), + OssInternalPutObjectRequest { + object_key: object_key.clone(), + content_type: Some("application/json".to_string()), + access: OssObjectAccess::Private, + metadata: Default::default(), + body, + }, + ) + .await + .map_err(|_| { + AppError::from_status(StatusCode::BAD_GATEWAY).with_message("项目快照清单上传失败") + })?; + // 清单写入成功之后再回收：任何时刻远端对象集合都是当前清单的超集， + // 不会出现清单引用了刚被删掉的对象。 + reclaim_unreferenced_objects(&state, oss, &user_id, previous.as_ref(), &payload).await; + + Ok(json_success_body( + Some(&ctx), + AgcProjectSnapshotManifestResponse { + project_id: payload.project_id, + sync_revision: payload.sync_revision, + object_key, + file_count: u32::try_from(payload.files.len()).unwrap_or(u32::MAX), + total_bytes, + }, + )) +} + +/// 单个用户在窗口内的上传计数。进程内计数，用于抑制异常客户端与失控重试； +/// 跨节点配额由"单项目累计体积上限 + 清单引用回收"保证。 +#[derive(Clone, Copy, Default)] +struct ProjectSnapshotUserWindow { + started_at: Option, + files: usize, + manifests: usize, +} + +#[derive(Clone, Copy)] +enum ProjectSnapshotUploadKind { + File, + Manifest, +} + +static PROJECT_SNAPSHOT_USER_WINDOWS: OnceLock>> = + OnceLock::new(); + +fn consume_user_upload_quota( + user_id: &str, + kind: ProjectSnapshotUploadKind, +) -> Result<(), AppError> { + let windows = PROJECT_SNAPSHOT_USER_WINDOWS.get_or_init(|| Mutex::new(HashMap::new())); + let mut windows = windows + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let window = windows.entry(user_id.to_string()).or_default(); + let now = Instant::now(); + let expired = window + .started_at + .is_none_or(|started_at| now.duration_since(started_at) >= Duration::from_secs(3_600)); + if expired { + *window = ProjectSnapshotUserWindow { + started_at: Some(now), + ..ProjectSnapshotUserWindow::default() + }; + } + let (counter, limit) = match kind { + ProjectSnapshotUploadKind::File => (&mut window.files, MAX_FILE_UPLOADS_PER_USER_PER_HOUR), + ProjectSnapshotUploadKind::Manifest => ( + &mut window.manifests, + MAX_MANIFEST_UPLOADS_PER_USER_PER_HOUR, + ), + }; + if *counter >= limit { + return Err(AppError::from_status(StatusCode::TOO_MANY_REQUESTS) + .with_message("项目快照上传次数超出当前小时上限，请稍后重试") + .with_header("retry-after", axum::http::HeaderValue::from_static("60"))); + } + *counter += 1; + Ok(()) +} + +fn unix_millis_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +/// 读取该项目上一版清单。只在对象不存在时返回 `None`；读取或解析失败同样返回 +/// `None`，调用方据此跳过回收（fail-closed：不确定就不删）。 +async fn read_project_snapshot_manifest( + state: &AppState, + object_key: &str, +) -> Option { + let oss = project_snapshot_oss(state).ok()?; + match oss + .get_object( + state.editor_oss_http_client(), + OssGetObjectRequest { + object_key: object_key.to_string(), + max_bytes: MAX_MANIFEST_REQUEST_BODY_BYTES, + }, + ) + .await + { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map_err(|error| { + warn!(object_key = %object_key, error = %error, "项目快照上一版清单解析失败，本轮跳过回收"); + error + }) + .ok(), + Err(error) => { + debug!(object_key = %object_key, error = %error, "项目快照上一版清单不可读，本轮跳过回收"); + None + } + } +} + +/// 回收不再被当前清单引用、且确实由上一版清单登记过的对象。 +/// +/// 只按上一版清单里的 `(路径, 字节数, 摘要)` 反推出确定的对象键，不做 LIST， +/// 因此不可能误删其它项目或其它功能的对象。任何单个删除失败都只记日志： +/// 清单已经写入成功，回收失败不影响本次同步的语义，下一次写入会再试。 +async fn reclaim_unreferenced_objects( + state: &AppState, + oss: &platform_oss::OssClient, + user_id: &str, + previous: Option<&AgcProjectSnapshotManifestRequest>, + next: &AgcProjectSnapshotManifestRequest, +) { + let Some(previous) = previous else { + return; + }; + let retained = next + .files + .iter() + .map(|file| { + ( + file.relative_path.as_str(), + file.size_bytes, + file.checksum.as_str(), + ) + }) + .collect::>(); + let mut reclaimed = 0_usize; + let mut skipped = 0_usize; + for file in &previous.files { + if retained.contains(&( + file.relative_path.as_str(), + file.size_bytes, + file.checksum.as_str(), + )) { + continue; + } + if reclaimed >= MAX_OBJECTS_RECLAIMED_PER_MANIFEST { + // 剩下的留给下一次清单写入继续回收，不在这里无限删除。 + skipped += 1; + continue; + } + let Some(digest) = file.checksum.strip_prefix("fnv1a64:") else { + continue; + }; + let Ok(object_key) = agc_project_snapshot_file_object_key( + user_id, + &previous.project_id, + file.size_bytes, + digest, + &file.relative_path, + ) else { + continue; + }; + match oss + .delete_object( + state.editor_oss_http_client(), + OssDeleteObjectRequest { + object_key: object_key.clone(), + }, + ) + .await + { + Ok(()) => reclaimed += 1, + Err(error) => { + warn!(object_key = %object_key, error = %error, "项目快照旧版本对象回收失败"); + } + } + } + if reclaimed > 0 || skipped > 0 { + info!( + project_id = %previous.project_id, + reclaimed, + skipped, + "项目快照旧版本对象回收完成" + ); + } +} + +fn validate_manifest(payload: &AgcProjectSnapshotManifestRequest) -> Result<(), AppError> { + if payload.schema_version != AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION { + return Err(bad_request("项目快照清单版本不受支持")); + } + validate_agc_project_snapshot_project_id(&payload.project_id).map_err(bad_request)?; + if payload.synced_at_ms == 0 { + return Err(bad_request("项目快照清单缺少同步时刻")); + } + if payload.files.len() > AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES { + return Err(bad_request("项目快照清单文件数量超过上限")); + } + let mut seen = HashSet::with_capacity(payload.files.len()); + let mut total_bytes = 0_u64; + for file in &payload.files { + if file.relative_path.chars().count() > MAX_MANIFEST_PATH_CHARS { + return Err(bad_request("项目快照清单路径过长")); + } + validate_agc_project_snapshot_relative_path(&file.relative_path).map_err(bad_request)?; + validate_agc_project_snapshot_checksum(&file.checksum).map_err(bad_request)?; + if file.size_bytes > AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES { + return Err(bad_request("项目快照清单包含超过单文件上限的条目")); + } + if !seen.insert(file.relative_path.as_str()) { + return Err(bad_request("项目快照清单包含重复路径")); + } + total_bytes = total_bytes.saturating_add(file.size_bytes); + } + // 清单描述的是项目当前全量文件，所以这里的累计体积就是该项目的常驻占用； + // 单项目上限同时约束了远端对象回收之后的实际存储量。 + if total_bytes > AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES { + return Err(AppError::from_status(StatusCode::PAYLOAD_TOO_LARGE) + .with_message("项目快照累计体积超过单项目上限")); + } + Ok(()) +} + +fn project_snapshot_oss(state: &AppState) -> Result<&platform_oss::OssClient, AppError> { + state.project_snapshot_oss_client().ok_or_else(|| { + AppError::from_status(StatusCode::SERVICE_UNAVAILABLE) + .with_message("AGC 项目快照 OSS 未配置") + }) +} + +fn bad_request(m: impl Into) -> AppError { + AppError::from_status(StatusCode::BAD_REQUEST).with_message(m) +} + +fn internal(e: E) -> AppError { + AppError::from_status(StatusCode::INTERNAL_SERVER_ERROR).with_message(e.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use shared_contracts::agc_project_snapshots::AgcProjectSnapshotManifestFile; + + fn manifest_file(relative_path: &str, size_bytes: u64) -> AgcProjectSnapshotManifestFile { + AgcProjectSnapshotManifestFile { + relative_path: relative_path.to_string(), + size_bytes, + checksum: "fnv1a64:0123456789abcdef".to_string(), + } + } + + fn manifest(files: Vec) -> AgcProjectSnapshotManifestRequest { + AgcProjectSnapshotManifestRequest { + schema_version: AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION, + project_id: "gameagent-1a2b3c4d".to_string(), + sync_revision: 1, + synced_at_ms: 1_700_000_000_000, + files, + } + } + + #[test] + fn project_snapshot_manifest_accepts_a_bounded_payload() { + assert!( + validate_manifest(&manifest(vec![ + manifest_file("game/index.html", 1024), + manifest_file("assets/Hero.png", 2048), + ])) + .is_ok() + ); + } + + #[test] + fn project_snapshot_manifest_rejects_duplicates_and_out_of_range_values() { + assert!( + validate_manifest(&manifest(vec![ + manifest_file("game/index.html", 1), + manifest_file("game/index.html", 1), + ])) + .is_err(), + "重复路径会让远端清单产生歧义" + ); + + let mut traversal = manifest(vec![manifest_file("game/index.html", 1)]); + traversal.files[0].relative_path = "../outside.txt".to_string(); + assert!(validate_manifest(&traversal).is_err()); + + let mut unsafe_project = manifest(vec![manifest_file("game/index.html", 1)]); + unsafe_project.project_id = "../escape".to_string(); + assert!(validate_manifest(&unsafe_project).is_err()); + + let mut bad_checksum = manifest(vec![manifest_file("game/index.html", 1)]); + bad_checksum.files[0].checksum = "md5:not-hex".to_string(); + assert!(validate_manifest(&bad_checksum).is_err()); + + let mut oversized = manifest(vec![manifest_file("game/index.html", 1)]); + oversized.files[0].size_bytes = AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES + 1; + assert!(validate_manifest(&oversized).is_err()); + + let mut stale_schema = manifest(vec![manifest_file("game/index.html", 1)]); + stale_schema.schema_version = AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION + 1; + assert!(validate_manifest(&stale_schema).is_err()); + + let mut missing_timestamp = manifest(vec![manifest_file("game/index.html", 1)]); + missing_timestamp.synced_at_ms = 0; + assert!(validate_manifest(&missing_timestamp).is_err()); + } + + #[test] + fn project_snapshot_manifest_rejects_too_many_entries() { + let files = (0..=AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES) + .map(|index| manifest_file(&format!("game/file-{index}.txt"), 1)) + .collect::>(); + assert!(validate_manifest(&manifest(files)).is_err()); + } + + #[test] + fn project_snapshot_manifest_rejects_a_project_over_the_total_cap() { + // 单文件与条目数都在上限内，只有项目累计体积越界：这必须按 413 拒绝， + // 否则远端回收之后仍会长期占住超大配额。 + let files = (0..40) + .map(|index| { + manifest_file( + &format!("assets/blob-{index}.bin"), + AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES, + ) + }) + .collect::>(); + let error = validate_manifest(&manifest(files)).expect_err("超额项目必须被拒绝"); + assert_eq!(error.status_code(), StatusCode::PAYLOAD_TOO_LARGE); + } + + #[test] + fn project_snapshot_user_quota_stops_after_the_hourly_file_limit() { + let user = "user-quota-fixture"; + for _ in 0..MAX_FILE_UPLOADS_PER_USER_PER_HOUR { + consume_user_upload_quota(user, ProjectSnapshotUploadKind::File) + .expect("配额内的上传必须放行"); + } + let error = consume_user_upload_quota(user, ProjectSnapshotUploadKind::File) + .expect_err("超出小时上限必须被拒绝"); + assert_eq!(error.status_code(), StatusCode::TOO_MANY_REQUESTS); + // 清单与文件是两条独立配额，不受文件配额影响。 + consume_user_upload_quota(user, ProjectSnapshotUploadKind::Manifest) + .expect("清单配额独立计数"); + } +} diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index b4030cd62..91c58792f 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -273,6 +273,8 @@ pub struct AppStateInner { test_external_background_removal_enqueue: Arc>>, oss_client: Option, + /// AGC 项目快照专用 OSS 客户端：bucket 与凭据可以独立于资源 bucket。 + project_snapshot_oss_client: Option, #[cfg_attr(test, allow(dead_code))] auth_store: InMemoryAuthStore, /// 当前进程工作集所基于的正式认证投影版本；跨节点写入使用它做 CAS。 @@ -339,6 +341,10 @@ impl fmt::Debug for AppStateInner { ) .field("admin_runtime_enabled", &self.admin_runtime.is_some()) .field("oss_client_enabled", &self.oss_client.is_some()) + .field( + "project_snapshot_oss_client_enabled", + &self.project_snapshot_oss_client.is_some(), + ) .field("spacetime_client", &self.spacetime_client) .field("tracking_outbox_enabled", &self.tracking_outbox.is_some()) .field( @@ -546,6 +552,7 @@ impl AppState { config.refresh_session_ttl_days, )?; let oss_client = build_oss_client(&config)?; + let project_snapshot_oss_client = build_project_snapshot_oss_client(&config)?; let sms_provider = SmsAuthProvider::new(SmsAuthConfig::new( SmsAuthProviderKind::parse(&config.sms_auth_provider).ok_or_else(|| { SmsProviderError::InvalidConfig("短信 provider 配置非法".to_string()) @@ -663,6 +670,7 @@ impl AppState { #[cfg(test)] test_external_background_removal_enqueue: Arc::new(Mutex::new(None)), oss_client, + project_snapshot_oss_client, auth_store, auth_projection_version: AtomicI64::new(auth_projection_version), auth_projection_synced_revision: AtomicU64::new(initial_auth_store_revision), @@ -1321,6 +1329,10 @@ impl AppState { self.oss_client.as_ref() } + pub fn project_snapshot_oss_client(&self) -> Option<&OssClient> { + self.project_snapshot_oss_client.as_ref() + } + pub fn password_entry_service(&self) -> &PasswordEntryService { &self.password_entry_service } @@ -2309,6 +2321,65 @@ impl AdminRuntime { } } +/// AGC 项目快照专用 OSS 客户端。 +/// +/// 目标 bucket 独立于资源 bucket：专用凭据未配置时回退 `ALIYUN_OSS_*`，而 bucket 与 +/// endpoint 默认指向 AGC 发行 bucket。凭据缺失或只配置一半时返回 `None`；路由层把 +/// "未配置" 当作失败关闭，不写空对象也不推进客户端索引。 +fn build_project_snapshot_oss_client( + config: &AppConfig, +) -> Result, AppStateInitError> { + let bucket = config.project_snapshot_oss_bucket.trim(); + let endpoint = config.project_snapshot_oss_endpoint.trim(); + if bucket.is_empty() || endpoint.is_empty() { + warn!("AGC 项目快照 OSS bucket/endpoint 未配置，跳过项目快照 OSS 客户端初始化"); + return Ok(None); + } + let access_key_id = config + .project_snapshot_oss_access_key_id + .as_deref() + .map(str::trim) + .unwrap_or(""); + let access_key_secret = config + .project_snapshot_oss_access_key_secret + .as_deref() + .map(str::trim) + .unwrap_or(""); + if access_key_id.is_empty() && access_key_secret.is_empty() { + return Ok(None); + } + if access_key_id.is_empty() || access_key_secret.is_empty() { + warn!("AGC 项目快照 OSS AccessKey 配置不完整，跳过项目快照 OSS 客户端初始化"); + return Ok(None); + } + let oss_config = OssConfig::new( + bucket.to_string(), + endpoint.to_string(), + access_key_id.to_string(), + access_key_secret.to_string(), + config.oss_read_expire_seconds, + config.oss_post_expire_seconds, + config.oss_post_max_size_bytes, + config.oss_success_action_status, + )?; + let credential_source = if config + .project_snapshot_oss_access_key_id + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + { + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID" + } else { + "ALIYUN_OSS_ACCESS_KEY_ID" + }; + info!( + bucket = %bucket, + endpoint = %endpoint, + credential_source, + "AGC 项目快照 OSS 客户端已启用" + ); + Ok(Some(OssClient::new(oss_config))) +} + fn build_oss_client(config: &AppConfig) -> Result, AppStateInitError> { let oss_fields = [ ("ALIYUN_OSS_BUCKET", config.oss_bucket.as_deref()), diff --git a/server-rs/crates/platform-oss/examples/agc_project_snapshot_live_smoke.rs b/server-rs/crates/platform-oss/examples/agc_project_snapshot_live_smoke.rs new file mode 100644 index 000000000..2ebc8f144 --- /dev/null +++ b/server-rs/crates/platform-oss/examples/agc_project_snapshot_live_smoke.rs @@ -0,0 +1,278 @@ +//! AGC 项目快照真实 OSS 冒烟：直接验证 `agc-dev` bucket 上的内部前缀读写与清理。 +//! +//! 默认从仓库根目录的 `.env`、`.env.local`、`.env.secrets.local` 读取 OSS 配置， +//! 非空 shell 环境变量优先；凭据优先使用 +//! `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID`，未配置时回退 `ALIYUN_OSS_ACCESS_KEY_ID`。 +//! +//! ```text +//! cargo run -p platform-oss --example agc_project_snapshot_live_smoke --manifest-path server-rs/Cargo.toml +//! ``` +//! +//! 冒烟只写入 `agc/project-snapshots/v1/` 下的固定探针对象，并在结束时删除； +//! 任何一步失败都会打印 `[FAIL]` 并以非 0 退出码结束，方便 CI 或人工判定。 + +use std::{ + collections::{HashMap, HashSet}, + env, fs, + path::Path, + time::Duration, +}; + +use platform_oss::{ + DEFAULT_POST_EXPIRE_SECONDS, DEFAULT_POST_MAX_SIZE_BYTES, DEFAULT_READ_EXPIRE_SECONDS, + DEFAULT_SUCCESS_ACTION_STATUS, OssClient, OssConfig, OssDeleteObjectRequest, OssError, + OssInternalPutObjectRequest, OssObjectAccess, agc_project_snapshot_file_object_key, + agc_project_snapshot_manifest_object_key, +}; + +const DEFAULT_BUCKET: &str = "agc-dev"; +const DEFAULT_ENDPOINT: &str = "oss-rg-china-mainland.aliyuncs.com"; +const SMOKE_USER_ID: &str = "smoke-user"; +const SMOKE_PROJECT_ID: &str = "smoke-project"; +const SMOKE_RELATIVE_PATH: &str = "smoke/README.txt"; +const SMOKE_BODY: &[u8] = b"agc project snapshot live smoke\n"; +const SMOKE_CHECKSUM_DIGEST: &str = "0123456789abcdef"; + +type SmokeResult = Result; + +#[tokio::main(flavor = "current_thread")] +async fn main() { + match run().await { + Ok(()) => { + println!("[PASS] AGC 项目快照 OSS 冒烟全部通过"); + } + Err(error) => { + println!("[FAIL] {error}"); + std::process::exit(1); + } + } +} + +async fn run() -> SmokeResult<()> { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let local_env = load_local_env(&repo_root)?; + let bucket = non_empty_env(&local_env, "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET") + .unwrap_or_else(|| DEFAULT_BUCKET.to_string()); + let endpoint = non_empty_env(&local_env, "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT") + .unwrap_or_else(|| DEFAULT_ENDPOINT.to_string()); + let access_key_id = non_empty_env( + &local_env, + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID", + ) + .or_else(|| non_empty_env(&local_env, "ALIYUN_OSS_ACCESS_KEY_ID")) + .ok_or_else(|| { + "缺少 AGC 项目快照 OSS AccessKey ID（或 ALIYUN_OSS_ACCESS_KEY_ID）".to_string() + })?; + let access_key_secret = non_empty_env( + &local_env, + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET", + ) + .or_else(|| non_empty_env(&local_env, "ALIYUN_OSS_ACCESS_KEY_SECRET")) + .ok_or_else(|| { + "缺少 AGC 项目快照 OSS AccessKey Secret（或 ALIYUN_OSS_ACCESS_KEY_SECRET）".to_string() + })?; + let config = OssConfig::new( + bucket.clone(), + endpoint.clone(), + access_key_id, + access_key_secret, + DEFAULT_READ_EXPIRE_SECONDS, + DEFAULT_POST_EXPIRE_SECONDS, + DEFAULT_POST_MAX_SIZE_BYTES, + DEFAULT_SUCCESS_ACTION_STATUS, + ) + .map_err(|error| format!("OSS 配置无效（{}）", oss_error_label(&error)))?; + let client = OssClient::new(config); + println!("目标：bucket={bucket} endpoint={endpoint}"); + + let http = reqwest::Client::builder() + .timeout(Duration::from_secs(60)) + .no_proxy() + .build() + .map_err(|error| format!("创建 HTTP 客户端失败：{error}"))?; + + // 1. 内部前缀白名单：非内部键必须在本地被拒绝，不产生网络请求。 + match client + .put_internal_object( + &http, + OssInternalPutObjectRequest { + object_key: "generated-characters/smoke/master.png".to_string(), + content_type: Some("application/octet-stream".to_string()), + access: OssObjectAccess::Private, + metadata: Default::default(), + body: SMOKE_BODY.to_vec(), + }, + ) + .await + { + Err(OssError::InvalidRequest(_)) => println!("[PASS] 非内部前缀被本地拒绝，未写入 OSS"), + Ok(_) => return Err("非内部前缀竟然写入成功，内部前缀白名单失效".to_string()), + Err(error) => { + return Err(format!( + "非内部前缀返回了非预期错误（{}）", + oss_error_label(&error) + )); + } + } + + // 3. 项目快照文件键：写入 → 读回。 + let file_key = agc_project_snapshot_file_object_key( + SMOKE_USER_ID, + SMOKE_PROJECT_ID, + SMOKE_BODY.len() as u64, + SMOKE_CHECKSUM_DIGEST, + SMOKE_RELATIVE_PATH, + ) + .map_err(|error| format!("构造项目快照文件键失败（{}）", oss_error_label(&error)))?; + let manifest_key = agc_project_snapshot_manifest_object_key(SMOKE_USER_ID, SMOKE_PROJECT_ID) + .map_err(|error| format!("构造项目快照清单键失败（{}）", oss_error_label(&error)))?; + + let result = write_and_verify(&client, &http, &bucket, &file_key, &manifest_key).await; + for key in [&file_key, &manifest_key] { + let _ = client + .delete_object( + &http, + OssDeleteObjectRequest { + object_key: key.clone(), + }, + ) + .await; + } + match client.head_internal_object(&http, &file_key).await { + Ok(None) => println!("[PASS] 探针对象已清理"), + Ok(Some(_)) => println!("[WARN] 探针对象仍存在，请手工清理 {file_key}"), + Err(error) => println!( + "[WARN] 清理后复核失败（{}），请手工确认 {file_key}", + oss_error_label(&error) + ), + } + result +} + +async fn write_and_verify( + client: &OssClient, + http: &reqwest::Client, + bucket: &str, + file_key: &str, + manifest_key: &str, +) -> SmokeResult<()> { + client + .put_internal_object( + http, + OssInternalPutObjectRequest { + object_key: file_key.to_string(), + content_type: Some("application/octet-stream".to_string()), + access: OssObjectAccess::Private, + metadata: Default::default(), + body: SMOKE_BODY.to_vec(), + }, + ) + .await + .map_err(|error| { + format!( + "写入项目快照文件失败（{}）：凭据可能没有 {bucket} 的 PutObject 权限", + oss_error_label(&error) + ) + })?; + println!("[PASS] 项目快照文件已写入 OSS：{file_key}"); + + let existing = client + .head_internal_object(http, file_key) + .await + .map_err(|error| format!("读回项目快照文件失败（{}）", oss_error_label(&error)))? + .ok_or_else(|| "写入成功但对象读回不存在".to_string())?; + if existing.content_length != SMOKE_BODY.len() as u64 { + return Err(format!( + "对象长度不一致：期望 {}，实际 {}", + SMOKE_BODY.len(), + existing.content_length + )); + } + println!( + "[PASS] 对象读回一致，服务端跳过判据成立：contentLength={} etag={}", + existing.content_length, + existing.etag.as_deref().unwrap_or("-") + ); + + client + .put_internal_object( + http, + OssInternalPutObjectRequest { + object_key: manifest_key.to_string(), + content_type: Some("application/json".to_string()), + access: OssObjectAccess::Private, + metadata: Default::default(), + body: br#"{"schemaVersion":1,"projectId":"smoke-project"}"#.to_vec(), + }, + ) + .await + .map_err(|error| format!("写入项目快照清单失败（{}）", oss_error_label(&error)))?; + println!("[PASS] 项目快照清单已覆盖写入 OSS：{manifest_key}"); + Ok(()) +} + +fn oss_error_label(error: &OssError) -> String { + format!("{error}") +} + +fn load_local_env(repo_root: &Path) -> SmokeResult> { + let shell = env::vars().collect::>(); + let protected = shell + .iter() + .filter(|(_, value)| !value.trim().is_empty()) + .map(|(key, _)| key.clone()) + .collect::>(); + let mut merged = shell; + for file_name in [".env", ".env.local", ".env.secrets.local"] { + let path = repo_root.join(file_name); + if !path.is_file() { + continue; + } + let contents = + fs::read_to_string(&path).map_err(|_| format!("无法读取本地配置文件 {file_name}"))?; + for raw_line in contents.lines() { + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let Some((key, raw_value)) = line.split_once('=') else { + continue; + }; + if !valid_env_key(key) || protected.contains(key) { + continue; + } + merged.insert( + key.to_string(), + trim_env_quotes(raw_value.trim()).to_string(), + ); + } + } + Ok(merged) +} + +fn valid_env_key(key: &str) -> bool { + let mut chars = key.chars(); + chars + .next() + .is_some_and(|value| value == '_' || value.is_ascii_alphabetic()) + && chars.all(|value| value == '_' || value.is_ascii_alphanumeric()) +} + +fn trim_env_quotes(value: &str) -> &str { + if value.len() >= 2 + && ((value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\''))) + { + &value[1..value.len() - 1] + } else { + value + } +} + +fn non_empty_env(local_env: &HashMap, name: &str) -> Option { + local_env + .get(name) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} diff --git a/server-rs/crates/platform-oss/src/lib.rs b/server-rs/crates/platform-oss/src/lib.rs index 4a542a8c6..e74d511c5 100644 --- a/server-rs/crates/platform-oss/src/lib.rs +++ b/server-rs/crates/platform-oss/src/lib.rs @@ -114,6 +114,20 @@ pub struct OssDeleteObjectRequest { pub object_key: String, } +/// 服务端专用内部对象写入请求。 +/// +/// 与 `OssPutObjectRequest` 的区别是对象键由调用方按内部前缀完整给出，不再走 +/// `path_segments`/`file_name` 的低位规范化，因此可以保留项目内的原始大小写与 +/// 目录层级。键必须落在 `normalize_internal_object_key` 允许的内部前缀下。 +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OssInternalPutObjectRequest { + pub object_key: String, + pub content_type: Option, + pub access: OssObjectAccess, + pub metadata: BTreeMap, + pub body: Vec, +} + #[derive(Clone, Debug, PartialEq, Eq)] pub struct OssPutObjectRequest { pub prefix: LegacyAssetPrefix, @@ -796,11 +810,7 @@ impl OssClient { } let headers = response.headers(); - let content_length = headers - .get(reqwest::header::CONTENT_LENGTH) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); + let content_length = head_object_content_length(headers); let content_type = headers .get(reqwest::header::CONTENT_TYPE) .and_then(|value| value.to_str().ok()) @@ -933,6 +943,129 @@ impl OssClient { )) } + /// 按内部前缀写入服务端专用对象。内容为空、键越界都直接失败，不写空对象。 + pub async fn put_internal_object( + &self, + client: &reqwest::Client, + request: OssInternalPutObjectRequest, + ) -> Result { + if request.body.is_empty() { + return Err(OssError::InvalidRequest( + "服务端内部对象内容不能为空".to_string(), + )); + } + let object_key = normalize_internal_object_key(&request.object_key)?; + let content_type = normalize_optional_value(request.content_type); + let headers = build_put_object_headers(request.metadata)?; + let target_url = build_object_url(&self.config.bucket, &self.config.endpoint, &object_key) + .map_err(|error| { + request_error( + OssRequestOperation::Put, + &format!("构造 OSS 对象 URL 失败：{error}"), + ) + })?; + let content_length = u64::try_from(request.body.len()) + .map_err(|_| OssError::InvalidRequest("上传对象大小超出可支持范围".to_string()))?; + let builder = signed_request_builder( + client, + &self.config, + Method::PUT, + Some(&object_key), + target_url, + content_type.as_deref(), + &headers, + )? + .header(reqwest::header::CONTENT_LENGTH, content_length) + .body(request.body); + let response = builder + .send() + .await + .map_err(|error| request_error_from_reqwest(OssRequestOperation::Put, error))?; + if !response.status().is_success() { + return Err(request_status_error( + OssRequestOperation::Put, + response.status().as_u16(), + format!("OSS PutObject 失败，状态码：{}", response.status()), + )); + } + let headers = response.headers(); + let etag = headers + .get(reqwest::header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim_matches('"').to_string()); + let last_modified = headers + .get(reqwest::header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_string()); + Ok(OssPutObjectResponse { + provider: OSS_PROVIDER, + bucket: self.config.bucket.clone(), + endpoint: self.config.endpoint.clone(), + host: self.config.upload_host(), + legacy_public_path: format!("/{object_key}"), + object_key, + content_type, + content_length, + access: request.access, + etag, + last_modified, + }) + } + + /// 探测内部对象是否存在。`Ok(None)` 表示确定不存在，其余失败都按上游错误返回， + /// 调用方不能把不确定当成"不存在"。 + pub async fn head_internal_object( + &self, + client: &reqwest::Client, + object_key: &str, + ) -> Result, OssError> { + let object_key = normalize_internal_object_key(object_key)?; + let target_url = build_object_url(&self.config.bucket, &self.config.endpoint, &object_key) + .map_err(|error| { + request_error( + OssRequestOperation::Head, + &format!("构造 OSS 对象 URL 失败：{error}"), + ) + })?; + let response = send_signed_request( + client, + &self.config, + Method::HEAD, + Some(&object_key), + target_url, + OssRequestOperation::Head, + ) + .await?; + if response.status() == reqwest::StatusCode::NOT_FOUND { + return Ok(None); + } + if !response.status().is_success() { + return Err(request_status_error( + OssRequestOperation::Head, + response.status().as_u16(), + format!("OSS HEAD Object 失败，状态码：{}", response.status()), + )); + } + let headers = response.headers(); + Ok(Some(OssHeadObjectResponse { + bucket: self.config.bucket.clone(), + object_key, + content_length: head_object_content_length(headers), + content_type: headers + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_string()), + etag: headers + .get(reqwest::header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim_matches('"').to_string()), + last_modified: headers + .get(reqwest::header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .map(|value| value.to_string()), + })) + } + // AI 生成资源默认由服务端上传 OSS，Web 端只拿签名读地址，不直接持有写权限。 pub async fn put_object( &self, @@ -1706,6 +1839,16 @@ fn build_policy_json( }) } +/// HEAD 响应没有正文，`reqwest::Response::content_length()` 对 HEAD 恒为 0； +/// 对象大小只能读响应头，两个 HEAD 入口共用这一处解析。 +fn head_object_content_length(headers: &reqwest::header::HeaderMap) -> u64 { + headers + .get(reqwest::header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0) +} + fn build_object_url( bucket: &str, endpoint: &str, @@ -1778,10 +1921,105 @@ fn normalize_editor_agent_messages_object_key(raw: &str) -> Result Result { + let user = validate_internal_key_segment(user_id, "用户标识")?; + let project = validate_internal_key_segment(project_id, "项目标识")?; + let digest = validate_internal_checksum_digest(checksum_digest)?; + let relative_path = validate_internal_relative_path(relative_path)?; + Ok(format!( + "{AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX}{user}/{project}/files/{size_bytes}-{digest}/{relative_path}" + )) +} + +/// 项目快照清单对象键：`agc/project-snapshots/v1/{user}/{project}/manifest.json`。 +pub fn agc_project_snapshot_manifest_object_key( + user_id: &str, + project_id: &str, +) -> Result { + let user = validate_internal_key_segment(user_id, "用户标识")?; + let project = validate_internal_key_segment(project_id, "项目标识")?; + Ok(format!( + "{AGC_PROJECT_SNAPSHOT_INTERNAL_PREFIX}{user}/{project}/manifest.json" + )) +} + +fn validate_internal_key_segment(raw: &str, label: &str) -> Result { + let trimmed = raw.trim(); + let allowed = !trimmed.is_empty() + && trimmed.len() <= 128 + && trimmed != "." + && trimmed != ".." + && trimmed.chars().all(|character| { + character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.') + }); + if !allowed { + return Err(OssError::InvalidRequest(format!( + "{label}不能作为 OSS 对象键片段" + ))); + } + Ok(trimmed.to_string()) +} + +fn validate_internal_checksum_digest(raw: &str) -> Result { + let trimmed = raw.trim(); + if trimmed.is_empty() + || trimmed.len() > 64 + || !trimmed + .chars() + .all(|character| character.is_ascii_hexdigit()) + { + return Err(OssError::InvalidRequest( + "对象摘要必须是 1 到 64 位十六进制".to_string(), + )); + } + Ok(trimmed.to_ascii_lowercase()) +} + +fn validate_internal_relative_path(raw: &str) -> Result { + if raw.is_empty() || raw.len() > 1024 || raw.starts_with('/') || raw.contains('\\') { + return Err(OssError::InvalidRequest( + "对象相对路径必须是 1 到 1024 字节的正斜杠相对路径".to_string(), + )); + } + for part in raw.split('/') { + if part.is_empty() || part == "." || part == ".." || part.chars().any(char::is_control) { + return Err(OssError::InvalidRequest( + "对象相对路径包含非法片段".to_string(), + )); + } + } + Ok(raw.to_string()) +} + fn normalize_internal_object_key(raw: &str) -> Result { let normalized = raw.trim().trim_start_matches('/').trim().to_string(); validate_object_key_segments(&normalized)?; - if normalized.starts_with("agc/error-reports/v1/") { + if AGC_INTERNAL_OBJECT_PREFIXES + .iter() + .any(|prefix| normalized.starts_with(prefix)) + { Ok(normalized) } else { Err(OssError::InvalidRequest( @@ -3243,6 +3481,100 @@ mod tests { LegacyAssetPrefix::from_object_key("workflow-cache/demo.json"), None ); + assert_eq!( + LegacyAssetPrefix::from_object_key( + "agc/project-snapshots/v1/user-1/project-1/manifest.json" + ), + None, + "AGC 内部前缀不能经由通用对象键解析变成客户端可写前缀" + ); + } + + #[test] + fn head_object_content_length_reads_the_response_header_not_the_head_body() { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::CONTENT_LENGTH, + "32".parse().expect("content length header value"), + ); + assert_eq!( + head_object_content_length(&headers), + 32, + "HEAD 的对象大小必须来自响应头；reqwest 对 HEAD 的 body 长度恒为 0" + ); + assert_eq!( + head_object_content_length(&reqwest::header::HeaderMap::new()), + 0 + ); + } + + #[test] + fn agc_project_snapshot_object_keys_preserve_case_and_reject_traversal() { + let file_key = agc_project_snapshot_file_object_key( + "user-1", + "gameagent-1a2b3c4d", + 1234, + "0123456789ABCDEF", + "Game/Scenes/Main.HTML", + ) + .expect("file key"); + assert_eq!( + file_key, + "agc/project-snapshots/v1/user-1/gameagent-1a2b3c4d/files/1234-0123456789abcdef/Game/Scenes/Main.HTML" + ); + assert_eq!( + agc_project_snapshot_manifest_object_key("user-1", "gameagent-1a2b3c4d") + .expect("manifest key"), + "agc/project-snapshots/v1/user-1/gameagent-1a2b3c4d/manifest.json" + ); + + for (user, project, path) in [ + ("../escape", "project-1", "game/index.html"), + ("user-1", "../escape", "game/index.html"), + ("user-1", "project-1", "../outside.txt"), + ("user-1", "project-1", "game/../../outside.txt"), + ("user-1", "project-1", "game\\index.html"), + ] { + assert!( + agc_project_snapshot_file_object_key(user, project, 1, "abcdef", path).is_err(), + "越界键片段必须被拒绝：{user} {project} {path}" + ); + } + assert!( + agc_project_snapshot_file_object_key("user-1", "project-1", 1, "not-hex", "game/a.txt") + .is_err(), + "摘要必须是十六进制" + ); + } + + #[test] + fn internal_object_prefixes_cover_agc_snapshots_but_reject_everything_else() { + let file_key = agc_project_snapshot_file_object_key( + "user-1", + "project-1", + 7, + "abcdef", + "game/index.html", + ) + .expect("file key"); + assert_eq!( + normalize_internal_object_key(&file_key).expect("snapshot key is internal"), + file_key + ); + assert_eq!( + normalize_internal_object_key("agc/error-reports/v1/batch.zip") + .expect("error report key stays internal"), + "agc/error-reports/v1/batch.zip" + ); + for key in [ + "generated-characters/hero/master.png", + "agc/other-purpose/v1/file.json", + ] { + assert!( + normalize_internal_object_key(key).is_err(), + "非内部前缀不能走服务端内部写入：{key}" + ); + } } #[test] diff --git a/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs b/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs new file mode 100644 index 000000000..7bc6c7dde --- /dev/null +++ b/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs @@ -0,0 +1,139 @@ +//! AGC 项目定时快照上传的客户端 ↔ api-server 契约。 +//! +//! 客户端只负责把项目内发生变化的内容和当前清单交给 api-server；对象键、bucket +//! 与存储凭据全部留在服务端。因此这里只描述可达字段，不描述 OSS 细节。 + +use serde::{Deserialize, Serialize}; + +/// 清单格式版本。客户端与 api-server 必须一致，服务端拒绝其它版本。 +pub const AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION: u32 = 1; + +/// 单个文件的硬上限，与客户端扫描口径和 api-server 请求体上限保持一致。 +pub const AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES: u64 = 64 * 1024 * 1024; + +/// 单次清单包含的文件数量上限，防止越界请求把服务端内存拖垮。 +pub const AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES: usize = 20_000; + +/// 单次同步允许上传的累计字节上限（客户端预算）。超出部分留到下一次同步， +/// 与"单项目总上限"是两个不同概念。 +pub const AGC_PROJECT_SNAPSHOT_MAX_SYNC_BYTES: u64 = 512 * 1024 * 1024; + +/// 单个项目快照的累计体积上限。远端对象按当前清单做回收，所以这个上限同时约束 +/// 了该项目在 OSS 上的常驻占用。 +pub const AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// 单文件上传查询参数。文件正文走请求体，元数据走查询串，避免 base64 膨胀。 +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AgcProjectSnapshotFileUploadQuery { + pub project_id: String, + pub relative_path: String, + pub checksum: String, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AgcProjectSnapshotFileUploadResponse { + pub project_id: String, + pub relative_path: String, + pub object_key: String, + /// 服务端按对象元数据判定内容已存在时为 true；此时不写入新对象。 + pub skipped: bool, + pub checksum: String, + pub size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AgcProjectSnapshotManifestFile { + pub relative_path: String, + pub size_bytes: u64, + pub checksum: String, +} + +/// 一次成功同步后的远端清单。删除文件只在这里消失，本期不删除远端对象。 +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AgcProjectSnapshotManifestRequest { + pub schema_version: u32, + pub project_id: String, + pub sync_revision: u64, + pub synced_at_ms: u64, + pub files: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct AgcProjectSnapshotManifestResponse { + pub project_id: String, + pub sync_revision: u64, + pub object_key: String, + pub file_count: u32, + pub total_bytes: u64, +} + +/// 项目身份校验。同一个值同时用作 AppData 目录名与 OSS 键段，因此必须拒绝路径 +/// 分隔符、相对段、控制字符与前导点。 +pub fn validate_agc_project_snapshot_project_id(value: &str) -> Result<(), String> { + let trimmed = value.trim(); + if trimmed.is_empty() || trimmed.len() > 128 || trimmed != value { + return Err("项目 ID 必须是 1 到 128 字节且不含首尾空白".to_string()); + } + let mut characters = trimmed.chars(); + let first = characters.next().unwrap_or_default(); + if !first.is_ascii_alphanumeric() { + return Err("项目 ID 必须以字母或数字开头".to_string()); + } + if !trimmed + .chars() + .all(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_' | '.')) + { + return Err("项目 ID 只能包含字母、数字、连字符、下划线和点".to_string()); + } + Ok(()) +} + +/// 相对路径校验：正斜杠分隔、逐段非空、拒绝绝对路径、相对段、控制字符与 +/// Windows 保留字符，与客户端 `normalize_relative_path` 同口径。 +pub fn validate_agc_project_snapshot_relative_path(value: &str) -> Result<(), String> { + if value.is_empty() || value.len() > 1024 { + return Err("项目快照路径必须是 1 到 1024 字节".to_string()); + } + if value.starts_with('/') || value.contains('\\') { + return Err("项目快照路径必须是相对路径并使用正斜杠".to_string()); + } + if value.chars().any(char::is_control) { + return Err("项目快照路径不能包含控制字符".to_string()); + } + for part in value.split('/') { + if part.is_empty() || part == "." || part == ".." { + return Err("项目快照路径不能包含空段或相对段".to_string()); + } + if part.ends_with('.') || part.ends_with(' ') { + return Err("项目快照路径组件不能以点或空格结尾".to_string()); + } + if part + .chars() + .any(|character| matches!(character, ':' | '<' | '>' | '"' | '|' | '?' | '*')) + { + return Err("项目快照路径不能包含 Windows 保留字符".to_string()); + } + } + Ok(()) +} + +/// 校验和校验：只接受与项目索引同口径的 `fnv1a64:<16 位十六进制>`。 +pub fn validate_agc_project_snapshot_checksum(value: &str) -> Result<(), String> { + let Some(digest) = value.strip_prefix("fnv1a64:") else { + return Err("项目快照校验和必须使用 fnv1a64 前缀".to_string()); + }; + if digest.len() != 16 + || !digest + .chars() + .all(|character| character.is_ascii_hexdigit()) + { + return Err("项目快照校验和必须是 16 位十六进制摘要".to_string()); + } + Ok(()) +} diff --git a/server-rs/crates/shared-contracts/src/lib.rs b/server-rs/crates/shared-contracts/src/lib.rs index bc32ee4b9..c3cc5ecaa 100644 --- a/server-rs/crates/shared-contracts/src/lib.rs +++ b/server-rs/crates/shared-contracts/src/lib.rs @@ -1,4 +1,5 @@ pub mod admin; +pub mod agc_project_snapshots; pub mod ai; pub mod api; #[cfg(feature = "oss-contracts")]