From e8a723c04e723b1122bd00cb1528443801749591 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:51:31 +0800 Subject: [PATCH] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20AGC=20=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E5=BF=AB=E7=85=A7=E7=9A=84=E5=9B=9E=E6=94=B6=E3=80=81=E9=85=8D?= =?UTF-8?q?=E9=A2=9D=E3=80=81=E5=8F=AF=E8=A7=81=E6=80=A7=E4=B8=8E=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E6=80=A7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 服务端在清单写入成功后回收上一版清单不再引用的对象，单次最多 2000 个，上一版不可读时整轮跳过 - 服务端新增单项目 2 GiB 上限（413）、同一项目 5 秒最小清单间隔与用户小时配额（429 + Retry-After） - 客户端新增单项目上限预判，并把单次同步预算改名为 MAX_SYNC_BYTES，避免与项目上限混淆 - 客户端在读取摘要后与上传前各复核一次文件元数据，同步期间被改写的文件不上传、不推进索引、也不被误判成删除 - 客户端新增 file-changed 与 throttled 两类可重试失败，429 计入可重试而不是终态失败 - 新增工作区「项目快照」面板：显示上传状态、上次同步时间、同步序号、已纳管文件数与最近一次同步计数，支持刷新与立即同步 - 新增面板前端测试，并把两条快照命令从 check-config 的 native-only 白名单移入前端 invoke 名单 - 部署模板补充 AGC 项目快照 OSS 变量，api-server 启动时打印启用状态与凭据来源 - 同步主规范、里程碑、开发运维文档 --- .../scripts/check-config.mjs | 4 - .../src-tauri/src/project_snapshot/diff.rs | 17 ++ .../src-tauri/src/project_snapshot/mod.rs | 19 +- .../src-tauri/src/project_snapshot/scan.rs | 22 ++ .../src-tauri/src/project_snapshot/tests.rs | 94 ++++++- .../src/project_snapshot/transport.rs | 30 ++- apps/ai-game-creator-shell/src/App.tsx | 10 + apps/ai-game-creator-shell/src/app/types.ts | 34 +++ .../ProjectSnapshotDialog.tsx | 222 ++++++++++++++++ .../ProjectWorkspaceChatPane.tsx | 9 + apps/ai-game-creator-shell/src/styles.css | 46 ++++ .../tests/projectSnapshotDialog.test.tsx | 117 ++++++++ deploy/env/api-server.env.example | 10 + ...里程碑】AGC项目定时快照上传-2026-09-17.md | 9 +- ...案】AI游戏创作智能体App实施计划-2026-06-24.md | 27 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 13 +- .../api-server/src/project_snapshots.rs | 250 +++++++++++++++++- server-rs/crates/api-server/src/state.rs | 15 ++ .../src/agc_project_snapshots.rs | 9 +- 19 files changed, 921 insertions(+), 36 deletions(-) create mode 100644 apps/ai-game-creator-shell/src/features/project-workspace/ProjectSnapshotDialog.tsx create mode 100644 apps/ai-game-creator-shell/tests/projectSnapshotDialog.test.tsx diff --git a/apps/ai-game-creator-shell/scripts/check-config.mjs b/apps/ai-game-creator-shell/scripts/check-config.mjs index 0cd3251a7..e32dfb4b7 100644 --- a/apps/ai-game-creator-shell/scripts/check-config.mjs +++ b/apps/ai-game-creator-shell/scripts/check-config.mjs @@ -121,10 +121,6 @@ const allowedUncalledTauriCommands = [ 'open_game_creator_launcher_window', 'open_game_creator_workspace_window', 'read_direct_project_conversation', - // 项目定时快照上传：当前只有 Rust 生命周期触发（周期 / 项目关闭 / 退出）与 - // 联调冒烟在调，界面还没有入口；同 `open_game_creator_*_window` 一样按 native-only 登记。 - 'read_local_project_snapshot_state', - 'sync_local_project_snapshot', 'reset_design_agent_session', 'stop_local_game_preview_if_matches', 'start_game_creator_external_mcp', diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs index 7a59bc85a..c6de3f6f1 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/diff.rs @@ -21,6 +21,8 @@ pub(crate) struct ProjectSnapshotDiff { pub(crate) skipped: Vec, /// 本轮因为累计上限没有上传、留给下一次同步的路径。 pub(crate) deferred: Vec, + /// 同步期间被改写、本轮不参与上传与索引推进的路径。 + pub(crate) pending: Vec, pub(crate) upload_bytes: u64, /// 扫描后的完整清单（尚未扣除上传失败与延后项），成功同步后就是新的索引。 pub(crate) current: BTreeMap, @@ -59,6 +61,21 @@ pub(crate) fn compute_project_snapshot_diff( } let bytes = read_project_snapshot_file_bytes(&file.absolute_path)?; + // 同步不持项目写锁：读完立刻复核一次，文件在读取期间被改写就留给下一轮， + // 既不把这份内容写进索引，也不上传它。 + if !project_snapshot_file_matches(&file.absolute_path, file.size_bytes, file.modified_ms) { + diff.pending.push(ProjectSnapshotSkippedPath { + relative_path: file.relative_path.clone(), + reason: "同步期间文件发生变化，留到下一次".to_string(), + }); + // 上一轮已同步过这条路径时沿用旧记录：文件仍然存在，只是这一轮读不到 + // 一致版本，所以既不能算删除（会被远端 GC 掉），也不能记成本轮已同步。 + if let Some(entry) = previous_entry { + diff.current + .insert(file.relative_path.clone(), entry.clone()); + } + continue; + } let checksum = project_snapshot_checksum(&bytes); match previous_entry { Some(entry) if entry.checksum == checksum => { diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs index 0bc44dc52..bcdf2ce21 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/mod.rs @@ -31,7 +31,10 @@ const PROJECT_SNAPSHOT_INDEX_MAX_BYTES: u64 = 32 * 1024 * 1024; pub(crate) const PROJECT_SNAPSHOT_MAX_FILE_BYTES: u64 = shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES; pub(crate) const PROJECT_SNAPSHOT_MAX_SYNC_BYTES: u64 = - shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_BYTES; + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_SYNC_BYTES; +/// 单项目常驻占用上限。超过时不再尝试上传，直接给出明确失败而不是反复被服务端拒绝。 +pub(crate) const PROJECT_SNAPSHOT_MAX_PROJECT_BYTES: u64 = + shared_contracts::agc_project_snapshots::AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES; const PROJECT_SNAPSHOT_FILES_ENDPOINT: &str = "/api/agc/project-snapshots/files"; const PROJECT_SNAPSHOT_MANIFEST_ENDPOINT: &str = "/api/agc/project-snapshots/manifest"; @@ -85,6 +88,8 @@ pub(crate) struct ProjectSnapshotSyncReport { pub(crate) deferred_files: usize, pub(crate) metadata_only_files: usize, pub(crate) skipped_files: Vec, + /// 同步期间被改写、本轮未参与上传与索引推进的路径。 + pub(crate) pending_files: Vec, pub(crate) failed_files: Vec, pub(crate) synced_at_ms: u64, } @@ -279,6 +284,16 @@ async fn sync_project_snapshot_async( }; let scan = scan_project_snapshot_files(project_root, PROJECT_SNAPSHOT_MAX_FILE_BYTES)?; + // 项目总量上限在本地先判：超限时明确失败，不再逐个文件上传后被服务端整体拒绝。 + let scan_total_bytes = scan + .files + .iter() + .fold(0_u64, |total, file| total.saturating_add(file.size_bytes)); + if scan_total_bytes > PROJECT_SNAPSHOT_MAX_PROJECT_BYTES { + return Err(format!( + "项目体积 {scan_total_bytes} 字节超过项目快照单项目上限 {PROJECT_SNAPSHOT_MAX_PROJECT_BYTES} 字节" + )); + } let diff = compute_project_snapshot_diff(&scan, &previous.files, PROJECT_SNAPSHOT_MAX_SYNC_BYTES)?; if !diff.has_changes() { @@ -294,6 +309,7 @@ async fn sync_project_snapshot_async( deferred_files: 0, metadata_only_files: diff.metadata_only.len(), skipped_files: failure_views(&diff.skipped), + pending_files: failure_views(&diff.pending), failed_files: Vec::new(), synced_at_ms: previous.synced_at_ms, }); @@ -352,6 +368,7 @@ async fn sync_project_snapshot_async( deferred_files: diff.deferred.len(), metadata_only_files: diff.metadata_only.len(), skipped_files: failure_views(&diff.skipped), + pending_files: failure_views(&diff.pending), failed_files: upload .failures .iter() diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs index 44f2140fe..2163c4b24 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/scan.rs @@ -133,3 +133,25 @@ pub(crate) fn read_project_snapshot_file_bytes(path: &Path) -> Result, S pub(crate) fn project_snapshot_checksum(bytes: &[u8]) -> String { format!("fnv1a64:{:016x}", fnv1a64(bytes)) } + +/// 复核文件是否仍是扫描时刻的那一份内容。 +/// +/// 同步不持有项目写锁：Agent 或编辑器可能在扫描之后、读取或上传之前改写文件。 +/// 读取前后都按 `(字节数, 修改时间)` 比对，任一处不一致就判定为"同步期间发生变化"， +/// 该文件不计入索引、留给下一次同步，而不是上传一份自相矛盾的快照。 +pub(crate) fn project_snapshot_file_matches( + path: &Path, + size_bytes: u64, + modified_ms: u64, +) -> bool { + let Ok(metadata) = fs::symlink_metadata(path) else { + return false; + }; + if metadata.file_type().is_symlink() || windows_metadata_is_reparse_point(&metadata) { + return false; + } + if !metadata.is_file() { + return false; + } + metadata.len() == size_bytes && project_snapshot_modified_ms(&metadata) == modified_ms +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs index 8e9335412..ff57bdfae 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/tests.rs @@ -350,12 +350,15 @@ fn fixture_candidate( bytes: &[u8], ) -> ProjectSnapshotUploadCandidate { write_fixture_file(root, relative_path, bytes); + let path = root.join(relative_path); + let metadata = fs::symlink_metadata(&path).expect("read fixture metadata"); ProjectSnapshotUploadCandidate { relative_path: relative_path.to_string(), size_bytes: bytes.len() as u64, - modified_ms: 1, + // 用真实修改时间：上传前的一致性复核会拿它比对，写死常量会把请求提前挡掉。 + modified_ms: project_snapshot_modified_ms(&metadata), checksum: project_snapshot_checksum(bytes), - absolute_path: root.join(relative_path), + absolute_path: path, } } @@ -401,6 +404,93 @@ fn project_snapshot_upload_marks_remote_duplicates_and_sends_marker_headers() { assert!(request.ends_with("alpha")); } +#[test] +fn project_snapshot_file_matches_detects_metadata_drift() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + let path = root.path().join("game/a.txt"); + let metadata = fs::symlink_metadata(&path).expect("fixture metadata"); + let size = metadata.len(); + let modified = project_snapshot_modified_ms(&metadata); + + assert!(project_snapshot_file_matches(&path, size, modified)); + assert!( + !project_snapshot_file_matches(&path, size + 1, modified), + "字节数变化必须被判为已改写" + ); + assert!( + !project_snapshot_file_matches(&path, size, modified + 1), + "修改时间变化必须被判为已改写" + ); + assert!( + !project_snapshot_file_matches(&root.path().join("game/missing.txt"), size, modified), + "文件消失必须被判为已改写" + ); +} + +#[test] +fn project_snapshot_diff_defers_files_that_changed_while_being_read() { + let root = fixture_root(); + write_fixture_file(root.path(), "game/a.txt", b"alpha"); + let mut previous = BTreeMap::new(); + previous.insert( + "game/a.txt".to_string(), + ProjectSnapshotIndexedFile { + size_bytes: 5, + modified_ms: 42, + checksum: "fnv1a64:00000000000000aa".to_string(), + }, + ); + // 扫描时刻的元数据与磁盘现状不一致：等价于"扫描之后文件被改写"。 + let scan = ProjectSnapshotScanResult { + files: vec![ProjectSnapshotScannedFile { + relative_path: "game/a.txt".to_string(), + size_bytes: 5, + modified_ms: 41, + absolute_path: root.path().join("game/a.txt"), + }], + skipped: Vec::new(), + }; + + let diff = compute_project_snapshot_diff(&scan, &previous, PROJECT_SNAPSHOT_MAX_SYNC_BYTES) + .expect("diff"); + assert!(diff.uploads.is_empty(), "{:?}", diff.uploads); + assert_eq!(diff.pending.len(), 1); + assert_eq!(diff.pending[0].relative_path, "game/a.txt"); + assert!( + diff.deleted.is_empty(), + "同步期间被改写的文件不能被当成删除，否则远端 GC 会误删" + ); + assert!( + diff.current.contains_key("game/a.txt"), + "应沿用上一轮记录，保持清单与远端对象一致" + ); +} + +#[test] +fn project_snapshot_upload_skips_files_that_changed_after_the_diff() { + let root = fixture_root(); + let mut candidate = fixture_candidate(root.path(), "game/a.txt", b"alpha"); + candidate.modified_ms = candidate.modified_ms.saturating_sub(1_000); + // 服务地址故意指向未监听端口：只要请求真的发出去就会出现 transport 失败， + // 因此这里同时证明"没有发出请求"和"分类是 file-changed"。 + let session = fixture_session("http://127.0.0.1:9".to_string()); + let diff = ProjectSnapshotDiff { + uploads: vec![candidate], + ..ProjectSnapshotDiff::default() + }; + + let report = + tauri::async_runtime::block_on(upload_project_snapshot_diff(&session, "project-1", &diff)); + assert!(report.uploaded_paths.is_empty()); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].code, "file-changed"); + assert!( + ProjectSnapshotUploadErrorKind::FileChanged.is_retryable(), + "文件被改写不是终态失败，下一次触发应重试" + ); +} + #[test] fn project_snapshot_upload_stops_after_a_deterministic_authentication_failure() { let root = fixture_root(); diff --git a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs index d24eb4f12..6ff4c226a 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/project_snapshot/transport.rs @@ -6,6 +6,10 @@ use std::collections::BTreeSet; pub(crate) enum ProjectSnapshotUploadErrorKind { Authentication, Permission, + /// 同步期间文件被改写：本轮跳过该文件，下一次重算即可，不是终态失败。 + FileChanged, + /// 服务端按配额或频率拒绝：等下一次触发再试。 + Throttled, Rejected, Upstream, Transport, @@ -16,6 +20,8 @@ impl ProjectSnapshotUploadErrorKind { match self { Self::Authentication => "authentication-required", Self::Permission => "permission-denied", + Self::FileChanged => "file-changed", + Self::Throttled => "throttled", Self::Rejected => "request-rejected", Self::Upstream => "upstream-failed", Self::Transport => "transport-failed", @@ -24,7 +30,10 @@ impl ProjectSnapshotUploadErrorKind { /// 鉴权、权限与请求被拒都是确定性失败；重复提交同样的内容不会变好。 pub(crate) fn is_retryable(self) -> bool { - matches!(self, Self::Upstream | Self::Transport) + matches!( + self, + Self::Upstream | Self::Transport | Self::FileChanged | Self::Throttled + ) } } @@ -166,7 +175,22 @@ async fn upload_project_snapshot_file( })?; if bytes.len() as u64 != candidate.size_bytes { return Err(ProjectSnapshotUploadError::new( - ProjectSnapshotUploadErrorKind::Rejected, + ProjectSnapshotUploadErrorKind::FileChanged, + format!( + "项目快照文件在同步期间发生变化：{}", + candidate.relative_path + ), + )); + } + // 上传前再复核一次元数据：只比长度会漏掉"改完又改回同样大小"的情况， + // 这种文件本轮不传，也不进入索引。 + if !project_snapshot_file_matches( + &candidate.absolute_path, + candidate.size_bytes, + candidate.modified_ms, + ) { + return Err(ProjectSnapshotUploadError::new( + ProjectSnapshotUploadErrorKind::FileChanged, format!( "项目快照文件在同步期间发生变化：{}", candidate.relative_path @@ -290,6 +314,8 @@ async fn project_snapshot_response_error( let kind = match status.as_u16() { 401 => ProjectSnapshotUploadErrorKind::Authentication, 403 => ProjectSnapshotUploadErrorKind::Permission, + // 服务端配额与频率闸门：等下一次触发，不做本轮内重试。 + 429 => ProjectSnapshotUploadErrorKind::Throttled, code if (400..500).contains(&code) => ProjectSnapshotUploadErrorKind::Rejected, _ => ProjectSnapshotUploadErrorKind::Upstream, }; diff --git a/apps/ai-game-creator-shell/src/App.tsx b/apps/ai-game-creator-shell/src/App.tsx index be33bfe93..85ddd7052 100644 --- a/apps/ai-game-creator-shell/src/App.tsx +++ b/apps/ai-game-creator-shell/src/App.tsx @@ -275,6 +275,7 @@ import { resolveChatProjectPath, resolvePendingCommandProjectPath, } from './features/project-workspace/projectCommandPolicy'; +import { ProjectSnapshotDialog } from './features/project-workspace/ProjectSnapshotDialog'; import { handleProjectSummaryChatCommand } from './features/project-workspace/projectSummaryCommands'; import { ProjectSupervisorView } from './features/project-workspace/ProjectSupervisorView'; import { ProjectWorkspaceChatPane } from './features/project-workspace/ProjectWorkspaceChatPane'; @@ -1919,6 +1920,7 @@ export function App({ const [professionalAgentResultsById, setProfessionalAgentResultsById] = useState>({}); const [runtimeConfigOpen, setRuntimeConfigOpen] = useState(false); + const [projectSnapshotOpen, setProjectSnapshotOpen] = useState(false); const [llmConfigStatus, setLlmConfigStatus] = useState(null); const [workspaceStatus, setWorkspaceStatus] = useState( @@ -13259,6 +13261,7 @@ export function App({ handleRevealCurrentProjectDirectory } handleRuntimeConfigOpen={handleRuntimeConfigOpen} + handleProjectSnapshotOpen={() => setProjectSnapshotOpen(true)} hiddenConversationCount={hiddenConversationCount} hasEarlierConversationMessages={hasEarlierConversationMessages} llmConfigStatus={llmConfigStatus} @@ -13307,6 +13310,13 @@ export function App({ /> ) : null} + {projectSnapshotOpen ? ( + setProjectSnapshotOpen(false)} + /> + ) : null} + {selectedAgent ? ( ; } +/** 项目快照同步中被跳过、延后或失败的单个路径。 */ +export interface LocalProjectSnapshotFailure { + relativePath: string; + code: string; + detail: string; +} + +export interface LocalProjectSnapshotState { + projectId: string; + indexPath: string; + indexPresent: boolean; + fileCount: number; + syncRevision: number; + syncedAtMs: number; + enabled: boolean; +} + +export interface LocalProjectSnapshotSyncResult { + projectId: string; + trigger: string; + status: string; + syncRevision: number; + uploadedFiles: number; + uploadedBytes: number; + remoteSkippedFiles: number; + deletedFiles: number; + deferredFiles: number; + metadataOnlyFiles: number; + skippedFiles: LocalProjectSnapshotFailure[]; + pendingFiles: LocalProjectSnapshotFailure[]; + failedFiles: LocalProjectSnapshotFailure[]; + syncedAtMs: number; +} + export interface LocalProjectCheckpointResult { checkpointId: string; checkpointPath: string; diff --git a/apps/ai-game-creator-shell/src/features/project-workspace/ProjectSnapshotDialog.tsx b/apps/ai-game-creator-shell/src/features/project-workspace/ProjectSnapshotDialog.tsx new file mode 100644 index 000000000..d1b5a27d6 --- /dev/null +++ b/apps/ai-game-creator-shell/src/features/project-workspace/ProjectSnapshotDialog.tsx @@ -0,0 +1,222 @@ +import { CloudUpload, Loader2, RefreshCw } from 'lucide-react'; +import { useCallback, useEffect, useState } from 'react'; +import { createPortal } from 'react-dom'; + +import { + closeDialogOnBackdropMouseDown, + closeDialogOnEscape, + useEscapeToClose, +} from '../../app/dialogs'; +import { resolveTauriInvoke } from '../../app/tauri'; +import type { + LocalProjectSnapshotFailure, + LocalProjectSnapshotState, + LocalProjectSnapshotSyncResult, +} from '../../app/types'; + +type ProjectSnapshotDialogProps = { + projectPath: string | null; + onClose: () => void; +}; + +const SNAPSHOT_STATUS_LABELS: Record = { + synced: '已同步', + 'no-op': '无改动', + partial: '部分成功', + failed: '同步失败', +}; + +function syncStatusLabel(status: string) { + return SNAPSHOT_STATUS_LABELS[status] ?? status; +} + +function formatSyncTime(value: number) { + if (!value) return '尚无记录'; + return new Date(value).toLocaleString('zh-CN', { hour12: false }); +} + +function describePending( + label: string, + entries: LocalProjectSnapshotFailure[], +): string | null { + const first = entries.at(0); + if (!first) return null; + return entries.length === 1 + ? `${label}：${first.relativePath}（${first.detail}）` + : `${label}：${entries.length} 个，例如 ${first.relativePath}（${first.detail}）`; +} + +/** + * 项目快照面板：显示本机索引状态与最近一次同步结果，并提供手动触发入口。 + */ +export function ProjectSnapshotDialog({ + projectPath, + onClose, +}: ProjectSnapshotDialogProps) { + const [state, setState] = useState(null); + const [result, setResult] = useState( + null, + ); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + const [loading, setLoading] = useState(false); + + const refreshState = useCallback(async () => { + const invoke = resolveTauriInvoke(); + const trimmed = projectPath?.trim(); + if (!invoke || !trimmed) { + setState(null); + return; + } + setLoading(true); + try { + const next = await invoke( + 'read_local_project_snapshot_state', + { projectPath: trimmed }, + ); + setState(next); + setError(null); + } catch (readError) { + setState(null); + setError( + readError instanceof Error ? readError.message : String(readError), + ); + } finally { + setLoading(false); + } + }, [projectPath]); + + useEffect(() => { + void refreshState(); + }, [refreshState]); + + useEscapeToClose(onClose, !busy); + + async function runSync() { + const invoke = resolveTauriInvoke(); + const trimmed = projectPath?.trim(); + if (!invoke || !trimmed) { + setError('需要在 Tauri App 内打开项目'); + return; + } + setBusy(true); + setError(null); + try { + setResult( + await invoke( + 'sync_local_project_snapshot', + { projectPath: trimmed }, + ), + ); + await refreshState(); + } catch (syncError) { + setResult(null); + setError( + syncError instanceof Error ? syncError.message : String(syncError), + ); + } finally { + setBusy(false); + } + } + + const details = result + ? [ + describePending('失败', result.failedFiles), + describePending('本次跳过', result.pendingFiles), + describePending('未纳入同步', result.skippedFiles), + ].filter((value): value is string => Boolean(value)) + : []; + + return createPortal( +
{ + if (busy) return; + closeDialogOnBackdropMouseDown(event, onClose); + }} + > +
{ + if (busy) return; + closeDialogOnEscape(event, onClose); + }} + > +

项目快照

+ {error ? ( +

+ {error} +

+ ) : null} +
+
上传状态
+
{state?.enabled === false ? '已停用' : '已启用'}
+
上次同步
+
{formatSyncTime(state?.syncedAtMs ?? 0)}
+
同步序号
+
{state?.syncRevision ?? 0}
+
已纳管文件
+
{state?.fileCount ?? 0}
+
本地索引
+
{state?.indexPresent ? '已写入' : '尚未写入'}
+ {result ? ( + <> +
最近一次
+
{syncStatusLabel(result.status)}
+
上传/字节
+
+ {result.uploadedFiles} 个 / {result.uploadedBytes} 字节 +
+
远端已有
+
{result.remoteSkippedFiles} 个
+
删除/延后
+
+ {result.deletedFiles} / {result.deferredFiles} +
+ + ) : null} +
+ {details.map((line) => ( +

+ {line} +

+ ))} +
+ + + +
+
+
, + document.body, + ); +} diff --git a/apps/ai-game-creator-shell/src/features/project-workspace/ProjectWorkspaceChatPane.tsx b/apps/ai-game-creator-shell/src/features/project-workspace/ProjectWorkspaceChatPane.tsx index 05e7c1056..f02073226 100644 --- a/apps/ai-game-creator-shell/src/features/project-workspace/ProjectWorkspaceChatPane.tsx +++ b/apps/ai-game-creator-shell/src/features/project-workspace/ProjectWorkspaceChatPane.tsx @@ -156,6 +156,7 @@ type ProjectWorkspaceChatPaneProps = { ) => Promise; handleRevealCurrentProjectDirectory: () => Promise; handleRuntimeConfigOpen: () => void; + handleProjectSnapshotOpen: () => void; hiddenConversationCount: number; hasEarlierConversationMessages?: boolean; llmConfigStatus: GameCreatorLlmConfigStatus | null; @@ -256,6 +257,7 @@ export function ProjectWorkspaceChatPane({ handleProjectSupervisorUserInput, handleRevealCurrentProjectDirectory, handleRuntimeConfigOpen, + handleProjectSnapshotOpen, hiddenConversationCount, hasEarlierConversationMessages = false, llmConfigStatus, @@ -322,6 +324,13 @@ export function ProjectWorkspaceChatPane({ + diff --git a/apps/ai-game-creator-shell/src/styles.css b/apps/ai-game-creator-shell/src/styles.css index 5b0352492..c9a192ae0 100644 --- a/apps/ai-game-creator-shell/src/styles.css +++ b/apps/ai-game-creator-shell/src/styles.css @@ -4979,6 +4979,52 @@ h2 { gap: 6px; } +.project-snapshot-dialog { + display: grid; + gap: 12px; +} + +.project-snapshot-dialog-error { + color: #b45309; +} + +.project-snapshot-dialog-facts { + display: grid; + grid-template-columns: auto 1fr; + gap: 6px 16px; + margin: 0; + font-size: 13px; +} + +.project-snapshot-dialog-facts dt { + color: #6b7280; +} + +.project-snapshot-dialog-facts dd { + margin: 0; + overflow-wrap: anywhere; +} + +.project-snapshot-dialog-detail { + margin: 0; + color: #6b7280; + font-size: 12px; + overflow-wrap: anywhere; +} + +.project-snapshot-dialog .launcher-dialog-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; + justify-content: flex-end; +} + +.project-snapshot-dialog .launcher-dialog-actions button { + display: inline-flex; + align-items: center; + gap: 6px; +} + .resource-reference-picker-scopes { padding: 0 12px 4px; } diff --git a/apps/ai-game-creator-shell/tests/projectSnapshotDialog.test.tsx b/apps/ai-game-creator-shell/tests/projectSnapshotDialog.test.tsx new file mode 100644 index 000000000..cb4b1811f --- /dev/null +++ b/apps/ai-game-creator-shell/tests/projectSnapshotDialog.test.tsx @@ -0,0 +1,117 @@ +/** @vitest-environment jsdom */ + +import { cleanup, render, screen, waitFor } from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { ProjectSnapshotDialog } from '../src/features/project-workspace/ProjectSnapshotDialog'; + +const PROJECT_PATH = 'C:\\games\\demo'; + +const state = { + projectId: 'gameagent-033b6cf397094f3e8d4e48c380ff4629', + indexPath: 'C:\\AppData\\project-snapshots\\gameagent\\index.json', + indexPresent: true, + fileCount: 6, + syncRevision: 2, + syncedAtMs: 1_789_629_339_000, + enabled: true, +}; + +const syncResult = { + projectId: state.projectId, + trigger: 'manual', + status: 'synced', + syncRevision: 3, + uploadedFiles: 1, + uploadedBytes: 2048, + remoteSkippedFiles: 0, + deletedFiles: 0, + deferredFiles: 0, + metadataOnlyFiles: 0, + skippedFiles: [], + pendingFiles: [], + failedFiles: [], + syncedAtMs: 1_789_629_400_000, +}; + +function installInvoke( + implementation: (command: string, args?: Record) => unknown, +) { + const invoke = vi.fn(implementation); + (window as unknown as { __TAURI__?: unknown }).__TAURI__ = { + core: { invoke }, + }; + return invoke; +} + +describe('ProjectSnapshotDialog', () => { + afterEach(() => { + cleanup(); + delete (window as unknown as { __TAURI__?: unknown }).__TAURI__; + }); + + it('打开时读取本机索引状态并显示同步序号与文件数', async () => { + const invoke = installInvoke((command) => { + if (command === 'read_local_project_snapshot_state') return state; + throw new Error(`unexpected command: ${command}`); + }); + + render( + , + ); + + expect(await screen.findByText('2')).toBeTruthy(); + expect(screen.getByText('6')).toBeTruthy(); + expect(screen.getByText('已写入')).toBeTruthy(); + expect(invoke).toHaveBeenCalledWith('read_local_project_snapshot_state', { + projectPath: PROJECT_PATH, + }); + }); + + it('手动同步后显示上传计数与状态，并刷新索引状态', async () => { + let stateReads = 0; + const invoke = installInvoke((command) => { + if (command === 'read_local_project_snapshot_state') { + stateReads += 1; + return state; + } + if (command === 'sync_local_project_snapshot') return syncResult; + throw new Error(`unexpected command: ${command}`); + }); + + render( + , + ); + await waitFor(() => expect(stateReads).toBe(1)); + + await userEvent.click(screen.getByRole('button', { name: /立即同步/u })); + + expect(await screen.findByText('已同步')).toBeTruthy(); + expect(screen.getByText('1 个 / 2048 字节')).toBeTruthy(); + await waitFor(() => expect(stateReads).toBe(2)); + expect(invoke).toHaveBeenCalledWith('sync_local_project_snapshot', { + projectPath: PROJECT_PATH, + }); + }); + + it('同步失败时显示错误而不是伪造成功状态', async () => { + const invoke = installInvoke((command) => { + if (command === 'read_local_project_snapshot_state') return state; + if (command === 'sync_local_project_snapshot') { + throw new Error('authentication-required: 请先登录陶泥儿账号'); + } + throw new Error(`unexpected command: ${command}`); + }); + + render( + , + ); + await waitFor(() => expect(invoke).toHaveBeenCalled()); + + await userEvent.click(screen.getByRole('button', { name: /立即同步/u })); + + expect(await screen.findByText(/authentication-required/u)).toBeTruthy(); + expect(screen.queryByText('已同步')).toBeNull(); + }); +}); diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 9b508dac1..eefb1ae55 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -162,6 +162,16 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS=600 ALIYUN_OSS_POST_MAX_SIZE_BYTES=20971520 ALIYUN_OSS_SUCCESS_ACTION_STATUS=200 +# AGC 项目定时快照上传目标。对象只落在服务端私有前缀 +# agc/project-snapshots/v1/{user}/{project}/ 下；AccessKey 为空时回退 ALIYUN_OSS_ACCESS_KEY_*， +# 因此回退凭据必须对目标 bucket 具备该前缀的 PutObject/GetObject/DeleteObject 权限。 +# bucket 未单独配置时默认 agc-dev，未配置凭据时 api-server 跳过该客户端， +# 接口返回 503 且客户端失败关闭（不写空对象、不推进本地索引）。 +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET=agc-dev +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ENDPOINT=oss-rg-china-mainland.aliyuncs.com +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID= +GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET= + # SpacetimeDB 数据目录 OSS 冷备份配置。可由 cron / Jenkins 调用发布包内 scripts/database-backup-to-oss.mjs。 GENARRATIVE_DATABASE_BACKUP_DATA_DIR=/stdb GENARRATIVE_DATABASE_BACKUP_WORK_DIR=/var/lib/genarrative/database-backups diff --git a/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md b/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md index 2f2135058..24310e7b3 100644 --- a/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md +++ b/docs/project-memory/plans/【里程碑】AGC项目定时快照上传-2026-09-17.md @@ -22,9 +22,10 @@ AGC 在项目打开期间按周期把用户项目增量上传到 OSS `agc-dev` ## 不做 - 不做云端下载/恢复、跨设备合并、版本回滚。 -- 不做远端多余对象清理与生命周期策略下发。 +- 不保留多版本历史：清单写入成功后回收不再被引用的旧对象，同一路径只保留当前内容。 +- 不下发 bucket 生命周期策略；不做跨节点的用户级总量配额与计费口径。 - 不新增 SpacetimeDB 表或 procedure，不修改 `/api/external/v1` 与 External OpenAPI。 -- 不新增面向用户的上传设置面板与进度 UI。 +- 不新增上传设置项与进度条；界面只提供状态与手动触发。 ## 验收标准 @@ -37,6 +38,10 @@ AGC 在项目打开期间按周期把用户项目增量上传到 OSS `agc-dev` 7. 工作区窗口关闭与应用退出都会触发一次同步，且关闭路径不因同步失败而阻塞退出超过超时上限。 8. 服务端拒绝越界 `projectId`、相对路径与摘要；相同摘要重复提交走跳过分支且不写入新对象。 9. 新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。 +10. 清单写入成功后，上一版清单里不再被引用的对象被回收；上一版清单不可读时整轮不删除任何对象。 +11. 单项目超过 2 GiB 时客户端明确失败、服务端按 413 拒绝；超过服务端小时配额或 5 秒最小间隔时返回 429 且带 `Retry-After`。 +12. 同步期间被改写的文件既不上传也不推进索引，沿用上一轮记录，且不会被误判成删除。 +13. 工作区「项目快照」面板能显示上传状态、上次同步时间、同步序号与已纳管文件数，并能手动触发同步与显示失败原因。 ## 依赖 diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index d92740c0b..c20eafbc4 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -1529,8 +1529,8 @@ Direct 回合的所有权属于进程内项目身份锁，不属于当前页面 ### 目标与非目标 -- 目标：AGC 在项目工作区打开期间按固定周期把用户项目增量上传到 OSS `agc-dev`，并在项目关闭（工作区窗口关闭、切回启动器、应用退出）时立即补一次同步；重复内容不重复上传。 -- 非目标：不做云端下载/恢复、不做跨设备合并、不做远端多余对象清理、不新增 UI 面板、不修改 `/api/external/v1` 与 OpenAPI、不新增 SpacetimeDB 表。 +- 目标：AGC 在项目工作区打开期间按固定周期把用户项目增量上传到 OSS `agc-dev`，并在项目关闭时立即补一次同步；重复内容不重复上传，远端占用跟随当前清单收敛，用户可在项目里查看同步状态并手动触发。 +- 非目标：不做云端下载/恢复、不做跨设备合并、不保留多版本历史、不修改 `/api/external/v1` 与 OpenAPI、不新增 SpacetimeDB 表。 - 非目标：不把 OSS AccessKey 放进客户端；客户端不直连 OSS。 ### 参与入口、状态与跨模块边界 @@ -1538,7 +1538,8 @@ Direct 回合的所有权属于进程内项目身份锁，不属于当前页面 - 触发入口有两个：工作区窗口 `main` 存活期间的周期定时器、工作区窗口关闭事件（`CloseRequested`）。两者共用同一个进程内同步器，同一项目的同步串行执行，周期触发在已有同步进行时直接让位，不排队堆积。 - 应用退出（`RunEvent::Exit`）不重复发起同步：该时刻窗口已销毁，按窗口重新枚举项目只会得到空集；退出路径只负责在有界预算（15 秒）内等待在途同步收尾，让关窗触发的那一次同步能写完索引再退出。 - 客户端扫描、差异对比、索引持久化与上传编排都在 Tauri Rust 进程（`src-tauri/src/project_snapshot/`）；WebView 只读状态，不参与差异计算。 -- 本地索引是增量对比的唯一依据：`/project-snapshots//index.json` 保存上次成功同步的相对路径、`sha256`、字节数和修改时间。项目根使用现有 manifest 的稳定 `project_id` 作为远端身份，路径不再作为身份。 +- 本地索引是增量对比的唯一依据：`/project-snapshots//index.json` 保存上次成功同步的相对路径、校验和、字节数和修改时间。项目根使用现有 manifest 的稳定 `project_id` 作为远端身份，路径不再作为身份。 +- 界面入口：工作区聊天头部的「项目快照」按钮打开独立面板（`ProjectSnapshotDialog`），显示上传状态、上次同步时间、同步序号、已纳管文件数、本地索引是否写入，以及最近一次同步的上传/远端已有/删除/延后计数与失败明细；面板提供「刷新」与「立即同步」。面板不写功能说明文案，只用标签与数值。 - 远端写入经 `api-server`，客户端只持平台登录态 Access Token。两条登录态路由：`POST /api/agc/project-snapshots/files`（单文件，正文为原始字节，元数据走查询串）与 `POST /api/agc/project-snapshots/manifest`（本次同步后的完整清单）。 - 对象键与清单由服务端决定：文件键为 `agc/project-snapshots/v1/{userId}/{projectId}/files/{sizeBytes}-{checksumDigest}/{relPath}`，清单键为 `agc/project-snapshots/v1/{userId}/{projectId}/manifest.json`。键里带字节数与摘要，因此"对象已存在且长度一致"可以作为内容一致的判据；路径按原始大小写保留，不走 `put_object` 的低位规范化。`agc` 前缀继续是服务端专用私有前缀，通用对象键解析与客户端直传票据都不覆盖它。 - 目标 bucket 使用独立配置 `GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_BUCKET` / `_ENDPOINT` / `_ACCESS_KEY_ID` / `_ACCESS_KEY_SECRET`，默认 `agc-dev` + `oss-rg-china-mainland.aliyuncs.com`，未配置时回退 `ALIYUN_OSS_*`；与"资源 bucket 与备份 bucket 分离"的既有口径一致。 @@ -1546,27 +1547,33 @@ Direct 回合的所有权属于进程内项目身份锁，不属于当前页面 ### 正常、失败、重试与幂等行为 - 差异对比口径：先按 `相对路径 + 字节数 + 修改时间` 判定是否候选变更，命中旧记录则复用已存 `sha256`，只有 `(size, mtime)` 变化才重算摘要。产出新增、修改、删除三类集合，只上传新增与修改的文件。 -- 每次成功同步的最后一步上传该项目的 `manifest.json`（当前全量文件清单：相对路径、摘要、字节数、同步序号）。删除文件只在清单中消失，本期不删除远端对象；远端清理留给后续里程碑。 +- 每次成功同步的最后一步上传该项目的 `manifest.json`（当前全量文件清单：相对路径、摘要、字节数、同步序号）。清单描述的是项目当前全量内容，因此清单体积就是该项目在 OSS 上的常驻占用。 +- 远端回收：清单写入成功后，服务端读取上一版清单，按 `(路径, 字节数, 摘要)` 反推出不再被当前清单引用的对象键并删除。只处理上一版清单登记过的键，不做 LIST，因此不可能误删其它项目或其它功能的对象；单次最多回收 2000 个对象，剩余部分留到下一次清单写入继续；上一版清单读不到或解析失败时整轮跳过回收（fail-closed）。单个删除失败只记日志，不影响本次同步语义。 +- 因此本功能是"当前状态镜像 + 清单"，不保留历史版本：同一路径的内容变化会覆盖式替换远端对象，回滚能力不在本轮范围内。 - 幂等：同一摘要与字节数的对象重复提交由服务端 HEAD 校验后跳过；探测失败按"未存在"处理并照常 PUT，宁可多传一次也不漏传。索引只在清单写入成功后推进，失败时保留旧索引以便下次重算。 -- 失败关闭：单个文件失败不推进整次同步的完成位，失败文件与剩余文件在下一次周期或下次关闭时重试。鉴权失败（401/403）、权限、额度与身份类失败不做自动重试，只记录分类结果并等待用户重新登录后的下一次触发。 +- 与项目写锁解耦：同步不持有项目写锁，也不阻塞 Agent 写入。读取摘要后与上传前各按 `(字节数, 修改时间)` 复核一次，任一处不一致就判定该文件"同步期间发生变化"：本轮不上传、不写入索引；若该路径上一轮已同步过则沿用旧记录，避免被误判成删除而触发远端回收。这类文件在下一个周期或下次关窗时重算重试。 +- 失败关闭：单个文件失败不推进该文件的索引项，失败文件与剩余文件在下一次周期或下次关闭时重试。鉴权失败（401/403）与格式类拒绝（400/413）是确定性失败，停止本轮剩余请求并等待用户处理后重试；服务端配额或频率拒绝（429）与传输类失败按可重试处理。 +- 配额与限流：单文件 64 MiB、单次同步上传预算 512 MiB（超出部分延后到下一次）、单项目常驻上限 2 GiB（客户端在扫描后先判，超限直接给出明确失败；服务端按清单累计体积复核并返回 413）；服务端按用户做进程内小时配额（文件 3000 次、清单 120 次）并对同一项目强制 5 秒最小清单间隔，超限返回 429 且带 `Retry-After`。进程内配额只用于抑制异常客户端与失控重试，跨节点配额由"单项目上限 + 清单引用回收"保证。 - 生命周期：同步有界超时（单文件与整次同步分别设上限），项目关闭与应用退出路径不因同步失败而阻塞或延迟退出超过超时上限。 - 上传内容边界：复用项目索引与 checkpoint 同一份 `should_skip_project_snapshot_path` 口径——整个 `.agent`（含 runtime、logs、checkpoint、manifest、project.lock）、版本控制目录、`node_modules`/`target`/`dist`/`build`/`coverage`/`.cache`、凭据目录与 `.pem`/`.key` 等敏感后缀都不参与同步；符号链接与重解析点同样跳过。单文件（64 MiB）与单次同步总量（512 MiB）各有上限，超限文件进入跳过或延后清单而不是静默丢弃。 ### 契约与兼容 - 新增登录态内部路由 `POST /api/agc/project-snapshots/files`，请求 DTO 放在 `shared-contracts`；不属于 `/api/external/v1`，因此不更新 External OpenAPI，与 `/api/error-reports` 同类。 -- 服务端校验 `projectId` 形态（拒绝路径分隔符、`..`、控制字符与超长值）、相对路径规范（正斜杠、拒绝绝对路径与穿越）、摘要形态（64 位十六进制）和字节数上限，任何越界返回 4xx 而不是写入 OSS。 +- 服务端校验 `projectId` 形态（拒绝路径分隔符、`..`、控制字符与超长值）、相对路径规范（正斜杠、拒绝绝对路径与穿越）、摘要形态（`fnv1a64:` + 16 位十六进制）和字节数上限，任何越界返回 4xx 而不是写入 OSS。 - 不改变客户端与 Runner 的本机协议、平台会话语义、项目写锁与 manifest 结构；新增索引文件位于 AppData，不进入用户项目目录。 ### 验收标准与证据来源 - 定向 Rust 测试：首次同步全量、仅改一个文件时只产生一个修改项、删除文件只体现在清单、`(size,mtime)` 未变时复用旧摘要、排除规则与上限跳过、同步失败不推进索引、同一项目并发触发串行化。 -- 服务端测试：越界 `projectId`/相对路径/摘要被拒；相同摘要重复提交走跳过分支；鉴权缺失返回 401；OSS 未配置返回明确的 5xx 而不是写入空对象。 +- 服务端测试：越界 `projectId`/相对路径/摘要被拒；相同摘要重复提交走跳过分支；超过单项目上限返回 413；超过用户小时配额返回 429；鉴权缺失返回 401；OSS 未配置返回明确的 5xx 而不是写入空对象。 +- 前端测试：面板打开即读取索引状态并显示同步序号与文件数；手动同步后显示上传计数并刷新状态；同步失败显示错误而不是伪造成功。 - 运行时 smoke：AGC 开发态打开项目、观察索引写入与同步日志、关闭工作区窗口后确认关闭触发的那次同步执行；报告为"客户端 diff 已验证 / 服务端已配置环境联调"两层，不合并成一句"已通"。 - 边界：新增日志与错误文案不含 Access Token、AccessKey、绝对路径与项目内容。 ### 未决问题 -- 远端删除对象清理、配额与保留策略未定；本期只写清单，OSS 侧对象只增不减。 -- 目标 bucket 的私有前缀权限与生命周期规则（例如转低频/过期删除）需要在部署环境确认后单独收口。 -- 大项目（素材数量多、单文件大）的首轮全量上传耗时与带宽占用未实测；必要时后续里程碑引入并发上限与断点续传。 +- 历史版本：本轮只保留"当前状态镜像 + 清单"，旧内容对象在清单写入成功后即被回收，没有回滚能力；要保留历史版本需要先定"保留几个 revision + 由谁回收"的策略。 +- 用户级配额：跨节点的用户总量配额与计费口径未定；当前用单项目 2 GiB 上限 + 清单引用回收保证常驻占用有界，用户级总量只能靠项目数间接约束。 +- 目标 bucket 的生命周期规则（例如转低频/过期删除）需要在部署环境确认后单独收口；功能本身已不再依赖它来控制增长。 +- 大项目（素材数量多、单文件大）的首轮全量上传耗时与带宽占用未实测；单次预算 512 MiB 会把超出部分留到下一次同步，但并发上限与断点续传仍未引入。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 6b60d6bb2..88f047068 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -555,9 +555,16 @@ GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_SECRET= ``` 专用凭据为空时回退 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，bucket 与 endpoint -仍默认指向 AGC 发行 bucket，因此回退凭据必须具备目标 bucket 该前缀的 `PutObject` 权限；`api-server` -启动时凭据缺失或只配一半会跳过该客户端，接口返回 `503`，客户端按失败关闭处理：不写空对象，也不推进 -本地增量索引，下一次触发重算重试。远端对象只增不减，删除与生命周期规则尚未落地，需要单独收口。 +仍默认指向 AGC 发行 bucket，因此回退凭据必须具备目标 bucket 该前缀的 `PutObject` / `GetObject` / +`DeleteObject` 权限；`api-server` 启动时会打印一行 `AGC 项目快照 OSS 客户端已启用`（含 bucket、 +endpoint 与凭据来源，不含密钥），凭据缺失或只配一半则跳过该客户端、接口返回 `503`，客户端按失败关闭 +处理：不写空对象，也不推进本地增量索引，下一次触发重算重试。 + +远端占用按当前清单收敛：每次清单写入成功后，服务端用上一版清单反推不再被引用的对象键并删除，单次最多 +回收 2000 个，上一版清单不可读时整轮跳过（不会误删）。因此不需要额外配置 bucket 生命周期来防止无界 +增长；`agc/project-snapshots/v1/` 下同一路径只保留当前内容，历史版本不保留。配额口径：单文件 64 MiB、 +单次同步上传预算 512 MiB、单项目常驻 2 GiB；超限分别表现为跳过/延后/413。服务端另有进程内小时配额 +（文件 3000 次、清单 120 次）与同一项目 5 秒最小清单间隔，超限返回 `429` 并带 `Retry-After`。 两层可重复的现场验证： diff --git a/server-rs/crates/api-server/src/project_snapshots.rs b/server-rs/crates/api-server/src/project_snapshots.rs index cdbba94cb..d745d7493 100644 --- a/server-rs/crates/api-server/src/project_snapshots.rs +++ b/server-rs/crates/api-server/src/project_snapshots.rs @@ -14,19 +14,24 @@ use axum::{ http::StatusCode, }; use platform_oss::{ - OssInternalPutObjectRequest, OssObjectAccess, agc_project_snapshot_file_object_key, - agc_project_snapshot_manifest_object_key, + OssDeleteObjectRequest, OssGetObjectRequest, OssInternalPutObjectRequest, OssObjectAccess, + agc_project_snapshot_file_object_key, agc_project_snapshot_manifest_object_key, }; use serde_json::Value; use shared_contracts::agc_project_snapshots::{ - AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES, AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_BYTES, - AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES, AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION, + AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES, AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES, + AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES, AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION, AgcProjectSnapshotFileUploadQuery, AgcProjectSnapshotFileUploadResponse, AgcProjectSnapshotManifestRequest, AgcProjectSnapshotManifestResponse, validate_agc_project_snapshot_checksum, validate_agc_project_snapshot_project_id, validate_agc_project_snapshot_relative_path, }; -use std::collections::HashSet; +use std::{ + collections::{HashMap, HashSet}, + sync::{Mutex, OnceLock}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tracing::{debug, info, warn}; /// 单文件请求体上限比文件上限留一点余量，超限请求由 body limit 直接拒绝。 pub(crate) const MAX_FILE_REQUEST_BODY_BYTES: usize = @@ -35,6 +40,14 @@ pub(crate) const MAX_FILE_REQUEST_BODY_BYTES: usize = pub(crate) const MAX_MANIFEST_REQUEST_BODY_BYTES: usize = 32 * 1024 * 1024; /// 清单里单条路径的字段长度上限（与契约校验口径一致）。 const MAX_MANIFEST_PATH_CHARS: usize = 1024; +/// 单个用户每小时允许的文件上传次数。进程内计数，用于抑制异常客户端，不是计费级配额。 +const MAX_FILE_UPLOADS_PER_USER_PER_HOUR: usize = 3_000; +/// 单个用户每小时允许的清单写入次数。 +const MAX_MANIFEST_UPLOADS_PER_USER_PER_HOUR: usize = 120; +/// 同一项目两次清单写入的最小间隔，避免异常客户端高频覆盖清单。 +const MIN_MANIFEST_INTERVAL_MS: u64 = 5_000; +/// 单次清单写入最多回收多少个不再被引用的对象，避免一次请求做过量删除。 +const MAX_OBJECTS_RECLAIMED_PER_MANIFEST: usize = 2_000; /// 写入一次增量同步里的单个文件。 /// @@ -47,6 +60,7 @@ pub async fn upload_project_snapshot_file( Query(query): Query, body: Bytes, ) -> Result, AppError> { + consume_user_upload_quota(auth.claims().user_id(), ProjectSnapshotUploadKind::File)?; validate_agc_project_snapshot_project_id(&query.project_id).map_err(bad_request)?; validate_agc_project_snapshot_relative_path(&query.relative_path).map_err(bad_request)?; validate_agc_project_snapshot_checksum(&query.checksum).map_err(bad_request)?; @@ -121,10 +135,21 @@ pub async fn upload_project_snapshot_manifest( Extension(auth): Extension, Json(payload): Json, ) -> Result, AppError> { + consume_user_upload_quota(auth.claims().user_id(), ProjectSnapshotUploadKind::Manifest)?; validate_manifest(&payload)?; - let object_key = - agc_project_snapshot_manifest_object_key(auth.claims().user_id(), &payload.project_id) - .map_err(|error| bad_request(error.to_string()))?; + let user_id = auth.claims().user_id().to_string(); + let object_key = agc_project_snapshot_manifest_object_key(&user_id, &payload.project_id) + .map_err(|error| bad_request(error.to_string()))?; + // 上一版清单同时承担两个职责：项目级写入频率闸门，以及本轮远端对象回收的引用基线。 + // 读不到或解析失败时只跳过回收，绝不据此删除任何对象。 + let previous = read_project_snapshot_manifest(&state, &object_key).await; + if let Some(previous) = previous.as_ref() + && unix_millis_now().saturating_sub(previous.synced_at_ms) < MIN_MANIFEST_INTERVAL_MS + { + return Err(AppError::from_status(StatusCode::TOO_MANY_REQUESTS) + .with_message("同一项目的项目快照写入过于频繁，请稍后重试") + .with_header("retry-after", axum::http::HeaderValue::from_static("5"))); + } let body = serde_json::to_vec(&payload) .map_err(|error| internal(format!("序列化项目快照清单失败：{error}")))?; let total_bytes = payload @@ -146,6 +171,9 @@ pub async fn upload_project_snapshot_manifest( .map_err(|_| { AppError::from_status(StatusCode::BAD_GATEWAY).with_message("项目快照清单上传失败") })?; + // 清单写入成功之后再回收：任何时刻远端对象集合都是当前清单的超集， + // 不会出现清单引用了刚被删掉的对象。 + reclaim_unreferenced_objects(&state, oss, &user_id, previous.as_ref(), &payload).await; Ok(json_success_body( Some(&ctx), @@ -159,6 +187,174 @@ pub async fn upload_project_snapshot_manifest( )) } +/// 单个用户在窗口内的上传计数。进程内计数，用于抑制异常客户端与失控重试； +/// 跨节点配额由"单项目累计体积上限 + 清单引用回收"保证。 +#[derive(Clone, Copy, Default)] +struct ProjectSnapshotUserWindow { + started_at: Option, + files: usize, + manifests: usize, +} + +#[derive(Clone, Copy)] +enum ProjectSnapshotUploadKind { + File, + Manifest, +} + +static PROJECT_SNAPSHOT_USER_WINDOWS: OnceLock>> = + OnceLock::new(); + +fn consume_user_upload_quota( + user_id: &str, + kind: ProjectSnapshotUploadKind, +) -> Result<(), AppError> { + let windows = PROJECT_SNAPSHOT_USER_WINDOWS.get_or_init(|| Mutex::new(HashMap::new())); + let mut windows = windows + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let window = windows.entry(user_id.to_string()).or_default(); + let now = Instant::now(); + let expired = window + .started_at + .is_none_or(|started_at| now.duration_since(started_at) >= Duration::from_secs(3_600)); + if expired { + *window = ProjectSnapshotUserWindow { + started_at: Some(now), + ..ProjectSnapshotUserWindow::default() + }; + } + let (counter, limit) = match kind { + ProjectSnapshotUploadKind::File => (&mut window.files, MAX_FILE_UPLOADS_PER_USER_PER_HOUR), + ProjectSnapshotUploadKind::Manifest => ( + &mut window.manifests, + MAX_MANIFEST_UPLOADS_PER_USER_PER_HOUR, + ), + }; + if *counter >= limit { + return Err(AppError::from_status(StatusCode::TOO_MANY_REQUESTS) + .with_message("项目快照上传次数超出当前小时上限，请稍后重试") + .with_header("retry-after", axum::http::HeaderValue::from_static("60"))); + } + *counter += 1; + Ok(()) +} + +fn unix_millis_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +/// 读取该项目上一版清单。只在对象不存在时返回 `None`；读取或解析失败同样返回 +/// `None`，调用方据此跳过回收（fail-closed：不确定就不删）。 +async fn read_project_snapshot_manifest( + state: &AppState, + object_key: &str, +) -> Option { + let oss = project_snapshot_oss(state).ok()?; + match oss + .get_object( + state.editor_oss_http_client(), + OssGetObjectRequest { + object_key: object_key.to_string(), + max_bytes: MAX_MANIFEST_REQUEST_BODY_BYTES, + }, + ) + .await + { + Ok(bytes) => serde_json::from_slice::(&bytes) + .map_err(|error| { + warn!(object_key = %object_key, error = %error, "项目快照上一版清单解析失败，本轮跳过回收"); + error + }) + .ok(), + Err(error) => { + debug!(object_key = %object_key, error = %error, "项目快照上一版清单不可读，本轮跳过回收"); + None + } + } +} + +/// 回收不再被当前清单引用、且确实由上一版清单登记过的对象。 +/// +/// 只按上一版清单里的 `(路径, 字节数, 摘要)` 反推出确定的对象键，不做 LIST， +/// 因此不可能误删其它项目或其它功能的对象。任何单个删除失败都只记日志： +/// 清单已经写入成功，回收失败不影响本次同步的语义，下一次写入会再试。 +async fn reclaim_unreferenced_objects( + state: &AppState, + oss: &platform_oss::OssClient, + user_id: &str, + previous: Option<&AgcProjectSnapshotManifestRequest>, + next: &AgcProjectSnapshotManifestRequest, +) { + let Some(previous) = previous else { + return; + }; + let retained = next + .files + .iter() + .map(|file| { + ( + file.relative_path.as_str(), + file.size_bytes, + file.checksum.as_str(), + ) + }) + .collect::>(); + let mut reclaimed = 0_usize; + let mut skipped = 0_usize; + for file in &previous.files { + if retained.contains(&( + file.relative_path.as_str(), + file.size_bytes, + file.checksum.as_str(), + )) { + continue; + } + if reclaimed >= MAX_OBJECTS_RECLAIMED_PER_MANIFEST { + // 剩下的留给下一次清单写入继续回收，不在这里无限删除。 + skipped += 1; + continue; + } + let Some(digest) = file.checksum.strip_prefix("fnv1a64:") else { + continue; + }; + let Ok(object_key) = agc_project_snapshot_file_object_key( + user_id, + &previous.project_id, + file.size_bytes, + digest, + &file.relative_path, + ) else { + continue; + }; + match oss + .delete_object( + state.editor_oss_http_client(), + OssDeleteObjectRequest { + object_key: object_key.clone(), + }, + ) + .await + { + Ok(()) => reclaimed += 1, + Err(error) => { + warn!(object_key = %object_key, error = %error, "项目快照旧版本对象回收失败"); + } + } + } + if reclaimed > 0 || skipped > 0 { + info!( + project_id = %previous.project_id, + reclaimed, + skipped, + "项目快照旧版本对象回收完成" + ); + } +} + fn validate_manifest(payload: &AgcProjectSnapshotManifestRequest) -> Result<(), AppError> { if payload.schema_version != AGC_PROJECT_SNAPSHOT_SCHEMA_VERSION { return Err(bad_request("项目快照清单版本不受支持")); @@ -186,8 +382,11 @@ fn validate_manifest(payload: &AgcProjectSnapshotManifestRequest) -> Result<(), } total_bytes = total_bytes.saturating_add(file.size_bytes); } - if total_bytes > AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_BYTES { - return Err(bad_request("项目快照清单累计体积超过上限")); + // 清单描述的是项目当前全量文件，所以这里的累计体积就是该项目的常驻占用； + // 单项目上限同时约束了远端对象回收之后的实际存储量。 + if total_bytes > AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES { + return Err(AppError::from_status(StatusCode::PAYLOAD_TOO_LARGE) + .with_message("项目快照累计体积超过单项目上限")); } Ok(()) } @@ -284,4 +483,35 @@ mod tests { .collect::>(); assert!(validate_manifest(&manifest(files)).is_err()); } + + #[test] + fn project_snapshot_manifest_rejects_a_project_over_the_total_cap() { + // 单文件与条目数都在上限内，只有项目累计体积越界：这必须按 413 拒绝， + // 否则远端回收之后仍会长期占住超大配额。 + let files = (0..40) + .map(|index| { + manifest_file( + &format!("assets/blob-{index}.bin"), + AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES, + ) + }) + .collect::>(); + let error = validate_manifest(&manifest(files)).expect_err("超额项目必须被拒绝"); + assert_eq!(error.status_code(), StatusCode::PAYLOAD_TOO_LARGE); + } + + #[test] + fn project_snapshot_user_quota_stops_after_the_hourly_file_limit() { + let user = "user-quota-fixture"; + for _ in 0..MAX_FILE_UPLOADS_PER_USER_PER_HOUR { + consume_user_upload_quota(user, ProjectSnapshotUploadKind::File) + .expect("配额内的上传必须放行"); + } + let error = consume_user_upload_quota(user, ProjectSnapshotUploadKind::File) + .expect_err("超出小时上限必须被拒绝"); + assert_eq!(error.status_code(), StatusCode::TOO_MANY_REQUESTS); + // 清单与文件是两条独立配额，不受文件配额影响。 + consume_user_upload_quota(user, ProjectSnapshotUploadKind::Manifest) + .expect("清单配额独立计数"); + } } diff --git a/server-rs/crates/api-server/src/state.rs b/server-rs/crates/api-server/src/state.rs index 58b6819f9..91c58792f 100644 --- a/server-rs/crates/api-server/src/state.rs +++ b/server-rs/crates/api-server/src/state.rs @@ -2362,6 +2362,21 @@ fn build_project_snapshot_oss_client( config.oss_post_max_size_bytes, config.oss_success_action_status, )?; + let credential_source = if config + .project_snapshot_oss_access_key_id + .as_deref() + .is_some_and(|value| !value.trim().is_empty()) + { + "GENARRATIVE_AGC_PROJECT_SNAPSHOT_OSS_ACCESS_KEY_ID" + } else { + "ALIYUN_OSS_ACCESS_KEY_ID" + }; + info!( + bucket = %bucket, + endpoint = %endpoint, + credential_source, + "AGC 项目快照 OSS 客户端已启用" + ); Ok(Some(OssClient::new(oss_config))) } diff --git a/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs b/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs index 97ada43fa..7bc6c7dde 100644 --- a/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs +++ b/server-rs/crates/shared-contracts/src/agc_project_snapshots.rs @@ -14,8 +14,13 @@ pub const AGC_PROJECT_SNAPSHOT_MAX_FILE_BYTES: u64 = 64 * 1024 * 1024; /// 单次清单包含的文件数量上限，防止越界请求把服务端内存拖垮。 pub const AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_FILES: usize = 20_000; -/// 单次清单包含的累计字节上限。 -pub const AGC_PROJECT_SNAPSHOT_MAX_MANIFEST_BYTES: u64 = 512 * 1024 * 1024; +/// 单次同步允许上传的累计字节上限（客户端预算）。超出部分留到下一次同步， +/// 与"单项目总上限"是两个不同概念。 +pub const AGC_PROJECT_SNAPSHOT_MAX_SYNC_BYTES: u64 = 512 * 1024 * 1024; + +/// 单个项目快照的累计体积上限。远端对象按当前清单做回收，所以这个上限同时约束 +/// 了该项目在 OSS 上的常驻占用。 +pub const AGC_PROJECT_SNAPSHOT_MAX_PROJECT_BYTES: u64 = 2 * 1024 * 1024 * 1024; /// 单文件上传查询参数。文件正文走请求体，元数据走查询串，避免 base64 膨胀。 #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]