From 1997dfeb802359b6357a60fac0e8d51189a732ba Mon Sep 17 00:00:00 2001 From: Suzumiya Date: Mon, 21 Sep 2026 08:31:39 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=9B=BE=E9=9B=86=E5=88=87=E7=89=87?= =?UTF-8?q?=E4=B8=8A=E9=99=90=E5=AF=B9=E9=BD=90=E5=B9=B3=E5=8F=B0=E5=A5=91?= =?UTF-8?q?=E7=BA=A6=EF=BC=9A=E5=AE=A2=E6=88=B7=E7=AB=AF=E7=BB=93=E6=9E=9C?= =?UTF-8?q?=E9=97=A8=2064=20=E2=86=92=20256?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 现场（gameagent-6e53c9e8）「AI 生成图标素材」失败：平台已生成并切完图，客户端在绑定结果这一步把整条结果判失败（assetId 为空、付费产物被丢）。 - 根因：同一条链路存在两个上限——平台切分 / Agent 工具 schema sliceCount / 持久化产物批次都是 256，客户端两处结果绑定门仍是 64（external_generation_state.rs、canvas_generation.rs）。 - 修复：两处门统一到 PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256，判据与拒绝文案各只留一份（数字由常量插值），注释点名三处同值权威。 - 用例：新增上限值/边界/文案用例与「64/65/256 片必须能落盘、257 片必须拒」的回归用例；两条都做过变异验证（常量改回 64 即红）。 - 文档：decision-log 与 pitfalls 各补一条（客户端门不得比平台产品上限更严；上限引同一常量）。 --- .../src/agent/generation/canvas_generation.rs | 54 +++++++++- .../generation/external_generation_state.rs | 98 ++++++++++++++++++- .../shared-memory/decision-log.md | 9 ++ docs/project-memory/shared-memory/pitfalls.md | 6 ++ 4 files changed, 163 insertions(+), 4 deletions(-) diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent/generation/canvas_generation.rs b/apps/ai-game-creator-shell/src-tauri/src/agent/generation/canvas_generation.rs index b607dc7f1..21f212f11 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent/generation/canvas_generation.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent/generation/canvas_generation.rs @@ -32,6 +32,33 @@ const PLATFORM_ART_SPRITESHEET_MAX_DIMENSION: u32 = 4_096; const PLATFORM_ART_SPRITESHEET_TOTAL_PIXELS: u64 = 16 * 1024 * 1024; const PLATFORM_ART_SPRITESHEET_MAX_DECODE_ALLOC: u64 = 64 * 1024 * 1024; +/// 客户端接受的图集切片数量上限。 +/// +/// 这个值**不是**客户端自己的预算，而是与另外三处取同一个值的一致性约束： +/// 平台切分上限（`server-rs/crates/api-server/src/editor_project_icon.rs` 的 +/// `EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`）、Agent 工具 schema 的 `sliceCount`（1..256， +/// 见 `agent_native_tools.rs` 与 `direct_tool_bridge.rs`），以及公开契约 +/// （`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`：单边最多 4096 像素、总像素最多 +/// 2048×2048、最多 256 个输出）。 +/// +/// 客户端一旦比平台更严，平台**合法**产出的 65~256 片结果会在绑定结果这一步被整条丢弃： +/// 用户只会看到「异步生成完成结果无法绑定到 operationId」加一句切片超限，既拿不到图也没法自查。 +/// 现场（`gameagent-6e53c9e8`，`connected-components` 自动切分）就是被旧值 64 挡下的。 +pub(crate) const PLATFORM_ART_SPRITESHEET_MAX_SLICES: usize = 256; + +/// 平台返回的图集切片数量是否超出上限（两个结果绑定入口共用同一条判据）。 +pub(crate) fn platform_art_spritesheet_slice_count_exceeds_limit(slice_count: usize) -> bool { + slice_count > PLATFORM_ART_SPRITESHEET_MAX_SLICES +} + +/// 切片数量超限的拒绝文案（两处入口同一句话，数字来自常量，避免再和上限漂移）。 +pub(crate) fn platform_art_spritesheet_slice_limit_error( + source_label: &str, + suffix: &str, +) -> String { + format!("{source_label}的图集切片超过 {PLATFORM_ART_SPRITESHEET_MAX_SLICES} 个{suffix}") +} + type ExternalCanvasBindingAsyncLock = tokio::sync::Mutex<()>; const EXTERNAL_CANVAS_BINDING_PARTIAL_SCHEMA_VERSION: &str = "game-creator-external-editor-project-binding-partial.v2"; @@ -2365,8 +2392,11 @@ async fn prepare_platform_art_spritesheet_slices( else { return Ok(Vec::new()); }; - if icons.len() > 64 { - return Err("External Editor 返回的图集切片超过 64 个，已拒绝同步".to_string()); + if platform_art_spritesheet_slice_count_exceeds_limit(icons.len()) { + return Err(platform_art_spritesheet_slice_limit_error( + "External Editor 返回", + "，已拒绝同步", + )); } let mut prepared = Vec::with_capacity(icons.len()); if initial_download_bytes > PLATFORM_ART_SPRITESHEET_TOTAL_DOWNLOAD_BYTES @@ -8394,6 +8424,26 @@ pub(crate) fn build_platform_art_asset_prompt( #[cfg(test)] mod canvas_generation_tests { + #[test] + fn spritesheet_slice_limit_matches_platform_and_tool_contract() { + // 现场回归（`gameagent-6e53c9e8`）：平台按 `connected-components` 自动切图，客户端旧门 + // （>64 就拒）把 65~256 片的**合法**产出整条判失败。这里把上限钉在与平台切分、Agent + // 工具 schema `sliceCount`（1..256）和公开契约同一个值上。 + assert_eq!(PLATFORM_ART_SPRITESHEET_MAX_SLICES, 256); + assert!(!platform_art_spritesheet_slice_count_exceeds_limit(0)); + assert!(!platform_art_spritesheet_slice_count_exceeds_limit(64)); + assert!(!platform_art_spritesheet_slice_count_exceeds_limit( + PLATFORM_ART_SPRITESHEET_MAX_SLICES + )); + assert!(platform_art_spritesheet_slice_count_exceeds_limit( + PLATFORM_ART_SPRITESHEET_MAX_SLICES + 1 + )); + // 拒绝文案里的数字来自常量本身，不会和上限再次漂移。 + assert_eq!( + platform_art_spritesheet_slice_limit_error("External Editor 返回", "，已拒绝同步"), + "External Editor 返回的图集切片超过 256 个，已拒绝同步" + ); + } use super::*; use image::{ codecs::png::{CompressionType, FilterType, PngEncoder}, diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent/generation/external_generation_state.rs b/apps/ai-game-creator-shell/src-tauri/src/agent/generation/external_generation_state.rs index c88a8a19c..f2332c009 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent/generation/external_generation_state.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent/generation/external_generation_state.rs @@ -912,8 +912,14 @@ fn durable_legacy_generation_result( .get("iconImageSrcs") .and_then(serde_json::Value::as_array) { - if icons.len() > 64 { - return Err("External Editor 旧同步结果的图集切片超过 64 个".to_string()); + if super::canvas_generation::platform_art_spritesheet_slice_count_exceeds_limit(icons.len()) + { + return Err( + super::canvas_generation::platform_art_spritesheet_slice_limit_error( + "External Editor 旧同步结果", + "", + ), + ); } let mut durable_icons = Vec::with_capacity(icons.len()); for (index, icon) in icons.iter().enumerate() { @@ -1995,6 +2001,94 @@ mod external_generation_state_tests { ); } + /// 现场回归（`gameagent-6e53c9e8`，2026-09-21）：平台按 `connected-components` 自动切图， + /// 切出的数量落在 65~256 之间是**合法**产出；客户端旧门（>64 就拒）会把整条结果判失败， + /// 用户只看到「异步生成完成结果无法绑定到 operationId」——图没落盘，付费结果被丢弃。 + /// + /// 上限必须与平台切分（`EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`）和 Agent 工具 schema + /// 的 `sliceCount`（1..256）同值；超过上限仍然拒，且文案里的数字来自常量本身。 + #[test] + fn legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond() { + fn slice(index: usize) -> serde_json::Value { + serde_json::json!({ + "name": format!("素材 {index}"), + "width": 64, + "height": 64, + "resource": { + "resourceId": format!("slice-{index}"), + "objectKey": format!("generated/slice-{index}.png") + } + }) + } + + fn persist_legacy_slice_result( + root: &Path, + count: usize, + ) -> Result { + let pending = pending_canvas_generation(root); + let context = platform_art_generation_runtime_context_from_pending(&pending); + let access = developer_access("https://editor.example.test", "test-api-key"); + let request_body = serde_json::json!({ + "prompt": "生成统一视觉规范图", + "kind": "spec", + "projectId": "canvas-project", + "assetFolderId": "asset-folder", + "referenceImageSrcs": [] + }); + let (state, _) = prepare_platform_art_generation_runtime_state( + root, + &context, + "/api/external/v1/editor/images/generations", + "legacy-generation-canvas", + "生成统一视觉规范图", + &request_body, + &access, + ) + .expect("prepare legacy generation ledger"); + let completed = mark_platform_art_generation_runtime_legacy_completed( + root, + state, + &serde_json::json!({ + "resource": { + "resourceId": "legacy-resource", + "objectKey": "generated/legacy.png" + }, + "iconImageSrcs": (0..count).map(slice).collect::>() + }), + )?; + remove_platform_art_generation_runtime_state_at( + root, + &pending.agent_id, + &pending.run_id, + ) + .expect("reset legacy ledger between slice counts"); + platform_art_generation_runtime_legacy_result(&completed) + .map_err(|error| error.to_string()) + } + + let temporary = crate::tests::canonical_test_tempdir("legacy-generation-slice-limit-"); + let root = temporary.path(); + init_local_game_project_at(root, "legacy-slice-limit", "旧同步切片上限回归") + .expect("init project"); + + // 64（旧上限）到 256（平台上限）之间都是合法产出：必须能持久化，切片一条不少。 + for count in [64_usize, 65, 256] { + let durable = + persist_legacy_slice_result(root, count).expect("persist legal slice count"); + assert_eq!( + durable["iconImageSrcs"].as_array().map(Vec::len), + Some(count) + ); + } + + // 257 片仍然拒：与平台同一句口径，数字来自常量。 + let error = persist_legacy_slice_result(root, 257).expect_err("reject beyond limit"); + assert!( + error.contains("图集切片超过 256 个"), + "unexpected rejection message: {error}" + ); + } + #[cfg(unix)] #[test] fn dangling_generation_ledger_symlink_is_reconciliation_evidence() { diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 411f0a3c8..54bb65a08 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -1,5 +1,14 @@ # 决策记录 +## 2026-09-21 图集切片上限：客户端结果门从 64 对齐到平台契约的 256 + +- 背景：现场（项目 `gameagent-6e53c9e8`，2026-09-21 07:54）「AI 生成图标素材」失败：`platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationId：External Editor 旧同步结果的图集切片超过 64 个`。任务账本（`.agent/runtime/asset-generation-tasks/tasks.json`）显示它跑了 99 秒、`assetId` 为空、没有落任何素材；对应的持久化请求（`canvas-generation-requests/manual-canvas-asset-generate/slot-560175669f….json`）是 `sliceMode: connected-components` + `sliceCount: null`（自动切分）。也就是**平台已经生成并切完图了，是客户端在绑定结果这一步把整条结果判失败**，付费产物被丢弃。 +- 根因：同一条链路里存在两个不同的切片上限。平台切分是 256（`server-rs/crates/api-server/src/editor_project_icon.rs` 的 `EDITOR_ICON_SPRITESHEET_MAX_SLICES`），Agent 工具 schema 的 `sliceCount` 是 1..256（`agent_native_tools.rs` / `direct_tool_bridge.rs`），持久化产物批次也是 1..256，公开契约（`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`）写的更是「最多 256 个输出」；唯独客户端**两处结果绑定门**还是 `> 64` 就拒（`agent/generation/external_generation_state.rs` 与 `agent/generation/canvas_generation.rs`，由 `602723ea0` 于 2026-08-03 引入）。自动切分落在这个窗口里（65~256 片）时，客户端比平台更严，于是把合法产出整条丢掉。 +- 决策：两处门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`，并抽成同一条判据 `platform_art_spritesheet_slice_count_exceeds_limit` 与同一句拒绝文案 `platform_art_spritesheet_slice_limit_error`（数字由常量插值，不再手写）。注释里点名三处同值权威（平台切分常量、工具 schema `sliceCount`、公开契约），客户端不得比平台更严。 +- 原因：客户端这两处门的作用是「防止把不可信/超预算的结果写进本地」，不是产品上限；真正的产品上限属于平台切分契约。两处各写一个字面量就会再次漂移，所以值只留一份、判据只留一条。 +- 验证：新增 `canvas_generation_tests::spritesheet_slice_limit_matches_platform_and_tool_contract`（上限值、边界判据与文案）与 `external_generation_state_tests::legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond`（64/65/256 片必须能持久化且切片一条不少、257 片必须按同一句文案拒绝），两条都用**变异验证**确认过：把常量改回 64，回归用例立刻变红。定向执行 `cargo test -- spritesheet`（22 passed）、`cargo test -- external_generation_state_tests::`（10 passed）与两条新用例；`cargo fmt --check` 干净。 +- 影响范围：`apps/ai-game-creator-shell/src-tauri/src/agent/generation/{canvas_generation.rs,external_generation_state.rs}`（+用例）。未动平台切分、OpenAPI、数据库或前端。 +- 已知未覆盖：真实客户端复验（重新生成一次图标素材）与远程 CI 未跑；256 片时的累计下载/像素预算未实测——平台自己的总像素上限是 2048×2048，客户端预算是 4096²，按切片是整图互不重叠子矩形推算不会先撞预算，且真撞了也只是给出明确错误而不是损坏数据。 ## 2026-09-21 本批自查（PR #441）：三处修正 - 背景：推 PR 后按「局部到整体」自查这一批（三需求 + 验收修正），查出三条：①拖动到对话的落点在 `pointermove` 上每帧都 `setState` 一个新对象；②替换面板相对 **stage** 写死 `top: 8.5rem`（与刚修的任务开关同一类隐患：工具条换行会压上去），且它和「生成任务」面板抢画布右上角同一个位置；③替换面板不显示「在替换哪张源素材」，而非模态化之后那点线索（画布上的源素材光环）会被一次空白点击清掉。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 4136d4b7b..61ada3b18 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,11 @@ # 踩坑与排障记录 +## 同一条链路两处上限不一致：平台合法产出被客户端整条丢弃 + +- 现象：客户端报「生成素材失败：platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationId：External Editor 旧同步结果的图集切片超过 64 个」，而平台侧这次生成**其实已经成功并切完图**（任务账本耗时正常、`assetId` 为空、没有任何素材落盘，付费产物被丢）。 +- 成因：图集切片上限在链路里存在两份字面量——平台切分、Agent 工具 schema `sliceCount` 与持久化产物批次都是 256，客户端结果绑定门写着 64（`agent/generation/{canvas_generation.rs,external_generation_state.rs}`）。自动切分（`connected-components` + `sliceCount=null`）切出 65~256 片是合法产出，客户端比平台更严就会把结果整条判失败。 +- 处理：客户端门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`，判据与文案各只留一份（数字由常量插值），并在注释里点名三处同值权威（平台切分常量、工具 schema、公开契约）。 +- 复用判据：凡是「平台产出 → 客户端校验后落盘」的链路，客户端门只能表达**安全 / 预算**约束，不得比平台的产品上限更严；两边上限要引同一个常量或同一份文档，改一边时必须同时改另一边，并补一条「上限之内必须能落盘」的回归用例。 ## Jenkins Windows 节点的 PATH 白名单决定 Godot 原生扩展能否构建 `Genarrative-Agc-Windows-Build` 在阶段里用 `AGC_WINDOWS_PATH` 整体替换 PATH、不继承节点机器的 PATH，所以 Godot C++ 引导需要的 CMake 与 Python 必须显式写进这份白名单，装在机器 PATH 上并不生效。2026-09-21 的 #97–#99 连续失败都停在 `Get-Command cmake.exe`（#93–#96 是更早的手写 C ABI 在 MSVC C 模式下的对齐问题）：节点只有 Visual Studio Build Tools（`C:\BuildTools`）自带的 CMake 3.31，缺 Python 3。修复后白名单包含 `C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin`、`C:\Python312`、`C:\Python312\Scripts`，preflight 校验 CMake ≥3.25、Python 3 和 Visual Studio 17 2022 生成器；把 `cmake.exe` 单独复制到别的目录会丢掉 `share/cmake-*/Modules`，不能替代加入安装目录。新节点的 Python 用 `python-3.12.10-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python312 PrependPath=1 Include_launcher=1 InstallLauncherAllUsers=1` 静默安装即可，CMake 不必另装。 From 117d482f93e9bd4f48491cdc0cf94792ccad71c2 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 10:56:25 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=89=8D=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E7=A9=BA=E9=97=B4=E9=A2=84=E6=A3=80=E5=89=8D=E7=BD=AE?= =?UTF-8?q?=E5=B9=B6=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8=E9=99=8D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --check-space-only：archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛，空间不足以退出码 3 返回 - production-stdb-publish.sh 在进入维护与停服务之前先做空间预检，archive 不足时自动降级 files（files 不支持 defer，async 收敛为 sync），可用环境变量关闭 - 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护，只有真正开始发布之后的失败才保持维护态 - 备份检查脚本与生产运维门禁补上新口径与执行顺序断言，运维文档与共享记忆记录事故与规则 --- docs/project-memory/shared-memory/pitfalls.md | 4 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-database-backup-to-oss.mjs | 67 ++++++- scripts/check-production-ops-guardrails.mjs | 41 ++++- scripts/database-backup-to-oss.mjs | 98 ++++++++-- scripts/deploy/production-stdb-publish.sh | 170 ++++++++++++++++-- 6 files changed, 353 insertions(+), 29 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 51ed5f002..a82811675 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,9 @@ # 踩坑与排障记录 +## release 冷备空间不足会把生产留在维护态 + +`Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0596085fc..0dca6f394 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index 44060f394..f097b8e37 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -26,6 +26,7 @@ import { cleanupHistoryCandidates, collectDirectFileEntries, createUploadBandwidthLimiter, + describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, restoreDirectFilesBackup, @@ -64,6 +65,7 @@ async function main() { assertDeferredArchiveDiscoveryIsBoundedAndDeterministic(); assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); + assertCheckSpaceOnlyUsesFormatSpecificRequirement(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -1198,7 +1200,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { '999999999999999999', ]); - assertStatus(result, 1, '空间不足时必须失败。'); + assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。'); assertIncludes( result.stdout, '备份空间预检', @@ -1213,6 +1215,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() { assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); } +function assertCheckSpaceOnlyUsesFormatSpecificRequirement() { + const fixture = createFixture('check-space-only'); + + const filesOk = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' }, + ); + assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。'); + assertIncludes( + filesOk.stdout, + '备份空间预检(files)', + 'files 预检必须打印 files 口径的预检结果。', + ); + assertIncludes( + filesOk.stdout, + '空间预检通过(check-space-only)', + 'check-space-only 通过时必须给出显式成功标记。', + ); + assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。'); + assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。'); + + const filesInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'files'], + { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。'); + + const archiveInsufficient = runBackup( + fixture, + ['--check-space-only', '--storage-format', 'archive'], + { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' }, + ); + assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。'); + + const archive = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'archive', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' }, + }); + const files = describeBackupSpaceRequirement({ + dataDir: fixture.dataDir, + workDir: fixture.workDir, + storageFormat: 'files', + args: {}, + env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' }, + }); + if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) { + failures.push( + `空间口径覆盖参数应生效：archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`, + ); + } + if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') { + failures.push('空间预检结果必须回显实际 storage-format。'); + } +} + function assertStopFailureRetainsRecoveryMarker() { const fixture = createFixture('stop-failure-marker'); writeExecutable( @@ -2695,7 +2757,7 @@ exit 2 return { root, binDir, dataDir, workDir, systemctlLog, tarLog }; } -function runBackup(fixture, extraArgs = []) { +function runBackup(fixture, extraArgs = [], envOverrides = {}) { return spawnSync( process.execPath, [ @@ -2720,6 +2782,7 @@ function runBackup(fixture, extraArgs = []) { env: { ...process.env, PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`, + ...envOverrides, }, }, ); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 323a17e6a..bfb4bdbe9 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -258,6 +258,44 @@ const checks = [ reason: 'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径，保证首次 rollout 重启即可读取 secret。', }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"', + reason: + '备份空间预检必须先于进入维护模式执行：磁盘不足时不得停服务或把生产留在维护态。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--check-space-only', + reason: + '生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检，保证空间口径与真实备份一致。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK', + reason: + 'archive 冷备份空间不足时必须能自动降级为 files 存储格式（不落地本地归档）。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: 'restore_runtime_services_before_publish', + reason: + '尚未开始 publish 的失败必须自动恢复运行时服务并退出维护，避免生产停在维护态等人工。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: + 'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then', + reason: + '只有尚未开始 publish 的失败才允许自动恢复并退出维护，半发布状态必须保持维护。', + }, + { + file: 'scripts/database-backup-to-oss.mjs', + includes: 'SPACE_INSUFFICIENT_EXIT_CODE', + reason: + '备份空间不足必须使用独立退出码，调用方据此决定降级存储格式而不是一律失败。', + }, { file: 'scripts/deploy/production-stdb-publish.sh', includes: 'stop_runtime_services_for_rollout_gate', @@ -984,7 +1022,8 @@ const checks = [ }, { file: 'scripts/database-backup-to-oss.mjs', - includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})', + includes: + "assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })", normalizeWhitespace: true, reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。', }, diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index bc215509f..2202017df 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -55,6 +55,11 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( ); const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; +// files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; +const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; +// 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 +const SPACE_INSUFFICIENT_EXIT_CODE = 3; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; const OSS_SERVICE = 'oss'; const OSS_REQUEST = 'aliyun_v4_request'; @@ -80,6 +85,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir @@ -106,6 +112,8 @@ function usage() { GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT archive（默认）或 files GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算 + GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES files 模式的最小可用字节（默认 2GiB） + GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO files 模式按数据目录比例要求的可用空间（默认 0.05） GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE history 使用的 full baseline 与追加批次状态文件 GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET @@ -175,6 +183,7 @@ function parseArgs(argv) { restartServicesAfter: [], database: '', dryRun: false, + checkSpaceOnly: false, deferUpload: false, uploadArchive: '', uploadDeferredDir: '', @@ -282,6 +291,9 @@ function parseArgs(argv) { case '--publish-manifest': options.publishManifest = readValue(); break; + case '--check-space-only': + options.checkSpaceOnly = true; + break; case '--storage-format': options.storageFormat = readValue(); break; @@ -1013,6 +1025,20 @@ function parseSafetyRatio(rawValue) { return ratio; } +function parseFilesSpaceSafetyRatio(rawValue) { + const value = String(rawValue ?? '').trim(); + if (!value) { + return DEFAULT_FILES_SPACE_SAFETY_RATIO; + } + const ratio = Number(value); + if (!Number.isFinite(ratio) || ratio <= 0) { + throw new Error( + `GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO 必须是 > 0 的数字，实际: ${rawValue}`, + ); + } + return ratio; +} + function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { const explicitMinFreeBytes = parseByteSize( firstNonEmpty( @@ -1042,22 +1068,57 @@ function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) { : extraRequirement; } -function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { +function calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) { + const explicitMinFreeBytes = parseByteSize( + env.GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES, + 'GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES', + ); + if (explicitMinFreeBytes !== null) { + return explicitMinFreeBytes; + } + const ratio = parseFilesSpaceSafetyRatio( + env.GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO, + ); + const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000)); + const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n; + const floorRequirement = BigInt(DEFAULT_FILES_EXTRA_FREE_BYTES); + return ratioRequirement > floorRequirement ? ratioRequirement : floorRequirement; +} + +export function describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat = 'archive', + args = {}, + env = {}, +}) { mkdirSync(workDir, { recursive: true }); const dataSizeBytes = getDirectorySizeBytes(dataDir); const availableBytes = getAvailableBytes(workDir); - const requiredFreeBytes = calculateRequiredFreeBytes({ - dataSizeBytes, - args, - env, - }); + const requiredFreeBytes = + storageFormat === 'files' + ? calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) + : calculateRequiredFreeBytes({ dataSizeBytes, args, env }); + return { storageFormat, dataSizeBytes, availableBytes, requiredFreeBytes }; +} + +function assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat = 'archive' }) { + const { dataSizeBytes, availableBytes, requiredFreeBytes } = + describeBackupSpaceRequirement({ + dataDir, + workDir, + storageFormat, + args, + env, + }); console.log( - `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, + `[database-backup] 备份空间预检(${storageFormat}): data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, ); if (availableBytes < requiredFreeBytes) { - throw new Error( + const error = new Error( [ `备份工作目录所在文件系统剩余空间不足: ${workDir}`, + `storage-format=${storageFormat}`, `available=${formatBytes(availableBytes)}`, `required=${formatBytes(requiredFreeBytes)}`, `dataDir=${dataDir}`, @@ -1065,7 +1126,10 @@ function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) { '为避免停库后写满磁盘，本次备份已在停止服务前中止。', ].join('；'), ); + error.spaceInsufficient = true; + throw error; } + return { dataSizeBytes, availableBytes, requiredFreeBytes }; } function assertSufficientHistoryWorkDirSpace({ @@ -4154,6 +4218,20 @@ async function main() { ); } + if (args.checkSpaceOnly) { + assertSufficientWorkDirSpace({ + dataDir, + workDir, + args, + env, + storageFormat, + }); + console.log( + `[database-backup] 空间预检通过(check-space-only): storage-format=${storageFormat}, workDir=${workDir}`, + ); + return; + } + for (const [label, value] of Object.entries({ bucket, endpoint, @@ -4369,7 +4447,7 @@ async function main() { const restartServicesAfter = collectRestartServicesAfterBackup({ args, env }); const stopMarkerPath = databaseBackupStopMarkerPath(workDir); try { - assertSufficientWorkDirSpace({ dataDir, workDir, args, env }); + assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' }); serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); archivePath = createArchive({ dataDir, workDir, fileName }); } catch (error) { @@ -4558,6 +4636,6 @@ if ( } catch { // 参数解析或状态文件写入失败时，不能掩盖原始备份错误。 } - process.exit(1); + process.exit(error?.spaceInsufficient ? SPACE_INSUFFICIENT_EXIT_CODE : 1); }); } diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index e96210ecb..b44eccf9c 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -18,9 +18,16 @@ usage() { 如需强制等待备份完成并在失败时阻断 publish，传入 --backup-mode sync。 发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径，再重启并验活重启前 active 的服务。 --keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker，并在成功后保持维护态，交由后续 API deploy 恢复服务。 + 发布前先做备份空间预检（不进入维护、不停服务）；archive 空间不足且未显式关闭自动降级时， + 自动改用 files 存储格式（不落地本地归档，改为同步直传 OSS），避免磁盘不足把生产留在维护态。 + +环境变量： + GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） + GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 - 失败时保留维护模式。 + 失败时：尚未开始 publish 的失败会自动恢复运行时服务并退出维护；真正开始 publish 之后的失败保留维护模式。 EOF } @@ -55,7 +62,13 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}" KEEP_MAINTENANCE_MODE=0 BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" +BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" +AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" +AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" DEPLOY_COMPLETED=0 +PUBLISH_STARTED=0 +MAINTENANCE_ENTERED=0 +STOPPED_RUNTIME_SERVICES=() PUBLISH_TMP_DIR="" ASYNC_BACKUP_STATUS_FILE="" ASYNC_BACKUP_SCRIPT="" @@ -287,6 +300,106 @@ restart_runtime_services_after_bootstrap_secret_install() { fi } +backup_script_path() { + local candidate="" + for candidate in \ + "${SCRIPT_DIR}/../database-backup-to-oss.mjs" \ + "${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do + if [[ -f "${candidate}" ]]; then + printf '%s\n' "${candidate}" + return 0 + fi + done + return 1 +} + +run_backup_space_precheck() { + local storage_format="$1" + local backup_script="" + if ! backup_script="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本，无法做备份空间预检" >&2 + return 1 + fi + node -- "${backup_script}" \ + --env-file /etc/genarrative/api-server.env \ + --data-dir "${SPACETIME_ROOT_DIR}" \ + --database "${DATABASE}" \ + --storage-format "${storage_format}" \ + --check-space-only +} + +# 空间预检必须发生在进入维护模式与停服务之前：磁盘不够时不允许再动生产。 +precheck_backup_space_before_maintenance() { + if [[ "${BACKUP_MODE}" == "skip" ]]; then + echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" + return 0 + fi + + local status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -eq 0 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}（尚未进入维护模式、未停服务）" + return 0 + fi + if [[ "${status}" -ne 3 ]]; then + echo "[production-stdb-publish] 发布前备份空间预检失败（非空间原因），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then + echo "[production-stdb-publish] files 模式备份空间仍不足，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then + echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级（GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}），中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + + echo "[production-stdb-publish] archive 冷备份空间不足：自动降级为 files 存储格式（不落地本地归档，改为文件级 catalog 直传 OSS）。" >&2 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 模式不支持 --defer-upload，本次备份改为同步执行。" >&2 + BACKUP_MODE="sync" + fi + status=0 + run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? + if [[ "${status}" -ne 0 ]]; then + echo "[production-stdb-publish] 降级为 files 后空间预检仍失败，中止发布；未进入维护模式、未停服务。" >&2 + exit 1 + fi + echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。" +} + +# 仅在「尚未开始 publish」的失败路径调用：把本次停掉的运行时服务拉回来。 +restore_runtime_services_before_publish() { + if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then + return 0 + fi + local service="" + local state="" + local attempt=0 + echo "[production-stdb-publish] 发布尚未开始，恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}" + if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then + echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2 + return 1 + fi + for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do + state="" + for attempt in $(seq 1 15); do + state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)" + if [[ "${state}" == "active" ]]; then + break + fi + sleep 1 + done + if [[ "${state}" != "active" ]]; then + echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2 + return 1 + fi + echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}" + done + return 0 +} + stop_runtime_services_for_rollout_gate() { local api_state="" local controller_state="" @@ -326,6 +439,7 @@ stop_runtime_services_for_rollout_gate() { fi echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}" + STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}") run_privileged systemctl stop "${services_to_stop[@]}" for worker_service in "${services_to_stop[@]}"; do if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then @@ -402,6 +516,10 @@ while [[ $# -gt 0 ]]; do BACKUP_MODE="${2:?缺少 --backup-mode 的值}" shift 2 ;; + --backup-storage-format) + BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}" + shift 2 + ;; *) echo "[production-stdb-publish] 未知参数: $1" >&2 usage >&2 @@ -431,6 +549,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do fi done +if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then + echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2 + exit 1 +fi +if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] files 存储格式不支持 --defer-upload，备份模式由 async 调整为 sync" >&2 + BACKUP_MODE="sync" +fi if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2 exit 1 @@ -497,7 +623,22 @@ on_exit() { rm -rf "${PUBLISH_TMP_DIR}" fi if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then - echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then + # 尚未开始 publish 就失败（例如备份空间/备份执行失败）：本次没有任何发布变更， + # 必须把停掉的运行时服务拉回来并退出维护，避免生产停在维护态等人工救。 + if restore_runtime_services_before_publish; then + if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then + if ! "${SCRIPT_DIR}/maintenance-off.sh"; then + echo "[production-stdb-publish] 自动退出维护模式失败，请手工执行 maintenance-off.sh。" >&2 + fi + fi + echo "[production-stdb-publish] 发布尚未开始即失败，已自动恢复运行时服务并退出维护模式。" + else + echo "[production-stdb-publish] 自动恢复运行时服务失败，保持维护模式，请手工处理。" >&2 + fi + else + echo "[production-stdb-publish] 发布失败，保持维护模式。" >&2 + fi fi exit "${exit_code}" } @@ -506,12 +647,8 @@ trap on_exit EXIT prepare_async_backup() { local -a restart_service_args=() - ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2 + if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi @@ -527,6 +664,7 @@ prepare_async_backup() { --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${restart_service_args[@]}" \ --defer-upload \ @@ -676,7 +814,10 @@ wait_for_api_healthz_ready() { return 1 } +precheck_backup_space_before_maintenance + "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}" +MAINTENANCE_ENTERED=1 if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then stop_runtime_services_for_rollout_gate fi @@ -687,23 +828,20 @@ case "${BACKUP_MODE}" in ;; sync) SYNC_BACKUP_RESTART_SERVICE_ARGS=() - BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" - fi - if [[ ! -f "${BACKUP_SCRIPT}" ]]; then - echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2 + if ! BACKUP_SCRIPT="$(backup_script_path)"; then + echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 exit 1 fi if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布" + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" node -- "${BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ + --storage-format "${BACKUP_STORAGE_FORMAT}" \ --stop-service spacetimedb.service \ "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" ;; @@ -768,8 +906,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then else PUBLISH_ARGS+=(--server "${SERVER_ALIAS}") fi + PUBLISH_STARTED=1 runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}" else + PUBLISH_STARTED=1 spacetime "${PUBLISH_ARGS[@]}" fi From 2f248ea0c330d5b6c29ecab75d2d31ec99d75014 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:33:39 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20minimal=20=E5=A4=87?= =?UTF-8?q?=E4=BB=BD=E5=8F=A3=E5=BE=84=E5=B9=B6=E5=88=87=E6=8D=A2=20releas?= =?UTF-8?q?e=20=E5=AE=9A=E6=97=B6=E5=A4=87=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --minimal/--retain-snapshots：按上游 retention 语义只备份最近 N 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/配置/控制库/模块字节，且不停止任何服务 - 备份检查脚本补充 minimal 计划用例（保留次新+最新 snapshot、丢弃更早 snapshot 与 clog 段、状态路径齐全、保留数校验） - 发布前备份默认走 minimal 热备（GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0 可回到冷备），不再需要 44.7GiB 冷备空间 - Server-Provision 新增 DATABASE_BACKUP_PROFILE=files-minimal 与 systemd drop-in，release 推荐使用并继续禁止 files-history - 运维文档与共享记忆记录上游 retention 依据、实测 40G→3.6G 与不停服热备口径 --- ...rrative-database-backup-files-minimal.conf | 5 + docs/project-memory/shared-memory/pitfalls.md | 2 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- .../Jenkinsfile.production-server-provision | 8 +- scripts/check-database-backup-to-oss.mjs | 102 +++++++ scripts/check-production-ops-guardrails.mjs | 24 +- scripts/database-backup-to-oss.mjs | 256 +++++++++++++++++- scripts/deploy/production-stdb-publish.sh | 35 ++- scripts/jenkins-server-provision.sh | 41 ++- 9 files changed, 452 insertions(+), 23 deletions(-) create mode 100644 deploy/systemd/genarrative-database-backup-files-minimal.conf diff --git a/deploy/systemd/genarrative-database-backup-files-minimal.conf b/deploy/systemd/genarrative-database-backup-files-minimal.conf new file mode 100644 index 000000000..2124eb345 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup-files-minimal.conf @@ -0,0 +1,5 @@ +[Service] +Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4 +Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520 +ExecStart= +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index a82811675..5912a73b2 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -4,6 +4,8 @@ `Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker，再执行发布前冷备份；archive 口径要求 `data × 1.1`（40.6GiB 数据 → 44.7GiB，加上生产根盘只剩 13.5GiB），于是 2026-09-21 的 release 发布在停服后失败并保持维护态，站点 503 直到人工恢复。规则：空间预检必须先于 `maintenance-on` 与停服；archive 不足且未显式禁用降级时改用 files（`max(data × 0.05, 2GiB)`，不落地本地归档，但 files 不支持 `--defer-upload`，会从 async 收敛为 sync）；只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另：定时备份的失效锁在当前仓库版本会自动清理，但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁，需随 provision 更新。 +按上游 [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555) 的 retention 语义，只有最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段是重启所需，其余历史可丢；因此备份改为 `files + full + --minimal --retain-snapshots 2`（release 实测 40G → 3.6G，热备不停服），不再做增量差异计算，也不需要 44.7G 冷备空间。 + ## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异 Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权：只有当被复制 Job 的 `CopyArtifactPermissionProperty`（仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护）显式列出当前消费者，或者该 Job 对认证用户开放 Item.Read 时才放行；`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象（2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中，定时调度 #104+ 正常）。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`；改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins，只改仓库文件不生效。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 0dca6f394..801261b85 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -449,7 +449,7 @@ UI 相关修改要重点验证： npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service --restart-service-after genarrative-api.service --restart-service-after genarrative-external-generation-worker@1.service --restart-service-after genarrative-external-generation-controller.service ``` -脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。备份使用同库进程锁：仍存活的备份进程会阻断并发执行；持有锁的进程已退出时，脚本会自动清理失效锁并重试获取，不需要人工删除锁文件。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS；因 `genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都依赖 `spacetimedb.service`，生产定时冷备份还必须传入对应的 `--restart-service-after`，确保备份后 API、保底 worker 和 controller 随数据库一起恢复。`2026-06-10` release 故障就是现场 unit 漏掉 API 重启参数，`03:20` 冷备份停止 SpacetimeDB 后 API 被依赖关系一并停止，备份脚本只恢复了 SpacetimeDB，API 直到人工重启前都不可用；`2026-06-24` release 又出现同类依赖停机后只恢复 API、未恢复外部生成 worker/controller，导致图片画布生成任务长期停留在队列中。后续现场变更、provision 模板和 Jenkins 归档都必须通过 `npm run check:production-ops` 防止回退。由于 OSS 上传可能受服务器带宽限制，`Genarrative-Stdb-Module-Publish` 默认使用 `DATABASE_BACKUP_MODE=async`：先在 publish 前用 `--defer-upload` 生成本地冷备份和 `.manifest.json`，随后继续执行 publish；发布脚本退出前会用独立 `systemd-run` transient service 执行 `--upload-deferred-dir `，串行补传该目录内同库的 `deferred/pending` 归档，不依赖 Jenkins 作业进程树存活。任一归档只有在 OSS archive、manifest 和 baseline state 全部上传并验真后，才按 `keep-local` 规则删除；失败归档保留原 manifest，由下次 publish 重试。`Genarrative-Full-Build-And-Deploy` 必须显式暴露并透传同一个 `DATABASE_BACKUP_MODE`，不得静默使用下游 `async`；release 已有验真冷备且明确禁止再上传时，Full 必须选择 `skip`。发布脚本在校验 wasm 后、执行 `spacetime publish` 前会等待显式 `SPACETIME_SERVER_URL` 的 `/v1/ping` 就绪，默认最多等待 `60` 秒；如生产机器冷备份恢复 `spacetimedb.service` 较慢，可临时设置 `GENARRATIVE_STDB_PUBLISH_READY_TIMEOUT_SECONDS` 调整等待时间。需要强一致发布闸门时改用 `DATABASE_BACKUP_MODE=sync`（等价脚本参数 `--backup-mode sync`），备份会在 publish 前同步打包并上传，失败会阻断 publish；确认已有其他备份窗口时才使用 `DATABASE_BACKUP_MODE=skip`（兼容脚本参数 `--skip-backup`）。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 `2026-09-21` 又命中一次同类事故：release 冷备按 `archive` 口径要求 `data × 1.1`（40.6GiB → 44.7GiB），生产根盘只剩 13.5GiB，备份在停服后才失败，生产被留在维护态。现在 `production-stdb-publish.sh` 在进入维护模式与停服务之前先执行 `database-backup-to-oss.mjs --check-space-only`：archive 口径为 `max(data × 1.1, data + 512MiB)`，files 口径为 `max(data × 0.05, 2GiB)`，空间不足以退出码 `3` 返回。archive 空间不足且未显式设置 `GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=0` 时，脚本自动降级为 `--storage-format files`（不落地本地归档；files 不支持 `--defer-upload`，因此 `DATABASE_BACKUP_MODE=async` 会收敛为 `sync`，窗口内直传 OSS）；也可用 `GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=files` 直接指定。任何「尚未开始 `spacetime publish`」的失败（空间预检、备份执行等）都会自动把本次停掉的 API/controller/worker 拉回并退出维护，只有真正开始发布之后的失败才保持维护态等待人工；该自动恢复可用 `GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=0` 关闭。定时备份的失效锁：当前仓库版本在持有者进程退出后会自动清理并重试，而生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若仍是旧版（2026-07-16 前）会拒绝抢锁并需要人工删锁，必须随 provision 一起更新。 上游确认（[#5542](https://github.com/clockworklabs/SpacetimeDB/issues/5542) / [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555)）只保留最近 `retain-snapshots`（默认 2）份 snapshot 与覆盖它之后的 commitlog 段就足以重启数据库，更早的 snapshot/commitlog 属于可丢弃历史；因此备份不再需要"保留每一版 + 计算增量"，改为 `database-backup-to-oss.mjs --storage-format files --mode full --minimal --retain-snapshots 2`：每个 replica 取最近 2 份 snapshot、覆盖最老保留 snapshot 的 commitlog 边界段及其后全部段，外加身份/`data/config.toml`/`data/metadata.toml`/`data/control-db`/`data/program-bytes`，其余历史不备份。实测 release `replicas/2` 为 55 个 snapshot(18G)+108 个 clog 文件(22G)，minimal 口径只需约 3.6GiB（16 个候选），因此定时任务与发布前备份都可以在**不停服**的前提下完成，也不再需要 44.7GiB 的冷备空间；Server-Provision 新增 `DATABASE_BACKUP_PROFILE=files-minimal`（release 推荐，`files-history` 继续禁止），发布前备份默认走 minimal（`GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=0` 可回到冷备）。本地磁盘上的历史数据仍由 SpacetimeDB 自己保留（PR #5555 尚未合并），升级到含该特性的版本后可配置 `[retention] policy = "delete"` 回收。 生产环境变量模板在 `deploy/env/api-server.env.example`： diff --git a/jenkins/Jenkinsfile.production-server-provision b/jenkins/Jenkinsfile.production-server-provision index d2104e68c..28a940332 100644 --- a/jenkins/Jenkinsfile.production-server-provision +++ b/jenkins/Jenkinsfile.production-server-provision @@ -34,7 +34,7 @@ pipeline { string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接') string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件') string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口') - choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile；release 仅允许 archive-full，files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用') + choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history'], description: '数据库定时备份 profile；release 用 files-minimal（最近 snapshot + 其后 commitlog 的热备）或 archive-full，files-history 因内存峰值仅 development 使用') string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录；必须使用已建立 full baseline 的独立目录') choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式；开发服无域名时选 development-http，release 正式入口选 production-https') booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务') @@ -110,11 +110,11 @@ pipeline { error('release 目标禁止安装 development-http Nginx 配置；无证书初始化请使用 NGINX_CONFIG_MODE=none。') } def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim() - if (!(databaseBackupProfile in ['archive-full', 'files-history'])) { - error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") + if (!(databaseBackupProfile in ['archive-full', 'files-minimal', 'files-history'])) { + error("DATABASE_BACKUP_PROFILE 只能是 archive-full、files-minimal 或 files-history，当前值: ${params.DATABASE_BACKUP_PROFILE}") } if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') { - error('release 仅允许 archive-full；files-history 会把整棵历史目录加载到 Node 内存，需先完成流式 catalog 改造后才能重新启用。') + error('release 不允许 files-history；files-history 会把整棵历史目录加载到 Node 内存。release 请选择 files-minimal（最近 snapshot + 其后 commitlog）或 archive-full。') } def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim() if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) { diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs index f097b8e37..049d7d851 100644 --- a/scripts/check-database-backup-to-oss.mjs +++ b/scripts/check-database-backup-to-oss.mjs @@ -29,6 +29,7 @@ import { describeBackupSpaceRequirement, discoverDeferredArchiveUploads, discoverHistoryPlan, + discoverMinimalPlan, restoreDirectFilesBackup, restoreDirectFilesLatest, resumeUploadedHistoryBatch, @@ -66,6 +67,7 @@ async function main() { assertCanonicalQueryAndAuthorizationIncludeMultipartParameters(); assertInsufficientSpaceStopsBeforeServiceChanges(); assertCheckSpaceOnlyUsesFormatSpecificRequirement(); + assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog(); assertStopFailureRetainsRecoveryMarker(); assertArchiveFailureStillRestoresDependentServices(); await assertMultipartUploadRetriesAndVerifiesRemoteLength(); @@ -2562,6 +2564,106 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() { } } +function assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog() { + const fixture = createHistoryFixture('minimal-plan', { nestedData: true }); + const replicaDir = path.join(fixture.replicasDir, '2'); + const snapshotsDir = path.join(replicaDir, 'snapshots'); + const clogDir = path.join(replicaDir, 'clog'); + mkdirSync(snapshotsDir, { recursive: true }); + mkdirSync(clogDir, { recursive: true }); + for (const transaction of ['100', '200', '300']) { + const padded = transaction.padStart(20, '0'); + const snapshotDir = path.join(snapshotsDir, `${padded}.snapshot_dir`); + mkdirSync(snapshotDir, { recursive: true }); + writeFileSync( + path.join(snapshotDir, `${padded}.snapshot_bsatn`), + 'snapshot', + ); + } + for (const transaction of ['50', '150', '250', '350']) { + const padded = transaction.padStart(20, '0'); + writeFileSync(path.join(clogDir, `${padded}.stdb.log`), 'log'); + writeFileSync(path.join(clogDir, `${padded}.stdb.ofs`), 'ofs'); + } + for (const relativeDir of ['config', 'data/control-db', 'data/program-bytes']) { + const directory = path.join(fixture.dataDir, relativeDir); + mkdirSync(directory, { recursive: true }); + writeFileSync(path.join(directory, 'state.bin'), 'state'); + } + writeFileSync(path.join(fixture.dataDir, 'data/config.toml'), 'config'); + writeFileSync(path.join(fixture.dataDir, 'data/metadata.toml'), 'metadata'); + + const plan = discoverMinimalPlan({ dataDir: fixture.dataDir }); + const paths = plan.candidates.map((item) => item.path); + const expect = (condition, reason) => { + if (!condition) { + failures.push(reason); + } + }; + + expect( + plan.retainSnapshots === 2, + `minimal 默认应保留 2 份 snapshot，实际 ${plan.retainSnapshots}`, + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000200.snapshot_dir', + ), + 'minimal 必须保留次新 snapshot。', + ); + expect( + paths.includes( + 'data/replicas/2/snapshots/00000000000000000300.snapshot_dir', + ), + 'minimal 必须保留最新 snapshot。', + ); + expect( + !paths.includes( + 'data/replicas/2/snapshots/00000000000000000100.snapshot_dir', + ), + 'minimal 不得备份更早的 snapshot。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000150.stdb.log'), + 'minimal 必须保留覆盖最老保留 snapshot 的边界 commitlog 段。', + ); + expect( + paths.includes('data/replicas/2/clog/00000000000000000350.stdb.log'), + 'minimal 必须保留最新 commitlog 段。', + ); + expect( + !paths.includes('data/replicas/2/clog/00000000000000000050.stdb.log'), + 'minimal 不得备份更早的 commitlog 段。', + ); + for (const staticPath of [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', + ]) { + expect( + paths.includes(staticPath), + `minimal 必须保留状态路径 ${staticPath}。`, + ); + } + const replica = plan.replicas.find((item) => item.replicaId === '2'); + expect( + replica?.retainedSnapshots === 2 && replica?.droppedSnapshots === 1, + 'minimal 必须报告保留/丢弃的 snapshot 数量。', + ); + expect( + replica?.droppedSegments === 1, + `minimal 必须报告丢弃的 commitlog 段数量，实际 ${replica?.droppedSegments}`, + ); + + assertThrows( + () => discoverMinimalPlan({ dataDir: fixture.dataDir, retainSnapshots: 0 }), + '--retain-snapshots 必须是 >= 1 的整数', + 'minimal 必须校验保留 snapshot 数量。', + ); +} + function createHistoryFixture(name, { nestedData }) { const root = path.join(tmpRoot, name); const dataDir = path.join(root, 'stdb'); diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index bfb4bdbe9..4c6e65d85 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -938,7 +938,7 @@ const checks = [ }, { file: 'scripts/jenkins-server-provision.sh', - includes: 'archive-full|files-history)', + includes: 'archive-full|files-history|files-minimal)', reason: 'Server-Provision 必须拒绝未知数据库备份 profile。', }, { @@ -963,7 +963,7 @@ const checks = [ { file: 'jenkins/Jenkinsfile.production-server-provision', includes: - "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']", + "choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history']", reason: 'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。', }, @@ -1016,10 +1016,28 @@ const checks = [ }, { file: 'jenkins/Jenkinsfile.production-server-provision', - includes: 'release 仅允许 archive-full；files-history', + includes: 'release 不允许 files-history', reason: 'release 必须拒绝 files-history，避免逐文件 catalog 扫描再次触发生产内存峰值。', }, + { + file: 'jenkins/Jenkinsfile.production-server-provision', + includes: 'files-minimal', + reason: + 'release 定时备份必须提供 files-minimal profile：只备最近 snapshot 与其后 commitlog，热备不停服。', + }, + { + file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf', + includes: '--mode full --minimal --retain-snapshots 2', + reason: + 'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。', + }, + { + file: 'scripts/deploy/production-stdb-publish.sh', + includes: '--minimal --retain-snapshots', + reason: + '发布前备份默认使用 minimal 热备，避免 40G 级冷备空间门槛与停服。', + }, { file: 'scripts/database-backup-to-oss.mjs', includes: diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index 2202017df..ba6e236fd 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -56,6 +56,16 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join( const DEFAULT_SPACE_SAFETY_RATIO = 1.1; const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; // files 模式不落地本地归档（只写 catalog/state），所需空间远小于数据目录本身。 +// minimal 备份保留的最近 snapshot 数（与上游 retention 默认值一致）。 +const DEFAULT_RETAIN_SNAPSHOTS = 2; +// minimal 备份额外保留的小体积状态：身份、standalone 配置、控制库与模块字节。 +const MINIMAL_STATIC_PATHS = [ + 'config', + 'data/config.toml', + 'data/metadata.toml', + 'data/control-db', + 'data/program-bytes', +]; const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05; const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024; // 空间不足用独立退出码，调用方可据此决定是否降级存储格式。 @@ -84,7 +94,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1; function usage() { console.log(`用法: - npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + npm run database:backup:oss -- [--mode full|history] [--minimal] [--retain-snapshots 2] [--storage-format archive|files] [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir ] [--work-dir ] node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] node -- scripts/database-backup-to-oss.mjs --upload-archive @@ -97,6 +107,7 @@ function usage() { 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + --minimal 只备份上游 retention 语义下仍需要的部分（最近 --retain-snapshots 份 snapshot + 其后 commitlog 段 + 身份/配置/控制库/模块字节）；只支持 files 格式，且不停止任何服务。 archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 --upload-deferred-dir 串行收敛目录内 deferred/pending 及已上传未清理归档；只有 OSS 上传与验真完成后才按 keep-local 规则删除。 @@ -192,6 +203,8 @@ function parseArgs(argv) { resultFile: '', minFreeBytes: '', mode: 'full', + minimal: false, + retainSnapshots: DEFAULT_RETAIN_SNAPSHOTS, baselineState: '', baselineManifest: '', publishManifest: '', @@ -282,6 +295,12 @@ function parseArgs(argv) { case '--mode': options.mode = readValue(); break; + case '--minimal': + options.minimal = true; + break; + case '--retain-snapshots': + options.retainSnapshots = readValue(); + break; case '--baseline-state': options.baselineState = readValue(); break; @@ -1383,6 +1402,217 @@ function candidateKey(candidate) { return `${candidate.kind}\0${candidate.path}`; } +/** + * minimal 备份只保留上游 retention 语义下仍然需要的部分： + * 每个 replica 最近 retainSnapshots 份 snapshot、覆盖最老保留 snapshot 的 + * commitlog 边界段及其后的全部 commitlog（段边界与上游 pruner 一致）， + * 外加身份、standalone 配置、控制库与模块字节等小体积状态。 + * 历史 snapshot 与更早的 commitlog 段不再备份，因此也不需要计算增量差异。 + */ +export function discoverMinimalPlan({ + dataDir, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, +}) { + const retainCount = Number(retainSnapshots); + if (!Number.isInteger(retainCount) || retainCount < 1) { + throw new Error( + `--retain-snapshots 必须是 >= 1 的整数，实际: ${retainSnapshots}`, + ); + } + const resolvedDataDir = resolvePath(dataDir); + const replicasDir = findReplicasDir(resolvedDataDir); + const replicas = []; + const candidates = []; + + const replicaIds = readdirSync(replicasDir, { withFileTypes: true }) + .sort((left, right) => left.name.localeCompare(right.name)) + .flatMap((entry) => { + if (!entry.isDirectory()) { + return []; + } + if (!/^\d+$/u.test(entry.name)) { + throw new Error(`replica 目录名不符合预期: ${entry.name}`); + } + return [entry.name]; + }); + + for (const replicaId of replicaIds) { + const replicaDir = join(replicasDir, replicaId); + const snapshotsDir = join(replicaDir, 'snapshots'); + const clogDir = join(replicaDir, 'clog'); + if (!existsSync(snapshotsDir) || !lstatSync(snapshotsDir).isDirectory()) { + replicas.push({ + replicaId, + status: 'skipped', + reason: 'no-snapshots-directory', + }); + continue; + } + const snapshots = readdirSync(snapshotsDir, { withFileTypes: true }) + .flatMap((entry) => { + const match = /^(\d{20})[.]snapshot_dir$/u.exec(entry.name); + if (!match) { + return []; + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`snapshot transaction 超出 u64: ${entry.name}`); + } + if (!entry.isDirectory()) { + throw new Error( + `snapshot 候选必须是目录: ${join(snapshotsDir, entry.name)}`, + ); + } + const snapshotDir = join(snapshotsDir, entry.name); + const lockPath = join(snapshotsDir, `${match[1]}.lock`); + const snapshotFile = join(snapshotDir, `${match[1]}.snapshot_bsatn`); + if ( + existsSync(lockPath) || + !existsSync(snapshotFile) || + !lstatSync(snapshotFile).isFile() + ) { + return []; + } + return [{ name: entry.name, transaction }]; + }) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + if (snapshots.length === 0) { + replicas.push({ replicaId, status: 'skipped', reason: 'no-snapshot' }); + continue; + } + if (!existsSync(clogDir) || !lstatSync(clogDir).isDirectory()) { + throw new Error(`replica ${replicaId} 缺少 clog 目录。`); + } + const segmentFiles = new Map(); + for (const entry of readdirSync(clogDir, { withFileTypes: true })) { + const match = /^(\d{20})[.]stdb[.](log|ofs)$/u.exec(entry.name); + if (!match) { + throw new Error(`commitlog 文件名不符合预期: ${entry.name}`); + } + if (!entry.isFile()) { + throw new Error( + `commitlog 候选必须是普通文件: ${join(clogDir, entry.name)}`, + ); + } + const transaction = BigInt(match[1]); + if (transaction > 0xffff_ffff_ffff_ffffn) { + throw new Error(`commitlog transaction 超出 u64: ${entry.name}`); + } + const key = transaction.toString(); + const group = segmentFiles.get(key) ?? { transaction }; + group[match[2]] = entry.name; + segmentFiles.set(key, group); + } + for (const group of segmentFiles.values()) { + if (group.ofs && !group.log) { + throw new Error( + `commitlog offset 缺少对应 log: replica=${replicaId}, transaction=${group.transaction}`, + ); + } + } + const segments = [...segmentFiles.values()] + .filter((group) => group.log) + .sort((left, right) => + left.transaction < right.transaction + ? -1 + : left.transaction > right.transaction + ? 1 + : 0, + ); + + const retainedSnapshots = snapshots.slice(-retainCount); + const oldestRetainedSnapshot = retainedSnapshots[0].transaction; + const boundarySegment = segments + .filter((segment) => segment.transaction <= oldestRetainedSnapshot) + .at(-1); + if (!boundarySegment) { + throw new Error( + `replica ${replicaId} 无法找到覆盖最老保留 snapshot ${oldestRetainedSnapshot} 的 commitlog 边界。`, + ); + } + const retainedSegments = segments.filter( + (segment) => segment.transaction >= boundarySegment.transaction, + ); + + for (const snapshot of retainedSnapshots) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(snapshotsDir, snapshot.name), + kind: 'snapshot', + replicaId, + transaction: snapshot.transaction, + }), + ); + } + for (const segment of retainedSegments) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.log), + kind: 'commitlog', + replicaId, + transaction: segment.transaction, + }), + ); + if (segment.ofs) { + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath: join(clogDir, segment.ofs), + kind: 'commitlog-offset', + replicaId, + transaction: segment.transaction, + }), + ); + } + } + replicas.push({ + replicaId, + status: 'ready', + retainedSnapshots: retainedSnapshots.length, + droppedSnapshots: snapshots.length - retainedSnapshots.length, + oldestRetainedSnapshot: oldestRetainedSnapshot.toString(), + boundarySegment: boundarySegment.transaction.toString(), + droppedSegments: segments.length - retainedSegments.length, + }); + } + + for (const relativePath of MINIMAL_STATIC_PATHS) { + const absolutePath = resolve(resolvedDataDir, relativePath); + if (!existsSync(absolutePath)) { + continue; + } + candidates.push( + historyCandidate({ + dataDir: resolvedDataDir, + absolutePath, + kind: 'state', + replicaId: 'state', + transaction: 0n, + }), + ); + } + + candidates.sort((left, right) => left.path.localeCompare(right.path)); + return { + dataDir: resolvedDataDir, + replicasDir: assertSafeRelativePath(resolvedDataDir, replicasDir), + replicas, + retainSnapshots: retainCount, + candidates, + totalSizeBytes: candidates + .reduce((sum, item) => sum + BigInt(item.sizeBytes), 0n) + .toString(), + }; +} + export function cleanupHistoryCandidates({ dataDir, candidates }) { const currentPlan = discoverHistoryPlan({ dataDir }); const eligible = new Map( @@ -2114,6 +2344,8 @@ async function publishDirectFilesLatest({ export async function runDirectFilesBackup({ mode, + minimal = false, + retainSnapshots = DEFAULT_RETAIN_SNAPSHOTS, dataDir, workDir, database, @@ -2139,7 +2371,17 @@ export async function runDirectFilesBackup({ `files history 模式缺少已发布 full baseline catalog: ${statePath}`, ); } - const plan = mode === 'history' ? discoverHistoryPlan({ dataDir }) : null; + const plan = + mode === 'history' + ? discoverHistoryPlan({ dataDir }) + : minimal + ? discoverMinimalPlan({ dataDir, retainSnapshots }) + : null; + if (minimal) { + console.log( + `[database-backup] files minimal: retainSnapshots=${plan.retainSnapshots}, candidates=${plan.candidates.length}, size=${plan.totalSizeBytes}`, + ); + } const collected = await collectDirectFileEntries({ dataDir, candidates: plan?.candidates ?? null, @@ -4217,6 +4459,12 @@ async function main() { `--storage-format 只能是 archive 或 files，实际: ${storageFormat}`, ); } + if (args.minimal && storageFormat !== 'files') { + throw new Error('--minimal 只支持 --storage-format files。'); + } + if (args.minimal && args.mode === 'history') { + throw new Error('--minimal 只支持 --mode full：minimal 自身就是自包含快照。'); + } if (args.checkSpaceOnly) { assertSufficientWorkDirSpace({ @@ -4351,11 +4599,13 @@ async function main() { let backupError = null; let restoreError = null; try { - if (args.mode === 'full' && !args.dryRun) { + if (args.mode === 'full' && !args.minimal && !args.dryRun) { serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath); } await runDirectFilesBackup({ mode: args.mode, + minimal: args.minimal, + retainSnapshots: args.retainSnapshots, dataDir, workDir, database, diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index b44eccf9c..3b16188f8 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -24,6 +24,8 @@ usage() { 环境变量： GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files（默认 archive） GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0（默认 1：archive 空间不足自动降级 files） + GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=1|0（默认 1：发布前备份只保留最近 N 份 snapshot + 其后 commitlog，热备不停服） + GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS=N（默认 2，仅在 minimal 模式下生效） GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0（默认 1：尚未开始 publish 的失败自动恢复服务并退出维护） migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供，不从构建 artifact 读取。 如果 API 重启前为 active，会在退出维护模式前等待本机 /healthz readiness 通过。 @@ -65,6 +67,8 @@ BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}" BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}" AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}" AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}" +BACKUP_MINIMAL="${GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL:-1}" +BACKUP_RETAIN_SNAPSHOTS="${GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS:-2}" DEPLOY_COMPLETED=0 PUBLISH_STARTED=0 MAINTENANCE_ENTERED=0 @@ -334,6 +338,14 @@ precheck_backup_space_before_maintenance() { echo "[production-stdb-publish] 已跳过发布前备份空间预检（--backup-mode skip）" return 0 fi + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + # minimal 备份是热备：只保留最近 N 份 snapshot + 其后 commitlog，不落地归档也不停服务。 + BACKUP_STORAGE_FORMAT="files" + if [[ "${BACKUP_MODE}" == "async" ]]; then + echo "[production-stdb-publish] minimal 备份为同步热备（无本地归档），备份模式由 async 调整为 sync。" >&2 + BACKUP_MODE="sync" + fi + fi local status=0 run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$? @@ -836,14 +848,21 @@ case "${BACKUP_MODE}" in SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service) fi - echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" - node -- "${BACKUP_SCRIPT}" \ - --env-file /etc/genarrative/api-server.env \ - --data-dir "${SPACETIME_ROOT_DIR}" \ - --database "${DATABASE}" \ - --storage-format "${BACKUP_STORAGE_FORMAT}" \ - --stop-service spacetimedb.service \ - "${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}" + SYNC_BACKUP_ARGS=( + --env-file /etc/genarrative/api-server.env + --data-dir "${SPACETIME_ROOT_DIR}" + --database "${DATABASE}" + --storage-format "${BACKUP_STORAGE_FORMAT}" + ) + if [[ "${BACKUP_MINIMAL}" == "1" ]]; then + echo "[production-stdb-publish] publish 前执行 minimal 热备（最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog），不停服务" + SYNC_BACKUP_ARGS+=(--mode full --minimal --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}") + else + echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" + SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service) + SYNC_BACKUP_ARGS+=("${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}") + fi + node -- "${BACKUP_SCRIPT}" "${SYNC_BACKUP_ARGS[@]}" ;; skip) echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份" diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 259b6f6c8..9d2bda38a 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -15,6 +15,8 @@ DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}" DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}" DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d" DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf" +DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR="${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR:-/var/lib/genarrative/database-backups/files-minimal}" +DATABASE_BACKUP_FILES_MINIMAL_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-minimal.conf" DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf" require_non_root_relative_path() { @@ -71,10 +73,10 @@ validate_server_names() { validate_database_backup_profile() { case "${DATABASE_BACKUP_PROFILE}" in - archive-full|files-history) + archive-full|files-history|files-minimal) ;; *) - echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 + echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full、files-history 或 files-minimal，当前值: ${DATABASE_BACKUP_PROFILE}" >&2 exit 1 ;; esac @@ -1323,12 +1325,29 @@ render_database_backup_files_history_drop_in() { deploy/systemd/genarrative-database-backup-files-history.conf } +render_database_backup_files_minimal_drop_in() { + local current_escaped env_escaped work_dir_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \ + -e "s|/var/lib/genarrative/database-backups/files-minimal|${work_dir_escaped}|g" \ + deploy/systemd/genarrative-database-backup-files-minimal.conf +} + configure_database_backup_profile() { local rendered_drop_in if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then echo "[server-provision] 数据库备份 profile=archive-full，保留主 service 的全量冷备行为。" - run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + return + fi + + if [[ "${DATABASE_BACKUP_PROFILE}" == "files-minimal" ]]; then + configure_database_backup_files_minimal_profile return fi @@ -1350,13 +1369,26 @@ configure_database_backup_profile() { run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" - run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" rendered_drop_in="$(mktemp)" render_database_backup_files_history_drop_in >"${rendered_drop_in}" install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644 rm -f "${rendered_drop_in}" } +configure_database_backup_files_minimal_profile() { + local rendered_drop_in + + echo "[server-provision] 数据库备份 profile=files-minimal，只保留最近 snapshot 与其后 commitlog（热备、不停服）。" + run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}" + run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}" + run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}" + rendered_drop_in="$(mktemp)" + render_database_backup_files_minimal_drop_in >"${rendered_drop_in}" + install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" 0644 + rm -f "${rendered_drop_in}" +} + render_health_patrol_service() { local current_escaped current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" @@ -1372,6 +1404,7 @@ require_path deploy/systemd/genarrative-external-generation-controller.service require_path deploy/systemd/genarrative-bgfilter-worker.service require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup-files-history.conf +require_path deploy/systemd/genarrative-database-backup-files-minimal.conf require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer From 90207bc1d550e12d72972716aae80aa97c914c40 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:37:14 +0800 Subject: [PATCH 4/5] =?UTF-8?q?minimal=20=E5=A4=87=E4=BB=BD=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20freeze=20=E5=89=AF=E6=9C=AC=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - database-backup-to-oss.mjs 新增 --freeze-dir：minimal 模式先把选中子集 rsync 成冻结副本（保持原始相对路径）再从副本上传，避免热备时最新 commitlog 持续追加触发 stat 漂移失败 - files-minimal systemd 模板与发布前 minimal 备份默认传入 freeze 目录 - 生产运维门禁同步 freeze 口径断言 --- ...rrative-database-backup-files-minimal.conf | 2 +- scripts/check-production-ops-guardrails.mjs | 4 +- scripts/database-backup-to-oss.mjs | 52 ++++++++++++++++++- scripts/deploy/production-stdb-publish.sh | 7 ++- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/deploy/systemd/genarrative-database-backup-files-minimal.conf b/deploy/systemd/genarrative-database-backup-files-minimal.conf index 2124eb345..c8d2e43fe 100644 --- a/deploy/systemd/genarrative-database-backup-files-minimal.conf +++ b/deploy/systemd/genarrative-database-backup-files-minimal.conf @@ -2,4 +2,4 @@ Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4 Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520 ExecStart= -ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --freeze-dir /var/lib/genarrative/database-backups/files-minimal-freeze --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 4c6e65d85..93d81f106 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -1028,13 +1028,13 @@ const checks = [ }, { file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf', - includes: '--mode full --minimal --retain-snapshots 2', + includes: '--mode full --minimal --retain-snapshots 2 --freeze-dir', reason: 'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。', }, { file: 'scripts/deploy/production-stdb-publish.sh', - includes: '--minimal --retain-snapshots', + includes: '--freeze-dir', reason: '发布前备份默认使用 minimal 热备，避免 40G 级冷备空间门槛与停服。', }, diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index ba6e236fd..26d8cf232 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -107,6 +107,7 @@ function usage() { 将 SpacetimeDB 数据目录以 .tar.gz 或逐文件 catalog 形式上传到阿里云 OSS 指定 bucket。 默认 full 模式保持原有全量冷备行为；history 模式只归档已被最新 snapshot 覆盖的历史 commitlog 与旧 snapshot。 --storage-format files 不打包：按原相对路径建立 catalog，文件内容以 SHA-256 不可变对象上传；重复运行只上传新增或变化内容。 + --freeze-dir 仅用于 --minimal：先把选中子集 rsync 成冻结副本再上传，避免热备时最新 commitlog 追加导致 stat 漂移；副本内保持原始相对路径。 --minimal 只备份上游 retention 语义下仍需要的部分（最近 --retain-snapshots 份 snapshot + 其后 commitlog 段 + 身份/配置/控制库/模块字节）；只支持 files 格式，且不停止任何服务。 archive history 必须有已验真的 full baseline state；files history 必须复用同一 work-dir 中已发布的 full catalog state。 --defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。 @@ -204,6 +205,7 @@ function parseArgs(argv) { minFreeBytes: '', mode: 'full', minimal: false, + freezeDir: '', retainSnapshots: DEFAULT_RETAIN_SNAPSHOTS, baselineState: '', baselineManifest: '', @@ -301,6 +303,9 @@ function parseArgs(argv) { case '--retain-snapshots': options.retainSnapshots = readValue(); break; + case '--freeze-dir': + options.freezeDir = readValue(); + break; case '--baseline-state': options.baselineState = readValue(); break; @@ -4387,7 +4392,7 @@ async function main() { const env = loadEffectiveEnv(args.envFiles); const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; - const dataDir = resolvePath( + let dataDir = resolvePath( firstNonEmpty( args.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, @@ -4466,6 +4471,51 @@ async function main() { throw new Error('--minimal 只支持 --mode full：minimal 自身就是自包含快照。'); } + if (args.minimal && args.freezeDir) { + if (storageFormat !== 'files') { + throw new Error('--freeze-dir 只支持 --minimal --storage-format files。'); + } + const freezeDir = resolvePath(args.freezeDir); + const plan = discoverMinimalPlan({ + dataDir, + retainSnapshots: args.retainSnapshots, + }); + console.log( + `[database-backup] minimal freeze: candidates=${plan.candidates.length}, size=${plan.totalSizeBytes}, freezeDir=${freezeDir}`, + ); + if (!args.dryRun) { + mkdirSync(freezeDir, { recursive: true }); + const listPath = join(workDir, 'minimal-freeze-files.txt'); + mkdirSync(workDir, { recursive: true }); + writeFileSync( + listPath, + `${plan.candidates.map((item) => item.path).join('\n')}\n`, + 'utf8', + ); + const result = spawnSync( + 'rsync', + [ + '-a', + '-r', + '--delete', + `--files-from=${listPath}`, + `${dataDir}/`, + `${freezeDir}/`, + ], + { stdio: 'inherit' }, + ); + if (result.status !== 0) { + throw new Error( + `minimal freeze 拷贝失败: rsync exit=${result.status ?? ''}`, + ); + } + console.log( + `[database-backup] minimal freeze 完成，后续从冻结副本上传: ${freezeDir}`, + ); + } + dataDir = freezeDir; + } + if (args.checkSpaceOnly) { assertSufficientWorkDirSpace({ dataDir, diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index 3b16188f8..115cf43cf 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -856,7 +856,12 @@ case "${BACKUP_MODE}" in ) if [[ "${BACKUP_MINIMAL}" == "1" ]]; then echo "[production-stdb-publish] publish 前执行 minimal 热备（最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog），不停服务" - SYNC_BACKUP_ARGS+=(--mode full --minimal --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}") + SYNC_BACKUP_ARGS+=( + --mode full + --minimal + --retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}" + --freeze-dir "${GENARRATIVE_STDB_PUBLISH_BACKUP_FREEZE_DIR:-/var/lib/genarrative/database-backups/publish-minimal-freeze}" + ) else echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份（storage-format=${BACKUP_STORAGE_FORMAT}），失败会阻断发布" SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service) From 9bf35e87457c338352706382ec3821fab179b622 Mon Sep 17 00:00:00 2001 From: kdletters Date: Mon, 21 Sep 2026 11:53:32 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20macOS=20=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E4=B8=8A=E7=9A=84=E6=B8=A0=E9=81=93=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E7=94=A8=E4=BE=8B=E5=AE=BF=E4=B8=BB=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build-release.test.mjs 的「packaged renderer receives the same channel as the updater manifest」用 windows target 却未注入 stageRuntime，真实 staging 要求宿主 Node 平台/架构与目标一致，导致 macOS(arm64)/Linux 上必然失败 - 该用例只断言传给 spawn 的 VITE_AGC_PLATFORM_CHANNEL，改为注入空 stageRuntime，与同文件其它用例一致 --- apps/ai-game-creator-shell/scripts/build-release.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/ai-game-creator-shell/scripts/build-release.test.mjs b/apps/ai-game-creator-shell/scripts/build-release.test.mjs index 732302eb1..b5f8a44da 100644 --- a/apps/ai-game-creator-shell/scripts/build-release.test.mjs +++ b/apps/ai-game-creator-shell/scripts/build-release.test.mjs @@ -215,6 +215,9 @@ test('packaged renderer receives the same channel as the updater manifest', () = spawnOptions = options; return { status: 0 }; }, + // 该用例只关心渠道环境变量；Node 运行时 staging 与宿主平台绑定， + // 真实 staging 会让同一用例在 macOS/Linux 上对 windows target 失败。 + stageRuntime: () => {}, }); assert.equal(spawnOptions?.env?.VITE_AGC_PLATFORM_CHANNEL, 'release'); });