合并最新 master 到埋点方案分支
Project CI / AI game creator shell Rust smoke (pull_request) Has been cancelled
Project CI / AI game creator shell Rust crates (pull_request) Has been cancelled
Project CI / Backend tests (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Has been cancelled
Project CI / Native shell tests (pull_request) Has been cancelled
Project CI / Frontend tests (pull_request) Has been cancelled
Project CI / Repository checks (pull_request) Has been cancelled
Project CI / AI game creator shell web tests (pull_request) Has been cancelled

同步主分支最新客户端和运维变更。
解决共享决策记录顶部冲突,保留双方新增记录。
This commit is contained in:
2026-09-21 04:16:32 +00:00
13 changed files with 1027 additions and 56 deletions
@@ -215,6 +215,9 @@ test('packaged renderer receives the same channel as the updater manifest', () =
spawnOptions = options;
return { status: 0 };
},
// 该用例只关心渠道环境变量;Node 运行时 staging 与宿主平台绑定,
// 真实 staging 会让同一用例在 macOS/Linux 上对 windows target 失败。
stageRuntime: () => {},
});
assert.equal(spawnOptions?.env?.VITE_AGC_PLATFORM_CHANNEL, 'release');
});
@@ -32,6 +32,33 @@ const PLATFORM_ART_SPRITESHEET_MAX_DIMENSION: u32 = 4_096;
const PLATFORM_ART_SPRITESHEET_TOTAL_PIXELS: u64 = 16 * 1024 * 1024;
const PLATFORM_ART_SPRITESHEET_MAX_DECODE_ALLOC: u64 = 64 * 1024 * 1024;
/// 客户端接受的图集切片数量上限。
///
/// 这个值**不是**客户端自己的预算,而是与另外三处取同一个值的一致性约束:
/// 平台切分上限(`server-rs/crates/api-server/src/editor_project_icon.rs` 的
/// `EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`)、Agent 工具 schema 的 `sliceCount`1..256
/// 见 `agent_native_tools.rs` 与 `direct_tool_bridge.rs`),以及公开契约
/// (`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`:单边最多 4096 像素、总像素最多
/// 2048×2048、最多 256 个输出)。
///
/// 客户端一旦比平台更严,平台**合法**产出的 65~256 片结果会在绑定结果这一步被整条丢弃:
/// 用户只会看到「异步生成完成结果无法绑定到 operationId」加一句切片超限,既拿不到图也没法自查。
/// 现场(`gameagent-6e53c9e8``connected-components` 自动切分)就是被旧值 64 挡下的。
pub(crate) const PLATFORM_ART_SPRITESHEET_MAX_SLICES: usize = 256;
/// 平台返回的图集切片数量是否超出上限(两个结果绑定入口共用同一条判据)。
pub(crate) fn platform_art_spritesheet_slice_count_exceeds_limit(slice_count: usize) -> bool {
slice_count > PLATFORM_ART_SPRITESHEET_MAX_SLICES
}
/// 切片数量超限的拒绝文案(两处入口同一句话,数字来自常量,避免再和上限漂移)。
pub(crate) fn platform_art_spritesheet_slice_limit_error(
source_label: &str,
suffix: &str,
) -> String {
format!("{source_label}的图集切片超过 {PLATFORM_ART_SPRITESHEET_MAX_SLICES}{suffix}")
}
type ExternalCanvasBindingAsyncLock = tokio::sync::Mutex<()>;
const EXTERNAL_CANVAS_BINDING_PARTIAL_SCHEMA_VERSION: &str =
"game-creator-external-editor-project-binding-partial.v2";
@@ -2365,8 +2392,11 @@ async fn prepare_platform_art_spritesheet_slices(
else {
return Ok(Vec::new());
};
if icons.len() > 64 {
return Err("External Editor 返回的图集切片超过 64 个,已拒绝同步".to_string());
if platform_art_spritesheet_slice_count_exceeds_limit(icons.len()) {
return Err(platform_art_spritesheet_slice_limit_error(
"External Editor 返回",
",已拒绝同步",
));
}
let mut prepared = Vec::with_capacity(icons.len());
if initial_download_bytes > PLATFORM_ART_SPRITESHEET_TOTAL_DOWNLOAD_BYTES
@@ -8394,6 +8424,26 @@ pub(crate) fn build_platform_art_asset_prompt(
#[cfg(test)]
mod canvas_generation_tests {
#[test]
fn spritesheet_slice_limit_matches_platform_and_tool_contract() {
// 现场回归(`gameagent-6e53c9e8`):平台按 `connected-components` 自动切图,客户端旧门
// >64 就拒)把 65~256 片的**合法**产出整条判失败。这里把上限钉在与平台切分、Agent
// 工具 schema `sliceCount`1..256)和公开契约同一个值上。
assert_eq!(PLATFORM_ART_SPRITESHEET_MAX_SLICES, 256);
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(0));
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(64));
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(
PLATFORM_ART_SPRITESHEET_MAX_SLICES
));
assert!(platform_art_spritesheet_slice_count_exceeds_limit(
PLATFORM_ART_SPRITESHEET_MAX_SLICES + 1
));
// 拒绝文案里的数字来自常量本身,不会和上限再次漂移。
assert_eq!(
platform_art_spritesheet_slice_limit_error("External Editor 返回", ",已拒绝同步"),
"External Editor 返回的图集切片超过 256 个,已拒绝同步"
);
}
use super::*;
use image::{
codecs::png::{CompressionType, FilterType, PngEncoder},
@@ -912,8 +912,14 @@ fn durable_legacy_generation_result(
.get("iconImageSrcs")
.and_then(serde_json::Value::as_array)
{
if icons.len() > 64 {
return Err("External Editor 旧同步结果的图集切片超过 64 个".to_string());
if super::canvas_generation::platform_art_spritesheet_slice_count_exceeds_limit(icons.len())
{
return Err(
super::canvas_generation::platform_art_spritesheet_slice_limit_error(
"External Editor 旧同步结果",
"",
),
);
}
let mut durable_icons = Vec::with_capacity(icons.len());
for (index, icon) in icons.iter().enumerate() {
@@ -1995,6 +2001,94 @@ mod external_generation_state_tests {
);
}
/// 现场回归(`gameagent-6e53c9e8`2026-09-21):平台按 `connected-components` 自动切图,
/// 切出的数量落在 65~256 之间是**合法**产出;客户端旧门(>64 就拒)会把整条结果判失败,
/// 用户只看到「异步生成完成结果无法绑定到 operationId」——图没落盘,付费结果被丢弃。
///
/// 上限必须与平台切分(`EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`)和 Agent 工具 schema
/// 的 `sliceCount`1..256)同值;超过上限仍然拒,且文案里的数字来自常量本身。
#[test]
fn legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond() {
fn slice(index: usize) -> serde_json::Value {
serde_json::json!({
"name": format!("素材 {index}"),
"width": 64,
"height": 64,
"resource": {
"resourceId": format!("slice-{index}"),
"objectKey": format!("generated/slice-{index}.png")
}
})
}
fn persist_legacy_slice_result(
root: &Path,
count: usize,
) -> Result<serde_json::Value, String> {
let pending = pending_canvas_generation(root);
let context = platform_art_generation_runtime_context_from_pending(&pending);
let access = developer_access("https://editor.example.test", "test-api-key");
let request_body = serde_json::json!({
"prompt": "生成统一视觉规范图",
"kind": "spec",
"projectId": "canvas-project",
"assetFolderId": "asset-folder",
"referenceImageSrcs": []
});
let (state, _) = prepare_platform_art_generation_runtime_state(
root,
&context,
"/api/external/v1/editor/images/generations",
"legacy-generation-canvas",
"生成统一视觉规范图",
&request_body,
&access,
)
.expect("prepare legacy generation ledger");
let completed = mark_platform_art_generation_runtime_legacy_completed(
root,
state,
&serde_json::json!({
"resource": {
"resourceId": "legacy-resource",
"objectKey": "generated/legacy.png"
},
"iconImageSrcs": (0..count).map(slice).collect::<Vec<_>>()
}),
)?;
remove_platform_art_generation_runtime_state_at(
root,
&pending.agent_id,
&pending.run_id,
)
.expect("reset legacy ledger between slice counts");
platform_art_generation_runtime_legacy_result(&completed)
.map_err(|error| error.to_string())
}
let temporary = crate::tests::canonical_test_tempdir("legacy-generation-slice-limit-");
let root = temporary.path();
init_local_game_project_at(root, "legacy-slice-limit", "旧同步切片上限回归")
.expect("init project");
// 64(旧上限)到 256(平台上限)之间都是合法产出:必须能持久化,切片一条不少。
for count in [64_usize, 65, 256] {
let durable =
persist_legacy_slice_result(root, count).expect("persist legal slice count");
assert_eq!(
durable["iconImageSrcs"].as_array().map(Vec::len),
Some(count)
);
}
// 257 片仍然拒:与平台同一句口径,数字来自常量。
let error = persist_legacy_slice_result(root, 257).expect_err("reject beyond limit");
assert!(
error.contains("图集切片超过 256 个"),
"unexpected rejection message: {error}"
);
}
#[cfg(unix)]
#[test]
fn dangling_generation_ledger_symlink_is_reconciliation_evidence() {
@@ -0,0 +1,5 @@
[Service]
Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4
Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520
ExecStart=
ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --freeze-dir /var/lib/genarrative/database-backups/files-minimal-freeze --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal
@@ -6,6 +6,16 @@
- 已确认本期只做本地事件采集和明文 JSONL 持久化,不做加密与上传;一个项目对应一个目标,事件按业务节点采集,5 分钟封存,7 天或 20 MiB 清理。后续上传周期为 15 分钟。
- 当前尚未实施,参数建议及剩余口径见方案第 11 节;文档入库不表示这些建议已获确认或功能已上线。
## 2026-09-21 图集切片上限:客户端结果门从 64 对齐到平台契约的 256
- 背景:现场(项目 `gameagent-6e53c9e8`2026-09-21 07:54)「AI 生成图标素材」失败:`platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationIdExternal Editor 旧同步结果的图集切片超过 64 个`。任务账本(`.agent/runtime/asset-generation-tasks/tasks.json`)显示它跑了 99 秒、`assetId` 为空、没有落任何素材;对应的持久化请求(`canvas-generation-requests/manual-canvas-asset-generate/slot-560175669f….json`)是 `sliceMode: connected-components` + `sliceCount: null`(自动切分)。也就是**平台已经生成并切完图了,是客户端在绑定结果这一步把整条结果判失败**,付费产物被丢弃。
- 根因:同一条链路里存在两个不同的切片上限。平台切分是 256(`server-rs/crates/api-server/src/editor_project_icon.rs``EDITOR_ICON_SPRITESHEET_MAX_SLICES`),Agent 工具 schema 的 `sliceCount` 是 1..256`agent_native_tools.rs` / `direct_tool_bridge.rs`),持久化产物批次也是 1..256,公开契约(`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`)写的更是「最多 256 个输出」;唯独客户端**两处结果绑定门**还是 `> 64` 就拒(`agent/generation/external_generation_state.rs``agent/generation/canvas_generation.rs`,由 `602723ea0` 于 2026-08-03 引入)。自动切分落在这个窗口里(65~256 片)时,客户端比平台更严,于是把合法产出整条丢掉。
- 决策:两处门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`,并抽成同一条判据 `platform_art_spritesheet_slice_count_exceeds_limit` 与同一句拒绝文案 `platform_art_spritesheet_slice_limit_error`(数字由常量插值,不再手写)。注释里点名三处同值权威(平台切分常量、工具 schema `sliceCount`、公开契约),客户端不得比平台更严。
- 原因:客户端这两处门的作用是「防止把不可信/超预算的结果写进本地」,不是产品上限;真正的产品上限属于平台切分契约。两处各写一个字面量就会再次漂移,所以值只留一份、判据只留一条。
- 验证:新增 `canvas_generation_tests::spritesheet_slice_limit_matches_platform_and_tool_contract`(上限值、边界判据与文案)与 `external_generation_state_tests::legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond`(64/65/256 片必须能持久化且切片一条不少、257 片必须按同一句文案拒绝),两条都用**变异验证**确认过:把常量改回 64,回归用例立刻变红。定向执行 `cargo test -- spritesheet`22 passed)、`cargo test -- external_generation_state_tests::`10 passed)与两条新用例;`cargo fmt --check` 干净。
- 影响范围:`apps/ai-game-creator-shell/src-tauri/src/agent/generation/{canvas_generation.rs,external_generation_state.rs}`(+用例)。未动平台切分、OpenAPI、数据库或前端。
- 已知未覆盖:真实客户端复验(重新生成一次图标素材)与远程 CI 未跑;256 片时的累计下载/像素预算未实测——平台自己的总像素上限是 2048×2048,客户端预算是 4096²,按切片是整图互不重叠子矩形推算不会先撞预算,且真撞了也只是给出明确错误而不是损坏数据。
## 2026-09-21 本批自查(PR #441):三处修正
- 背景:推 PR 后按「局部到整体」自查这一批(三需求 + 验收修正),查出三条:①拖动到对话的落点在 `pointermove` 上每帧都 `setState` 一个新对象;②替换面板相对 **stage** 写死 `top: 8.5rem`(与刚修的任务开关同一类隐患:工具条换行会压上去),且它和「生成任务」面板抢画布右上角同一个位置;③替换面板不显示「在替换哪张源素材」,而非模态化之后那点线索(画布上的源素材光环)会被一次空白点击清掉。
@@ -1,9 +1,22 @@
# 踩坑与排障记录
## release 冷备空间不足会把生产留在维护态
`Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker,再执行发布前冷备份;archive 口径要求 `data × 1.1`40.6GiB 数据 → 44.7GiB,加上生产根盘只剩 13.5GiB),于是 2026-09-21 的 release 发布在停服后失败并保持维护态,站点 503 直到人工恢复。规则:空间预检必须先于 `maintenance-on` 与停服;archive 不足且未显式禁用降级时改用 files(`max(data × 0.05, 2GiB)`,不落地本地归档,但 files 不支持 `--defer-upload`,会从 async 收敛为 sync);只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另:定时备份的失效锁在当前仓库版本会自动清理,但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁,需随 provision 更新。
按上游 [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555) 的 retention 语义,只有最近 `retain-snapshots`(默认 2)份 snapshot 与覆盖它之后的 commitlog 段是重启所需,其余历史可丢;因此备份改为 `files + full + --minimal --retain-snapshots 2`release 实测 40G → 3.6G,热备不停服),不再做增量差异计算,也不需要 44.7G 冷备空间。
## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异
Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权:只有当被复制 Job 的 `CopyArtifactPermissionProperty`(仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护)显式列出当前消费者,或者该 Job 对认证用户开放 Item.Read 时才放行;`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象(2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中,定时调度 #104+ 正常)。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger``Genarrative-Manual-Build-And-Deploy`;改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins,只改仓库文件不生效。
## 同一条链路两处上限不一致:平台合法产出被客户端整条丢弃
- 现象:客户端报「生成素材失败:platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationIdExternal Editor 旧同步结果的图集切片超过 64 个」,而平台侧这次生成**其实已经成功并切完图**(任务账本耗时正常、`assetId` 为空、没有任何素材落盘,付费产物被丢)。
- 成因:图集切片上限在链路里存在两份字面量——平台切分、Agent 工具 schema `sliceCount` 与持久化产物批次都是 256,客户端结果绑定门写着 64(`agent/generation/{canvas_generation.rs,external_generation_state.rs}`)。自动切分(`connected-components` + `sliceCount=null`)切出 65~256 片是合法产出,客户端比平台更严就会把结果整条判失败。
- 处理:客户端门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`,判据与文案各只留一份(数字由常量插值),并在注释里点名三处同值权威(平台切分常量、工具 schema、公开契约)。
- 复用判据:凡是「平台产出 → 客户端校验后落盘」的链路,客户端门只能表达**安全 / 预算**约束,不得比平台的产品上限更严;两边上限要引同一个常量或同一份文档,改一边时必须同时改另一边,并补一条「上限之内必须能落盘」的回归用例。
## Jenkins Windows 节点的 PATH 白名单决定 Godot 原生扩展能否构建
`Genarrative-Agc-Windows-Build` 在阶段里用 `AGC_WINDOWS_PATH` 整体替换 PATH、不继承节点机器的 PATH,所以 Godot C++ 引导需要的 CMake 与 Python 必须显式写进这份白名单,装在机器 PATH 上并不生效。2026-09-21 的 #97#99 连续失败都停在 `Get-Command cmake.exe`#93#96 是更早的手写 C ABI 在 MSVC C 模式下的对齐问题):节点只有 Visual Studio Build Tools`C:\BuildTools`)自带的 CMake 3.31,缺 Python 3。修复后白名单包含 `C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin``C:\Python312``C:\Python312\Scripts`preflight 校验 CMake ≥3.25、Python 3 和 Visual Studio 17 2022 生成器;把 `cmake.exe` 单独复制到别的目录会丢掉 `share/cmake-*/Modules`,不能替代加入安装目录。新节点的 Python 用 `python-3.12.10-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python312 PrependPath=1 Include_launcher=1 InstallLauncherAllUsers=1` 静默安装即可,CMake 不必另装。
File diff suppressed because one or more lines are too long
@@ -34,7 +34,7 @@ pipeline {
string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接')
string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件')
string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口')
choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profilerelease 仅允许 archive-fullfiles-history 仅 development 在指定 work-dir 已有完整 full baseline 后启用')
choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history'], description: '数据库定时备份 profilerelease 用 files-minimal(最近 snapshot + 其后 commitlog 的热备)或 archive-fullfiles-history 因内存峰值仅 development 使用')
string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录;必须使用已建立 full baseline 的独立目录')
choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式;开发服无域名时选 development-httprelease 正式入口选 production-https')
booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务')
@@ -110,11 +110,11 @@ pipeline {
error('release 目标禁止安装 development-http Nginx 配置;无证书初始化请使用 NGINX_CONFIG_MODE=none。')
}
def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim()
if (!(databaseBackupProfile in ['archive-full', 'files-history'])) {
error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history,当前值: ${params.DATABASE_BACKUP_PROFILE}")
if (!(databaseBackupProfile in ['archive-full', 'files-minimal', 'files-history'])) {
error("DATABASE_BACKUP_PROFILE 只能是 archive-full、files-minimal 或 files-history,当前值: ${params.DATABASE_BACKUP_PROFILE}")
}
if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') {
error('release 允许 archive-fullfiles-history 会把整棵历史目录加载到 Node 内存,需先完成流式 catalog 改造后才能重新启用。')
error('release 允许 files-historyfiles-history 会把整棵历史目录加载到 Node 内存。release 请选择 files-minimal(最近 snapshot + 其后 commitlog)或 archive-full。')
}
def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim()
if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) {
+167 -2
View File
@@ -26,8 +26,10 @@ import {
cleanupHistoryCandidates,
collectDirectFileEntries,
createUploadBandwidthLimiter,
describeBackupSpaceRequirement,
discoverDeferredArchiveUploads,
discoverHistoryPlan,
discoverMinimalPlan,
restoreDirectFilesBackup,
restoreDirectFilesLatest,
resumeUploadedHistoryBatch,
@@ -64,6 +66,8 @@ async function main() {
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
assertInsufficientSpaceStopsBeforeServiceChanges();
assertCheckSpaceOnlyUsesFormatSpecificRequirement();
assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog();
assertStopFailureRetainsRecoveryMarker();
assertArchiveFailureStillRestoresDependentServices();
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
@@ -1198,7 +1202,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
'999999999999999999',
]);
assertStatus(result, 1, '空间不足必须失败。');
assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。');
assertIncludes(
result.stdout,
'备份空间预检',
@@ -1213,6 +1217,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
}
function assertCheckSpaceOnlyUsesFormatSpecificRequirement() {
const fixture = createFixture('check-space-only');
const filesOk = runBackup(
fixture,
['--check-space-only', '--storage-format', 'files'],
{ GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' },
);
assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。');
assertIncludes(
filesOk.stdout,
'备份空间预检(files)',
'files 预检必须打印 files 口径的预检结果。',
);
assertIncludes(
filesOk.stdout,
'空间预检通过(check-space-only)',
'check-space-only 通过时必须给出显式成功标记。',
);
assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。');
assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。');
const filesInsufficient = runBackup(
fixture,
['--check-space-only', '--storage-format', 'files'],
{ GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' },
);
assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。');
const archiveInsufficient = runBackup(
fixture,
['--check-space-only', '--storage-format', 'archive'],
{ GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' },
);
assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。');
const archive = describeBackupSpaceRequirement({
dataDir: fixture.dataDir,
workDir: fixture.workDir,
storageFormat: 'archive',
args: {},
env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' },
});
const files = describeBackupSpaceRequirement({
dataDir: fixture.dataDir,
workDir: fixture.workDir,
storageFormat: 'files',
args: {},
env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' },
});
if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) {
failures.push(
`空间口径覆盖参数应生效:archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`,
);
}
if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') {
failures.push('空间预检结果必须回显实际 storage-format。');
}
}
function assertStopFailureRetainsRecoveryMarker() {
const fixture = createFixture('stop-failure-marker');
writeExecutable(
@@ -2500,6 +2564,106 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() {
}
}
function assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog() {
const fixture = createHistoryFixture('minimal-plan', { nestedData: true });
const replicaDir = path.join(fixture.replicasDir, '2');
const snapshotsDir = path.join(replicaDir, 'snapshots');
const clogDir = path.join(replicaDir, 'clog');
mkdirSync(snapshotsDir, { recursive: true });
mkdirSync(clogDir, { recursive: true });
for (const transaction of ['100', '200', '300']) {
const padded = transaction.padStart(20, '0');
const snapshotDir = path.join(snapshotsDir, `${padded}.snapshot_dir`);
mkdirSync(snapshotDir, { recursive: true });
writeFileSync(
path.join(snapshotDir, `${padded}.snapshot_bsatn`),
'snapshot',
);
}
for (const transaction of ['50', '150', '250', '350']) {
const padded = transaction.padStart(20, '0');
writeFileSync(path.join(clogDir, `${padded}.stdb.log`), 'log');
writeFileSync(path.join(clogDir, `${padded}.stdb.ofs`), 'ofs');
}
for (const relativeDir of ['config', 'data/control-db', 'data/program-bytes']) {
const directory = path.join(fixture.dataDir, relativeDir);
mkdirSync(directory, { recursive: true });
writeFileSync(path.join(directory, 'state.bin'), 'state');
}
writeFileSync(path.join(fixture.dataDir, 'data/config.toml'), 'config');
writeFileSync(path.join(fixture.dataDir, 'data/metadata.toml'), 'metadata');
const plan = discoverMinimalPlan({ dataDir: fixture.dataDir });
const paths = plan.candidates.map((item) => item.path);
const expect = (condition, reason) => {
if (!condition) {
failures.push(reason);
}
};
expect(
plan.retainSnapshots === 2,
`minimal 默认应保留 2 份 snapshot,实际 ${plan.retainSnapshots}`,
);
expect(
paths.includes(
'data/replicas/2/snapshots/00000000000000000200.snapshot_dir',
),
'minimal 必须保留次新 snapshot。',
);
expect(
paths.includes(
'data/replicas/2/snapshots/00000000000000000300.snapshot_dir',
),
'minimal 必须保留最新 snapshot。',
);
expect(
!paths.includes(
'data/replicas/2/snapshots/00000000000000000100.snapshot_dir',
),
'minimal 不得备份更早的 snapshot。',
);
expect(
paths.includes('data/replicas/2/clog/00000000000000000150.stdb.log'),
'minimal 必须保留覆盖最老保留 snapshot 的边界 commitlog 段。',
);
expect(
paths.includes('data/replicas/2/clog/00000000000000000350.stdb.log'),
'minimal 必须保留最新 commitlog 段。',
);
expect(
!paths.includes('data/replicas/2/clog/00000000000000000050.stdb.log'),
'minimal 不得备份更早的 commitlog 段。',
);
for (const staticPath of [
'config',
'data/config.toml',
'data/metadata.toml',
'data/control-db',
'data/program-bytes',
]) {
expect(
paths.includes(staticPath),
`minimal 必须保留状态路径 ${staticPath}`,
);
}
const replica = plan.replicas.find((item) => item.replicaId === '2');
expect(
replica?.retainedSnapshots === 2 && replica?.droppedSnapshots === 1,
'minimal 必须报告保留/丢弃的 snapshot 数量。',
);
expect(
replica?.droppedSegments === 1,
`minimal 必须报告丢弃的 commitlog 段数量,实际 ${replica?.droppedSegments}`,
);
assertThrows(
() => discoverMinimalPlan({ dataDir: fixture.dataDir, retainSnapshots: 0 }),
'--retain-snapshots 必须是 >= 1 的整数',
'minimal 必须校验保留 snapshot 数量。',
);
}
function createHistoryFixture(name, { nestedData }) {
const root = path.join(tmpRoot, name);
const dataDir = path.join(root, 'stdb');
@@ -2695,7 +2859,7 @@ exit 2
return { root, binDir, dataDir, workDir, systemctlLog, tarLog };
}
function runBackup(fixture, extraArgs = []) {
function runBackup(fixture, extraArgs = [], envOverrides = {}) {
return spawnSync(
process.execPath,
[
@@ -2720,6 +2884,7 @@ function runBackup(fixture, extraArgs = []) {
env: {
...process.env,
PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
...envOverrides,
},
},
);
+62 -6
View File
@@ -44,8 +44,7 @@ const checks = [
},
{
file: 'jenkins/Jenkinsfile.agc-global-version-issue',
includes:
"copyArtifactPermission('Genarrative-Scheduled-Revision-Trigger,",
includes: "copyArtifactPermission('Genarrative-Scheduled-Revision-Trigger,",
reason:
'Copy Artifact Production 模式下,AGC 发号 Job 必须显式授权调度管线读取 agc-global-version.txt,否则整轮调度会在 copyArtifacts 处失败。',
},
@@ -259,6 +258,44 @@ const checks = [
reason:
'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径,保证首次 rollout 重启即可读取 secret。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes:
'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"',
reason:
'备份空间预检必须先于进入维护模式执行:磁盘不足时不得停服务或把生产留在维护态。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--check-space-only',
reason:
'生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检,保证空间口径与真实备份一致。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK',
reason:
'archive 冷备份空间不足时必须能自动降级为 files 存储格式(不落地本地归档)。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: 'restore_runtime_services_before_publish',
reason:
'尚未开始 publish 的失败必须自动恢复运行时服务并退出维护,避免生产停在维护态等人工。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes:
'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then',
reason:
'只有尚未开始 publish 的失败才允许自动恢复并退出维护,半发布状态必须保持维护。',
},
{
file: 'scripts/database-backup-to-oss.mjs',
includes: 'SPACE_INSUFFICIENT_EXIT_CODE',
reason:
'备份空间不足必须使用独立退出码,调用方据此决定降级存储格式而不是一律失败。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: 'stop_runtime_services_for_rollout_gate',
@@ -901,7 +938,7 @@ const checks = [
},
{
file: 'scripts/jenkins-server-provision.sh',
includes: 'archive-full|files-history)',
includes: 'archive-full|files-history|files-minimal)',
reason: 'Server-Provision 必须拒绝未知数据库备份 profile。',
},
{
@@ -926,7 +963,7 @@ const checks = [
{
file: 'jenkins/Jenkinsfile.production-server-provision',
includes:
"choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']",
"choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history']",
reason:
'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。',
},
@@ -979,13 +1016,32 @@ const checks = [
},
{
file: 'jenkins/Jenkinsfile.production-server-provision',
includes: 'release 允许 archive-fullfiles-history',
includes: 'release 允许 files-history',
reason:
'release 必须拒绝 files-history,避免逐文件 catalog 扫描再次触发生产内存峰值。',
},
{
file: 'jenkins/Jenkinsfile.production-server-provision',
includes: 'files-minimal',
reason:
'release 定时备份必须提供 files-minimal profile:只备最近 snapshot 与其后 commitlog,热备不停服。',
},
{
file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf',
includes: '--mode full --minimal --retain-snapshots 2 --freeze-dir',
reason:
'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。',
},
{
file: 'scripts/deploy/production-stdb-publish.sh',
includes: '--freeze-dir',
reason:
'发布前备份默认使用 minimal 热备,避免 40G 级冷备空间门槛与停服。',
},
{
file: 'scripts/database-backup-to-oss.mjs',
includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})',
includes:
"assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })",
normalizeWhitespace: true,
reason: '生产冷备份必须先做工作目录剩余空间预检,避免停库后写满磁盘。',
},
File diff suppressed because it is too large Load Diff
+185 -21
View File
@@ -18,9 +18,18 @@ usage() {
如需强制等待备份完成并在失败时阻断 publish,传入 --backup-mode sync。
发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径,再重启并验活重启前 active 的服务。
--keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker,并在成功后保持维护态,交由后续 API deploy 恢复服务。
发布前先做备份空间预检(不进入维护、不停服务);archive 空间不足且未显式关闭自动降级时,
自动改用 files 存储格式(不落地本地归档,改为同步直传 OSS),避免磁盘不足把生产留在维护态。
环境变量:
GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files(默认 archive
GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0(默认 1archive 空间不足自动降级 files
GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=1|0(默认 1:发布前备份只保留最近 N 份 snapshot + 其后 commitlog,热备不停服)
GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS=N(默认 2,仅在 minimal 模式下生效)
GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0(默认 1:尚未开始 publish 的失败自动恢复服务并退出维护)
migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供,不从构建 artifact 读取。
如果 API 重启前为 active,会在退出维护模式前等待本机 /healthz readiness 通过。
失败时保留维护模式。
失败时:尚未开始 publish 的失败会自动恢复运行时服务并退出维护;真正开始 publish 之后的失败保留维护模式。
EOF
}
@@ -55,7 +64,15 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv
WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
KEEP_MAINTENANCE_MODE=0
BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}"
AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}"
AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}"
BACKUP_MINIMAL="${GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL:-1}"
BACKUP_RETAIN_SNAPSHOTS="${GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS:-2}"
DEPLOY_COMPLETED=0
PUBLISH_STARTED=0
MAINTENANCE_ENTERED=0
STOPPED_RUNTIME_SERVICES=()
PUBLISH_TMP_DIR=""
ASYNC_BACKUP_STATUS_FILE=""
ASYNC_BACKUP_SCRIPT=""
@@ -287,6 +304,114 @@ restart_runtime_services_after_bootstrap_secret_install() {
fi
}
backup_script_path() {
local candidate=""
for candidate in \
"${SCRIPT_DIR}/../database-backup-to-oss.mjs" \
"${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do
if [[ -f "${candidate}" ]]; then
printf '%s\n' "${candidate}"
return 0
fi
done
return 1
}
run_backup_space_precheck() {
local storage_format="$1"
local backup_script=""
if ! backup_script="$(backup_script_path)"; then
echo "[production-stdb-publish] 缺少数据库备份脚本,无法做备份空间预检" >&2
return 1
fi
node -- "${backup_script}" \
--env-file /etc/genarrative/api-server.env \
--data-dir "${SPACETIME_ROOT_DIR}" \
--database "${DATABASE}" \
--storage-format "${storage_format}" \
--check-space-only
}
# 空间预检必须发生在进入维护模式与停服务之前:磁盘不够时不允许再动生产。
precheck_backup_space_before_maintenance() {
if [[ "${BACKUP_MODE}" == "skip" ]]; then
echo "[production-stdb-publish] 已跳过发布前备份空间预检(--backup-mode skip"
return 0
fi
if [[ "${BACKUP_MINIMAL}" == "1" ]]; then
# minimal 备份是热备:只保留最近 N 份 snapshot + 其后 commitlog,不落地归档也不停服务。
BACKUP_STORAGE_FORMAT="files"
if [[ "${BACKUP_MODE}" == "async" ]]; then
echo "[production-stdb-publish] minimal 备份为同步热备(无本地归档),备份模式由 async 调整为 sync。" >&2
BACKUP_MODE="sync"
fi
fi
local status=0
run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
if [[ "${status}" -eq 0 ]]; then
echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}(尚未进入维护模式、未停服务)"
return 0
fi
if [[ "${status}" -ne 3 ]]; then
echo "[production-stdb-publish] 发布前备份空间预检失败(非空间原因),中止发布;未进入维护模式、未停服务。" >&2
exit 1
fi
if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then
echo "[production-stdb-publish] files 模式备份空间仍不足,中止发布;未进入维护模式、未停服务。" >&2
exit 1
fi
if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then
echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级(GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}),中止发布;未进入维护模式、未停服务。" >&2
exit 1
fi
echo "[production-stdb-publish] archive 冷备份空间不足:自动降级为 files 存储格式(不落地本地归档,改为文件级 catalog 直传 OSS)。" >&2
BACKUP_STORAGE_FORMAT="files"
if [[ "${BACKUP_MODE}" == "async" ]]; then
echo "[production-stdb-publish] files 模式不支持 --defer-upload,本次备份改为同步执行。" >&2
BACKUP_MODE="sync"
fi
status=0
run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
if [[ "${status}" -ne 0 ]]; then
echo "[production-stdb-publish] 降级为 files 后空间预检仍失败,中止发布;未进入维护模式、未停服务。" >&2
exit 1
fi
echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。"
}
# 仅在「尚未开始 publish」的失败路径调用:把本次停掉的运行时服务拉回来。
restore_runtime_services_before_publish() {
if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then
return 0
fi
local service=""
local state=""
local attempt=0
echo "[production-stdb-publish] 发布尚未开始,恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}"
if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then
echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2
return 1
fi
for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do
state=""
for attempt in $(seq 1 15); do
state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)"
if [[ "${state}" == "active" ]]; then
break
fi
sleep 1
done
if [[ "${state}" != "active" ]]; then
echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2
return 1
fi
echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}"
done
return 0
}
stop_runtime_services_for_rollout_gate() {
local api_state=""
local controller_state=""
@@ -326,6 +451,7 @@ stop_runtime_services_for_rollout_gate() {
fi
echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}")
run_privileged systemctl stop "${services_to_stop[@]}"
for worker_service in "${services_to_stop[@]}"; do
if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
@@ -402,6 +528,10 @@ while [[ $# -gt 0 ]]; do
BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
shift 2
;;
--backup-storage-format)
BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}"
shift 2
;;
*)
echo "[production-stdb-publish] 未知参数: $1" >&2
usage >&2
@@ -431,6 +561,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
fi
done
if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then
echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2
exit 1
fi
if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then
echo "[production-stdb-publish] files 存储格式不支持 --defer-upload,备份模式由 async 调整为 sync" >&2
BACKUP_MODE="sync"
fi
if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
exit 1
@@ -497,7 +635,22 @@ on_exit() {
rm -rf "${PUBLISH_TMP_DIR}"
fi
if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
echo "[production-stdb-publish] 发布失败,保持维护模式。" >&2
if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then
# 尚未开始 publish 就失败(例如备份空间/备份执行失败):本次没有任何发布变更,
# 必须把停掉的运行时服务拉回来并退出维护,避免生产停在维护态等人工救。
if restore_runtime_services_before_publish; then
if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then
if ! "${SCRIPT_DIR}/maintenance-off.sh"; then
echo "[production-stdb-publish] 自动退出维护模式失败,请手工执行 maintenance-off.sh。" >&2
fi
fi
echo "[production-stdb-publish] 发布尚未开始即失败,已自动恢复运行时服务并退出维护模式。"
else
echo "[production-stdb-publish] 自动恢复运行时服务失败,保持维护模式,请手工处理。" >&2
fi
else
echo "[production-stdb-publish] 发布失败,保持维护模式。" >&2
fi
fi
exit "${exit_code}"
}
@@ -506,12 +659,8 @@ trap on_exit EXIT
prepare_async_backup() {
local -a restart_service_args=()
ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
fi
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then
echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
exit 1
fi
@@ -527,6 +676,7 @@ prepare_async_backup() {
--env-file /etc/genarrative/api-server.env \
--data-dir "${SPACETIME_ROOT_DIR}" \
--database "${DATABASE}" \
--storage-format "${BACKUP_STORAGE_FORMAT}" \
--stop-service spacetimedb.service \
"${restart_service_args[@]}" \
--defer-upload \
@@ -676,7 +826,10 @@ wait_for_api_healthz_ready() {
return 1
}
precheck_backup_space_before_maintenance
"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
MAINTENANCE_ENTERED=1
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
stop_runtime_services_for_rollout_gate
fi
@@ -687,25 +840,34 @@ case "${BACKUP_MODE}" in
;;
sync)
SYNC_BACKUP_RESTART_SERVICE_ARGS=()
BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
fi
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
if ! BACKUP_SCRIPT="$(backup_script_path)"; then
echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
exit 1
fi
if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
fi
echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份,失败会阻断发布"
node -- "${BACKUP_SCRIPT}" \
--env-file /etc/genarrative/api-server.env \
--data-dir "${SPACETIME_ROOT_DIR}" \
--database "${DATABASE}" \
--stop-service spacetimedb.service \
"${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
SYNC_BACKUP_ARGS=(
--env-file /etc/genarrative/api-server.env
--data-dir "${SPACETIME_ROOT_DIR}"
--database "${DATABASE}"
--storage-format "${BACKUP_STORAGE_FORMAT}"
)
if [[ "${BACKUP_MINIMAL}" == "1" ]]; then
echo "[production-stdb-publish] publish 前执行 minimal 热备(最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog),不停服务"
SYNC_BACKUP_ARGS+=(
--mode full
--minimal
--retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}"
--freeze-dir "${GENARRATIVE_STDB_PUBLISH_BACKUP_FREEZE_DIR:-/var/lib/genarrative/database-backups/publish-minimal-freeze}"
)
else
echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份(storage-format=${BACKUP_STORAGE_FORMAT}),失败会阻断发布"
SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service)
SYNC_BACKUP_ARGS+=("${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}")
fi
node -- "${BACKUP_SCRIPT}" "${SYNC_BACKUP_ARGS[@]}"
;;
skip)
echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份"
@@ -768,8 +930,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
else
PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
fi
PUBLISH_STARTED=1
runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
else
PUBLISH_STARTED=1
spacetime "${PUBLISH_ARGS[@]}"
fi
+37 -4
View File
@@ -15,6 +15,8 @@ DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}"
DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}"
DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d"
DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf"
DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR="${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR:-/var/lib/genarrative/database-backups/files-minimal}"
DATABASE_BACKUP_FILES_MINIMAL_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-minimal.conf"
DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf"
require_non_root_relative_path() {
@@ -71,10 +73,10 @@ validate_server_names() {
validate_database_backup_profile() {
case "${DATABASE_BACKUP_PROFILE}" in
archive-full|files-history)
archive-full|files-history|files-minimal)
;;
*)
echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-fullfiles-history,当前值: ${DATABASE_BACKUP_PROFILE}" >&2
echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-fullfiles-history 或 files-minimal,当前值: ${DATABASE_BACKUP_PROFILE}" >&2
exit 1
;;
esac
@@ -1323,12 +1325,29 @@ render_database_backup_files_history_drop_in() {
deploy/systemd/genarrative-database-backup-files-history.conf
}
render_database_backup_files_minimal_drop_in() {
local current_escaped env_escaped work_dir_escaped
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}")"
sed \
-e "s|/opt/genarrative/current|${current_escaped}|g" \
-e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \
-e "s|/var/lib/genarrative/database-backups/files-minimal|${work_dir_escaped}|g" \
deploy/systemd/genarrative-database-backup-files-minimal.conf
}
configure_database_backup_profile() {
local rendered_drop_in
if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then
echo "[server-provision] 数据库备份 profile=archive-full,保留主 service 的全量冷备行为。"
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
return
fi
if [[ "${DATABASE_BACKUP_PROFILE}" == "files-minimal" ]]; then
configure_database_backup_files_minimal_profile
return
fi
@@ -1350,13 +1369,26 @@ configure_database_backup_profile() {
run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}"
run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}"
run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
run_cmd rm -f "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
rendered_drop_in="$(mktemp)"
render_database_backup_files_history_drop_in >"${rendered_drop_in}"
install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644
rm -f "${rendered_drop_in}"
}
configure_database_backup_files_minimal_profile() {
local rendered_drop_in
echo "[server-provision] 数据库备份 profile=files-minimal,只保留最近 snapshot 与其后 commitlog(热备、不停服)。"
run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}"
run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}"
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
rendered_drop_in="$(mktemp)"
render_database_backup_files_minimal_drop_in >"${rendered_drop_in}"
install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" 0644
rm -f "${rendered_drop_in}"
}
render_health_patrol_service() {
local current_escaped
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
@@ -1372,6 +1404,7 @@ require_path deploy/systemd/genarrative-external-generation-controller.service
require_path deploy/systemd/genarrative-bgfilter-worker.service
require_path deploy/systemd/genarrative-database-backup.service
require_path deploy/systemd/genarrative-database-backup-files-history.conf
require_path deploy/systemd/genarrative-database-backup-files-minimal.conf
require_path deploy/systemd/genarrative-database-backup.timer
require_path deploy/systemd/genarrative-health-patrol.service
require_path deploy/systemd/genarrative-health-patrol.timer