合并最新 master 到埋点方案分支
Project CI / AI game creator shell Rust smoke (pull_request) Has been cancelled
Project CI / AI game creator shell Rust crates (pull_request) Has been cancelled
Project CI / Backend tests (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Has been cancelled
Project CI / Native shell tests (pull_request) Has been cancelled
Project CI / Frontend tests (pull_request) Has been cancelled
Project CI / Repository checks (pull_request) Has been cancelled
Project CI / AI game creator shell web tests (pull_request) Has been cancelled
Project CI / AI game creator shell Rust smoke (pull_request) Has been cancelled
Project CI / AI game creator shell Rust crates (pull_request) Has been cancelled
Project CI / Backend tests (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 1/2 (pull_request) Has been cancelled
Project CI / AI game creator shell Rust lane 2/2 (pull_request) Has been cancelled
Project CI / Native shell tests (pull_request) Has been cancelled
Project CI / Frontend tests (pull_request) Has been cancelled
Project CI / Repository checks (pull_request) Has been cancelled
Project CI / AI game creator shell web tests (pull_request) Has been cancelled
同步主分支最新客户端和运维变更。 解决共享决策记录顶部冲突,保留双方新增记录。
This commit is contained in:
@@ -215,6 +215,9 @@ test('packaged renderer receives the same channel as the updater manifest', () =
|
||||
spawnOptions = options;
|
||||
return { status: 0 };
|
||||
},
|
||||
// 该用例只关心渠道环境变量;Node 运行时 staging 与宿主平台绑定,
|
||||
// 真实 staging 会让同一用例在 macOS/Linux 上对 windows target 失败。
|
||||
stageRuntime: () => {},
|
||||
});
|
||||
assert.equal(spawnOptions?.env?.VITE_AGC_PLATFORM_CHANNEL, 'release');
|
||||
});
|
||||
|
||||
@@ -32,6 +32,33 @@ const PLATFORM_ART_SPRITESHEET_MAX_DIMENSION: u32 = 4_096;
|
||||
const PLATFORM_ART_SPRITESHEET_TOTAL_PIXELS: u64 = 16 * 1024 * 1024;
|
||||
const PLATFORM_ART_SPRITESHEET_MAX_DECODE_ALLOC: u64 = 64 * 1024 * 1024;
|
||||
|
||||
/// 客户端接受的图集切片数量上限。
|
||||
///
|
||||
/// 这个值**不是**客户端自己的预算,而是与另外三处取同一个值的一致性约束:
|
||||
/// 平台切分上限(`server-rs/crates/api-server/src/editor_project_icon.rs` 的
|
||||
/// `EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`)、Agent 工具 schema 的 `sliceCount`(1..256,
|
||||
/// 见 `agent_native_tools.rs` 与 `direct_tool_bridge.rs`),以及公开契约
|
||||
/// (`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`:单边最多 4096 像素、总像素最多
|
||||
/// 2048×2048、最多 256 个输出)。
|
||||
///
|
||||
/// 客户端一旦比平台更严,平台**合法**产出的 65~256 片结果会在绑定结果这一步被整条丢弃:
|
||||
/// 用户只会看到「异步生成完成结果无法绑定到 operationId」加一句切片超限,既拿不到图也没法自查。
|
||||
/// 现场(`gameagent-6e53c9e8`,`connected-components` 自动切分)就是被旧值 64 挡下的。
|
||||
pub(crate) const PLATFORM_ART_SPRITESHEET_MAX_SLICES: usize = 256;
|
||||
|
||||
/// 平台返回的图集切片数量是否超出上限(两个结果绑定入口共用同一条判据)。
|
||||
pub(crate) fn platform_art_spritesheet_slice_count_exceeds_limit(slice_count: usize) -> bool {
|
||||
slice_count > PLATFORM_ART_SPRITESHEET_MAX_SLICES
|
||||
}
|
||||
|
||||
/// 切片数量超限的拒绝文案(两处入口同一句话,数字来自常量,避免再和上限漂移)。
|
||||
pub(crate) fn platform_art_spritesheet_slice_limit_error(
|
||||
source_label: &str,
|
||||
suffix: &str,
|
||||
) -> String {
|
||||
format!("{source_label}的图集切片超过 {PLATFORM_ART_SPRITESHEET_MAX_SLICES} 个{suffix}")
|
||||
}
|
||||
|
||||
type ExternalCanvasBindingAsyncLock = tokio::sync::Mutex<()>;
|
||||
const EXTERNAL_CANVAS_BINDING_PARTIAL_SCHEMA_VERSION: &str =
|
||||
"game-creator-external-editor-project-binding-partial.v2";
|
||||
@@ -2365,8 +2392,11 @@ async fn prepare_platform_art_spritesheet_slices(
|
||||
else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if icons.len() > 64 {
|
||||
return Err("External Editor 返回的图集切片超过 64 个,已拒绝同步".to_string());
|
||||
if platform_art_spritesheet_slice_count_exceeds_limit(icons.len()) {
|
||||
return Err(platform_art_spritesheet_slice_limit_error(
|
||||
"External Editor 返回",
|
||||
",已拒绝同步",
|
||||
));
|
||||
}
|
||||
let mut prepared = Vec::with_capacity(icons.len());
|
||||
if initial_download_bytes > PLATFORM_ART_SPRITESHEET_TOTAL_DOWNLOAD_BYTES
|
||||
@@ -8394,6 +8424,26 @@ pub(crate) fn build_platform_art_asset_prompt(
|
||||
|
||||
#[cfg(test)]
|
||||
mod canvas_generation_tests {
|
||||
#[test]
|
||||
fn spritesheet_slice_limit_matches_platform_and_tool_contract() {
|
||||
// 现场回归(`gameagent-6e53c9e8`):平台按 `connected-components` 自动切图,客户端旧门
|
||||
// (>64 就拒)把 65~256 片的**合法**产出整条判失败。这里把上限钉在与平台切分、Agent
|
||||
// 工具 schema `sliceCount`(1..256)和公开契约同一个值上。
|
||||
assert_eq!(PLATFORM_ART_SPRITESHEET_MAX_SLICES, 256);
|
||||
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(0));
|
||||
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(64));
|
||||
assert!(!platform_art_spritesheet_slice_count_exceeds_limit(
|
||||
PLATFORM_ART_SPRITESHEET_MAX_SLICES
|
||||
));
|
||||
assert!(platform_art_spritesheet_slice_count_exceeds_limit(
|
||||
PLATFORM_ART_SPRITESHEET_MAX_SLICES + 1
|
||||
));
|
||||
// 拒绝文案里的数字来自常量本身,不会和上限再次漂移。
|
||||
assert_eq!(
|
||||
platform_art_spritesheet_slice_limit_error("External Editor 返回", ",已拒绝同步"),
|
||||
"External Editor 返回的图集切片超过 256 个,已拒绝同步"
|
||||
);
|
||||
}
|
||||
use super::*;
|
||||
use image::{
|
||||
codecs::png::{CompressionType, FilterType, PngEncoder},
|
||||
|
||||
+96
-2
@@ -912,8 +912,14 @@ fn durable_legacy_generation_result(
|
||||
.get("iconImageSrcs")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
{
|
||||
if icons.len() > 64 {
|
||||
return Err("External Editor 旧同步结果的图集切片超过 64 个".to_string());
|
||||
if super::canvas_generation::platform_art_spritesheet_slice_count_exceeds_limit(icons.len())
|
||||
{
|
||||
return Err(
|
||||
super::canvas_generation::platform_art_spritesheet_slice_limit_error(
|
||||
"External Editor 旧同步结果",
|
||||
"",
|
||||
),
|
||||
);
|
||||
}
|
||||
let mut durable_icons = Vec::with_capacity(icons.len());
|
||||
for (index, icon) in icons.iter().enumerate() {
|
||||
@@ -1995,6 +2001,94 @@ mod external_generation_state_tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// 现场回归(`gameagent-6e53c9e8`,2026-09-21):平台按 `connected-components` 自动切图,
|
||||
/// 切出的数量落在 65~256 之间是**合法**产出;客户端旧门(>64 就拒)会把整条结果判失败,
|
||||
/// 用户只看到「异步生成完成结果无法绑定到 operationId」——图没落盘,付费结果被丢弃。
|
||||
///
|
||||
/// 上限必须与平台切分(`EDITOR_ICON_SPRITESHEET_MAX_SLICES = 256`)和 Agent 工具 schema
|
||||
/// 的 `sliceCount`(1..256)同值;超过上限仍然拒,且文案里的数字来自常量本身。
|
||||
#[test]
|
||||
fn legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond() {
|
||||
fn slice(index: usize) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"name": format!("素材 {index}"),
|
||||
"width": 64,
|
||||
"height": 64,
|
||||
"resource": {
|
||||
"resourceId": format!("slice-{index}"),
|
||||
"objectKey": format!("generated/slice-{index}.png")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn persist_legacy_slice_result(
|
||||
root: &Path,
|
||||
count: usize,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let pending = pending_canvas_generation(root);
|
||||
let context = platform_art_generation_runtime_context_from_pending(&pending);
|
||||
let access = developer_access("https://editor.example.test", "test-api-key");
|
||||
let request_body = serde_json::json!({
|
||||
"prompt": "生成统一视觉规范图",
|
||||
"kind": "spec",
|
||||
"projectId": "canvas-project",
|
||||
"assetFolderId": "asset-folder",
|
||||
"referenceImageSrcs": []
|
||||
});
|
||||
let (state, _) = prepare_platform_art_generation_runtime_state(
|
||||
root,
|
||||
&context,
|
||||
"/api/external/v1/editor/images/generations",
|
||||
"legacy-generation-canvas",
|
||||
"生成统一视觉规范图",
|
||||
&request_body,
|
||||
&access,
|
||||
)
|
||||
.expect("prepare legacy generation ledger");
|
||||
let completed = mark_platform_art_generation_runtime_legacy_completed(
|
||||
root,
|
||||
state,
|
||||
&serde_json::json!({
|
||||
"resource": {
|
||||
"resourceId": "legacy-resource",
|
||||
"objectKey": "generated/legacy.png"
|
||||
},
|
||||
"iconImageSrcs": (0..count).map(slice).collect::<Vec<_>>()
|
||||
}),
|
||||
)?;
|
||||
remove_platform_art_generation_runtime_state_at(
|
||||
root,
|
||||
&pending.agent_id,
|
||||
&pending.run_id,
|
||||
)
|
||||
.expect("reset legacy ledger between slice counts");
|
||||
platform_art_generation_runtime_legacy_result(&completed)
|
||||
.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
let temporary = crate::tests::canonical_test_tempdir("legacy-generation-slice-limit-");
|
||||
let root = temporary.path();
|
||||
init_local_game_project_at(root, "legacy-slice-limit", "旧同步切片上限回归")
|
||||
.expect("init project");
|
||||
|
||||
// 64(旧上限)到 256(平台上限)之间都是合法产出:必须能持久化,切片一条不少。
|
||||
for count in [64_usize, 65, 256] {
|
||||
let durable =
|
||||
persist_legacy_slice_result(root, count).expect("persist legal slice count");
|
||||
assert_eq!(
|
||||
durable["iconImageSrcs"].as_array().map(Vec::len),
|
||||
Some(count)
|
||||
);
|
||||
}
|
||||
|
||||
// 257 片仍然拒:与平台同一句口径,数字来自常量。
|
||||
let error = persist_legacy_slice_result(root, 257).expect_err("reject beyond limit");
|
||||
assert!(
|
||||
error.contains("图集切片超过 256 个"),
|
||||
"unexpected rejection message: {error}"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn dangling_generation_ledger_symlink_is_reconciliation_evidence() {
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
[Service]
|
||||
Environment=GENARRATIVE_DATABASE_BACKUP_FILES_CONCURRENCY=4
|
||||
Environment=GENARRATIVE_DATABASE_BACKUP_UPLOAD_MAX_BYTES_PER_SECOND=20971520
|
||||
ExecStart=
|
||||
ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --storage-format files --mode full --minimal --retain-snapshots 2 --freeze-dir /var/lib/genarrative/database-backups/files-minimal-freeze --data-dir /stdb --work-dir /var/lib/genarrative/database-backups/files-minimal
|
||||
@@ -6,6 +6,16 @@
|
||||
- 已确认本期只做本地事件采集和明文 JSONL 持久化,不做加密与上传;一个项目对应一个目标,事件按业务节点采集,5 分钟封存,7 天或 20 MiB 清理。后续上传周期为 15 分钟。
|
||||
- 当前尚未实施,参数建议及剩余口径见方案第 11 节;文档入库不表示这些建议已获确认或功能已上线。
|
||||
|
||||
## 2026-09-21 图集切片上限:客户端结果门从 64 对齐到平台契约的 256
|
||||
|
||||
- 背景:现场(项目 `gameagent-6e53c9e8`,2026-09-21 07:54)「AI 生成图标素材」失败:`platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationId:External Editor 旧同步结果的图集切片超过 64 个`。任务账本(`.agent/runtime/asset-generation-tasks/tasks.json`)显示它跑了 99 秒、`assetId` 为空、没有落任何素材;对应的持久化请求(`canvas-generation-requests/manual-canvas-asset-generate/slot-560175669f….json`)是 `sliceMode: connected-components` + `sliceCount: null`(自动切分)。也就是**平台已经生成并切完图了,是客户端在绑定结果这一步把整条结果判失败**,付费产物被丢弃。
|
||||
- 根因:同一条链路里存在两个不同的切片上限。平台切分是 256(`server-rs/crates/api-server/src/editor_project_icon.rs` 的 `EDITOR_ICON_SPRITESHEET_MAX_SLICES`),Agent 工具 schema 的 `sliceCount` 是 1..256(`agent_native_tools.rs` / `direct_tool_bridge.rs`),持久化产物批次也是 1..256,公开契约(`docs/【编辑器】画板图标素材生成入口设计-2026-06-15.md`)写的更是「最多 256 个输出」;唯独客户端**两处结果绑定门**还是 `> 64` 就拒(`agent/generation/external_generation_state.rs` 与 `agent/generation/canvas_generation.rs`,由 `602723ea0` 于 2026-08-03 引入)。自动切分落在这个窗口里(65~256 片)时,客户端比平台更严,于是把合法产出整条丢掉。
|
||||
- 决策:两处门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`,并抽成同一条判据 `platform_art_spritesheet_slice_count_exceeds_limit` 与同一句拒绝文案 `platform_art_spritesheet_slice_limit_error`(数字由常量插值,不再手写)。注释里点名三处同值权威(平台切分常量、工具 schema `sliceCount`、公开契约),客户端不得比平台更严。
|
||||
- 原因:客户端这两处门的作用是「防止把不可信/超预算的结果写进本地」,不是产品上限;真正的产品上限属于平台切分契约。两处各写一个字面量就会再次漂移,所以值只留一份、判据只留一条。
|
||||
- 验证:新增 `canvas_generation_tests::spritesheet_slice_limit_matches_platform_and_tool_contract`(上限值、边界判据与文案)与 `external_generation_state_tests::legacy_result_accepts_slice_counts_up_to_platform_limit_and_rejects_beyond`(64/65/256 片必须能持久化且切片一条不少、257 片必须按同一句文案拒绝),两条都用**变异验证**确认过:把常量改回 64,回归用例立刻变红。定向执行 `cargo test -- spritesheet`(22 passed)、`cargo test -- external_generation_state_tests::`(10 passed)与两条新用例;`cargo fmt --check` 干净。
|
||||
- 影响范围:`apps/ai-game-creator-shell/src-tauri/src/agent/generation/{canvas_generation.rs,external_generation_state.rs}`(+用例)。未动平台切分、OpenAPI、数据库或前端。
|
||||
- 已知未覆盖:真实客户端复验(重新生成一次图标素材)与远程 CI 未跑;256 片时的累计下载/像素预算未实测——平台自己的总像素上限是 2048×2048,客户端预算是 4096²,按切片是整图互不重叠子矩形推算不会先撞预算,且真撞了也只是给出明确错误而不是损坏数据。
|
||||
|
||||
## 2026-09-21 本批自查(PR #441):三处修正
|
||||
|
||||
- 背景:推 PR 后按「局部到整体」自查这一批(三需求 + 验收修正),查出三条:①拖动到对话的落点在 `pointermove` 上每帧都 `setState` 一个新对象;②替换面板相对 **stage** 写死 `top: 8.5rem`(与刚修的任务开关同一类隐患:工具条换行会压上去),且它和「生成任务」面板抢画布右上角同一个位置;③替换面板不显示「在替换哪张源素材」,而非模态化之后那点线索(画布上的源素材光环)会被一次空白点击清掉。
|
||||
|
||||
@@ -1,9 +1,22 @@
|
||||
# 踩坑与排障记录
|
||||
|
||||
## release 冷备空间不足会把生产留在维护态
|
||||
|
||||
`Genarrative-Stdb-Module-Publish` 先进入维护模式、停掉 API/controller/worker,再执行发布前冷备份;archive 口径要求 `data × 1.1`(40.6GiB 数据 → 44.7GiB,加上生产根盘只剩 13.5GiB),于是 2026-09-21 的 release 发布在停服后失败并保持维护态,站点 503 直到人工恢复。规则:空间预检必须先于 `maintenance-on` 与停服;archive 不足且未显式禁用降级时改用 files(`max(data × 0.05, 2GiB)`,不落地本地归档,但 files 不支持 `--defer-upload`,会从 async 收敛为 sync);只有真正开始 `spacetime publish` 之后的失败才允许保持维护态。另:定时备份的失效锁在当前仓库版本会自动清理,但生产机 `/var/lib/genarrative/backup-tools/database-backup-to-oss.mjs` 若是旧版会拒绝抢锁并要求人工删锁,需随 provision 更新。
|
||||
|
||||
按上游 [#5555](https://github.com/clockworklabs/SpacetimeDB/pull/5555) 的 retention 语义,只有最近 `retain-snapshots`(默认 2)份 snapshot 与覆盖它之后的 commitlog 段是重启所需,其余历史可丢;因此备份改为 `files + full + --minimal --retain-snapshots 2`(release 实测 40G → 3.6G,热备不停服),不再做增量差异计算,也不需要 44.7G 冷备空间。
|
||||
|
||||
## copyArtifacts 报「Unable to find project for artifact copy」的用户触发构建差异
|
||||
|
||||
Copy Artifact 插件在**非 SYSTEM 认证**下按「认证用户」判权:只有当被复制 Job 的 `CopyArtifactPermissionProperty`(仓库里由 Declarative 的 `copyArtifactPermission(...)` 维护)显式列出当前消费者,或者该 Job 对认证用户开放 Item.Read 时才放行;`ACL.SYSTEM2` 的定时构建会短路通过。因此会出现「定时调度一路成功、手动发布必挂」的现象(2026-09-21 手动发布 #6/#7 与同期的用户触发探测全部命中,定时调度 #104+ 正常)。`Genarrative-Agc-Global-Version-Issue` 生产权限模式的授权名单必须同时包含 `Genarrative-Scheduled-Revision-Trigger` 与 `Genarrative-Manual-Build-And-Deploy`;改完 `copyArtifactPermission` 后要先跑一次发号 Job 把 Job property 写回 Jenkins,只改仓库文件不生效。
|
||||
|
||||
## 同一条链路两处上限不一致:平台合法产出被客户端整条丢弃
|
||||
|
||||
- 现象:客户端报「生成素材失败:platform-generation-result-unknown: 异步生成完成结果无法绑定到 operationId:External Editor 旧同步结果的图集切片超过 64 个」,而平台侧这次生成**其实已经成功并切完图**(任务账本耗时正常、`assetId` 为空、没有任何素材落盘,付费产物被丢)。
|
||||
- 成因:图集切片上限在链路里存在两份字面量——平台切分、Agent 工具 schema `sliceCount` 与持久化产物批次都是 256,客户端结果绑定门写着 64(`agent/generation/{canvas_generation.rs,external_generation_state.rs}`)。自动切分(`connected-components` + `sliceCount=null`)切出 65~256 片是合法产出,客户端比平台更严就会把结果整条判失败。
|
||||
- 处理:客户端门统一到 `PLATFORM_ART_SPRITESHEET_MAX_SLICES = 256`,判据与文案各只留一份(数字由常量插值),并在注释里点名三处同值权威(平台切分常量、工具 schema、公开契约)。
|
||||
- 复用判据:凡是「平台产出 → 客户端校验后落盘」的链路,客户端门只能表达**安全 / 预算**约束,不得比平台的产品上限更严;两边上限要引同一个常量或同一份文档,改一边时必须同时改另一边,并补一条「上限之内必须能落盘」的回归用例。
|
||||
|
||||
## Jenkins Windows 节点的 PATH 白名单决定 Godot 原生扩展能否构建
|
||||
|
||||
`Genarrative-Agc-Windows-Build` 在阶段里用 `AGC_WINDOWS_PATH` 整体替换 PATH、不继承节点机器的 PATH,所以 Godot C++ 引导需要的 CMake 与 Python 必须显式写进这份白名单,装在机器 PATH 上并不生效。2026-09-21 的 #97–#99 连续失败都停在 `Get-Command cmake.exe`(#93–#96 是更早的手写 C ABI 在 MSVC C 模式下的对齐问题):节点只有 Visual Studio Build Tools(`C:\BuildTools`)自带的 CMake 3.31,缺 Python 3。修复后白名单包含 `C:\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin`、`C:\Python312`、`C:\Python312\Scripts`,preflight 校验 CMake ≥3.25、Python 3 和 Visual Studio 17 2022 生成器;把 `cmake.exe` 单独复制到别的目录会丢掉 `share/cmake-*/Modules`,不能替代加入安装目录。新节点的 Python 用 `python-3.12.10-amd64.exe /quiet InstallAllUsers=1 TargetDir=C:\Python312 PrependPath=1 Include_launcher=1 InstallLauncherAllUsers=1` 静默安装即可,CMake 不必另装。
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -34,7 +34,7 @@ pipeline {
|
||||
string(name: 'WEB_LINK', defaultValue: '/srv/genarrative/web', description: 'Nginx 静态站点目录或软链接')
|
||||
string(name: 'API_ENV_FILE', defaultValue: '/etc/genarrative/api-server.env', description: 'api-server 环境文件')
|
||||
string(name: 'API_PORT', defaultValue: '8082', description: 'api-server 本机监听端口')
|
||||
choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history'], description: '数据库定时备份 profile;release 仅允许 archive-full,files-history 仅供 development 在指定 work-dir 已有完整 full baseline 后启用')
|
||||
choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history'], description: '数据库定时备份 profile;release 用 files-minimal(最近 snapshot + 其后 commitlog 的热备)或 archive-full,files-history 因内存峰值仅 development 使用')
|
||||
string(name: 'DATABASE_BACKUP_FILES_HISTORY_WORK_DIR', defaultValue: '/var/lib/genarrative/database-backups/files-history', description: 'development files-history 的本地 state/catalog 目录;必须使用已建立 full baseline 的独立目录')
|
||||
choice(name: 'NGINX_CONFIG_MODE', choices: ['none', 'production-https', 'development-http'], description: 'Nginx 配置模式;开发服无域名时选 development-http,release 正式入口选 production-https')
|
||||
booleanParam(name: 'ENABLE_SERVICES', defaultValue: true, description: '启用并启动 spacetimedb 与 api-server systemd 服务')
|
||||
@@ -110,11 +110,11 @@ pipeline {
|
||||
error('release 目标禁止安装 development-http Nginx 配置;无证书初始化请使用 NGINX_CONFIG_MODE=none。')
|
||||
}
|
||||
def databaseBackupProfile = params.DATABASE_BACKUP_PROFILE?.trim()
|
||||
if (!(databaseBackupProfile in ['archive-full', 'files-history'])) {
|
||||
error("DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history,当前值: ${params.DATABASE_BACKUP_PROFILE}")
|
||||
if (!(databaseBackupProfile in ['archive-full', 'files-minimal', 'files-history'])) {
|
||||
error("DATABASE_BACKUP_PROFILE 只能是 archive-full、files-minimal 或 files-history,当前值: ${params.DATABASE_BACKUP_PROFILE}")
|
||||
}
|
||||
if (params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history') {
|
||||
error('release 仅允许 archive-full;files-history 会把整棵历史目录加载到 Node 内存,需先完成流式 catalog 改造后才能重新启用。')
|
||||
error('release 不允许 files-history;files-history 会把整棵历史目录加载到 Node 内存。release 请选择 files-minimal(最近 snapshot + 其后 commitlog)或 archive-full。')
|
||||
}
|
||||
def databaseBackupFilesHistoryWorkDir = params.DATABASE_BACKUP_FILES_HISTORY_WORK_DIR?.trim()
|
||||
if (!(databaseBackupFilesHistoryWorkDir ==~ /^\/var\/lib\/genarrative\/database-backups\/[A-Za-z0-9._\/-]+$/) || databaseBackupFilesHistoryWorkDir.contains('..')) {
|
||||
|
||||
@@ -26,8 +26,10 @@ import {
|
||||
cleanupHistoryCandidates,
|
||||
collectDirectFileEntries,
|
||||
createUploadBandwidthLimiter,
|
||||
describeBackupSpaceRequirement,
|
||||
discoverDeferredArchiveUploads,
|
||||
discoverHistoryPlan,
|
||||
discoverMinimalPlan,
|
||||
restoreDirectFilesBackup,
|
||||
restoreDirectFilesLatest,
|
||||
resumeUploadedHistoryBatch,
|
||||
@@ -64,6 +66,8 @@ async function main() {
|
||||
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
|
||||
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
|
||||
assertInsufficientSpaceStopsBeforeServiceChanges();
|
||||
assertCheckSpaceOnlyUsesFormatSpecificRequirement();
|
||||
assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog();
|
||||
assertStopFailureRetainsRecoveryMarker();
|
||||
assertArchiveFailureStillRestoresDependentServices();
|
||||
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
|
||||
@@ -1198,7 +1202,7 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
||||
'999999999999999999',
|
||||
]);
|
||||
|
||||
assertStatus(result, 1, '空间不足时必须失败。');
|
||||
assertStatus(result, 3, '空间不足必须用独立退出码 3 失败。');
|
||||
assertIncludes(
|
||||
result.stdout,
|
||||
'备份空间预检',
|
||||
@@ -1213,6 +1217,66 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
||||
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
|
||||
}
|
||||
|
||||
function assertCheckSpaceOnlyUsesFormatSpecificRequirement() {
|
||||
const fixture = createFixture('check-space-only');
|
||||
|
||||
const filesOk = runBackup(
|
||||
fixture,
|
||||
['--check-space-only', '--storage-format', 'files'],
|
||||
{ GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1M' },
|
||||
);
|
||||
assertStatus(filesOk, 0, 'files 模式空间预检应在阈值满足时通过。');
|
||||
assertIncludes(
|
||||
filesOk.stdout,
|
||||
'备份空间预检(files)',
|
||||
'files 预检必须打印 files 口径的预检结果。',
|
||||
);
|
||||
assertIncludes(
|
||||
filesOk.stdout,
|
||||
'空间预检通过(check-space-only)',
|
||||
'check-space-only 通过时必须给出显式成功标记。',
|
||||
);
|
||||
assertFileMissing(fixture.systemctlLog, '空间预检不得调用 systemctl。');
|
||||
assertFileMissing(fixture.tarLog, '空间预检不得调用 tar。');
|
||||
|
||||
const filesInsufficient = runBackup(
|
||||
fixture,
|
||||
['--check-space-only', '--storage-format', 'files'],
|
||||
{ GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1000T' },
|
||||
);
|
||||
assertStatus(filesInsufficient, 3, 'files 空间不足同样使用退出码 3。');
|
||||
|
||||
const archiveInsufficient = runBackup(
|
||||
fixture,
|
||||
['--check-space-only', '--storage-format', 'archive'],
|
||||
{ GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1000T' },
|
||||
);
|
||||
assertStatus(archiveInsufficient, 3, 'archive 空间不足必须使用退出码 3。');
|
||||
|
||||
const archive = describeBackupSpaceRequirement({
|
||||
dataDir: fixture.dataDir,
|
||||
workDir: fixture.workDir,
|
||||
storageFormat: 'archive',
|
||||
args: {},
|
||||
env: { GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES: '1G' },
|
||||
});
|
||||
const files = describeBackupSpaceRequirement({
|
||||
dataDir: fixture.dataDir,
|
||||
workDir: fixture.workDir,
|
||||
storageFormat: 'files',
|
||||
args: {},
|
||||
env: { GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES: '1G' },
|
||||
});
|
||||
if (archive.requiredFreeBytes !== 1024n ** 3n || files.requiredFreeBytes !== 1024n ** 3n) {
|
||||
failures.push(
|
||||
`空间口径覆盖参数应生效:archive=${archive.requiredFreeBytes} files=${files.requiredFreeBytes}`,
|
||||
);
|
||||
}
|
||||
if (archive.storageFormat !== 'archive' || files.storageFormat !== 'files') {
|
||||
failures.push('空间预检结果必须回显实际 storage-format。');
|
||||
}
|
||||
}
|
||||
|
||||
function assertStopFailureRetainsRecoveryMarker() {
|
||||
const fixture = createFixture('stop-failure-marker');
|
||||
writeExecutable(
|
||||
@@ -2500,6 +2564,106 @@ async function assertHistoryResumeReverifiesArchiveAndManifest() {
|
||||
}
|
||||
}
|
||||
|
||||
function assertMinimalPlanKeepsOnlyRetainedSnapshotAndTrailingCommitlog() {
|
||||
const fixture = createHistoryFixture('minimal-plan', { nestedData: true });
|
||||
const replicaDir = path.join(fixture.replicasDir, '2');
|
||||
const snapshotsDir = path.join(replicaDir, 'snapshots');
|
||||
const clogDir = path.join(replicaDir, 'clog');
|
||||
mkdirSync(snapshotsDir, { recursive: true });
|
||||
mkdirSync(clogDir, { recursive: true });
|
||||
for (const transaction of ['100', '200', '300']) {
|
||||
const padded = transaction.padStart(20, '0');
|
||||
const snapshotDir = path.join(snapshotsDir, `${padded}.snapshot_dir`);
|
||||
mkdirSync(snapshotDir, { recursive: true });
|
||||
writeFileSync(
|
||||
path.join(snapshotDir, `${padded}.snapshot_bsatn`),
|
||||
'snapshot',
|
||||
);
|
||||
}
|
||||
for (const transaction of ['50', '150', '250', '350']) {
|
||||
const padded = transaction.padStart(20, '0');
|
||||
writeFileSync(path.join(clogDir, `${padded}.stdb.log`), 'log');
|
||||
writeFileSync(path.join(clogDir, `${padded}.stdb.ofs`), 'ofs');
|
||||
}
|
||||
for (const relativeDir of ['config', 'data/control-db', 'data/program-bytes']) {
|
||||
const directory = path.join(fixture.dataDir, relativeDir);
|
||||
mkdirSync(directory, { recursive: true });
|
||||
writeFileSync(path.join(directory, 'state.bin'), 'state');
|
||||
}
|
||||
writeFileSync(path.join(fixture.dataDir, 'data/config.toml'), 'config');
|
||||
writeFileSync(path.join(fixture.dataDir, 'data/metadata.toml'), 'metadata');
|
||||
|
||||
const plan = discoverMinimalPlan({ dataDir: fixture.dataDir });
|
||||
const paths = plan.candidates.map((item) => item.path);
|
||||
const expect = (condition, reason) => {
|
||||
if (!condition) {
|
||||
failures.push(reason);
|
||||
}
|
||||
};
|
||||
|
||||
expect(
|
||||
plan.retainSnapshots === 2,
|
||||
`minimal 默认应保留 2 份 snapshot,实际 ${plan.retainSnapshots}`,
|
||||
);
|
||||
expect(
|
||||
paths.includes(
|
||||
'data/replicas/2/snapshots/00000000000000000200.snapshot_dir',
|
||||
),
|
||||
'minimal 必须保留次新 snapshot。',
|
||||
);
|
||||
expect(
|
||||
paths.includes(
|
||||
'data/replicas/2/snapshots/00000000000000000300.snapshot_dir',
|
||||
),
|
||||
'minimal 必须保留最新 snapshot。',
|
||||
);
|
||||
expect(
|
||||
!paths.includes(
|
||||
'data/replicas/2/snapshots/00000000000000000100.snapshot_dir',
|
||||
),
|
||||
'minimal 不得备份更早的 snapshot。',
|
||||
);
|
||||
expect(
|
||||
paths.includes('data/replicas/2/clog/00000000000000000150.stdb.log'),
|
||||
'minimal 必须保留覆盖最老保留 snapshot 的边界 commitlog 段。',
|
||||
);
|
||||
expect(
|
||||
paths.includes('data/replicas/2/clog/00000000000000000350.stdb.log'),
|
||||
'minimal 必须保留最新 commitlog 段。',
|
||||
);
|
||||
expect(
|
||||
!paths.includes('data/replicas/2/clog/00000000000000000050.stdb.log'),
|
||||
'minimal 不得备份更早的 commitlog 段。',
|
||||
);
|
||||
for (const staticPath of [
|
||||
'config',
|
||||
'data/config.toml',
|
||||
'data/metadata.toml',
|
||||
'data/control-db',
|
||||
'data/program-bytes',
|
||||
]) {
|
||||
expect(
|
||||
paths.includes(staticPath),
|
||||
`minimal 必须保留状态路径 ${staticPath}。`,
|
||||
);
|
||||
}
|
||||
const replica = plan.replicas.find((item) => item.replicaId === '2');
|
||||
expect(
|
||||
replica?.retainedSnapshots === 2 && replica?.droppedSnapshots === 1,
|
||||
'minimal 必须报告保留/丢弃的 snapshot 数量。',
|
||||
);
|
||||
expect(
|
||||
replica?.droppedSegments === 1,
|
||||
`minimal 必须报告丢弃的 commitlog 段数量,实际 ${replica?.droppedSegments}`,
|
||||
);
|
||||
|
||||
assertThrows(
|
||||
() => discoverMinimalPlan({ dataDir: fixture.dataDir, retainSnapshots: 0 }),
|
||||
'--retain-snapshots 必须是 >= 1 的整数',
|
||||
'minimal 必须校验保留 snapshot 数量。',
|
||||
);
|
||||
}
|
||||
|
||||
function createHistoryFixture(name, { nestedData }) {
|
||||
const root = path.join(tmpRoot, name);
|
||||
const dataDir = path.join(root, 'stdb');
|
||||
@@ -2695,7 +2859,7 @@ exit 2
|
||||
return { root, binDir, dataDir, workDir, systemctlLog, tarLog };
|
||||
}
|
||||
|
||||
function runBackup(fixture, extraArgs = []) {
|
||||
function runBackup(fixture, extraArgs = [], envOverrides = {}) {
|
||||
return spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
@@ -2720,6 +2884,7 @@ function runBackup(fixture, extraArgs = []) {
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
|
||||
...envOverrides,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@@ -44,8 +44,7 @@ const checks = [
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.agc-global-version-issue',
|
||||
includes:
|
||||
"copyArtifactPermission('Genarrative-Scheduled-Revision-Trigger,",
|
||||
includes: "copyArtifactPermission('Genarrative-Scheduled-Revision-Trigger,",
|
||||
reason:
|
||||
'Copy Artifact Production 模式下,AGC 发号 Job 必须显式授权调度管线读取 agc-global-version.txt,否则整轮调度会在 copyArtifacts 处失败。',
|
||||
},
|
||||
@@ -259,6 +258,44 @@ const checks = [
|
||||
reason:
|
||||
'Stdb 先于 API 发布时必须先补齐 api-server env 的 FILE 路径,保证首次 rollout 重启即可读取 secret。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes:
|
||||
'precheck_backup_space_before_maintenance\n\n"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"',
|
||||
reason:
|
||||
'备份空间预检必须先于进入维护模式执行:磁盘不足时不得停服务或把生产留在维护态。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes: '--check-space-only',
|
||||
reason:
|
||||
'生产 Stdb publish 必须复用备份脚本的 --check-space-only 预检,保证空间口径与真实备份一致。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes: 'GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK',
|
||||
reason:
|
||||
'archive 冷备份空间不足时必须能自动降级为 files 存储格式(不落地本地归档)。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes: 'restore_runtime_services_before_publish',
|
||||
reason:
|
||||
'尚未开始 publish 的失败必须自动恢复运行时服务并退出维护,避免生产停在维护态等人工。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes:
|
||||
'if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then',
|
||||
reason:
|
||||
'只有尚未开始 publish 的失败才允许自动恢复并退出维护,半发布状态必须保持维护。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/database-backup-to-oss.mjs',
|
||||
includes: 'SPACE_INSUFFICIENT_EXIT_CODE',
|
||||
reason:
|
||||
'备份空间不足必须使用独立退出码,调用方据此决定降级存储格式而不是一律失败。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes: 'stop_runtime_services_for_rollout_gate',
|
||||
@@ -901,7 +938,7 @@ const checks = [
|
||||
},
|
||||
{
|
||||
file: 'scripts/jenkins-server-provision.sh',
|
||||
includes: 'archive-full|files-history)',
|
||||
includes: 'archive-full|files-history|files-minimal)',
|
||||
reason: 'Server-Provision 必须拒绝未知数据库备份 profile。',
|
||||
},
|
||||
{
|
||||
@@ -926,7 +963,7 @@ const checks = [
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-server-provision',
|
||||
includes:
|
||||
"choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-history']",
|
||||
"choice(name: 'DATABASE_BACKUP_PROFILE', choices: ['archive-full', 'files-minimal', 'files-history']",
|
||||
reason:
|
||||
'Server-Provision Job 必须显式暴露 archive-first 的数据库备份 profile。',
|
||||
},
|
||||
@@ -979,13 +1016,32 @@ const checks = [
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-server-provision',
|
||||
includes: 'release 仅允许 archive-full;files-history',
|
||||
includes: 'release 不允许 files-history',
|
||||
reason:
|
||||
'release 必须拒绝 files-history,避免逐文件 catalog 扫描再次触发生产内存峰值。',
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-server-provision',
|
||||
includes: 'files-minimal',
|
||||
reason:
|
||||
'release 定时备份必须提供 files-minimal profile:只备最近 snapshot 与其后 commitlog,热备不停服。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup-files-minimal.conf',
|
||||
includes: '--mode full --minimal --retain-snapshots 2 --freeze-dir',
|
||||
reason:
|
||||
'files-minimal 定时备份必须使用 minimal 口径并保留上游默认的最近 2 份 snapshot。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/deploy/production-stdb-publish.sh',
|
||||
includes: '--freeze-dir',
|
||||
reason:
|
||||
'发布前备份默认使用 minimal 热备,避免 40G 级冷备空间门槛与停服。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/database-backup-to-oss.mjs',
|
||||
includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})',
|
||||
includes:
|
||||
"assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' })",
|
||||
normalizeWhitespace: true,
|
||||
reason: '生产冷备份必须先做工作目录剩余空间预检,避免停库后写满磁盘。',
|
||||
},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -18,9 +18,18 @@ usage() {
|
||||
如需强制等待备份完成并在失败时阻断 publish,传入 --backup-mode sync。
|
||||
发布成功后会补齐生产 API/worker env 的固定 bootstrap secret FILE 路径,再重启并验活重启前 active 的服务。
|
||||
--keep-maintenance-mode 会在 publish 前停止旧 API/controller/worker,并在成功后保持维护态,交由后续 API deploy 恢复服务。
|
||||
发布前先做备份空间预检(不进入维护、不停服务);archive 空间不足且未显式关闭自动降级时,
|
||||
自动改用 files 存储格式(不落地本地归档,改为同步直传 OSS),避免磁盘不足把生产留在维护态。
|
||||
|
||||
环境变量:
|
||||
GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT=archive|files(默认 archive)
|
||||
GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=1|0(默认 1:archive 空间不足自动降级 files)
|
||||
GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL=1|0(默认 1:发布前备份只保留最近 N 份 snapshot + 其后 commitlog,热备不停服)
|
||||
GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS=N(默认 2,仅在 minimal 模式下生效)
|
||||
GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE=1|0(默认 1:尚未开始 publish 的失败自动恢复服务并退出维护)
|
||||
migration bootstrap secret 必须由 Jenkins Secret File credential 或等价的受保护文件提供,不从构建 artifact 读取。
|
||||
如果 API 重启前为 active,会在退出维护模式前等待本机 /healthz readiness 通过。
|
||||
失败时保留维护模式。
|
||||
失败时:尚未开始 publish 的失败会自动恢复运行时服务并退出维护;真正开始 publish 之后的失败保留维护模式。
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -55,7 +64,15 @@ API_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_API_ENV_FILE:-/etc/genarrative/api-serv
|
||||
WORKER_ENV_FILE="${GENARRATIVE_STDB_PUBLISH_WORKER_ENV_FILE:-/etc/genarrative/external-generation-worker.env}"
|
||||
KEEP_MAINTENANCE_MODE=0
|
||||
BACKUP_MODE="${GENARRATIVE_STDB_PUBLISH_BACKUP_MODE:-async}"
|
||||
BACKUP_STORAGE_FORMAT="${GENARRATIVE_STDB_PUBLISH_BACKUP_STORAGE_FORMAT:-archive}"
|
||||
AUTO_FILES_FALLBACK="${GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK:-1}"
|
||||
AUTO_RECOVER_BEFORE_PUBLISH="${GENARRATIVE_STDB_PUBLISH_AUTO_RECOVER_ON_PREPUBLISH_FAILURE:-1}"
|
||||
BACKUP_MINIMAL="${GENARRATIVE_STDB_PUBLISH_BACKUP_MINIMAL:-1}"
|
||||
BACKUP_RETAIN_SNAPSHOTS="${GENARRATIVE_STDB_PUBLISH_BACKUP_RETAIN_SNAPSHOTS:-2}"
|
||||
DEPLOY_COMPLETED=0
|
||||
PUBLISH_STARTED=0
|
||||
MAINTENANCE_ENTERED=0
|
||||
STOPPED_RUNTIME_SERVICES=()
|
||||
PUBLISH_TMP_DIR=""
|
||||
ASYNC_BACKUP_STATUS_FILE=""
|
||||
ASYNC_BACKUP_SCRIPT=""
|
||||
@@ -287,6 +304,114 @@ restart_runtime_services_after_bootstrap_secret_install() {
|
||||
fi
|
||||
}
|
||||
|
||||
backup_script_path() {
|
||||
local candidate=""
|
||||
for candidate in \
|
||||
"${SCRIPT_DIR}/../database-backup-to-oss.mjs" \
|
||||
"${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"; do
|
||||
if [[ -f "${candidate}" ]]; then
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
run_backup_space_precheck() {
|
||||
local storage_format="$1"
|
||||
local backup_script=""
|
||||
if ! backup_script="$(backup_script_path)"; then
|
||||
echo "[production-stdb-publish] 缺少数据库备份脚本,无法做备份空间预检" >&2
|
||||
return 1
|
||||
fi
|
||||
node -- "${backup_script}" \
|
||||
--env-file /etc/genarrative/api-server.env \
|
||||
--data-dir "${SPACETIME_ROOT_DIR}" \
|
||||
--database "${DATABASE}" \
|
||||
--storage-format "${storage_format}" \
|
||||
--check-space-only
|
||||
}
|
||||
|
||||
# 空间预检必须发生在进入维护模式与停服务之前:磁盘不够时不允许再动生产。
|
||||
precheck_backup_space_before_maintenance() {
|
||||
if [[ "${BACKUP_MODE}" == "skip" ]]; then
|
||||
echo "[production-stdb-publish] 已跳过发布前备份空间预检(--backup-mode skip)"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${BACKUP_MINIMAL}" == "1" ]]; then
|
||||
# minimal 备份是热备:只保留最近 N 份 snapshot + 其后 commitlog,不落地归档也不停服务。
|
||||
BACKUP_STORAGE_FORMAT="files"
|
||||
if [[ "${BACKUP_MODE}" == "async" ]]; then
|
||||
echo "[production-stdb-publish] minimal 备份为同步热备(无本地归档),备份模式由 async 调整为 sync。" >&2
|
||||
BACKUP_MODE="sync"
|
||||
fi
|
||||
fi
|
||||
|
||||
local status=0
|
||||
run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
echo "[production-stdb-publish] 发布前备份空间预检通过: storage-format=${BACKUP_STORAGE_FORMAT}(尚未进入维护模式、未停服务)"
|
||||
return 0
|
||||
fi
|
||||
if [[ "${status}" -ne 3 ]]; then
|
||||
echo "[production-stdb-publish] 发布前备份空间预检失败(非空间原因),中止发布;未进入维护模式、未停服务。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${BACKUP_STORAGE_FORMAT}" == "files" ]]; then
|
||||
echo "[production-stdb-publish] files 模式备份空间仍不足,中止发布;未进入维护模式、未停服务。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${AUTO_FILES_FALLBACK}" != "1" ]]; then
|
||||
echo "[production-stdb-publish] archive 备份空间不足且已禁用自动降级(GENARRATIVE_STDB_PUBLISH_AUTO_FILES_FALLBACK=${AUTO_FILES_FALLBACK}),中止发布;未进入维护模式、未停服务。" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[production-stdb-publish] archive 冷备份空间不足:自动降级为 files 存储格式(不落地本地归档,改为文件级 catalog 直传 OSS)。" >&2
|
||||
BACKUP_STORAGE_FORMAT="files"
|
||||
if [[ "${BACKUP_MODE}" == "async" ]]; then
|
||||
echo "[production-stdb-publish] files 模式不支持 --defer-upload,本次备份改为同步执行。" >&2
|
||||
BACKUP_MODE="sync"
|
||||
fi
|
||||
status=0
|
||||
run_backup_space_precheck "${BACKUP_STORAGE_FORMAT}" || status=$?
|
||||
if [[ "${status}" -ne 0 ]]; then
|
||||
echo "[production-stdb-publish] 降级为 files 后空间预检仍失败,中止发布;未进入维护模式、未停服务。" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[production-stdb-publish] 已降级为 files 存储格式且空间预检通过。"
|
||||
}
|
||||
|
||||
# 仅在「尚未开始 publish」的失败路径调用:把本次停掉的运行时服务拉回来。
|
||||
restore_runtime_services_before_publish() {
|
||||
if [[ "${#STOPPED_RUNTIME_SERVICES[@]}" -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
local service=""
|
||||
local state=""
|
||||
local attempt=0
|
||||
echo "[production-stdb-publish] 发布尚未开始,恢复本次停掉的运行时服务: ${STOPPED_RUNTIME_SERVICES[*]}"
|
||||
if ! run_privileged systemctl start "${STOPPED_RUNTIME_SERVICES[@]}"; then
|
||||
echo "[production-stdb-publish] 启动运行时服务失败: ${STOPPED_RUNTIME_SERVICES[*]}" >&2
|
||||
return 1
|
||||
fi
|
||||
for service in "${STOPPED_RUNTIME_SERVICES[@]}"; do
|
||||
state=""
|
||||
for attempt in $(seq 1 15); do
|
||||
state="$(get_runtime_service_active_state "${service}" 2>/dev/null || true)"
|
||||
if [[ "${state}" == "active" ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if [[ "${state}" != "active" ]]; then
|
||||
echo "[production-stdb-publish] 运行时服务未恢复 active: ${service}, state=${state}" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "[production-stdb-publish] 运行时服务已恢复 active: ${service}"
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
stop_runtime_services_for_rollout_gate() {
|
||||
local api_state=""
|
||||
local controller_state=""
|
||||
@@ -326,6 +451,7 @@ stop_runtime_services_for_rollout_gate() {
|
||||
fi
|
||||
|
||||
echo "[production-stdb-publish] 停止旧运行时服务并保持维护态: ${services_to_stop[*]}"
|
||||
STOPPED_RUNTIME_SERVICES=("${services_to_stop[@]}")
|
||||
run_privileged systemctl stop "${services_to_stop[@]}"
|
||||
for worker_service in "${services_to_stop[@]}"; do
|
||||
if [[ "$(get_runtime_service_active_state "${worker_service}")" == "active" ]]; then
|
||||
@@ -402,6 +528,10 @@ while [[ $# -gt 0 ]]; do
|
||||
BACKUP_MODE="${2:?缺少 --backup-mode 的值}"
|
||||
shift 2
|
||||
;;
|
||||
--backup-storage-format)
|
||||
BACKUP_STORAGE_FORMAT="${2:?缺少 --backup-storage-format 的值}"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "[production-stdb-publish] 未知参数: $1" >&2
|
||||
usage >&2
|
||||
@@ -431,6 +561,14 @@ for runtime_env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}"; do
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! "${BACKUP_STORAGE_FORMAT}" =~ ^(archive|files)$ ]]; then
|
||||
echo "[production-stdb-publish] --backup-storage-format 只能是 archive 或 files: ${BACKUP_STORAGE_FORMAT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${BACKUP_STORAGE_FORMAT}" == "files" && "${BACKUP_MODE}" == "async" ]]; then
|
||||
echo "[production-stdb-publish] files 存储格式不支持 --defer-upload,备份模式由 async 调整为 sync" >&2
|
||||
BACKUP_MODE="sync"
|
||||
fi
|
||||
if [[ ! "${BACKUP_MODE}" =~ ^(async|sync|skip)$ ]]; then
|
||||
echo "[production-stdb-publish] --backup-mode 只能是 async、sync 或 skip: ${BACKUP_MODE}" >&2
|
||||
exit 1
|
||||
@@ -497,7 +635,22 @@ on_exit() {
|
||||
rm -rf "${PUBLISH_TMP_DIR}"
|
||||
fi
|
||||
if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then
|
||||
echo "[production-stdb-publish] 发布失败,保持维护模式。" >&2
|
||||
if [[ "${PUBLISH_STARTED}" -ne 1 && "${AUTO_RECOVER_BEFORE_PUBLISH}" == "1" ]]; then
|
||||
# 尚未开始 publish 就失败(例如备份空间/备份执行失败):本次没有任何发布变更,
|
||||
# 必须把停掉的运行时服务拉回来并退出维护,避免生产停在维护态等人工救。
|
||||
if restore_runtime_services_before_publish; then
|
||||
if [[ "${MAINTENANCE_ENTERED}" -eq 1 ]]; then
|
||||
if ! "${SCRIPT_DIR}/maintenance-off.sh"; then
|
||||
echo "[production-stdb-publish] 自动退出维护模式失败,请手工执行 maintenance-off.sh。" >&2
|
||||
fi
|
||||
fi
|
||||
echo "[production-stdb-publish] 发布尚未开始即失败,已自动恢复运行时服务并退出维护模式。"
|
||||
else
|
||||
echo "[production-stdb-publish] 自动恢复运行时服务失败,保持维护模式,请手工处理。" >&2
|
||||
fi
|
||||
else
|
||||
echo "[production-stdb-publish] 发布失败,保持维护模式。" >&2
|
||||
fi
|
||||
fi
|
||||
exit "${exit_code}"
|
||||
}
|
||||
@@ -506,12 +659,8 @@ trap on_exit EXIT
|
||||
|
||||
prepare_async_backup() {
|
||||
local -a restart_service_args=()
|
||||
ASYNC_BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
|
||||
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
|
||||
ASYNC_BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
||||
fi
|
||||
if [[ ! -f "${ASYNC_BACKUP_SCRIPT}" ]]; then
|
||||
echo "[production-stdb-publish] 缺少数据库备份脚本: ${ASYNC_BACKUP_SCRIPT}" >&2
|
||||
if ! ASYNC_BACKUP_SCRIPT="$(backup_script_path)"; then
|
||||
echo "[production-stdb-publish] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -527,6 +676,7 @@ prepare_async_backup() {
|
||||
--env-file /etc/genarrative/api-server.env \
|
||||
--data-dir "${SPACETIME_ROOT_DIR}" \
|
||||
--database "${DATABASE}" \
|
||||
--storage-format "${BACKUP_STORAGE_FORMAT}" \
|
||||
--stop-service spacetimedb.service \
|
||||
"${restart_service_args[@]}" \
|
||||
--defer-upload \
|
||||
@@ -676,7 +826,10 @@ wait_for_api_healthz_ready() {
|
||||
return 1
|
||||
}
|
||||
|
||||
precheck_backup_space_before_maintenance
|
||||
|
||||
"${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}"
|
||||
MAINTENANCE_ENTERED=1
|
||||
if [[ "${KEEP_MAINTENANCE_MODE}" -eq 1 ]]; then
|
||||
stop_runtime_services_for_rollout_gate
|
||||
fi
|
||||
@@ -687,25 +840,34 @@ case "${BACKUP_MODE}" in
|
||||
;;
|
||||
sync)
|
||||
SYNC_BACKUP_RESTART_SERVICE_ARGS=()
|
||||
BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs"
|
||||
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
|
||||
BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs"
|
||||
fi
|
||||
if [[ ! -f "${BACKUP_SCRIPT}" ]]; then
|
||||
echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2
|
||||
if ! BACKUP_SCRIPT="$(backup_script_path)"; then
|
||||
echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${KEEP_MAINTENANCE_MODE}" -ne 1 ]]; then
|
||||
SYNC_BACKUP_RESTART_SERVICE_ARGS+=(--restart-service-after genarrative-api.service)
|
||||
fi
|
||||
|
||||
echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份,失败会阻断发布"
|
||||
node -- "${BACKUP_SCRIPT}" \
|
||||
--env-file /etc/genarrative/api-server.env \
|
||||
--data-dir "${SPACETIME_ROOT_DIR}" \
|
||||
--database "${DATABASE}" \
|
||||
--stop-service spacetimedb.service \
|
||||
"${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}"
|
||||
SYNC_BACKUP_ARGS=(
|
||||
--env-file /etc/genarrative/api-server.env
|
||||
--data-dir "${SPACETIME_ROOT_DIR}"
|
||||
--database "${DATABASE}"
|
||||
--storage-format "${BACKUP_STORAGE_FORMAT}"
|
||||
)
|
||||
if [[ "${BACKUP_MINIMAL}" == "1" ]]; then
|
||||
echo "[production-stdb-publish] publish 前执行 minimal 热备(最近 ${BACKUP_RETAIN_SNAPSHOTS} 份 snapshot + 其后 commitlog),不停服务"
|
||||
SYNC_BACKUP_ARGS+=(
|
||||
--mode full
|
||||
--minimal
|
||||
--retain-snapshots "${BACKUP_RETAIN_SNAPSHOTS}"
|
||||
--freeze-dir "${GENARRATIVE_STDB_PUBLISH_BACKUP_FREEZE_DIR:-/var/lib/genarrative/database-backups/publish-minimal-freeze}"
|
||||
)
|
||||
else
|
||||
echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份(storage-format=${BACKUP_STORAGE_FORMAT}),失败会阻断发布"
|
||||
SYNC_BACKUP_ARGS+=(--stop-service spacetimedb.service)
|
||||
SYNC_BACKUP_ARGS+=("${SYNC_BACKUP_RESTART_SERVICE_ARGS[@]}")
|
||||
fi
|
||||
node -- "${BACKUP_SCRIPT}" "${SYNC_BACKUP_ARGS[@]}"
|
||||
;;
|
||||
skip)
|
||||
echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份"
|
||||
@@ -768,8 +930,10 @@ if [[ -n "${RUN_AS_USER}" && "$(id -u)" -eq 0 ]]; then
|
||||
else
|
||||
PUBLISH_ARGS+=(--server "${SERVER_ALIAS}")
|
||||
fi
|
||||
PUBLISH_STARTED=1
|
||||
runuser -u "${RUN_AS_USER}" -- spacetime "${PUBLISH_ARGS[@]}"
|
||||
else
|
||||
PUBLISH_STARTED=1
|
||||
spacetime "${PUBLISH_ARGS[@]}"
|
||||
fi
|
||||
|
||||
|
||||
@@ -15,6 +15,8 @@ DATABASE_BACKUP_PROFILE="${DATABASE_BACKUP_PROFILE:-archive-full}"
|
||||
DATABASE_BACKUP_FILES_HISTORY_WORK_DIR="${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR:-/var/lib/genarrative/database-backups/files-history}"
|
||||
DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR="/etc/systemd/system/genarrative-database-backup.service.d"
|
||||
DATABASE_BACKUP_FILES_HISTORY_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-history.conf"
|
||||
DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR="${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR:-/var/lib/genarrative/database-backups/files-minimal}"
|
||||
DATABASE_BACKUP_FILES_MINIMAL_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-files-minimal.conf"
|
||||
DATABASE_BACKUP_LEGACY_DEV_DROP_IN="${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}/10-dev-files.conf"
|
||||
|
||||
require_non_root_relative_path() {
|
||||
@@ -71,10 +73,10 @@ validate_server_names() {
|
||||
|
||||
validate_database_backup_profile() {
|
||||
case "${DATABASE_BACKUP_PROFILE}" in
|
||||
archive-full|files-history)
|
||||
archive-full|files-history|files-minimal)
|
||||
;;
|
||||
*)
|
||||
echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full 或 files-history,当前值: ${DATABASE_BACKUP_PROFILE}" >&2
|
||||
echo "[server-provision] DATABASE_BACKUP_PROFILE 只能是 archive-full、files-history 或 files-minimal,当前值: ${DATABASE_BACKUP_PROFILE}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1323,12 +1325,29 @@ render_database_backup_files_history_drop_in() {
|
||||
deploy/systemd/genarrative-database-backup-files-history.conf
|
||||
}
|
||||
|
||||
render_database_backup_files_minimal_drop_in() {
|
||||
local current_escaped env_escaped work_dir_escaped
|
||||
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
|
||||
env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")"
|
||||
work_dir_escaped="$(escape_sed_replacement "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}")"
|
||||
sed \
|
||||
-e "s|/opt/genarrative/current|${current_escaped}|g" \
|
||||
-e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \
|
||||
-e "s|/var/lib/genarrative/database-backups/files-minimal|${work_dir_escaped}|g" \
|
||||
deploy/systemd/genarrative-database-backup-files-minimal.conf
|
||||
}
|
||||
|
||||
configure_database_backup_profile() {
|
||||
local rendered_drop_in
|
||||
|
||||
if [[ "${DATABASE_BACKUP_PROFILE}" == "archive-full" ]]; then
|
||||
echo "[server-provision] 数据库备份 profile=archive-full,保留主 service 的全量冷备行为。"
|
||||
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
|
||||
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "${DATABASE_BACKUP_PROFILE}" == "files-minimal" ]]; then
|
||||
configure_database_backup_files_minimal_profile
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -1350,13 +1369,26 @@ configure_database_backup_profile() {
|
||||
|
||||
run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}"
|
||||
run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}"
|
||||
run_cmd rm -f "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
|
||||
run_cmd rm -f "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
|
||||
rendered_drop_in="$(mktemp)"
|
||||
render_database_backup_files_history_drop_in >"${rendered_drop_in}"
|
||||
install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" 0644
|
||||
rm -f "${rendered_drop_in}"
|
||||
}
|
||||
|
||||
configure_database_backup_files_minimal_profile() {
|
||||
local rendered_drop_in
|
||||
|
||||
echo "[server-provision] 数据库备份 profile=files-minimal,只保留最近 snapshot 与其后 commitlog(热备、不停服)。"
|
||||
run_cmd install -d -o genarrative -g genarrative -m 0750 "${DATABASE_BACKUP_FILES_MINIMAL_WORK_DIR}"
|
||||
run_cmd install -d -o root -g root -m 0755 "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN_DIR}"
|
||||
run_cmd rm -f "${DATABASE_BACKUP_FILES_HISTORY_DROP_IN}" "${DATABASE_BACKUP_LEGACY_DEV_DROP_IN}"
|
||||
rendered_drop_in="$(mktemp)"
|
||||
render_database_backup_files_minimal_drop_in >"${rendered_drop_in}"
|
||||
install_file "${rendered_drop_in}" "${DATABASE_BACKUP_FILES_MINIMAL_DROP_IN}" 0644
|
||||
rm -f "${rendered_drop_in}"
|
||||
}
|
||||
|
||||
render_health_patrol_service() {
|
||||
local current_escaped
|
||||
current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")"
|
||||
@@ -1372,6 +1404,7 @@ require_path deploy/systemd/genarrative-external-generation-controller.service
|
||||
require_path deploy/systemd/genarrative-bgfilter-worker.service
|
||||
require_path deploy/systemd/genarrative-database-backup.service
|
||||
require_path deploy/systemd/genarrative-database-backup-files-history.conf
|
||||
require_path deploy/systemd/genarrative-database-backup-files-minimal.conf
|
||||
require_path deploy/systemd/genarrative-database-backup.timer
|
||||
require_path deploy/systemd/genarrative-health-patrol.service
|
||||
require_path deploy/systemd/genarrative-health-patrol.timer
|
||||
|
||||
Reference in New Issue
Block a user