发布前备份空间预检前置并支持自动降级
Project CI / AI game creator shell Rust smoke (push) Successful in 2m13s
Project CI / AI game creator shell Rust crates (push) Successful in 1m7s
Project CI / AI game creator shell Rust lane 1/2 (push) Failing after 7m9s
Project CI / AI game creator shell Rust lane 2/2 (push) Failing after 6m57s
Project CI / Native shell tests (push) Successful in 7m58s
Project CI / Repository checks (push) Successful in 5m13s
Project CI / Frontend tests (push) Successful in 7m19s
Project CI / Backend tests (push) Successful in 10m10s
Project CI / AI game creator shell web tests (push) Successful in 4m47s

- database-backup-to-oss.mjs 新增 --check-space-only:archive 按 data×1.1、files 按 max(data×0.05, 2GiB) 计算门槛,空间不足以退出码 3 返回
- production-stdb-publish.sh 在进入维护与停服务之前先做空间预检,archive 不足时自动降级 files(files 不支持 defer,async 收敛为 sync),可用环境变量关闭
- 尚未开始 publish 的失败自动恢复本次停掉的 API/controller/worker 并退出维护,只有真正开始发布之后的失败才保持维护态
- 备份检查脚本与生产运维门禁补上新口径与执行顺序断言,运维文档与共享记忆记录事故与规则
This commit is contained in:
2026-09-21 10:56:25 +08:00
parent 959d367ed4
commit 117d482f93
6 changed files with 353 additions and 29 deletions
+88 -10
View File
@@ -55,6 +55,11 @@ const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(
);
const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
// files 模式不落地本地归档(只写 catalog/state),所需空间远小于数据目录本身。
const DEFAULT_FILES_SPACE_SAFETY_RATIO = 0.05;
const DEFAULT_FILES_EXTRA_FREE_BYTES = 2 * 1024 * 1024 * 1024;
// 空间不足用独立退出码,调用方可据此决定是否降级存储格式。
const SPACE_INSUFFICIENT_EXIT_CODE = 3;
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
const OSS_SERVICE = 'oss';
const OSS_REQUEST = 'aliyun_v4_request';
@@ -80,6 +85,7 @@ const DIRECT_FILES_LATEST_SCHEMA_VERSION = 1;
function usage() {
console.log(`用法:
npm run database:backup:oss -- [--mode full|history] [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
npm run database:backup:oss -- --check-space-only [--storage-format archive|files] [--data-dir <path>] [--work-dir <path>]
node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload]
node -- scripts/database-backup-to-oss.mjs --upload-archive <path>
node -- scripts/database-backup-to-oss.mjs --upload-deferred-dir <path>
@@ -106,6 +112,8 @@ function usage() {
GENARRATIVE_DATABASE_BACKUP_STORAGE_FORMAT archive(默认)或 files
GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz
GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节;未设置时按数据目录大小估算
GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES files 模式的最小可用字节(默认 2GiB)
GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO files 模式按数据目录比例要求的可用空间(默认 0.05)
GENARRATIVE_DATABASE_BACKUP_BASELINE_STATE history 使用的 full baseline 与追加批次状态文件
GENARRATIVE_DATABASE_BACKUP_BASELINE_MANIFEST 首次初始化 history state 的 uploaded full manifest
ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
@@ -175,6 +183,7 @@ function parseArgs(argv) {
restartServicesAfter: [],
database: '',
dryRun: false,
checkSpaceOnly: false,
deferUpload: false,
uploadArchive: '',
uploadDeferredDir: '',
@@ -282,6 +291,9 @@ function parseArgs(argv) {
case '--publish-manifest':
options.publishManifest = readValue();
break;
case '--check-space-only':
options.checkSpaceOnly = true;
break;
case '--storage-format':
options.storageFormat = readValue();
break;
@@ -1013,6 +1025,20 @@ function parseSafetyRatio(rawValue) {
return ratio;
}
function parseFilesSpaceSafetyRatio(rawValue) {
const value = String(rawValue ?? '').trim();
if (!value) {
return DEFAULT_FILES_SPACE_SAFETY_RATIO;
}
const ratio = Number(value);
if (!Number.isFinite(ratio) || ratio <= 0) {
throw new Error(
`GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO 必须是 > 0 的数字,实际: ${rawValue}`,
);
}
return ratio;
}
function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) {
const explicitMinFreeBytes = parseByteSize(
firstNonEmpty(
@@ -1042,22 +1068,57 @@ function calculateRequiredFreeBytes({ dataSizeBytes, args, env }) {
: extraRequirement;
}
function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) {
function calculateFilesRequiredFreeBytes({ dataSizeBytes, env }) {
const explicitMinFreeBytes = parseByteSize(
env.GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES,
'GENARRATIVE_DATABASE_BACKUP_FILES_MIN_FREE_BYTES',
);
if (explicitMinFreeBytes !== null) {
return explicitMinFreeBytes;
}
const ratio = parseFilesSpaceSafetyRatio(
env.GENARRATIVE_DATABASE_BACKUP_FILES_SPACE_SAFETY_RATIO,
);
const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000));
const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n;
const floorRequirement = BigInt(DEFAULT_FILES_EXTRA_FREE_BYTES);
return ratioRequirement > floorRequirement ? ratioRequirement : floorRequirement;
}
export function describeBackupSpaceRequirement({
dataDir,
workDir,
storageFormat = 'archive',
args = {},
env = {},
}) {
mkdirSync(workDir, { recursive: true });
const dataSizeBytes = getDirectorySizeBytes(dataDir);
const availableBytes = getAvailableBytes(workDir);
const requiredFreeBytes = calculateRequiredFreeBytes({
dataSizeBytes,
args,
env,
});
const requiredFreeBytes =
storageFormat === 'files'
? calculateFilesRequiredFreeBytes({ dataSizeBytes, env })
: calculateRequiredFreeBytes({ dataSizeBytes, args, env });
return { storageFormat, dataSizeBytes, availableBytes, requiredFreeBytes };
}
function assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat = 'archive' }) {
const { dataSizeBytes, availableBytes, requiredFreeBytes } =
describeBackupSpaceRequirement({
dataDir,
workDir,
storageFormat,
args,
env,
});
console.log(
`[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
`[database-backup] 备份空间预检(${storageFormat}): data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
);
if (availableBytes < requiredFreeBytes) {
throw new Error(
const error = new Error(
[
`备份工作目录所在文件系统剩余空间不足: ${workDir}`,
`storage-format=${storageFormat}`,
`available=${formatBytes(availableBytes)}`,
`required=${formatBytes(requiredFreeBytes)}`,
`dataDir=${dataDir}`,
@@ -1065,7 +1126,10 @@ function assertSufficientWorkDirSpace({ dataDir, workDir, args, env }) {
'为避免停库后写满磁盘,本次备份已在停止服务前中止。',
].join(';'),
);
error.spaceInsufficient = true;
throw error;
}
return { dataSizeBytes, availableBytes, requiredFreeBytes };
}
function assertSufficientHistoryWorkDirSpace({
@@ -4154,6 +4218,20 @@ async function main() {
);
}
if (args.checkSpaceOnly) {
assertSufficientWorkDirSpace({
dataDir,
workDir,
args,
env,
storageFormat,
});
console.log(
`[database-backup] 空间预检通过(check-space-only): storage-format=${storageFormat}, workDir=${workDir}`,
);
return;
}
for (const [label, value] of Object.entries({
bucket,
endpoint,
@@ -4369,7 +4447,7 @@ async function main() {
const restartServicesAfter = collectRestartServicesAfterBackup({ args, env });
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
try {
assertSufficientWorkDirSpace({ dataDir, workDir, args, env });
assertSufficientWorkDirSpace({ dataDir, workDir, args, env, storageFormat: 'archive' });
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
archivePath = createArchive({ dataDir, workDir, fileName });
} catch (error) {
@@ -4558,6 +4636,6 @@ if (
} catch {
// 参数解析或状态文件写入失败时,不能掩盖原始备份错误。
}
process.exit(1);
process.exit(error?.spaceInsufficient ? SPACE_INSUFFICIENT_EXIT_CODE : 1);
});
}