Files
Genarrative/scripts/check-database-backup-to-oss.mjs
T
kdletters 4e3c26d5d8 修复 release 发布与冷备份恢复
冷备份增加工作目录空间预检并确保失败后恢复依赖服务

API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理

Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数

补充备份与 API deploy 回归检查和生产运维护栏

更新生产运维文档与项目记忆的事故处理口径
2026-07-08 11:04:48 +08:00

168 lines
4.9 KiB
JavaScript

#!/usr/bin/env node
import {spawnSync} from 'node:child_process';
import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
import {tmpdir} from 'node:os';
import path from 'node:path';
const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs');
const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-'));
const failures = [];
try {
main();
} finally {
rmSync(tmpRoot, {recursive: true, force: true});
}
if (failures.length > 0) {
console.error('[check:database-backup] FAILED');
for (const failure of failures) {
console.error(`- ${failure}`);
}
process.exit(1);
}
console.log('[check:database-backup] OK');
function main() {
assertInsufficientSpaceStopsBeforeServiceChanges();
assertArchiveFailureStillRestoresDependentServices();
}
function assertInsufficientSpaceStopsBeforeServiceChanges() {
const fixture = createFixture('insufficient-space');
const result = runBackup(fixture, [
'--stop-service',
'spacetimedb.service',
'--restart-service-after',
'genarrative-api.service',
'--min-free-bytes',
'999999999999999999',
]);
assertStatus(result, 1, '空间不足时必须失败。');
assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。');
assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。');
assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。');
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
}
function assertArchiveFailureStillRestoresDependentServices() {
const fixture = createFixture('tar-failure');
const result = runBackup(fixture, [
'--stop-service',
'spacetimedb.service',
'--restart-service-after',
'genarrative-api.service',
'--restart-service-after',
'genarrative-external-generation-worker@1.service',
'--restart-service-after',
'genarrative-external-generation-controller.service',
'--min-free-bytes',
'1',
]);
assertStatus(result, 1, 'tar 失败时备份脚本必须失败。');
assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。');
const systemctlLog = readFile(fixture.systemctlLog);
const expectedCommands = [
'systemctl stop spacetimedb.service',
'systemctl start spacetimedb.service',
'systemctl restart genarrative-api.service',
'systemctl restart genarrative-external-generation-worker@1.service',
'systemctl restart genarrative-external-generation-controller.service',
];
for (const command of expectedCommands) {
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
}
}
function createFixture(name) {
const root = path.join(tmpRoot, name);
const binDir = path.join(root, 'bin');
const dataDir = path.join(root, 'data');
const workDir = path.join(root, 'work');
const systemctlLog = path.join(root, 'systemctl.log');
const tarLog = path.join(root, 'tar.log');
mkdirSync(binDir, {recursive: true});
mkdirSync(dataDir, {recursive: true});
writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8');
writeExecutable(
path.join(binDir, 'systemctl'),
`#!/usr/bin/env bash
printf 'systemctl %s\\n' "$*" >> "${systemctlLog}"
exit 0
`,
);
writeExecutable(
path.join(binDir, 'tar'),
`#!/usr/bin/env bash
printf 'tar %s\\n' "$*" >> "${tarLog}"
echo 'fake tar failure' >&2
exit 2
`,
);
return {root, binDir, dataDir, workDir, systemctlLog, tarLog};
}
function runBackup(fixture, extraArgs = []) {
return spawnSync(
process.execPath,
[
BACKUP_SCRIPT,
'--data-dir',
fixture.dataDir,
'--work-dir',
fixture.workDir,
'--bucket',
'genarrative-test',
'--endpoint',
'oss-cn-shanghai.aliyuncs.com',
'--access-key-id',
'test',
'--access-key-secret',
'test',
...extraArgs,
],
{
cwd: process.cwd(),
encoding: 'utf8',
env: {
...process.env,
PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
},
},
);
}
function writeExecutable(filePath, content) {
writeFileSync(filePath, content, 'utf8');
spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'});
}
function readFile(filePath) {
return existsSync(filePath) ? readFileSync(filePath, 'utf8') : '';
}
function assertStatus(result, expected, reason) {
const actual = result.status ?? 0;
if (actual !== expected) {
failures.push(
`${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
);
}
}
function assertIncludes(content, expected, reason) {
if (!String(content).includes(expected)) {
failures.push(`${reason} 缺少: ${expected}`);
}
}
function assertFileMissing(filePath, reason) {
if (existsSync(filePath)) {
failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`);
}
}