4e3c26d5d8
冷备份增加工作目录空间预检并确保失败后恢复依赖服务 API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理 Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数 补充备份与 API deploy 回归检查和生产运维护栏 更新生产运维文档与项目记忆的事故处理口径
168 lines
4.9 KiB
JavaScript
168 lines
4.9 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import {spawnSync} from 'node:child_process';
|
|
import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
|
|
import {tmpdir} from 'node:os';
|
|
import path from 'node:path';
|
|
|
|
const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs');
|
|
const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-'));
|
|
const failures = [];
|
|
|
|
try {
|
|
main();
|
|
} finally {
|
|
rmSync(tmpRoot, {recursive: true, force: true});
|
|
}
|
|
|
|
if (failures.length > 0) {
|
|
console.error('[check:database-backup] FAILED');
|
|
for (const failure of failures) {
|
|
console.error(`- ${failure}`);
|
|
}
|
|
process.exit(1);
|
|
}
|
|
|
|
console.log('[check:database-backup] OK');
|
|
|
|
function main() {
|
|
assertInsufficientSpaceStopsBeforeServiceChanges();
|
|
assertArchiveFailureStillRestoresDependentServices();
|
|
}
|
|
|
|
function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
|
const fixture = createFixture('insufficient-space');
|
|
const result = runBackup(fixture, [
|
|
'--stop-service',
|
|
'spacetimedb.service',
|
|
'--restart-service-after',
|
|
'genarrative-api.service',
|
|
'--min-free-bytes',
|
|
'999999999999999999',
|
|
]);
|
|
|
|
assertStatus(result, 1, '空间不足时必须失败。');
|
|
assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。');
|
|
assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。');
|
|
assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。');
|
|
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
|
|
}
|
|
|
|
function assertArchiveFailureStillRestoresDependentServices() {
|
|
const fixture = createFixture('tar-failure');
|
|
const result = runBackup(fixture, [
|
|
'--stop-service',
|
|
'spacetimedb.service',
|
|
'--restart-service-after',
|
|
'genarrative-api.service',
|
|
'--restart-service-after',
|
|
'genarrative-external-generation-worker@1.service',
|
|
'--restart-service-after',
|
|
'genarrative-external-generation-controller.service',
|
|
'--min-free-bytes',
|
|
'1',
|
|
]);
|
|
|
|
assertStatus(result, 1, 'tar 失败时备份脚本必须失败。');
|
|
assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。');
|
|
const systemctlLog = readFile(fixture.systemctlLog);
|
|
const expectedCommands = [
|
|
'systemctl stop spacetimedb.service',
|
|
'systemctl start spacetimedb.service',
|
|
'systemctl restart genarrative-api.service',
|
|
'systemctl restart genarrative-external-generation-worker@1.service',
|
|
'systemctl restart genarrative-external-generation-controller.service',
|
|
];
|
|
for (const command of expectedCommands) {
|
|
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
|
|
}
|
|
}
|
|
|
|
function createFixture(name) {
|
|
const root = path.join(tmpRoot, name);
|
|
const binDir = path.join(root, 'bin');
|
|
const dataDir = path.join(root, 'data');
|
|
const workDir = path.join(root, 'work');
|
|
const systemctlLog = path.join(root, 'systemctl.log');
|
|
const tarLog = path.join(root, 'tar.log');
|
|
mkdirSync(binDir, {recursive: true});
|
|
mkdirSync(dataDir, {recursive: true});
|
|
writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8');
|
|
writeExecutable(
|
|
path.join(binDir, 'systemctl'),
|
|
`#!/usr/bin/env bash
|
|
printf 'systemctl %s\\n' "$*" >> "${systemctlLog}"
|
|
exit 0
|
|
`,
|
|
);
|
|
writeExecutable(
|
|
path.join(binDir, 'tar'),
|
|
`#!/usr/bin/env bash
|
|
printf 'tar %s\\n' "$*" >> "${tarLog}"
|
|
echo 'fake tar failure' >&2
|
|
exit 2
|
|
`,
|
|
);
|
|
return {root, binDir, dataDir, workDir, systemctlLog, tarLog};
|
|
}
|
|
|
|
function runBackup(fixture, extraArgs = []) {
|
|
return spawnSync(
|
|
process.execPath,
|
|
[
|
|
BACKUP_SCRIPT,
|
|
'--data-dir',
|
|
fixture.dataDir,
|
|
'--work-dir',
|
|
fixture.workDir,
|
|
'--bucket',
|
|
'genarrative-test',
|
|
'--endpoint',
|
|
'oss-cn-shanghai.aliyuncs.com',
|
|
'--access-key-id',
|
|
'test',
|
|
'--access-key-secret',
|
|
'test',
|
|
...extraArgs,
|
|
],
|
|
{
|
|
cwd: process.cwd(),
|
|
encoding: 'utf8',
|
|
env: {
|
|
...process.env,
|
|
PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`,
|
|
},
|
|
},
|
|
);
|
|
}
|
|
|
|
function writeExecutable(filePath, content) {
|
|
writeFileSync(filePath, content, 'utf8');
|
|
spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'});
|
|
}
|
|
|
|
function readFile(filePath) {
|
|
return existsSync(filePath) ? readFileSync(filePath, 'utf8') : '';
|
|
}
|
|
|
|
function assertStatus(result, expected, reason) {
|
|
const actual = result.status ?? 0;
|
|
if (actual !== expected) {
|
|
failures.push(
|
|
`${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function assertIncludes(content, expected, reason) {
|
|
if (!String(content).includes(expected)) {
|
|
failures.push(`${reason} 缺少: ${expected}`);
|
|
}
|
|
}
|
|
|
|
function assertFileMissing(filePath, reason) {
|
|
if (existsSync(filePath)) {
|
|
failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`);
|
|
}
|
|
}
|