修复生产内存工作集回收与 OOM 恢复
回收持续队列中的已完成 worker 句柄并收紧脱管许可生命周期 统一 AI 任务终态写入的文本、结构化输出和 warning 内存上限 限制认证投影恢复的 retained refresh session 数量 为备份停库增加 marker 与 systemd OOM 兜底恢复 补充回归测试并同步后端、运维和项目决策文档
This commit is contained in:
@@ -50,6 +50,7 @@ async function main() {
|
||||
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
|
||||
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
|
||||
assertInsufficientSpaceStopsBeforeServiceChanges();
|
||||
assertStopFailureRetainsRecoveryMarker();
|
||||
assertArchiveFailureStillRestoresDependentServices();
|
||||
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
|
||||
await assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors();
|
||||
@@ -748,6 +749,27 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
||||
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
|
||||
}
|
||||
|
||||
function assertStopFailureRetainsRecoveryMarker() {
|
||||
const fixture = createFixture('stop-failure-marker');
|
||||
writeExecutable(
|
||||
path.join(fixture.binDir, 'systemctl'),
|
||||
`#!/usr/bin/env bash
|
||||
printf 'systemctl %s\\n' "$*" >> "${fixture.systemctlLog}"
|
||||
if [ "$1" = stop ]; then
|
||||
exit 9
|
||||
fi
|
||||
exit 0
|
||||
`,
|
||||
);
|
||||
const result = runBackup(fixture, ['--stop-service', 'spacetimedb.service']);
|
||||
|
||||
assertStatus(result, 1, '停止服务失败时备份必须失败。');
|
||||
assertTrue(
|
||||
existsSync(path.join(fixture.workDir, '.spacetimedb-stopped')),
|
||||
'停止服务命令失败时必须保留 marker，供 systemd ExecStopPost 兜底恢复。',
|
||||
);
|
||||
}
|
||||
|
||||
function assertArchiveFailureStillRestoresDependentServices() {
|
||||
const fixture = createFixture('tar-failure');
|
||||
const result = runBackup(fixture, [
|
||||
@@ -776,6 +798,10 @@ function assertArchiveFailureStillRestoresDependentServices() {
|
||||
for (const command of expectedCommands) {
|
||||
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
|
||||
}
|
||||
assertFileMissing(
|
||||
path.join(fixture.workDir, '.spacetimedb-stopped'),
|
||||
'正常执行 finally 恢复全部服务后必须清理停库 marker。',
|
||||
);
|
||||
}
|
||||
|
||||
async function assertMultipartUploadRetriesAndVerifiesRemoteLength() {
|
||||
|
||||
@@ -827,12 +827,25 @@ const checks = [
|
||||
reason:
|
||||
'备份 Node 进程必须设置独立 heap 上限，避免目录扫描异常拖垮 release 主机。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes:
|
||||
'Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped',
|
||||
reason:
|
||||
'备份停库 marker 必须固定在受保护的 release work-dir，供 OOM 后 systemd 兜底恢复服务。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes: 'MemoryMax=1G',
|
||||
reason:
|
||||
'备份 service 必须设置 systemd 内存硬上限，避免异常进程消耗整机内存。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes: 'ExecStopPost=/bin/sh -c',
|
||||
reason:
|
||||
'备份主进程被 OOM kill 后必须由 systemd 兜底恢复停掉的 SpacetimeDB、API、worker 和 controller。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
excludes: '--storage-format files',
|
||||
@@ -965,7 +978,7 @@ const checks = [
|
||||
{
|
||||
file: 'scripts/database-backup-to-oss.mjs',
|
||||
includes:
|
||||
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})',
|
||||
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})',
|
||||
reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。',
|
||||
},
|
||||
{
|
||||
|
||||
@@ -35,6 +35,7 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/
|
||||
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
|
||||
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
|
||||
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
|
||||
const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(DEFAULT_PRODUCTION_WORK_DIR, '.spacetimedb-stopped');
|
||||
const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
|
||||
const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
|
||||
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
|
||||
@@ -890,11 +891,37 @@ function collectRestartServicesAfterBackup({args, env}) {
|
||||
return [...new Set(serviceNames.filter(Boolean))];
|
||||
}
|
||||
|
||||
function stopServiceIfNeeded(serviceName) {
|
||||
function databaseBackupStopMarkerPath(workDir) {
|
||||
return resolvePath(firstNonEmpty(
|
||||
process.env.GENARRATIVE_DATABASE_BACKUP_STOP_MARKER,
|
||||
workDir === DEFAULT_PRODUCTION_WORK_DIR
|
||||
? DEFAULT_DATABASE_BACKUP_STOP_MARKER
|
||||
: join(workDir, '.spacetimedb-stopped'),
|
||||
));
|
||||
}
|
||||
|
||||
function writeDatabaseBackupStopMarker(markerPath, serviceName) {
|
||||
atomicWriteJson(markerPath, {
|
||||
serviceName,
|
||||
pid: process.pid,
|
||||
stoppedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
function clearDatabaseBackupStopMarker(markerPath) {
|
||||
if (markerPath) {
|
||||
rmSync(markerPath, {force: true});
|
||||
}
|
||||
}
|
||||
|
||||
function stopServiceIfNeeded(serviceName, stopMarkerPath) {
|
||||
if (!serviceName) {
|
||||
return false;
|
||||
}
|
||||
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
|
||||
writeDatabaseBackupStopMarker(stopMarkerPath, serviceName);
|
||||
// stop 命令失败时仍保留 marker：systemd 的 ExecStopPost 需要它判断是否要
|
||||
// 兜底恢复，不能因为当前进程还能捕获异常就抹掉上一次停库证据。
|
||||
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
|
||||
return true;
|
||||
}
|
||||
@@ -925,7 +952,7 @@ function restartServicesAfterBackup(serviceNames) {
|
||||
}
|
||||
}
|
||||
|
||||
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) {
|
||||
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}) {
|
||||
const errors = [];
|
||||
try {
|
||||
startServiceIfNeeded(stopService, serviceStopped);
|
||||
@@ -940,6 +967,7 @@ function restoreServicesAfterBackup({stopService, serviceStopped, restartService
|
||||
if (errors.length > 0) {
|
||||
throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`);
|
||||
}
|
||||
clearDatabaseBackupStopMarker(stopMarkerPath);
|
||||
}
|
||||
|
||||
function createArchive({dataDir, workDir, fileName}) {
|
||||
@@ -3341,12 +3369,13 @@ async function main() {
|
||||
}
|
||||
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
|
||||
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
|
||||
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
|
||||
let serviceStopped = false;
|
||||
let backupError = null;
|
||||
let restoreError = null;
|
||||
try {
|
||||
if (args.mode === 'full' && !args.dryRun) {
|
||||
serviceStopped = stopServiceIfNeeded(stopService);
|
||||
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
|
||||
}
|
||||
await runDirectFilesBackup({
|
||||
mode: args.mode,
|
||||
@@ -3365,7 +3394,7 @@ async function main() {
|
||||
} finally {
|
||||
try {
|
||||
if (serviceStopped) {
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
|
||||
} else if (!backupError && args.mode === 'full' && !args.dryRun) {
|
||||
restartServicesAfterBackup(restartServicesAfter);
|
||||
}
|
||||
@@ -3419,16 +3448,17 @@ async function main() {
|
||||
let restoreError = null;
|
||||
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
|
||||
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
|
||||
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
|
||||
try {
|
||||
assertSufficientWorkDirSpace({dataDir, workDir, args, env});
|
||||
serviceStopped = stopServiceIfNeeded(stopService);
|
||||
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
|
||||
archivePath = createArchive({dataDir, workDir, fileName});
|
||||
} catch (error) {
|
||||
backupError = error;
|
||||
} finally {
|
||||
try {
|
||||
if (serviceStopped) {
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
|
||||
} else if (!backupError) {
|
||||
restartServicesAfterBackup(restartServicesAfter);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user