修复生产发布内存持续增长 (#203)
Project CI / Repository checks (push) Successful in 3m24s
Project CI / Frontend tests (push) Successful in 4m40s
Project CI / Backend tests (push) Successful in 10m5s
Project CI / Native shell tests (push) Successful in 15m8s

修复 release 内存持续增长问题。

本次范围:
- 收口备份扫描与历史维护的内存峰值。
- 限制外部生成 worker 脱管任务的实际并发。
- 为 API 内存态和历史数据增加有界留存。

验收:
- 定向测试、cargo 检查和生产运维门禁通过。
- 备份不再触发全局 OOM。
- worker/API/SpacetimeDB 内存曲线在空闲期停止单调增长。

Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/203
Co-authored-by: kdletters <kdletters@qq.com>
Co-committed-by: kdletters <kdletters@qq.com>
This commit was merged in pull request #203.
This commit is contained in:
2026-08-27 21:46:56 +08:00
committed by 段舒康
parent 94521af890
commit a7337c67a1
27 changed files with 1775 additions and 154 deletions
+26
View File
@@ -50,6 +50,7 @@ async function main() {
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
assertInsufficientSpaceStopsBeforeServiceChanges();
assertStopFailureRetainsRecoveryMarker();
assertArchiveFailureStillRestoresDependentServices();
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
await assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors();
@@ -748,6 +749,27 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
}
function assertStopFailureRetainsRecoveryMarker() {
const fixture = createFixture('stop-failure-marker');
writeExecutable(
path.join(fixture.binDir, 'systemctl'),
`#!/usr/bin/env bash
printf 'systemctl %s\\n' "$*" >> "${fixture.systemctlLog}"
if [ "$1" = stop ]; then
exit 9
fi
exit 0
`,
);
const result = runBackup(fixture, ['--stop-service', 'spacetimedb.service']);
assertStatus(result, 1, '停止服务失败时备份必须失败。');
assertTrue(
existsSync(path.join(fixture.workDir, '.spacetimedb-stopped')),
'停止服务命令失败时必须保留 marker,供 systemd ExecStopPost 兜底恢复。',
);
}
function assertArchiveFailureStillRestoresDependentServices() {
const fixture = createFixture('tar-failure');
const result = runBackup(fixture, [
@@ -776,6 +798,10 @@ function assertArchiveFailureStillRestoresDependentServices() {
for (const command of expectedCommands) {
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
}
assertFileMissing(
path.join(fixture.workDir, '.spacetimedb-stopped'),
'正常执行 finally 恢复全部服务后必须清理停库 marker。',
);
}
async function assertMultipartUploadRetriesAndVerifiesRemoteLength() {
+28 -4
View File
@@ -821,6 +821,31 @@ const checks = [
reason:
'生产冷备份 service 必须用 node -- 分隔脚本参数,避免 Node 22 抢占业务 --env-file。',
},
{
file: 'deploy/systemd/genarrative-database-backup.service',
includes: 'Environment=NODE_OPTIONS=--max-old-space-size=768',
reason:
'备份 Node 进程必须设置独立 heap 上限,避免目录扫描异常拖垮 release 主机。',
},
{
file: 'deploy/systemd/genarrative-database-backup.service',
includes:
'Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped',
reason:
'备份停库 marker 必须固定在受保护的 release work-dir,供 OOM 后 systemd 兜底恢复服务。',
},
{
file: 'deploy/systemd/genarrative-database-backup.service',
includes: 'MemoryMax=1G',
reason:
'备份 service 必须设置 systemd 内存硬上限,避免异常进程消耗整机内存。',
},
{
file: 'deploy/systemd/genarrative-database-backup.service',
includes: 'ExecStopPost=/bin/sh -c',
reason:
'备份主进程被 OOM kill 后必须由 systemd 兜底恢复停掉的 SpacetimeDB、API、worker 和 controller。',
},
{
file: 'deploy/systemd/genarrative-database-backup.service',
excludes: '--storage-format files',
@@ -941,10 +966,9 @@ const checks = [
},
{
file: 'jenkins/Jenkinsfile.production-server-provision',
excludes:
"params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history'",
includes: 'release 仅允许 archive-full;files-history',
reason:
'release 必须能在显式选择 profile 且 baseline 预检通过后启用 files-history。',
'release 必须拒绝 files-history,避免逐文件 catalog 扫描再次触发生产内存峰值。',
},
{
file: 'scripts/database-backup-to-oss.mjs',
@@ -954,7 +978,7 @@ const checks = [
{
file: 'scripts/database-backup-to-oss.mjs',
includes:
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})',
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})',
reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。',
},
{
+91 -22
View File
@@ -35,6 +35,7 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(DEFAULT_PRODUCTION_WORK_DIR, '.spacetimedb-stopped');
const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
@@ -555,7 +556,11 @@ function assertSafeRelativePath(dataDir, absolutePath) {
}
function statFingerprint(absolutePath, rootPath = absolutePath) {
const entries = [];
// 候选 snapshot 可能包含数十万条目录项;增量更新摘要,避免把每条
// fingerprint 字符串同时保存在 entries[] 后再 join,造成一次性内存峰值。
const fingerprintHash = createHash('sha256');
let isFirstEntry = true;
let entryCount = 0;
let totalSize = 0n;
const visit = (currentPath) => {
const stat = lstatSync(currentPath, {bigint: true});
@@ -567,7 +572,7 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
if (kind === 'other') {
throw new Error(`history 候选只允许普通文件或目录: ${currentPath}`);
}
entries.push([
const entry = [
entryPath,
kind,
stat.dev.toString(),
@@ -575,7 +580,13 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
stat.mode.toString(),
stat.size.toString(),
stat.mtimeNs.toString(),
].join('\0'));
].join('\0');
if (!isFirstEntry) {
fingerprintHash.update('\n');
}
fingerprintHash.update(entry);
isFirstEntry = false;
entryCount += 1;
if (stat.isFile()) {
totalSize += stat.size;
} else {
@@ -586,9 +597,9 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
};
visit(rootPath);
return {
fingerprint: sha256Hex(entries.join('\n')),
fingerprint: fingerprintHash.digest('hex'),
sizeBytes: totalSize.toString(),
entryCount: entries.length,
entryCount,
};
}
@@ -880,11 +891,37 @@ function collectRestartServicesAfterBackup({args, env}) {
return [...new Set(serviceNames.filter(Boolean))];
}
function stopServiceIfNeeded(serviceName) {
function databaseBackupStopMarkerPath(workDir) {
return resolvePath(firstNonEmpty(
process.env.GENARRATIVE_DATABASE_BACKUP_STOP_MARKER,
workDir === DEFAULT_PRODUCTION_WORK_DIR
? DEFAULT_DATABASE_BACKUP_STOP_MARKER
: join(workDir, '.spacetimedb-stopped'),
));
}
function writeDatabaseBackupStopMarker(markerPath, serviceName) {
atomicWriteJson(markerPath, {
serviceName,
pid: process.pid,
stoppedAt: new Date().toISOString(),
});
}
function clearDatabaseBackupStopMarker(markerPath) {
if (markerPath) {
rmSync(markerPath, {force: true});
}
}
function stopServiceIfNeeded(serviceName, stopMarkerPath) {
if (!serviceName) {
return false;
}
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
writeDatabaseBackupStopMarker(stopMarkerPath, serviceName);
// stop 命令失败时仍保留 marker:systemd 的 ExecStopPost 需要它判断是否要
// 兜底恢复,不能因为当前进程还能捕获异常就抹掉上一次停库证据。
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
return true;
}
@@ -915,7 +952,7 @@ function restartServicesAfterBackup(serviceNames) {
}
}
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) {
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}) {
const errors = [];
try {
startServiceIfNeeded(stopService, serviceStopped);
@@ -930,6 +967,7 @@ function restoreServicesAfterBackup({stopService, serviceStopped, restartService
if (errors.length > 0) {
throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`);
}
clearDatabaseBackupStopMarker(stopMarkerPath);
}
function createArchive({dataDir, workDir, fileName}) {
@@ -1285,14 +1323,17 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje
throw new Error(`files 扫描期间源文件发生变化: ${relativePath}`);
}
const basePrefix = normalizeObjectPrefix(objectPrefix, database);
files.set(relativePath, {
const file = {
path: relativePath,
sizeBytes: Number(after.size),
sha256,
mode: after.mode,
objectKey: `${basePrefix}/files/sha256/${sha256.slice(0, 2)}/${sha256}`,
sourceStat: after,
});
};
// 上传前后的 inode/stat 仍用于防止在线扫描漂移,但设为不可枚举,避免
// 把仅供本地校验的副本再次写入 catalog 或 result JSON。
Object.defineProperty(file, 'sourceStat', {value: after, enumerable: false});
files.set(relativePath, file);
};
for (const root of roots.sort((left, right) => left.relativePath.localeCompare(right.relativePath))) {
@@ -1309,14 +1350,40 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje
}
function directCatalogIdentity({mode, baselineCatalogId, rootName, directories, files, symlinks}) {
return sha256Hex(JSON.stringify({
mode,
baselineCatalogId: baselineCatalogId || '',
rootName,
directories,
files: files.map(({path, sizeBytes, sha256, mode, objectKey}) => ({path, sizeBytes, sha256, mode, objectKey})),
symlinks,
// 不把数十万条文件元数据先拼成一个巨型 JSON 字符串;分段写入 hash
// 保持与 JSON.stringify 同样的字段顺序和转义结果,同时把峰值降到单条记录。
const hash = createHash('sha256');
hash.update('{"mode":');
hash.update(JSON.stringify(mode));
hash.update(',"baselineCatalogId":');
hash.update(JSON.stringify(baselineCatalogId || ''));
hash.update(',"rootName":');
hash.update(JSON.stringify(rootName));
hash.update(',"directories":');
updateJsonArrayHash(hash, directories, (directory) => JSON.stringify(directory));
hash.update(',"files":');
updateJsonArrayHash(hash, files, (file) => JSON.stringify({
path: file.path,
sizeBytes: file.sizeBytes,
sha256: file.sha256,
mode: file.mode,
objectKey: file.objectKey,
}));
hash.update(',"symlinks":');
updateJsonArrayHash(hash, symlinks, (symlink) => JSON.stringify(symlink));
hash.update('}');
return hash.digest('hex');
}
function updateJsonArrayHash(hash, values, serialize) {
hash.update('[');
values.forEach((value, index) => {
if (index > 0) {
hash.update(',');
}
hash.update(serialize(value));
});
hash.update(']');
}
function readDirectFilesState(statePath, {database, bucket}) {
@@ -1621,7 +1688,7 @@ export async function runDirectFilesBackup({
baselineCatalogId,
rootName,
directories: collected.directories,
files: collected.files.map(({sourceStat: _sourceStat, ...file}) => file),
files: collected.files,
symlinks: collected.symlinks,
};
writeManifest({manifestPath: catalogPath, payload: catalog});
@@ -3302,12 +3369,13 @@ async function main() {
}
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
let serviceStopped = false;
let backupError = null;
let restoreError = null;
try {
if (args.mode === 'full' && !args.dryRun) {
serviceStopped = stopServiceIfNeeded(stopService);
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
}
await runDirectFilesBackup({
mode: args.mode,
@@ -3326,7 +3394,7 @@ async function main() {
} finally {
try {
if (serviceStopped) {
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
} else if (!backupError && args.mode === 'full' && !args.dryRun) {
restartServicesAfterBackup(restartServicesAfter);
}
@@ -3380,16 +3448,17 @@ async function main() {
let restoreError = null;
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
try {
assertSufficientWorkDirSpace({dataDir, workDir, args, env});
serviceStopped = stopServiceIfNeeded(stopService);
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
archivePath = createArchive({dataDir, workDir, fileName});
} catch (error) {
backupError = error;
} finally {
try {
if (serviceStopped) {
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
} else if (!backupError) {
restartServicesAfterBackup(restartServicesAfter);
}
+4
View File
@@ -78,6 +78,10 @@ validate_database_backup_profile() {
exit 1
;;
esac
if [[ "${DEPLOY_TARGET}" == "release" && "${DATABASE_BACKUP_PROFILE}" == "files-history" ]]; then
echo "[server-provision] release 仅允许 archive-full;files-history 会把整棵历史目录加载到 Node 内存,需先完成流式 catalog 改造后才能重新启用。" >&2
exit 1
fi
if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then
echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径,当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2
exit 1
@@ -8,23 +8,29 @@ import {
} from './spacetime-migration-common.mjs';
const MAX_BATCH_SIZE = 25;
const DEFAULT_RETENTION_DAYS = 30;
const MICROS_PER_DAY = 86_400_000_000;
function usage() {
return `用法:
node scripts/spacetime-maintain-external-generation-jobs.mjs --database <name> [选项]
默认只 dry-run 一批历史终态任务 payload 压缩,不修改数据库。
使用 --prune-history 时改为清理已确认通知且超过保留期的历史任务、摘要与事件。
公共选项:
--database <name> 目标数据库(必填,也可用 GENARRATIVE_SPACETIME_DATABASE)
--server <name-or-url> spacetime CLI server 名或 URL
--server-url <url> 显式 server URL
--limit <1-${MAX_BATCH_SIZE}> 单批任务数,默认 10
--limit <1-${MAX_BATCH_SIZE}> 单批任务数,默认 10
--cursor-job-id <jobId> 从上一批 next_cursor_job_id 继续
--apply 执行写入;省略时始终 dry-run
--backfill-summaries 改为回填轻量摘要投影
--prune-history 改为清理已确认通知的终态历史
--owner-user-id <userId> 仅摘要回填可选,限定 owner
--completed-before-micros <n> 仅 payload 压缩可选,限定终态完成时间
--source-module <module> 仅历史清理可选,默认 editor-canvas
--retention-days <n> 仅历史清理可选,默认 ${DEFAULT_RETENTION_DAYS} 天
--completed-before-micros <n> 限定终态完成时间;历史清理默认按 retention-days 计算
--help 显示帮助
必须使用已授权 migration operator 的 spacetime CLI 登录态。脚本每次只处理一批;
@@ -40,6 +46,9 @@ function parseOptions(argv) {
database: process.env.GENARRATIVE_SPACETIME_DATABASE || '',
limit: 10,
ownerUserId: '',
pruneHistory: false,
retentionDays: DEFAULT_RETENTION_DAYS,
sourceModule: 'editor-canvas',
passthrough: [],
server: process.env.GENARRATIVE_SPACETIME_SERVER || '',
serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '',
@@ -82,6 +91,15 @@ function parseOptions(argv) {
options.apply = true;
} else if (arg === '--backfill-summaries') {
options.backfillSummaries = true;
} else if (arg === '--prune-history') {
options.pruneHistory = true;
} else if (arg === '--source-module') {
options.sourceModule = readValue(arg).trim();
if (!options.sourceModule) {
throw new Error('--source-module 不能为空。');
}
} else if (arg === '--retention-days') {
options.retentionDays = parsePositiveInteger(readValue(arg), arg);
} else if (arg === '--help' || arg === '-h') {
options.help = true;
} else {
@@ -95,12 +113,49 @@ function parseOptions(argv) {
if (options.ownerUserId && !options.backfillSummaries) {
throw new Error('--owner-user-id 只能与 --backfill-summaries 一起使用。');
}
if (options.backfillSummaries && options.pruneHistory) {
throw new Error('--backfill-summaries 与 --prune-history 不能同时使用。');
}
if (options.sourceModule !== 'editor-canvas' && !options.pruneHistory) {
throw new Error('--source-module 只能与 --prune-history 一起使用。');
}
if (
options.retentionDays !== DEFAULT_RETENTION_DAYS &&
!options.pruneHistory
) {
throw new Error('--retention-days 只能与 --prune-history 一起使用。');
}
if (options.completedBeforeMicros !== null && options.backfillSummaries) {
throw new Error('--completed-before-micros 不能用于摘要回填。');
}
if (
options.completedBeforeMicros !== null &&
options.pruneHistory &&
options.retentionDays !== DEFAULT_RETENTION_DAYS
) {
throw new Error(
'--completed-before-micros 与 --retention-days 不能同时使用。',
);
}
return options;
}
function resolveRetentionCutoffMicros(options) {
if (!options.pruneHistory) {
return options.completedBeforeMicros;
}
if (options.completedBeforeMicros !== null) {
return options.completedBeforeMicros;
}
const cutoff = Date.now() * 1000 - options.retentionDays * MICROS_PER_DAY;
if (!Number.isSafeInteger(cutoff)) {
throw new Error(
'--retention-days 计算出的 completed_before_micros 超出安全整数范围。',
);
}
return cutoff;
}
try {
const options = parseOptions(process.argv.slice(2));
if (options.help) {
@@ -113,24 +168,36 @@ try {
);
}
const procedureName = options.backfillSummaries
? 'backfill_external_generation_job_summaries_and_return'
: 'compact_external_generation_job_payloads_and_return';
const input = options.backfillSummaries
const completedBeforeMicros = resolveRetentionCutoffMicros(options);
const procedureName = options.pruneHistory
? 'prune_external_generation_job_history_and_return'
: options.backfillSummaries
? 'backfill_external_generation_job_summaries_and_return'
: 'compact_external_generation_job_payloads_and_return';
const input = options.pruneHistory
? {
owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null),
source_module: options.sourceModule,
limit: options.limit,
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
completed_before_micros: completedBeforeMicros,
dry_run: !options.apply,
}
: {
dry_run: !options.apply,
limit: options.limit,
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
completed_before_micros: encodeSpacetimeCliOption(
options.completedBeforeMicros,
),
};
: options.backfillSummaries
? {
owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null),
limit: options.limit,
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
dry_run: !options.apply,
}
: {
dry_run: !options.apply,
limit: options.limit,
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
completed_before_micros: encodeSpacetimeCliOption(
completedBeforeMicros,
),
};
const result = await callSpacetimeProcedureViaCli(
options,
procedureName,
@@ -138,10 +205,29 @@ try {
);
ensureProcedureOk(result);
console.log(JSON.stringify({ procedure: procedureName, ...result }, null, 2));
const pendingApplyCount = options.backfillSummaries
? Number(result.selected_count ?? 0)
: Number(result.matched_count ?? 0);
console.log(
JSON.stringify(
{
procedure: procedureName,
...(options.pruneHistory
? {
source_module: options.sourceModule,
completed_before_micros: completedBeforeMicros,
...(options.completedBeforeMicros === null
? { retention_days: options.retentionDays }
: {}),
}
: {}),
...result,
},
null,
2,
),
);
const pendingApplyCount =
options.pruneHistory || options.backfillSummaries
? Number(result.selected_count ?? 0)
: Number(result.matched_count ?? 0);
if (result.has_more && options.apply) {
console.log(
`仍有后续批次;下一次追加 --cursor-job-id ${result.next_cursor_job_id ?? '<missing>'}。`,
@@ -150,8 +236,11 @@ try {
const currentCursor = options.cursorJobId
? `保留 --cursor-job-id ${options.cursorJobId}`
: '仍从首批开始';
const cutoffHint = options.pruneHistory
? `并固定 --completed-before-micros ${completedBeforeMicros}`
: '';
console.log(
`当前仅 dry-run;请${currentCursor}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`,
`当前仅 dry-run;请${currentCursor}${cutoffHint}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`,
);
}
} catch (error) {