修复生产发布内存持续增长 (#203)
修复 release 内存持续增长问题。 本次范围: - 收口备份扫描与历史维护的内存峰值。 - 限制外部生成 worker 脱管任务的实际并发。 - 为 API 内存态和历史数据增加有界留存。 验收: - 定向测试、cargo 检查和生产运维门禁通过。 - 备份不再触发全局 OOM。 - worker/API/SpacetimeDB 内存曲线在空闲期停止单调增长。 Reviewed-on: http://192.168.35.82/git/GenarrativeAI/Genarrative/pulls/203 Co-authored-by: kdletters <kdletters@qq.com> Co-committed-by: kdletters <kdletters@qq.com>
This commit was merged in pull request #203.
This commit is contained in:
@@ -50,6 +50,7 @@ async function main() {
|
||||
assertDeferredArchiveDiscoveryIsBoundedAndDeterministic();
|
||||
assertCanonicalQueryAndAuthorizationIncludeMultipartParameters();
|
||||
assertInsufficientSpaceStopsBeforeServiceChanges();
|
||||
assertStopFailureRetainsRecoveryMarker();
|
||||
assertArchiveFailureStillRestoresDependentServices();
|
||||
await assertMultipartUploadRetriesAndVerifiesRemoteLength();
|
||||
await assertUploadBandwidthLimiterSharesBudgetAndPropagatesErrors();
|
||||
@@ -748,6 +749,27 @@ function assertInsufficientSpaceStopsBeforeServiceChanges() {
|
||||
assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。');
|
||||
}
|
||||
|
||||
function assertStopFailureRetainsRecoveryMarker() {
|
||||
const fixture = createFixture('stop-failure-marker');
|
||||
writeExecutable(
|
||||
path.join(fixture.binDir, 'systemctl'),
|
||||
`#!/usr/bin/env bash
|
||||
printf 'systemctl %s\\n' "$*" >> "${fixture.systemctlLog}"
|
||||
if [ "$1" = stop ]; then
|
||||
exit 9
|
||||
fi
|
||||
exit 0
|
||||
`,
|
||||
);
|
||||
const result = runBackup(fixture, ['--stop-service', 'spacetimedb.service']);
|
||||
|
||||
assertStatus(result, 1, '停止服务失败时备份必须失败。');
|
||||
assertTrue(
|
||||
existsSync(path.join(fixture.workDir, '.spacetimedb-stopped')),
|
||||
'停止服务命令失败时必须保留 marker,供 systemd ExecStopPost 兜底恢复。',
|
||||
);
|
||||
}
|
||||
|
||||
function assertArchiveFailureStillRestoresDependentServices() {
|
||||
const fixture = createFixture('tar-failure');
|
||||
const result = runBackup(fixture, [
|
||||
@@ -776,6 +798,10 @@ function assertArchiveFailureStillRestoresDependentServices() {
|
||||
for (const command of expectedCommands) {
|
||||
assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`);
|
||||
}
|
||||
assertFileMissing(
|
||||
path.join(fixture.workDir, '.spacetimedb-stopped'),
|
||||
'正常执行 finally 恢复全部服务后必须清理停库 marker。',
|
||||
);
|
||||
}
|
||||
|
||||
async function assertMultipartUploadRetriesAndVerifiesRemoteLength() {
|
||||
|
||||
@@ -821,6 +821,31 @@ const checks = [
|
||||
reason:
|
||||
'生产冷备份 service 必须用 node -- 分隔脚本参数,避免 Node 22 抢占业务 --env-file。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes: 'Environment=NODE_OPTIONS=--max-old-space-size=768',
|
||||
reason:
|
||||
'备份 Node 进程必须设置独立 heap 上限,避免目录扫描异常拖垮 release 主机。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes:
|
||||
'Environment=GENARRATIVE_DATABASE_BACKUP_STOP_MARKER=/var/lib/genarrative/database-backups/.spacetimedb-stopped',
|
||||
reason:
|
||||
'备份停库 marker 必须固定在受保护的 release work-dir,供 OOM 后 systemd 兜底恢复服务。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes: 'MemoryMax=1G',
|
||||
reason:
|
||||
'备份 service 必须设置 systemd 内存硬上限,避免异常进程消耗整机内存。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
includes: 'ExecStopPost=/bin/sh -c',
|
||||
reason:
|
||||
'备份主进程被 OOM kill 后必须由 systemd 兜底恢复停掉的 SpacetimeDB、API、worker 和 controller。',
|
||||
},
|
||||
{
|
||||
file: 'deploy/systemd/genarrative-database-backup.service',
|
||||
excludes: '--storage-format files',
|
||||
@@ -941,10 +966,9 @@ const checks = [
|
||||
},
|
||||
{
|
||||
file: 'jenkins/Jenkinsfile.production-server-provision',
|
||||
excludes:
|
||||
"params.DEPLOY_TARGET == 'release' && databaseBackupProfile == 'files-history'",
|
||||
includes: 'release 仅允许 archive-full;files-history',
|
||||
reason:
|
||||
'release 必须能在显式选择 profile 且 baseline 预检通过后启用 files-history。',
|
||||
'release 必须拒绝 files-history,避免逐文件 catalog 扫描再次触发生产内存峰值。',
|
||||
},
|
||||
{
|
||||
file: 'scripts/database-backup-to-oss.mjs',
|
||||
@@ -954,7 +978,7 @@ const checks = [
|
||||
{
|
||||
file: 'scripts/database-backup-to-oss.mjs',
|
||||
includes:
|
||||
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})',
|
||||
'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath})',
|
||||
reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。',
|
||||
},
|
||||
{
|
||||
|
||||
@@ -35,6 +35,7 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/
|
||||
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
|
||||
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
|
||||
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
|
||||
const DEFAULT_DATABASE_BACKUP_STOP_MARKER = join(DEFAULT_PRODUCTION_WORK_DIR, '.spacetimedb-stopped');
|
||||
const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
|
||||
const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
|
||||
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
|
||||
@@ -555,7 +556,11 @@ function assertSafeRelativePath(dataDir, absolutePath) {
|
||||
}
|
||||
|
||||
function statFingerprint(absolutePath, rootPath = absolutePath) {
|
||||
const entries = [];
|
||||
// 候选 snapshot 可能包含数十万条目录项;增量更新摘要,避免把每条
|
||||
// fingerprint 字符串同时保存在 entries[] 后再 join,造成一次性内存峰值。
|
||||
const fingerprintHash = createHash('sha256');
|
||||
let isFirstEntry = true;
|
||||
let entryCount = 0;
|
||||
let totalSize = 0n;
|
||||
const visit = (currentPath) => {
|
||||
const stat = lstatSync(currentPath, {bigint: true});
|
||||
@@ -567,7 +572,7 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
|
||||
if (kind === 'other') {
|
||||
throw new Error(`history 候选只允许普通文件或目录: ${currentPath}`);
|
||||
}
|
||||
entries.push([
|
||||
const entry = [
|
||||
entryPath,
|
||||
kind,
|
||||
stat.dev.toString(),
|
||||
@@ -575,7 +580,13 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
|
||||
stat.mode.toString(),
|
||||
stat.size.toString(),
|
||||
stat.mtimeNs.toString(),
|
||||
].join('\0'));
|
||||
].join('\0');
|
||||
if (!isFirstEntry) {
|
||||
fingerprintHash.update('\n');
|
||||
}
|
||||
fingerprintHash.update(entry);
|
||||
isFirstEntry = false;
|
||||
entryCount += 1;
|
||||
if (stat.isFile()) {
|
||||
totalSize += stat.size;
|
||||
} else {
|
||||
@@ -586,9 +597,9 @@ function statFingerprint(absolutePath, rootPath = absolutePath) {
|
||||
};
|
||||
visit(rootPath);
|
||||
return {
|
||||
fingerprint: sha256Hex(entries.join('\n')),
|
||||
fingerprint: fingerprintHash.digest('hex'),
|
||||
sizeBytes: totalSize.toString(),
|
||||
entryCount: entries.length,
|
||||
entryCount,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -880,11 +891,37 @@ function collectRestartServicesAfterBackup({args, env}) {
|
||||
return [...new Set(serviceNames.filter(Boolean))];
|
||||
}
|
||||
|
||||
function stopServiceIfNeeded(serviceName) {
|
||||
function databaseBackupStopMarkerPath(workDir) {
|
||||
return resolvePath(firstNonEmpty(
|
||||
process.env.GENARRATIVE_DATABASE_BACKUP_STOP_MARKER,
|
||||
workDir === DEFAULT_PRODUCTION_WORK_DIR
|
||||
? DEFAULT_DATABASE_BACKUP_STOP_MARKER
|
||||
: join(workDir, '.spacetimedb-stopped'),
|
||||
));
|
||||
}
|
||||
|
||||
function writeDatabaseBackupStopMarker(markerPath, serviceName) {
|
||||
atomicWriteJson(markerPath, {
|
||||
serviceName,
|
||||
pid: process.pid,
|
||||
stoppedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
function clearDatabaseBackupStopMarker(markerPath) {
|
||||
if (markerPath) {
|
||||
rmSync(markerPath, {force: true});
|
||||
}
|
||||
}
|
||||
|
||||
function stopServiceIfNeeded(serviceName, stopMarkerPath) {
|
||||
if (!serviceName) {
|
||||
return false;
|
||||
}
|
||||
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
|
||||
writeDatabaseBackupStopMarker(stopMarkerPath, serviceName);
|
||||
// stop 命令失败时仍保留 marker:systemd 的 ExecStopPost 需要它判断是否要
|
||||
// 兜底恢复,不能因为当前进程还能捕获异常就抹掉上一次停库证据。
|
||||
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
|
||||
return true;
|
||||
}
|
||||
@@ -915,7 +952,7 @@ function restartServicesAfterBackup(serviceNames) {
|
||||
}
|
||||
}
|
||||
|
||||
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) {
|
||||
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath}) {
|
||||
const errors = [];
|
||||
try {
|
||||
startServiceIfNeeded(stopService, serviceStopped);
|
||||
@@ -930,6 +967,7 @@ function restoreServicesAfterBackup({stopService, serviceStopped, restartService
|
||||
if (errors.length > 0) {
|
||||
throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`);
|
||||
}
|
||||
clearDatabaseBackupStopMarker(stopMarkerPath);
|
||||
}
|
||||
|
||||
function createArchive({dataDir, workDir, fileName}) {
|
||||
@@ -1285,14 +1323,17 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje
|
||||
throw new Error(`files 扫描期间源文件发生变化: ${relativePath}`);
|
||||
}
|
||||
const basePrefix = normalizeObjectPrefix(objectPrefix, database);
|
||||
files.set(relativePath, {
|
||||
const file = {
|
||||
path: relativePath,
|
||||
sizeBytes: Number(after.size),
|
||||
sha256,
|
||||
mode: after.mode,
|
||||
objectKey: `${basePrefix}/files/sha256/${sha256.slice(0, 2)}/${sha256}`,
|
||||
sourceStat: after,
|
||||
});
|
||||
};
|
||||
// 上传前后的 inode/stat 仍用于防止在线扫描漂移,但设为不可枚举,避免
|
||||
// 把仅供本地校验的副本再次写入 catalog 或 result JSON。
|
||||
Object.defineProperty(file, 'sourceStat', {value: after, enumerable: false});
|
||||
files.set(relativePath, file);
|
||||
};
|
||||
|
||||
for (const root of roots.sort((left, right) => left.relativePath.localeCompare(right.relativePath))) {
|
||||
@@ -1309,14 +1350,40 @@ export async function collectDirectFileEntries({dataDir, candidates = null, obje
|
||||
}
|
||||
|
||||
function directCatalogIdentity({mode, baselineCatalogId, rootName, directories, files, symlinks}) {
|
||||
return sha256Hex(JSON.stringify({
|
||||
mode,
|
||||
baselineCatalogId: baselineCatalogId || '',
|
||||
rootName,
|
||||
directories,
|
||||
files: files.map(({path, sizeBytes, sha256, mode, objectKey}) => ({path, sizeBytes, sha256, mode, objectKey})),
|
||||
symlinks,
|
||||
// 不把数十万条文件元数据先拼成一个巨型 JSON 字符串;分段写入 hash
|
||||
// 保持与 JSON.stringify 同样的字段顺序和转义结果,同时把峰值降到单条记录。
|
||||
const hash = createHash('sha256');
|
||||
hash.update('{"mode":');
|
||||
hash.update(JSON.stringify(mode));
|
||||
hash.update(',"baselineCatalogId":');
|
||||
hash.update(JSON.stringify(baselineCatalogId || ''));
|
||||
hash.update(',"rootName":');
|
||||
hash.update(JSON.stringify(rootName));
|
||||
hash.update(',"directories":');
|
||||
updateJsonArrayHash(hash, directories, (directory) => JSON.stringify(directory));
|
||||
hash.update(',"files":');
|
||||
updateJsonArrayHash(hash, files, (file) => JSON.stringify({
|
||||
path: file.path,
|
||||
sizeBytes: file.sizeBytes,
|
||||
sha256: file.sha256,
|
||||
mode: file.mode,
|
||||
objectKey: file.objectKey,
|
||||
}));
|
||||
hash.update(',"symlinks":');
|
||||
updateJsonArrayHash(hash, symlinks, (symlink) => JSON.stringify(symlink));
|
||||
hash.update('}');
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
function updateJsonArrayHash(hash, values, serialize) {
|
||||
hash.update('[');
|
||||
values.forEach((value, index) => {
|
||||
if (index > 0) {
|
||||
hash.update(',');
|
||||
}
|
||||
hash.update(serialize(value));
|
||||
});
|
||||
hash.update(']');
|
||||
}
|
||||
|
||||
function readDirectFilesState(statePath, {database, bucket}) {
|
||||
@@ -1621,7 +1688,7 @@ export async function runDirectFilesBackup({
|
||||
baselineCatalogId,
|
||||
rootName,
|
||||
directories: collected.directories,
|
||||
files: collected.files.map(({sourceStat: _sourceStat, ...file}) => file),
|
||||
files: collected.files,
|
||||
symlinks: collected.symlinks,
|
||||
};
|
||||
writeManifest({manifestPath: catalogPath, payload: catalog});
|
||||
@@ -3302,12 +3369,13 @@ async function main() {
|
||||
}
|
||||
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
|
||||
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
|
||||
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
|
||||
let serviceStopped = false;
|
||||
let backupError = null;
|
||||
let restoreError = null;
|
||||
try {
|
||||
if (args.mode === 'full' && !args.dryRun) {
|
||||
serviceStopped = stopServiceIfNeeded(stopService);
|
||||
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
|
||||
}
|
||||
await runDirectFilesBackup({
|
||||
mode: args.mode,
|
||||
@@ -3326,7 +3394,7 @@ async function main() {
|
||||
} finally {
|
||||
try {
|
||||
if (serviceStopped) {
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
|
||||
} else if (!backupError && args.mode === 'full' && !args.dryRun) {
|
||||
restartServicesAfterBackup(restartServicesAfter);
|
||||
}
|
||||
@@ -3380,16 +3448,17 @@ async function main() {
|
||||
let restoreError = null;
|
||||
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
|
||||
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
|
||||
const stopMarkerPath = databaseBackupStopMarkerPath(workDir);
|
||||
try {
|
||||
assertSufficientWorkDirSpace({dataDir, workDir, args, env});
|
||||
serviceStopped = stopServiceIfNeeded(stopService);
|
||||
serviceStopped = stopServiceIfNeeded(stopService, stopMarkerPath);
|
||||
archivePath = createArchive({dataDir, workDir, fileName});
|
||||
} catch (error) {
|
||||
backupError = error;
|
||||
} finally {
|
||||
try {
|
||||
if (serviceStopped) {
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
|
||||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter, stopMarkerPath});
|
||||
} else if (!backupError) {
|
||||
restartServicesAfterBackup(restartServicesAfter);
|
||||
}
|
||||
|
||||
@@ -78,6 +78,10 @@ validate_database_backup_profile() {
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
if [[ "${DEPLOY_TARGET}" == "release" && "${DATABASE_BACKUP_PROFILE}" == "files-history" ]]; then
|
||||
echo "[server-provision] release 仅允许 archive-full;files-history 会把整棵历史目录加载到 Node 内存,需先完成流式 catalog 改造后才能重新启用。" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" =~ ^/var/lib/genarrative/database-backups/[A-Za-z0-9._/-]+$ || "${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" == *..* ]]; then
|
||||
echo "[server-provision] DATABASE_BACKUP_FILES_HISTORY_WORK_DIR 必须是 /var/lib/genarrative/database-backups/ 下不含连续点号的绝对路径,当前值: ${DATABASE_BACKUP_FILES_HISTORY_WORK_DIR}" >&2
|
||||
exit 1
|
||||
|
||||
@@ -8,23 +8,29 @@ import {
|
||||
} from './spacetime-migration-common.mjs';
|
||||
|
||||
const MAX_BATCH_SIZE = 25;
|
||||
const DEFAULT_RETENTION_DAYS = 30;
|
||||
const MICROS_PER_DAY = 86_400_000_000;
|
||||
|
||||
function usage() {
|
||||
return `用法:
|
||||
node scripts/spacetime-maintain-external-generation-jobs.mjs --database <name> [选项]
|
||||
|
||||
默认只 dry-run 一批历史终态任务 payload 压缩,不修改数据库。
|
||||
使用 --prune-history 时改为清理已确认通知且超过保留期的历史任务、摘要与事件。
|
||||
|
||||
公共选项:
|
||||
--database <name> 目标数据库(必填,也可用 GENARRATIVE_SPACETIME_DATABASE)
|
||||
--server <name-or-url> spacetime CLI server 名或 URL
|
||||
--server-url <url> 显式 server URL
|
||||
--limit <1-${MAX_BATCH_SIZE}> 单批任务数,默认 10
|
||||
--limit <1-${MAX_BATCH_SIZE}> 单批任务数,默认 10
|
||||
--cursor-job-id <jobId> 从上一批 next_cursor_job_id 继续
|
||||
--apply 执行写入;省略时始终 dry-run
|
||||
--backfill-summaries 改为回填轻量摘要投影
|
||||
--prune-history 改为清理已确认通知的终态历史
|
||||
--owner-user-id <userId> 仅摘要回填可选,限定 owner
|
||||
--completed-before-micros <n> 仅 payload 压缩可选,限定终态完成时间
|
||||
--source-module <module> 仅历史清理可选,默认 editor-canvas
|
||||
--retention-days <n> 仅历史清理可选,默认 ${DEFAULT_RETENTION_DAYS} 天
|
||||
--completed-before-micros <n> 限定终态完成时间;历史清理默认按 retention-days 计算
|
||||
--help 显示帮助
|
||||
|
||||
必须使用已授权 migration operator 的 spacetime CLI 登录态。脚本每次只处理一批;
|
||||
@@ -40,6 +46,9 @@ function parseOptions(argv) {
|
||||
database: process.env.GENARRATIVE_SPACETIME_DATABASE || '',
|
||||
limit: 10,
|
||||
ownerUserId: '',
|
||||
pruneHistory: false,
|
||||
retentionDays: DEFAULT_RETENTION_DAYS,
|
||||
sourceModule: 'editor-canvas',
|
||||
passthrough: [],
|
||||
server: process.env.GENARRATIVE_SPACETIME_SERVER || '',
|
||||
serverUrl: process.env.GENARRATIVE_SPACETIME_SERVER_URL || '',
|
||||
@@ -82,6 +91,15 @@ function parseOptions(argv) {
|
||||
options.apply = true;
|
||||
} else if (arg === '--backfill-summaries') {
|
||||
options.backfillSummaries = true;
|
||||
} else if (arg === '--prune-history') {
|
||||
options.pruneHistory = true;
|
||||
} else if (arg === '--source-module') {
|
||||
options.sourceModule = readValue(arg).trim();
|
||||
if (!options.sourceModule) {
|
||||
throw new Error('--source-module 不能为空。');
|
||||
}
|
||||
} else if (arg === '--retention-days') {
|
||||
options.retentionDays = parsePositiveInteger(readValue(arg), arg);
|
||||
} else if (arg === '--help' || arg === '-h') {
|
||||
options.help = true;
|
||||
} else {
|
||||
@@ -95,12 +113,49 @@ function parseOptions(argv) {
|
||||
if (options.ownerUserId && !options.backfillSummaries) {
|
||||
throw new Error('--owner-user-id 只能与 --backfill-summaries 一起使用。');
|
||||
}
|
||||
if (options.backfillSummaries && options.pruneHistory) {
|
||||
throw new Error('--backfill-summaries 与 --prune-history 不能同时使用。');
|
||||
}
|
||||
if (options.sourceModule !== 'editor-canvas' && !options.pruneHistory) {
|
||||
throw new Error('--source-module 只能与 --prune-history 一起使用。');
|
||||
}
|
||||
if (
|
||||
options.retentionDays !== DEFAULT_RETENTION_DAYS &&
|
||||
!options.pruneHistory
|
||||
) {
|
||||
throw new Error('--retention-days 只能与 --prune-history 一起使用。');
|
||||
}
|
||||
if (options.completedBeforeMicros !== null && options.backfillSummaries) {
|
||||
throw new Error('--completed-before-micros 不能用于摘要回填。');
|
||||
}
|
||||
if (
|
||||
options.completedBeforeMicros !== null &&
|
||||
options.pruneHistory &&
|
||||
options.retentionDays !== DEFAULT_RETENTION_DAYS
|
||||
) {
|
||||
throw new Error(
|
||||
'--completed-before-micros 与 --retention-days 不能同时使用。',
|
||||
);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function resolveRetentionCutoffMicros(options) {
|
||||
if (!options.pruneHistory) {
|
||||
return options.completedBeforeMicros;
|
||||
}
|
||||
if (options.completedBeforeMicros !== null) {
|
||||
return options.completedBeforeMicros;
|
||||
}
|
||||
const cutoff = Date.now() * 1000 - options.retentionDays * MICROS_PER_DAY;
|
||||
if (!Number.isSafeInteger(cutoff)) {
|
||||
throw new Error(
|
||||
'--retention-days 计算出的 completed_before_micros 超出安全整数范围。',
|
||||
);
|
||||
}
|
||||
return cutoff;
|
||||
}
|
||||
|
||||
try {
|
||||
const options = parseOptions(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
@@ -113,24 +168,36 @@ try {
|
||||
);
|
||||
}
|
||||
|
||||
const procedureName = options.backfillSummaries
|
||||
? 'backfill_external_generation_job_summaries_and_return'
|
||||
: 'compact_external_generation_job_payloads_and_return';
|
||||
const input = options.backfillSummaries
|
||||
const completedBeforeMicros = resolveRetentionCutoffMicros(options);
|
||||
|
||||
const procedureName = options.pruneHistory
|
||||
? 'prune_external_generation_job_history_and_return'
|
||||
: options.backfillSummaries
|
||||
? 'backfill_external_generation_job_summaries_and_return'
|
||||
: 'compact_external_generation_job_payloads_and_return';
|
||||
const input = options.pruneHistory
|
||||
? {
|
||||
owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null),
|
||||
source_module: options.sourceModule,
|
||||
limit: options.limit,
|
||||
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
|
||||
completed_before_micros: completedBeforeMicros,
|
||||
dry_run: !options.apply,
|
||||
}
|
||||
: {
|
||||
dry_run: !options.apply,
|
||||
limit: options.limit,
|
||||
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
|
||||
completed_before_micros: encodeSpacetimeCliOption(
|
||||
options.completedBeforeMicros,
|
||||
),
|
||||
};
|
||||
: options.backfillSummaries
|
||||
? {
|
||||
owner_user_id: encodeSpacetimeCliOption(options.ownerUserId || null),
|
||||
limit: options.limit,
|
||||
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
|
||||
dry_run: !options.apply,
|
||||
}
|
||||
: {
|
||||
dry_run: !options.apply,
|
||||
limit: options.limit,
|
||||
cursor_job_id: encodeSpacetimeCliOption(options.cursorJobId || null),
|
||||
completed_before_micros: encodeSpacetimeCliOption(
|
||||
completedBeforeMicros,
|
||||
),
|
||||
};
|
||||
const result = await callSpacetimeProcedureViaCli(
|
||||
options,
|
||||
procedureName,
|
||||
@@ -138,10 +205,29 @@ try {
|
||||
);
|
||||
ensureProcedureOk(result);
|
||||
|
||||
console.log(JSON.stringify({ procedure: procedureName, ...result }, null, 2));
|
||||
const pendingApplyCount = options.backfillSummaries
|
||||
? Number(result.selected_count ?? 0)
|
||||
: Number(result.matched_count ?? 0);
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
procedure: procedureName,
|
||||
...(options.pruneHistory
|
||||
? {
|
||||
source_module: options.sourceModule,
|
||||
completed_before_micros: completedBeforeMicros,
|
||||
...(options.completedBeforeMicros === null
|
||||
? { retention_days: options.retentionDays }
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
...result,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const pendingApplyCount =
|
||||
options.pruneHistory || options.backfillSummaries
|
||||
? Number(result.selected_count ?? 0)
|
||||
: Number(result.matched_count ?? 0);
|
||||
if (result.has_more && options.apply) {
|
||||
console.log(
|
||||
`仍有后续批次;下一次追加 --cursor-job-id ${result.next_cursor_job_id ?? '<missing>'}。`,
|
||||
@@ -150,8 +236,11 @@ try {
|
||||
const currentCursor = options.cursorJobId
|
||||
? `保留 --cursor-job-id ${options.cursorJobId}`
|
||||
: '仍从首批开始';
|
||||
const cutoffHint = options.pruneHistory
|
||||
? `并固定 --completed-before-micros ${completedBeforeMicros}`
|
||||
: '';
|
||||
console.log(
|
||||
`当前仅 dry-run;请${currentCursor}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`,
|
||||
`当前仅 dry-run;请${currentCursor}${cutoffHint}并追加 --apply 重跑同一批。apply 成功后再使用其 next_cursor_job_id 进入下一批。`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user