fix publish backup flow

This commit is contained in:
kdletters
2026-05-28 02:05:41 +08:00
parent 23f6317c8b
commit de8b82c575
5 changed files with 193 additions and 28 deletions

View File

@@ -20,10 +20,12 @@ const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
function usage() {
console.log(`用法:
npm run database:backup:oss -- [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service]
node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--defer-upload]
node scripts/database-backup-to-oss.mjs --upload-archive <path>
说明:
将 SpacetimeDB 数据目录打包成 .tar.gz并上传到阿里云 OSS 指定 bucket。
--defer-upload 只生成本地冷备份和 manifest不上传后续用 --upload-archive 异步上传。
默认读取 .env / .env.local / .env.secrets.local生产服务可传 --env-file /etc/genarrative/api-server.env。
shell 环境变量优先级最高,不会被 env 文件覆盖。
@@ -100,6 +102,11 @@ function parseArgs(argv) {
stopService: '',
database: '',
dryRun: false,
deferUpload: false,
uploadArchive: '',
manifestFile: '',
objectKey: '',
resultFile: '',
};
for (let index = 0; index < argv.length; index += 1) {
@@ -134,6 +141,9 @@ function parseArgs(argv) {
case '--object-prefix':
options.objectPrefix = readValue();
break;
case '--object-key':
options.objectKey = readValue();
break;
case '--access-key-id':
options.accessKeyId = readValue();
break;
@@ -155,6 +165,19 @@ function parseArgs(argv) {
case '--dry-run':
options.dryRun = true;
break;
case '--defer-upload':
options.deferUpload = true;
options.keepLocal = true;
break;
case '--upload-archive':
options.uploadArchive = readValue();
break;
case '--manifest-file':
options.manifestFile = readValue();
break;
case '--result-file':
options.resultFile = readValue();
break;
default:
throw new Error(`未知参数: ${arg}`);
}
@@ -260,6 +283,17 @@ function createArchive({dataDir, workDir, fileName}) {
return archivePath;
}
function writeManifest({manifestPath, payload}) {
writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}
function readManifest(manifestPath) {
if (!existsSync(manifestPath)) {
throw new Error(`备份清单不存在: ${manifestPath}`);
}
return JSON.parse(readFileSync(manifestPath, 'utf8'));
}
function hmac(key, content, encoding) {
return createHmac('sha256', key).update(content).digest(encoding);
}
@@ -372,6 +406,59 @@ async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKe
};
}
async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}) {
const archivePath = resolvePath(args.uploadArchive);
if (!existsSync(archivePath)) {
throw new Error(`待上传备份文件不存在: ${archivePath}`);
}
const manifestPath = resolvePath(args.manifestFile || `${archivePath}.manifest.json`);
const manifest = existsSync(manifestPath) ? readManifest(manifestPath) : {};
const dataDir = firstNonEmpty(manifest.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, DEFAULT_PRODUCTION_DATA_DIR);
const database = firstNonEmpty(args.database, manifest.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
const objectKey = firstNonEmpty(args.objectKey, manifest.objectKey, buildBackupNames({database, dataDir, objectPrefix}).objectKey);
console.log(`[database-backup] 上传已有备份: ${archivePath}`);
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
if (args.dryRun) {
console.log('[database-backup] dry-run仅校验上传配置。');
return;
}
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
const uploadedAt = new Date().toISOString();
writeManifest({
manifestPath,
payload: {
...manifest,
database,
bucket: result.bucket,
objectKey: result.objectKey,
contentLength: result.contentLength,
etag: result.etag,
uploadedAt,
uploadStatus: 'uploaded',
},
});
if (args.resultFile) {
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, ...result, uploadedAt}, null, 2)}\n`, 'utf8');
}
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
if (!keepLocal) {
rmSync(archivePath, {force: true});
rmSync(manifestPath, {force: true});
console.log('[database-backup] 已删除本地临时备份文件;如需保留请设置 --keep-local。');
} else {
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
}
}
async function main() {
const args = parseArgs(process.argv.slice(2));
const env = loadEffectiveEnv(args.envFiles);
@@ -400,6 +487,11 @@ async function main() {
}
}
if (args.uploadArchive) {
await uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix});
return;
}
const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix});
console.log(`[database-backup] 数据目录: ${dataDir}`);
console.log(`[database-backup] 本地临时目录: ${workDir}`);
@@ -419,22 +511,47 @@ async function main() {
startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped);
}
const manifestPath = `${archivePath}.manifest.json`;
writeManifest({
manifestPath,
payload: {
createdAt: new Date().toISOString(),
database,
dataDir,
bucket,
objectKey,
archivePath,
uploadStatus: args.deferUpload ? 'deferred' : 'pending',
},
});
if (args.deferUpload) {
console.log(`[database-backup] 已生成本地冷备份,延后上传: ${archivePath}`);
console.log(`[database-backup] 已写入备份清单: ${manifestPath}`);
if (args.resultFile) {
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, bucket, objectKey}, null, 2)}\n`, 'utf8');
}
return;
}
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
const manifestPath = `${archivePath}.manifest.json`;
writeFileSync(
writeManifest({
manifestPath,
`${JSON.stringify({
payload: {
createdAt: new Date().toISOString(),
database,
dataDir,
bucket: result.bucket,
objectKey: result.objectKey,
archivePath,
contentLength: result.contentLength,
etag: result.etag,
}, null, 2)}\n`,
'utf8',
);
uploadedAt: new Date().toISOString(),
uploadStatus: 'uploaded',
},
});
if (!keepLocal) {
rmSync(archivePath, {force: true});