Files
Genarrative/scripts/database-backup-to-oss.mjs
2026-05-28 02:05:41 +08:00

570 lines
19 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env node
import {spawnSync} from 'node:child_process';
import {createHash, createHmac} from 'node:crypto';
import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs';
import {basename, dirname, isAbsolute, resolve} from 'node:path';
import {fileURLToPath} from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const REPO_ROOT = resolve(__dirname, '..');
const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/data');
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
const OSS_SERVICE = 'oss';
const OSS_REQUEST = 'aliyun_v4_request';
const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
function usage() {
console.log(`用法:
npm run database:backup:oss -- [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--defer-upload]
node scripts/database-backup-to-oss.mjs --upload-archive <path>
说明:
将 SpacetimeDB 数据目录打包成 .tar.gz并上传到阿里云 OSS 指定 bucket。
--defer-upload 只生成本地冷备份和 manifest不上传后续用 --upload-archive 异步上传。
默认读取 .env / .env.local / .env.secrets.local生产服务可传 --env-file /etc/genarrative/api-server.env。
shell 环境变量优先级最高,不会被 env 文件覆盖。
常用环境变量:
GENARRATIVE_DATABASE_BACKUP_DATA_DIR 数据目录;生产建议 /stdb
GENARRATIVE_DATABASE_BACKUP_WORK_DIR 本地临时备份目录;生产建议 /var/lib/genarrative/database-backups
GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket未设置时回退 ALIYUN_OSS_BUCKET
GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀,默认 database-backups
GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint未设置时回退 ALIYUN_OSS_ENDPOINT
GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz
ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
`);
}
function loadEnvFile(filePath, target, protectedKeys) {
if (!existsSync(filePath)) {
return;
}
const rawText = readFileSync(filePath, 'utf8');
for (const rawLine of rawText.split(/\r?\n/u)) {
const line = rawLine.trim();
if (!line || line.startsWith('#')) {
continue;
}
const match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u);
if (!match) {
continue;
}
const [, key, rawValue] = match;
if (protectedKeys.has(key)) {
continue;
}
target[key] = rawValue.replace(/^['"]|['"]$/gu, '');
}
}
function loadRepoEnv() {
const env = {...process.env};
const protectedKeys = new Set(
Object.entries(process.env)
.filter(([, value]) => String(value ?? '').trim())
.map(([key]) => key),
);
for (const fileName of ['.env', '.env.local', '.env.secrets.local']) {
loadEnvFile(resolve(REPO_ROOT, fileName), env, protectedKeys);
}
return env;
}
function loadEffectiveEnv(envFiles) {
const env = loadRepoEnv();
const protectedKeys = new Set(
Object.entries(process.env)
.filter(([, value]) => String(value ?? '').trim())
.map(([key]) => key),
);
for (const filePath of envFiles) {
loadEnvFile(resolvePath(filePath), env, protectedKeys);
}
return env;
}
function parseArgs(argv) {
const options = {
dataDir: '',
workDir: '',
bucket: '',
endpoint: '',
objectPrefix: '',
accessKeyId: '',
accessKeySecret: '',
envFiles: [],
keepLocal: false,
stopService: '',
database: '',
dryRun: false,
deferUpload: false,
uploadArchive: '',
manifestFile: '',
objectKey: '',
resultFile: '',
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
const readValue = () => {
const value = argv[index + 1];
if (!value || value.startsWith('--')) {
throw new Error(`${arg} 缺少参数值`);
}
index += 1;
return value;
};
switch (arg) {
case '--help':
case '-h':
usage();
process.exit(0);
break;
case '--data-dir':
options.dataDir = readValue();
break;
case '--work-dir':
options.workDir = readValue();
break;
case '--bucket':
options.bucket = readValue();
break;
case '--endpoint':
options.endpoint = readValue();
break;
case '--object-prefix':
options.objectPrefix = readValue();
break;
case '--object-key':
options.objectKey = readValue();
break;
case '--access-key-id':
options.accessKeyId = readValue();
break;
case '--access-key-secret':
options.accessKeySecret = readValue();
break;
case '--env-file':
options.envFiles.push(readValue());
break;
case '--database':
options.database = readValue();
break;
case '--stop-service':
options.stopService = readValue();
break;
case '--keep-local':
options.keepLocal = true;
break;
case '--dry-run':
options.dryRun = true;
break;
case '--defer-upload':
options.deferUpload = true;
options.keepLocal = true;
break;
case '--upload-archive':
options.uploadArchive = readValue();
break;
case '--manifest-file':
options.manifestFile = readValue();
break;
case '--result-file':
options.resultFile = readValue();
break;
default:
throw new Error(`未知参数: ${arg}`);
}
}
return options;
}
function firstNonEmpty(...values) {
return values.map((value) => String(value ?? '').trim()).find(Boolean) ?? '';
}
function resolvePath(value) {
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
}
function normalizeEndpoint(raw) {
return String(raw ?? '')
.trim()
.replace(/^https?:\/\//u, '')
.replace(/\/+$/u, '');
}
function sanitizeObjectPart(value, fallback) {
const sanitized = String(value ?? '')
.trim()
.toLowerCase()
.replace(/[^a-z0-9._-]+/gu, '-')
.replace(/-+/gu, '-')
.replace(/^-|-$/gu, '');
return sanitized || fallback;
}
function timestampForFile(date = new Date()) {
const pad = (value) => String(value).padStart(2, '0');
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
}
function buildBackupNames({database, dataDir, objectPrefix}) {
const timestamp = timestampForFile();
const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb');
const fileName = `${databasePart}-${timestamp}.tar.gz`;
const prefix = String(objectPrefix || 'database-backups')
.trim()
.replace(/^\/+|\/+$/gu, '')
.split('/')
.filter(Boolean)
.map((part) => sanitizeObjectPart(part, 'backup'))
.join('/');
const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/');
return {fileName, objectKey};
}
function runCommand(command, args, options = {}) {
const result = spawnSync(command, args, {
cwd: options.cwd ?? REPO_ROOT,
env: options.env ?? process.env,
encoding: 'utf8',
stdio: options.stdio ?? 'pipe',
shell: process.platform === 'win32',
});
if (result.error) {
throw new Error(`${command} 启动失败: ${result.error.message}`);
}
if (result.status !== 0) {
const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim();
throw new Error(`${command} 退出码 ${result.status}: ${output}`);
}
return result;
}
function stopServiceIfNeeded(serviceName) {
if (!serviceName) {
return false;
}
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
return true;
}
function startServiceIfNeeded(serviceName, wasStopped) {
if (!serviceName || !wasStopped) {
return;
}
console.log(`[database-backup] 恢复服务: ${serviceName}`);
runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'});
}
function createArchive({dataDir, workDir, fileName}) {
if (!existsSync(dataDir)) {
throw new Error(`数据库数据目录不存在: ${dataDir}`);
}
const stat = statSync(dataDir);
if (!stat.isDirectory()) {
throw new Error(`数据库数据路径不是目录: ${dataDir}`);
}
mkdirSync(workDir, {recursive: true});
const archivePath = resolve(workDir, fileName);
const parentDir = dirname(dataDir);
const entryName = basename(dataDir);
console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`);
runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'});
return archivePath;
}
function writeManifest({manifestPath, payload}) {
writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
}
function readManifest(manifestPath) {
if (!existsSync(manifestPath)) {
throw new Error(`备份清单不存在: ${manifestPath}`);
}
return JSON.parse(readFileSync(manifestPath, 'utf8'));
}
function hmac(key, content, encoding) {
return createHmac('sha256', key).update(content).digest(encoding);
}
function sha256Hex(content) {
return createHash('sha256').update(content).digest('hex');
}
function regionFromEndpoint(endpoint) {
const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint);
if (!match) {
throw new Error(`无法从 OSS endpoint 推断 region: ${endpoint}`);
}
return match[1];
}
function formatScopeDate(date) {
return timestampForFile(date).slice(0, 8);
}
function formatOssDate(date) {
return timestampForFile(date).replace(/[-:]/gu, '');
}
function encodePath(path) {
return path
.split('/')
.map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`))
.join('/');
}
function canonicalHeaderValue(value) {
return String(value).trim().replace(/\s+/gu, ' ');
}
function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) {
const region = regionFromEndpoint(endpoint);
const scopeDate = formatScopeDate(date);
const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`;
const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`;
const signedHeaders = Object.fromEntries(
Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]),
);
const canonicalHeaders = Object.entries(signedHeaders)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, value]) => `${key}:${value}\n`)
.join('');
const additionalHeaders = 'host';
const canonicalRequest = [
method,
canonicalUri,
'',
canonicalHeaders,
additionalHeaders,
UNSIGNED_PAYLOAD,
].join('\n');
const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n');
const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate);
const regionKey = hmac(signature, region);
const serviceKey = hmac(regionKey, OSS_SERVICE);
const signingKey = hmac(serviceKey, OSS_REQUEST);
const finalSignature = hmac(signingKey, stringToSign, 'hex');
return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`;
}
async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) {
const fileStat = statSync(archivePath);
const now = new Date();
const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`;
const headers = {
host: `${bucket}.${endpoint}`,
'content-type': 'application/gzip',
'x-oss-content-sha256': UNSIGNED_PAYLOAD,
'x-oss-date': formatOssDate(now),
'x-oss-meta-backup-kind': 'spacetimedb-data-dir',
};
const authorization = buildAuthorization({
method: 'PUT',
bucket,
endpoint,
objectKey,
accessKeyId,
accessKeySecret,
headers,
date: now,
});
console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`);
const response = await fetch(targetUrl, {
method: 'PUT',
headers: {
...headers,
authorization,
'content-length': String(fileStat.size),
},
body: createReadStream(archivePath),
duplex: 'half',
});
const responseText = await response.text();
if (!response.ok) {
throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`);
}
return {
bucket,
objectKey,
contentLength: fileStat.size,
etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '',
};
}
async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}) {
const archivePath = resolvePath(args.uploadArchive);
if (!existsSync(archivePath)) {
throw new Error(`待上传备份文件不存在: ${archivePath}`);
}
const manifestPath = resolvePath(args.manifestFile || `${archivePath}.manifest.json`);
const manifest = existsSync(manifestPath) ? readManifest(manifestPath) : {};
const dataDir = firstNonEmpty(manifest.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, DEFAULT_PRODUCTION_DATA_DIR);
const database = firstNonEmpty(args.database, manifest.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
const objectKey = firstNonEmpty(args.objectKey, manifest.objectKey, buildBackupNames({database, dataDir, objectPrefix}).objectKey);
console.log(`[database-backup] 上传已有备份: ${archivePath}`);
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
if (args.dryRun) {
console.log('[database-backup] dry-run仅校验上传配置。');
return;
}
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
const uploadedAt = new Date().toISOString();
writeManifest({
manifestPath,
payload: {
...manifest,
database,
bucket: result.bucket,
objectKey: result.objectKey,
contentLength: result.contentLength,
etag: result.etag,
uploadedAt,
uploadStatus: 'uploaded',
},
});
if (args.resultFile) {
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, ...result, uploadedAt}, null, 2)}\n`, 'utf8');
}
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
if (!keepLocal) {
rmSync(archivePath, {force: true});
rmSync(manifestPath, {force: true});
console.log('[database-backup] 已删除本地临时备份文件;如需保留请设置 --keep-local。');
} else {
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
}
}
async function main() {
const args = parseArgs(process.argv.slice(2));
const env = loadEffectiveEnv(args.envFiles);
const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32';
const dataDir = resolvePath(firstNonEmpty(
args.dataDir,
env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR,
isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR,
));
const workDir = resolvePath(firstNonEmpty(
args.workDir,
env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR,
isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR,
));
const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET);
const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT));
const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID);
const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET);
const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups');
const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) {
if (!value) {
throw new Error(`缺少 ${label} 配置`);
}
}
if (args.uploadArchive) {
await uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix});
return;
}
const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix});
console.log(`[database-backup] 数据目录: ${dataDir}`);
console.log(`[database-backup] 本地临时目录: ${workDir}`);
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
if (args.dryRun) {
console.log('[database-backup] dry-run仅校验配置不打包上传。');
return;
}
let archivePath = '';
let serviceStopped = false;
try {
serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE));
archivePath = createArchive({dataDir, workDir, fileName});
} finally {
startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped);
}
const manifestPath = `${archivePath}.manifest.json`;
writeManifest({
manifestPath,
payload: {
createdAt: new Date().toISOString(),
database,
dataDir,
bucket,
objectKey,
archivePath,
uploadStatus: args.deferUpload ? 'deferred' : 'pending',
},
});
if (args.deferUpload) {
console.log(`[database-backup] 已生成本地冷备份,延后上传: ${archivePath}`);
console.log(`[database-backup] 已写入备份清单: ${manifestPath}`);
if (args.resultFile) {
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, bucket, objectKey}, null, 2)}\n`, 'utf8');
}
return;
}
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
writeManifest({
manifestPath,
payload: {
createdAt: new Date().toISOString(),
database,
dataDir,
bucket: result.bucket,
objectKey: result.objectKey,
archivePath,
contentLength: result.contentLength,
etag: result.etag,
uploadedAt: new Date().toISOString(),
uploadStatus: 'uploaded',
},
});
if (!keepLocal) {
rmSync(archivePath, {force: true});
rmSync(manifestPath, {force: true});
console.log('[database-backup] 已删除本地临时备份文件;如需保留请设置 --keep-local。');
} else {
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
}
}
main().catch((error) => {
console.error(`[database-backup] ${error instanceof Error ? error.message : String(error)}`);
process.exit(1);
});