feat: automate database OSS backups
This commit is contained in:
kdletters
452
scripts/database-backup-to-oss.mjs
Normal file
452
scripts/database-backup-to-oss.mjs
Normal file
@@ -0,0 +1,452 @@
|
||||
#!/usr/bin/env node
|
||||
import {spawnSync} from 'node:child_process';
|
||||
import {createHash, createHmac} from 'node:crypto';
|
||||
import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs';
|
||||
import {basename, dirname, isAbsolute, resolve} from 'node:path';
|
||||
import {fileURLToPath} from 'node:url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const REPO_ROOT = resolve(__dirname, '..');
|
||||
const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/data');
|
||||
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
|
||||
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
|
||||
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
|
||||
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
|
||||
const OSS_SERVICE = 'oss';
|
||||
const OSS_REQUEST = 'aliyun_v4_request';
|
||||
const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
|
||||
|
||||
function usage() {
|
||||
console.log(`用法:
|
||||
npm run database:backup:oss -- [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
|
||||
node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service]
|
||||
|
||||
说明:
|
||||
将 SpacetimeDB 数据目录打包成 .tar.gz，并上传到阿里云 OSS 指定 bucket。
|
||||
默认读取 .env / .env.local / .env.secrets.local；生产服务可传 --env-file /etc/genarrative/api-server.env。
|
||||
shell 环境变量优先级最高，不会被 env 文件覆盖。
|
||||
|
||||
常用环境变量:
|
||||
GENARRATIVE_DATABASE_BACKUP_DATA_DIR 数据目录；生产建议 /stdb
|
||||
GENARRATIVE_DATABASE_BACKUP_WORK_DIR 本地临时备份目录；生产建议 /var/lib/genarrative/database-backups
|
||||
GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket；未设置时回退 ALIYUN_OSS_BUCKET
|
||||
GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀，默认 database-backups
|
||||
GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT
|
||||
GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz
|
||||
ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
|
||||
`);
|
||||
}
|
||||
|
||||
function loadEnvFile(filePath, target, protectedKeys) {
|
||||
if (!existsSync(filePath)) {
|
||||
return;
|
||||
}
|
||||
const rawText = readFileSync(filePath, 'utf8');
|
||||
for (const rawLine of rawText.split(/\r?\n/u)) {
|
||||
const line = rawLine.trim();
|
||||
if (!line || line.startsWith('#')) {
|
||||
continue;
|
||||
}
|
||||
const match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const [, key, rawValue] = match;
|
||||
if (protectedKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
target[key] = rawValue.replace(/^['"]|['"]$/gu, '');
|
||||
}
|
||||
}
|
||||
|
||||
function loadRepoEnv() {
|
||||
const env = {...process.env};
|
||||
const protectedKeys = new Set(
|
||||
Object.entries(process.env)
|
||||
.filter(([, value]) => String(value ?? '').trim())
|
||||
.map(([key]) => key),
|
||||
);
|
||||
for (const fileName of ['.env', '.env.local', '.env.secrets.local']) {
|
||||
loadEnvFile(resolve(REPO_ROOT, fileName), env, protectedKeys);
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
function loadEffectiveEnv(envFiles) {
|
||||
const env = loadRepoEnv();
|
||||
const protectedKeys = new Set(
|
||||
Object.entries(process.env)
|
||||
.filter(([, value]) => String(value ?? '').trim())
|
||||
.map(([key]) => key),
|
||||
);
|
||||
for (const filePath of envFiles) {
|
||||
loadEnvFile(resolvePath(filePath), env, protectedKeys);
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
dataDir: '',
|
||||
workDir: '',
|
||||
bucket: '',
|
||||
endpoint: '',
|
||||
objectPrefix: '',
|
||||
accessKeyId: '',
|
||||
accessKeySecret: '',
|
||||
envFiles: [],
|
||||
keepLocal: false,
|
||||
stopService: '',
|
||||
database: '',
|
||||
dryRun: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const readValue = () => {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) {
|
||||
throw new Error(`${arg} 缺少参数值`);
|
||||
}
|
||||
index += 1;
|
||||
return value;
|
||||
};
|
||||
|
||||
switch (arg) {
|
||||
case '--help':
|
||||
case '-h':
|
||||
usage();
|
||||
process.exit(0);
|
||||
break;
|
||||
case '--data-dir':
|
||||
options.dataDir = readValue();
|
||||
break;
|
||||
case '--work-dir':
|
||||
options.workDir = readValue();
|
||||
break;
|
||||
case '--bucket':
|
||||
options.bucket = readValue();
|
||||
break;
|
||||
case '--endpoint':
|
||||
options.endpoint = readValue();
|
||||
break;
|
||||
case '--object-prefix':
|
||||
options.objectPrefix = readValue();
|
||||
break;
|
||||
case '--access-key-id':
|
||||
options.accessKeyId = readValue();
|
||||
break;
|
||||
case '--access-key-secret':
|
||||
options.accessKeySecret = readValue();
|
||||
break;
|
||||
case '--env-file':
|
||||
options.envFiles.push(readValue());
|
||||
break;
|
||||
case '--database':
|
||||
options.database = readValue();
|
||||
break;
|
||||
case '--stop-service':
|
||||
options.stopService = readValue();
|
||||
break;
|
||||
case '--keep-local':
|
||||
options.keepLocal = true;
|
||||
break;
|
||||
case '--dry-run':
|
||||
options.dryRun = true;
|
||||
break;
|
||||
default:
|
||||
throw new Error(`未知参数: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function firstNonEmpty(...values) {
|
||||
return values.map((value) => String(value ?? '').trim()).find(Boolean) ?? '';
|
||||
}
|
||||
|
||||
function resolvePath(value) {
|
||||
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
|
||||
}
|
||||
|
||||
function normalizeEndpoint(raw) {
|
||||
return String(raw ?? '')
|
||||
.trim()
|
||||
.replace(/^https?:\/\//u, '')
|
||||
.replace(/\/+$/u, '');
|
||||
}
|
||||
|
||||
function sanitizeObjectPart(value, fallback) {
|
||||
const sanitized = String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9._-]+/gu, '-')
|
||||
.replace(/-+/gu, '-')
|
||||
.replace(/^-|-$/gu, '');
|
||||
return sanitized || fallback;
|
||||
}
|
||||
|
||||
function timestampForFile(date = new Date()) {
|
||||
const pad = (value) => String(value).padStart(2, '0');
|
||||
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
|
||||
}
|
||||
|
||||
function buildBackupNames({database, dataDir, objectPrefix}) {
|
||||
const timestamp = timestampForFile();
|
||||
const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb');
|
||||
const fileName = `${databasePart}-${timestamp}.tar.gz`;
|
||||
const prefix = String(objectPrefix || 'database-backups')
|
||||
.trim()
|
||||
.replace(/^\/+|\/+$/gu, '')
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.map((part) => sanitizeObjectPart(part, 'backup'))
|
||||
.join('/');
|
||||
const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/');
|
||||
return {fileName, objectKey};
|
||||
}
|
||||
|
||||
function runCommand(command, args, options = {}) {
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: options.cwd ?? REPO_ROOT,
|
||||
env: options.env ?? process.env,
|
||||
encoding: 'utf8',
|
||||
stdio: options.stdio ?? 'pipe',
|
||||
shell: process.platform === 'win32',
|
||||
});
|
||||
if (result.error) {
|
||||
throw new Error(`${command} 启动失败: ${result.error.message}`);
|
||||
}
|
||||
if (result.status !== 0) {
|
||||
const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim();
|
||||
throw new Error(`${command} 退出码 ${result.status}: ${output}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function stopServiceIfNeeded(serviceName) {
|
||||
if (!serviceName) {
|
||||
return false;
|
||||
}
|
||||
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
|
||||
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
|
||||
return true;
|
||||
}
|
||||
|
||||
function startServiceIfNeeded(serviceName, wasStopped) {
|
||||
if (!serviceName || !wasStopped) {
|
||||
return;
|
||||
}
|
||||
console.log(`[database-backup] 恢复服务: ${serviceName}`);
|
||||
runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'});
|
||||
}
|
||||
|
||||
function createArchive({dataDir, workDir, fileName}) {
|
||||
if (!existsSync(dataDir)) {
|
||||
throw new Error(`数据库数据目录不存在: ${dataDir}`);
|
||||
}
|
||||
const stat = statSync(dataDir);
|
||||
if (!stat.isDirectory()) {
|
||||
throw new Error(`数据库数据路径不是目录: ${dataDir}`);
|
||||
}
|
||||
mkdirSync(workDir, {recursive: true});
|
||||
const archivePath = resolve(workDir, fileName);
|
||||
const parentDir = dirname(dataDir);
|
||||
const entryName = basename(dataDir);
|
||||
console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`);
|
||||
runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'});
|
||||
return archivePath;
|
||||
}
|
||||
|
||||
function hmac(key, content, encoding) {
|
||||
return createHmac('sha256', key).update(content).digest(encoding);
|
||||
}
|
||||
|
||||
function sha256Hex(content) {
|
||||
return createHash('sha256').update(content).digest('hex');
|
||||
}
|
||||
|
||||
function regionFromEndpoint(endpoint) {
|
||||
const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint);
|
||||
if (!match) {
|
||||
throw new Error(`无法从 OSS endpoint 推断 region: ${endpoint}`);
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function formatScopeDate(date) {
|
||||
return timestampForFile(date).slice(0, 8);
|
||||
}
|
||||
|
||||
function formatOssDate(date) {
|
||||
return timestampForFile(date).replace(/[-:]/gu, '');
|
||||
}
|
||||
|
||||
function encodePath(path) {
|
||||
return path
|
||||
.split('/')
|
||||
.map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
function canonicalHeaderValue(value) {
|
||||
return String(value).trim().replace(/\s+/gu, ' ');
|
||||
}
|
||||
|
||||
function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) {
|
||||
const region = regionFromEndpoint(endpoint);
|
||||
const scopeDate = formatScopeDate(date);
|
||||
const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`;
|
||||
const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`;
|
||||
const signedHeaders = Object.fromEntries(
|
||||
Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]),
|
||||
);
|
||||
const canonicalHeaders = Object.entries(signedHeaders)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, value]) => `${key}:${value}\n`)
|
||||
.join('');
|
||||
const additionalHeaders = 'host';
|
||||
const canonicalRequest = [
|
||||
method,
|
||||
canonicalUri,
|
||||
'',
|
||||
canonicalHeaders,
|
||||
additionalHeaders,
|
||||
UNSIGNED_PAYLOAD,
|
||||
].join('\n');
|
||||
const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n');
|
||||
const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate);
|
||||
const regionKey = hmac(signature, region);
|
||||
const serviceKey = hmac(regionKey, OSS_SERVICE);
|
||||
const signingKey = hmac(serviceKey, OSS_REQUEST);
|
||||
const finalSignature = hmac(signingKey, stringToSign, 'hex');
|
||||
return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`;
|
||||
}
|
||||
|
||||
async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) {
|
||||
const fileStat = statSync(archivePath);
|
||||
const now = new Date();
|
||||
const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`;
|
||||
const headers = {
|
||||
host: `${bucket}.${endpoint}`,
|
||||
'content-type': 'application/gzip',
|
||||
'x-oss-content-sha256': UNSIGNED_PAYLOAD,
|
||||
'x-oss-date': formatOssDate(now),
|
||||
'x-oss-meta-backup-kind': 'spacetimedb-data-dir',
|
||||
};
|
||||
const authorization = buildAuthorization({
|
||||
method: 'PUT',
|
||||
bucket,
|
||||
endpoint,
|
||||
objectKey,
|
||||
accessKeyId,
|
||||
accessKeySecret,
|
||||
headers,
|
||||
date: now,
|
||||
});
|
||||
|
||||
console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`);
|
||||
const response = await fetch(targetUrl, {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
...headers,
|
||||
authorization,
|
||||
'content-length': String(fileStat.size),
|
||||
},
|
||||
body: createReadStream(archivePath),
|
||||
duplex: 'half',
|
||||
});
|
||||
|
||||
const responseText = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
return {
|
||||
bucket,
|
||||
objectKey,
|
||||
contentLength: fileStat.size,
|
||||
etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '',
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const env = loadEffectiveEnv(args.envFiles);
|
||||
const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32';
|
||||
const dataDir = resolvePath(firstNonEmpty(
|
||||
args.dataDir,
|
||||
env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR,
|
||||
isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR,
|
||||
));
|
||||
const workDir = resolvePath(firstNonEmpty(
|
||||
args.workDir,
|
||||
env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR,
|
||||
isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR,
|
||||
));
|
||||
const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET);
|
||||
const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT));
|
||||
const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID);
|
||||
const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET);
|
||||
const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups');
|
||||
const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
|
||||
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
|
||||
|
||||
for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) {
|
||||
if (!value) {
|
||||
throw new Error(`缺少 ${label} 配置`);
|
||||
}
|
||||
}
|
||||
|
||||
const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix});
|
||||
console.log(`[database-backup] 数据目录: ${dataDir}`);
|
||||
console.log(`[database-backup] 本地临时目录: ${workDir}`);
|
||||
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
|
||||
|
||||
if (args.dryRun) {
|
||||
console.log('[database-backup] dry-run，仅校验配置，不打包上传。');
|
||||
return;
|
||||
}
|
||||
|
||||
let archivePath = '';
|
||||
let serviceStopped = false;
|
||||
try {
|
||||
serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE));
|
||||
archivePath = createArchive({dataDir, workDir, fileName});
|
||||
} finally {
|
||||
startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped);
|
||||
}
|
||||
|
||||
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
|
||||
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
|
||||
|
||||
const manifestPath = `${archivePath}.manifest.json`;
|
||||
writeFileSync(
|
||||
manifestPath,
|
||||
`${JSON.stringify({
|
||||
createdAt: new Date().toISOString(),
|
||||
dataDir,
|
||||
bucket: result.bucket,
|
||||
objectKey: result.objectKey,
|
||||
contentLength: result.contentLength,
|
||||
etag: result.etag,
|
||||
}, null, 2)}\n`,
|
||||
'utf8',
|
||||
);
|
||||
|
||||
if (!keepLocal) {
|
||||
rmSync(archivePath, {force: true});
|
||||
rmSync(manifestPath, {force: true});
|
||||
console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。');
|
||||
} else {
|
||||
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
|
||||
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`[database-backup] ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user