#!/usr/bin/env node import {spawnSync} from 'node:child_process'; import {createHash, createHmac} from 'node:crypto'; import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs'; import {basename, dirname, isAbsolute, resolve} from 'node:path'; import {fileURLToPath} from 'node:url'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const REPO_ROOT = resolve(__dirname, '..'); const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/data'); const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups'); const DEFAULT_PRODUCTION_DATA_DIR = '/stdb'; const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups'; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; const OSS_SERVICE = 'oss'; const OSS_REQUEST = 'aliyun_v4_request'; const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD'; function usage() { console.log(`用法: npm run database:backup:oss -- [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] 说明: 将 SpacetimeDB 数据目录打包成 .tar.gz,并上传到阿里云 OSS 指定 bucket。 默认读取 .env / .env.local / .env.secrets.local;生产服务可传 --env-file /etc/genarrative/api-server.env。 shell 环境变量优先级最高,不会被 env 文件覆盖。 常用环境变量: GENARRATIVE_DATABASE_BACKUP_DATA_DIR 数据目录;生产建议 /stdb GENARRATIVE_DATABASE_BACKUP_WORK_DIR 本地临时备份目录;生产建议 /var/lib/genarrative/database-backups GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket;未设置时回退 ALIYUN_OSS_BUCKET GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀,默认 database-backups GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint;未设置时回退 ALIYUN_OSS_ENDPOINT GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET `); } function loadEnvFile(filePath, target, protectedKeys) { if (!existsSync(filePath)) { return; } const rawText = readFileSync(filePath, 'utf8'); for (const rawLine of rawText.split(/\r?\n/u)) { const line = rawLine.trim(); if (!line || line.startsWith('#')) { continue; } const match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u); if (!match) { continue; } const [, key, rawValue] = match; if (protectedKeys.has(key)) { continue; } target[key] = rawValue.replace(/^['"]|['"]$/gu, ''); } } function loadRepoEnv() { const env = {...process.env}; const protectedKeys = new Set( Object.entries(process.env) .filter(([, value]) => String(value ?? '').trim()) .map(([key]) => key), ); for (const fileName of ['.env', '.env.local', '.env.secrets.local']) { loadEnvFile(resolve(REPO_ROOT, fileName), env, protectedKeys); } return env; } function loadEffectiveEnv(envFiles) { const env = loadRepoEnv(); const protectedKeys = new Set( Object.entries(process.env) .filter(([, value]) => String(value ?? '').trim()) .map(([key]) => key), ); for (const filePath of envFiles) { loadEnvFile(resolvePath(filePath), env, protectedKeys); } return env; } function parseArgs(argv) { const options = { dataDir: '', workDir: '', bucket: '', endpoint: '', objectPrefix: '', accessKeyId: '', accessKeySecret: '', envFiles: [], keepLocal: false, stopService: '', database: '', dryRun: false, }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; const readValue = () => { const value = argv[index + 1]; if (!value || value.startsWith('--')) { throw new Error(`${arg} 缺少参数值`); } index += 1; return value; }; switch (arg) { case '--help': case '-h': usage(); process.exit(0); break; case '--data-dir': options.dataDir = readValue(); break; case '--work-dir': options.workDir = readValue(); break; case '--bucket': options.bucket = readValue(); break; case '--endpoint': options.endpoint = readValue(); break; case '--object-prefix': options.objectPrefix = readValue(); break; case '--access-key-id': options.accessKeyId = readValue(); break; case '--access-key-secret': options.accessKeySecret = readValue(); break; case '--env-file': options.envFiles.push(readValue()); break; case '--database': options.database = readValue(); break; case '--stop-service': options.stopService = readValue(); break; case '--keep-local': options.keepLocal = true; break; case '--dry-run': options.dryRun = true; break; default: throw new Error(`未知参数: ${arg}`); } } return options; } function firstNonEmpty(...values) { return values.map((value) => String(value ?? '').trim()).find(Boolean) ?? ''; } function resolvePath(value) { return isAbsolute(value) ? value : resolve(REPO_ROOT, value); } function normalizeEndpoint(raw) { return String(raw ?? '') .trim() .replace(/^https?:\/\//u, '') .replace(/\/+$/u, ''); } function sanitizeObjectPart(value, fallback) { const sanitized = String(value ?? '') .trim() .toLowerCase() .replace(/[^a-z0-9._-]+/gu, '-') .replace(/-+/gu, '-') .replace(/^-|-$/gu, ''); return sanitized || fallback; } function timestampForFile(date = new Date()) { const pad = (value) => String(value).padStart(2, '0'); return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`; } function buildBackupNames({database, dataDir, objectPrefix}) { const timestamp = timestampForFile(); const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb'); const fileName = `${databasePart}-${timestamp}.tar.gz`; const prefix = String(objectPrefix || 'database-backups') .trim() .replace(/^\/+|\/+$/gu, '') .split('/') .filter(Boolean) .map((part) => sanitizeObjectPart(part, 'backup')) .join('/'); const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/'); return {fileName, objectKey}; } function runCommand(command, args, options = {}) { const result = spawnSync(command, args, { cwd: options.cwd ?? REPO_ROOT, env: options.env ?? process.env, encoding: 'utf8', stdio: options.stdio ?? 'pipe', shell: process.platform === 'win32', }); if (result.error) { throw new Error(`${command} 启动失败: ${result.error.message}`); } if (result.status !== 0) { const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim(); throw new Error(`${command} 退出码 ${result.status}: ${output}`); } return result; } function stopServiceIfNeeded(serviceName) { if (!serviceName) { return false; } console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`); runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'}); return true; } function startServiceIfNeeded(serviceName, wasStopped) { if (!serviceName || !wasStopped) { return; } console.log(`[database-backup] 恢复服务: ${serviceName}`); runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'}); } function createArchive({dataDir, workDir, fileName}) { if (!existsSync(dataDir)) { throw new Error(`数据库数据目录不存在: ${dataDir}`); } const stat = statSync(dataDir); if (!stat.isDirectory()) { throw new Error(`数据库数据路径不是目录: ${dataDir}`); } mkdirSync(workDir, {recursive: true}); const archivePath = resolve(workDir, fileName); const parentDir = dirname(dataDir); const entryName = basename(dataDir); console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`); runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'}); return archivePath; } function hmac(key, content, encoding) { return createHmac('sha256', key).update(content).digest(encoding); } function sha256Hex(content) { return createHash('sha256').update(content).digest('hex'); } function regionFromEndpoint(endpoint) { const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint); if (!match) { throw new Error(`无法从 OSS endpoint 推断 region: ${endpoint}`); } return match[1]; } function formatScopeDate(date) { return timestampForFile(date).slice(0, 8); } function formatOssDate(date) { return timestampForFile(date).replace(/[-:]/gu, ''); } function encodePath(path) { return path .split('/') .map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)) .join('/'); } function canonicalHeaderValue(value) { return String(value).trim().replace(/\s+/gu, ' '); } function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) { const region = regionFromEndpoint(endpoint); const scopeDate = formatScopeDate(date); const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`; const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`; const signedHeaders = Object.fromEntries( Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]), ); const canonicalHeaders = Object.entries(signedHeaders) .sort(([left], [right]) => left.localeCompare(right)) .map(([key, value]) => `${key}:${value}\n`) .join(''); const additionalHeaders = 'host'; const canonicalRequest = [ method, canonicalUri, '', canonicalHeaders, additionalHeaders, UNSIGNED_PAYLOAD, ].join('\n'); const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n'); const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate); const regionKey = hmac(signature, region); const serviceKey = hmac(regionKey, OSS_SERVICE); const signingKey = hmac(serviceKey, OSS_REQUEST); const finalSignature = hmac(signingKey, stringToSign, 'hex'); return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`; } async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) { const fileStat = statSync(archivePath); const now = new Date(); const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`; const headers = { host: `${bucket}.${endpoint}`, 'content-type': 'application/gzip', 'x-oss-content-sha256': UNSIGNED_PAYLOAD, 'x-oss-date': formatOssDate(now), 'x-oss-meta-backup-kind': 'spacetimedb-data-dir', }; const authorization = buildAuthorization({ method: 'PUT', bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date: now, }); console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`); const response = await fetch(targetUrl, { method: 'PUT', headers: { ...headers, authorization, 'content-length': String(fileStat.size), }, body: createReadStream(archivePath), duplex: 'half', }); const responseText = await response.text(); if (!response.ok) { throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`); } return { bucket, objectKey, contentLength: fileStat.size, etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '', }; } async function main() { const args = parseArgs(process.argv.slice(2)); const env = loadEffectiveEnv(args.envFiles); const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; const dataDir = resolvePath(firstNonEmpty( args.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR, )); const workDir = resolvePath(firstNonEmpty( args.workDir, env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR, isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR, )); const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET); const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT)); const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID); const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET); const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups'); const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) { if (!value) { throw new Error(`缺少 ${label} 配置`); } } const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix}); console.log(`[database-backup] 数据目录: ${dataDir}`); console.log(`[database-backup] 本地临时目录: ${workDir}`); console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`); if (args.dryRun) { console.log('[database-backup] dry-run,仅校验配置,不打包上传。'); return; } let archivePath = ''; let serviceStopped = false; try { serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE)); archivePath = createArchive({dataDir, workDir, fileName}); } finally { startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped); } const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}); console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); const manifestPath = `${archivePath}.manifest.json`; writeFileSync( manifestPath, `${JSON.stringify({ createdAt: new Date().toISOString(), dataDir, bucket: result.bucket, objectKey: result.objectKey, contentLength: result.contentLength, etag: result.etag, }, null, 2)}\n`, 'utf8', ); if (!keepLocal) { rmSync(archivePath, {force: true}); rmSync(manifestPath, {force: true}); console.log('[database-backup] 已删除本地临时备份文件;如需保留请设置 --keep-local。'); } else { console.log(`[database-backup] 已保留本地备份: ${archivePath}`); console.log(`[database-backup] 已保留备份清单: ${manifestPath}`); } } main().catch((error) => { console.error(`[database-backup] ${error instanceof Error ? error.message : String(error)}`); process.exit(1); });