kdletters
4e3c26d5d8
冷备份增加工作目录空间预检并确保失败后恢复依赖服务 API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理 Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数 补充备份与 API deploy 回归检查和生产运维护栏 更新生产运维文档与项目记忆的事故处理口径
800 lines
27 KiB
JavaScript
800 lines
27 KiB
JavaScript
#!/usr/bin/env node
|
||
import {spawnSync} from 'node:child_process';
|
||
import {createHash, createHmac} from 'node:crypto';
|
||
import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs';
|
||
import {basename, dirname, isAbsolute, resolve} from 'node:path';
|
||
import {fileURLToPath} from 'node:url';
|
||
|
||
const __filename = fileURLToPath(import.meta.url);
|
||
const __dirname = dirname(__filename);
|
||
const REPO_ROOT = resolve(__dirname, '..');
|
||
const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/data');
|
||
const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups');
|
||
const DEFAULT_PRODUCTION_DATA_DIR = '/stdb';
|
||
const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups';
|
||
const DEFAULT_SPACE_SAFETY_RATIO = 1.1;
|
||
const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024;
|
||
const OSS_ALGORITHM = 'OSS4-HMAC-SHA256';
|
||
const OSS_SERVICE = 'oss';
|
||
const OSS_REQUEST = 'aliyun_v4_request';
|
||
const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD';
|
||
|
||
function usage() {
|
||
console.log(`用法:
|
||
npm run database:backup:oss -- [--data-dir <path>] [--work-dir <path>] [--bucket <bucket>] [--object-prefix <prefix>] [--keep-local]
|
||
node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload]
|
||
node -- scripts/database-backup-to-oss.mjs --upload-archive <path>
|
||
|
||
说明:
|
||
将 SpacetimeDB 数据目录打包成 .tar.gz，并上传到阿里云 OSS 指定 bucket。
|
||
--defer-upload 只生成本地冷备份和 manifest，不上传；后续用 --upload-archive 异步上传。
|
||
默认读取 .env / .env.local / .env.secrets.local；生产服务可传 --env-file /etc/genarrative/api-server.env。
|
||
shell 环境变量优先级最高，不会被 env 文件覆盖。
|
||
|
||
常用环境变量:
|
||
GENARRATIVE_DATABASE_BACKUP_DATA_DIR 数据目录；生产建议 /stdb
|
||
GENARRATIVE_DATABASE_BACKUP_WORK_DIR 本地临时备份目录；生产建议 /var/lib/genarrative/database-backups
|
||
GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket；未设置时回退 ALIYUN_OSS_BUCKET
|
||
GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀，默认 database-backups
|
||
GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT
|
||
GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz
|
||
GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算
|
||
ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET
|
||
`);
|
||
}
|
||
|
||
function loadEnvFile(filePath, target, protectedKeys) {
|
||
if (!existsSync(filePath)) {
|
||
return;
|
||
}
|
||
const rawText = readFileSync(filePath, 'utf8');
|
||
for (const rawLine of rawText.split(/\r?\n/u)) {
|
||
const line = rawLine.trim();
|
||
if (!line || line.startsWith('#')) {
|
||
continue;
|
||
}
|
||
const match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u);
|
||
if (!match) {
|
||
continue;
|
||
}
|
||
const [, key, rawValue] = match;
|
||
if (protectedKeys.has(key)) {
|
||
continue;
|
||
}
|
||
target[key] = rawValue.replace(/^['"]|['"]$/gu, '');
|
||
}
|
||
}
|
||
|
||
function loadRepoEnv() {
|
||
const env = {...process.env};
|
||
const protectedKeys = new Set(
|
||
Object.entries(process.env)
|
||
.filter(([, value]) => String(value ?? '').trim())
|
||
.map(([key]) => key),
|
||
);
|
||
for (const fileName of ['.env', '.env.local', '.env.secrets.local']) {
|
||
loadEnvFile(resolve(REPO_ROOT, fileName), env, protectedKeys);
|
||
}
|
||
return env;
|
||
}
|
||
|
||
function loadEffectiveEnv(envFiles) {
|
||
const env = loadRepoEnv();
|
||
const protectedKeys = new Set(
|
||
Object.entries(process.env)
|
||
.filter(([, value]) => String(value ?? '').trim())
|
||
.map(([key]) => key),
|
||
);
|
||
for (const filePath of envFiles) {
|
||
loadEnvFile(resolvePath(filePath), env, protectedKeys);
|
||
}
|
||
return env;
|
||
}
|
||
|
||
function parseArgs(argv) {
|
||
const options = {
|
||
dataDir: '',
|
||
workDir: '',
|
||
bucket: '',
|
||
endpoint: '',
|
||
objectPrefix: '',
|
||
accessKeyId: '',
|
||
accessKeySecret: '',
|
||
envFiles: [],
|
||
keepLocal: false,
|
||
stopService: '',
|
||
restartServicesAfter: [],
|
||
database: '',
|
||
dryRun: false,
|
||
deferUpload: false,
|
||
uploadArchive: '',
|
||
manifestFile: '',
|
||
objectKey: '',
|
||
resultFile: '',
|
||
minFreeBytes: '',
|
||
};
|
||
|
||
for (let index = 0; index < argv.length; index += 1) {
|
||
const arg = argv[index];
|
||
const readValue = () => {
|
||
const value = argv[index + 1];
|
||
if (!value || value.startsWith('--')) {
|
||
throw new Error(`${arg} 缺少参数值`);
|
||
}
|
||
index += 1;
|
||
return value;
|
||
};
|
||
|
||
switch (arg) {
|
||
case '--help':
|
||
case '-h':
|
||
usage();
|
||
process.exit(0);
|
||
break;
|
||
case '--data-dir':
|
||
options.dataDir = readValue();
|
||
break;
|
||
case '--work-dir':
|
||
options.workDir = readValue();
|
||
break;
|
||
case '--bucket':
|
||
options.bucket = readValue();
|
||
break;
|
||
case '--endpoint':
|
||
options.endpoint = readValue();
|
||
break;
|
||
case '--object-prefix':
|
||
options.objectPrefix = readValue();
|
||
break;
|
||
case '--object-key':
|
||
options.objectKey = readValue();
|
||
break;
|
||
case '--access-key-id':
|
||
options.accessKeyId = readValue();
|
||
break;
|
||
case '--access-key-secret':
|
||
options.accessKeySecret = readValue();
|
||
break;
|
||
case '--env-file':
|
||
options.envFiles.push(readValue());
|
||
break;
|
||
case '--database':
|
||
options.database = readValue();
|
||
break;
|
||
case '--stop-service':
|
||
options.stopService = readValue();
|
||
break;
|
||
case '--restart-service-after':
|
||
options.restartServicesAfter.push(readValue());
|
||
break;
|
||
case '--keep-local':
|
||
options.keepLocal = true;
|
||
break;
|
||
case '--dry-run':
|
||
options.dryRun = true;
|
||
break;
|
||
case '--defer-upload':
|
||
options.deferUpload = true;
|
||
options.keepLocal = true;
|
||
break;
|
||
case '--upload-archive':
|
||
options.uploadArchive = readValue();
|
||
break;
|
||
case '--manifest-file':
|
||
options.manifestFile = readValue();
|
||
break;
|
||
case '--result-file':
|
||
options.resultFile = readValue();
|
||
break;
|
||
case '--min-free-bytes':
|
||
options.minFreeBytes = readValue();
|
||
break;
|
||
default:
|
||
throw new Error(`未知参数: ${arg}`);
|
||
}
|
||
}
|
||
|
||
return options;
|
||
}
|
||
|
||
function firstNonEmpty(...values) {
|
||
return values.map((value) => String(value ?? '').trim()).find(Boolean) ?? '';
|
||
}
|
||
|
||
function resolvePath(value) {
|
||
return isAbsolute(value) ? value : resolve(REPO_ROOT, value);
|
||
}
|
||
|
||
function normalizeEndpoint(raw) {
|
||
return String(raw ?? '')
|
||
.trim()
|
||
.replace(/^https?:\/\//u, '')
|
||
.replace(/\/+$/u, '');
|
||
}
|
||
|
||
function sanitizeObjectPart(value, fallback) {
|
||
const sanitized = String(value ?? '')
|
||
.trim()
|
||
.toLowerCase()
|
||
.replace(/[^a-z0-9._-]+/gu, '-')
|
||
.replace(/-+/gu, '-')
|
||
.replace(/^-|-$/gu, '');
|
||
return sanitized || fallback;
|
||
}
|
||
|
||
function timestampForFile(date = new Date()) {
|
||
const pad = (value) => String(value).padStart(2, '0');
|
||
return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`;
|
||
}
|
||
|
||
function buildBackupNames({database, dataDir, objectPrefix}) {
|
||
const timestamp = timestampForFile();
|
||
const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb');
|
||
const fileName = `${databasePart}-${timestamp}.tar.gz`;
|
||
const prefix = String(objectPrefix || 'database-backups')
|
||
.trim()
|
||
.replace(/^\/+|\/+$/gu, '')
|
||
.split('/')
|
||
.filter(Boolean)
|
||
.map((part) => sanitizeObjectPart(part, 'backup'))
|
||
.join('/');
|
||
const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/');
|
||
return {fileName, objectKey};
|
||
}
|
||
|
||
function runCommand(command, args, options = {}) {
|
||
const result = spawnSync(command, args, {
|
||
cwd: options.cwd ?? REPO_ROOT,
|
||
env: options.env ?? process.env,
|
||
encoding: 'utf8',
|
||
stdio: options.stdio ?? 'pipe',
|
||
shell: process.platform === 'win32',
|
||
});
|
||
if (result.error) {
|
||
throw new Error(`${command} 启动失败: ${result.error.message}`);
|
||
}
|
||
if (result.status !== 0) {
|
||
const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim();
|
||
throw new Error(`${command} 退出码 ${result.status}: ${output}`);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
function parseByteSize(rawValue, label) {
|
||
const value = String(rawValue ?? '').trim();
|
||
if (!value) {
|
||
return null;
|
||
}
|
||
const match = /^(\d+)(?:\s*([KMGTPE]?)(?:I?B?)?)?$/iu.exec(value);
|
||
if (!match) {
|
||
throw new Error(`${label} 必须是字节数或 K/M/G/T/P/E 后缀大小，实际: ${rawValue}`);
|
||
}
|
||
const [, amountText, unitText = ''] = match;
|
||
const multipliers = {
|
||
'': 1n,
|
||
K: 1024n,
|
||
M: 1024n ** 2n,
|
||
G: 1024n ** 3n,
|
||
T: 1024n ** 4n,
|
||
P: 1024n ** 5n,
|
||
E: 1024n ** 6n,
|
||
};
|
||
return BigInt(amountText) * multipliers[unitText.toUpperCase()];
|
||
}
|
||
|
||
function formatBytes(bytes) {
|
||
const value = BigInt(bytes);
|
||
const gib = Number(value) / (1024 ** 3);
|
||
if (gib >= 1) {
|
||
return `${gib.toFixed(1)}GiB`;
|
||
}
|
||
const mib = Number(value) / (1024 ** 2);
|
||
if (mib >= 1) {
|
||
return `${mib.toFixed(1)}MiB`;
|
||
}
|
||
return `${value}B`;
|
||
}
|
||
|
||
function getDirectorySizeBytes(dataDir) {
|
||
const result = runCommand('du', ['-sk', dataDir]);
|
||
const [sizeKbText] = String(result.stdout ?? '').trim().split(/\s+/u);
|
||
if (!sizeKbText || !/^\d+$/u.test(sizeKbText)) {
|
||
throw new Error(`无法解析数据目录大小: ${result.stdout}`);
|
||
}
|
||
return BigInt(sizeKbText) * 1024n;
|
||
}
|
||
|
||
function getAvailableBytes(fileSystemPath) {
|
||
const stat = statfsSync(fileSystemPath, {bigint: true});
|
||
return stat.bavail * stat.bsize;
|
||
}
|
||
|
||
function parseSafetyRatio(rawValue) {
|
||
const value = String(rawValue ?? '').trim();
|
||
if (!value) {
|
||
return DEFAULT_SPACE_SAFETY_RATIO;
|
||
}
|
||
const ratio = Number(value);
|
||
if (!Number.isFinite(ratio) || ratio < 1) {
|
||
throw new Error(`GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO 必须是 >= 1 的数字，实际: ${rawValue}`);
|
||
}
|
||
return ratio;
|
||
}
|
||
|
||
function calculateRequiredFreeBytes({dataSizeBytes, args, env}) {
|
||
const explicitMinFreeBytes = parseByteSize(
|
||
firstNonEmpty(args.minFreeBytes, env.GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES),
|
||
'GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES',
|
||
);
|
||
if (explicitMinFreeBytes !== null) {
|
||
return explicitMinFreeBytes;
|
||
}
|
||
const ratio = parseSafetyRatio(env.GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO);
|
||
const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000));
|
||
const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n;
|
||
const extraFreeBytes = parseByteSize(
|
||
firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES, String(DEFAULT_EXTRA_FREE_BYTES)),
|
||
'GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES',
|
||
);
|
||
const extraRequirement = dataSizeBytes + extraFreeBytes;
|
||
return ratioRequirement > extraRequirement ? ratioRequirement : extraRequirement;
|
||
}
|
||
|
||
function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) {
|
||
mkdirSync(workDir, {recursive: true});
|
||
const dataSizeBytes = getDirectorySizeBytes(dataDir);
|
||
const availableBytes = getAvailableBytes(workDir);
|
||
const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes, args, env});
|
||
console.log(
|
||
`[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`,
|
||
);
|
||
if (availableBytes < requiredFreeBytes) {
|
||
throw new Error(
|
||
[
|
||
`备份工作目录所在文件系统剩余空间不足: ${workDir}`,
|
||
`available=${formatBytes(availableBytes)}`,
|
||
`required=${formatBytes(requiredFreeBytes)}`,
|
||
`dataDir=${dataDir}`,
|
||
`dataSize=${formatBytes(dataSizeBytes)}`,
|
||
'为避免停库后写满磁盘，本次备份已在停止服务前中止。',
|
||
].join('；'),
|
||
);
|
||
}
|
||
}
|
||
|
||
function collectRestartServicesAfterBackup({args, env}) {
|
||
const serviceNames = [
|
||
...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '')
|
||
.split(',')
|
||
.map((value) => value.trim())
|
||
.filter(Boolean),
|
||
...args.restartServicesAfter,
|
||
];
|
||
return [...new Set(serviceNames.filter(Boolean))];
|
||
}
|
||
|
||
function stopServiceIfNeeded(serviceName) {
|
||
if (!serviceName) {
|
||
return false;
|
||
}
|
||
console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`);
|
||
runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'});
|
||
return true;
|
||
}
|
||
|
||
function startServiceIfNeeded(serviceName, wasStopped) {
|
||
if (!serviceName || !wasStopped) {
|
||
return;
|
||
}
|
||
console.log(`[database-backup] 恢复服务: ${serviceName}`);
|
||
runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'});
|
||
}
|
||
|
||
function restartServicesAfterBackup(serviceNames) {
|
||
const errors = [];
|
||
for (const serviceName of serviceNames) {
|
||
if (!serviceName) {
|
||
continue;
|
||
}
|
||
console.log(`[database-backup] 冷备份后重启依赖服务: ${serviceName}`);
|
||
try {
|
||
runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'});
|
||
} catch (error) {
|
||
errors.push(error);
|
||
}
|
||
}
|
||
if (errors.length > 0) {
|
||
throw new AggregateError(errors, `冷备份后重启依赖服务失败: ${errors.map((error) => error.message).join('; ')}`);
|
||
}
|
||
}
|
||
|
||
function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) {
|
||
const errors = [];
|
||
try {
|
||
startServiceIfNeeded(stopService, serviceStopped);
|
||
} catch (error) {
|
||
errors.push(error);
|
||
}
|
||
try {
|
||
restartServicesAfterBackup(restartServicesAfter);
|
||
} catch (error) {
|
||
errors.push(error);
|
||
}
|
||
if (errors.length > 0) {
|
||
throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`);
|
||
}
|
||
}
|
||
|
||
function createArchive({dataDir, workDir, fileName}) {
|
||
if (!existsSync(dataDir)) {
|
||
throw new Error(`数据库数据目录不存在: ${dataDir}`);
|
||
}
|
||
const stat = statSync(dataDir);
|
||
if (!stat.isDirectory()) {
|
||
throw new Error(`数据库数据路径不是目录: ${dataDir}`);
|
||
}
|
||
mkdirSync(workDir, {recursive: true});
|
||
const archivePath = resolve(workDir, fileName);
|
||
const parentDir = dirname(dataDir);
|
||
const entryName = basename(dataDir);
|
||
console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`);
|
||
runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'});
|
||
return archivePath;
|
||
}
|
||
|
||
function writeManifest({manifestPath, payload}) {
|
||
writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
|
||
}
|
||
|
||
function readManifest(manifestPath) {
|
||
if (!existsSync(manifestPath)) {
|
||
throw new Error(`备份清单不存在: ${manifestPath}`);
|
||
}
|
||
return JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||
}
|
||
|
||
function hmac(key, content, encoding) {
|
||
return createHmac('sha256', key).update(content).digest(encoding);
|
||
}
|
||
|
||
function sha256Hex(content) {
|
||
return createHash('sha256').update(content).digest('hex');
|
||
}
|
||
|
||
function regionFromEndpoint(endpoint) {
|
||
const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint);
|
||
if (!match) {
|
||
throw new Error(`无法从 OSS endpoint 推断 region: ${endpoint}`);
|
||
}
|
||
return match[1];
|
||
}
|
||
|
||
function formatScopeDate(date) {
|
||
return timestampForFile(date).slice(0, 8);
|
||
}
|
||
|
||
function formatOssDate(date) {
|
||
return timestampForFile(date).replace(/[-:]/gu, '');
|
||
}
|
||
|
||
function encodePath(path) {
|
||
return path
|
||
.split('/')
|
||
.map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`))
|
||
.join('/');
|
||
}
|
||
|
||
function canonicalHeaderValue(value) {
|
||
return String(value).trim().replace(/\s+/gu, ' ');
|
||
}
|
||
|
||
function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) {
|
||
const region = regionFromEndpoint(endpoint);
|
||
const scopeDate = formatScopeDate(date);
|
||
const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`;
|
||
const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`;
|
||
const signedHeaders = Object.fromEntries(
|
||
Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]),
|
||
);
|
||
const canonicalHeaders = Object.entries(signedHeaders)
|
||
.sort(([left], [right]) => left.localeCompare(right))
|
||
.map(([key, value]) => `${key}:${value}\n`)
|
||
.join('');
|
||
const additionalHeaders = 'host';
|
||
const canonicalRequest = [
|
||
method,
|
||
canonicalUri,
|
||
'',
|
||
canonicalHeaders,
|
||
additionalHeaders,
|
||
UNSIGNED_PAYLOAD,
|
||
].join('\n');
|
||
const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n');
|
||
const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate);
|
||
const regionKey = hmac(signature, region);
|
||
const serviceKey = hmac(regionKey, OSS_SERVICE);
|
||
const signingKey = hmac(serviceKey, OSS_REQUEST);
|
||
const finalSignature = hmac(signingKey, stringToSign, 'hex');
|
||
return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`;
|
||
}
|
||
|
||
async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) {
|
||
const fileStat = statSync(archivePath);
|
||
const now = new Date();
|
||
const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`;
|
||
const headers = {
|
||
host: `${bucket}.${endpoint}`,
|
||
'content-type': 'application/gzip',
|
||
'x-oss-content-sha256': UNSIGNED_PAYLOAD,
|
||
'x-oss-date': formatOssDate(now),
|
||
'x-oss-meta-backup-kind': 'spacetimedb-data-dir',
|
||
};
|
||
const authorization = buildAuthorization({
|
||
method: 'PUT',
|
||
bucket,
|
||
endpoint,
|
||
objectKey,
|
||
accessKeyId,
|
||
accessKeySecret,
|
||
headers,
|
||
date: now,
|
||
});
|
||
|
||
console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`);
|
||
let response;
|
||
try {
|
||
response = await fetch(targetUrl, {
|
||
method: 'PUT',
|
||
headers: {
|
||
...headers,
|
||
authorization,
|
||
'content-length': String(fileStat.size),
|
||
},
|
||
body: createReadStream(archivePath),
|
||
duplex: 'half',
|
||
});
|
||
} catch (error) {
|
||
throw new Error(`OSS 上传请求失败: oss://${bucket}/${objectKey}`, {cause: error});
|
||
}
|
||
|
||
const responseText = await response.text();
|
||
if (!response.ok) {
|
||
throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`);
|
||
}
|
||
|
||
return {
|
||
bucket,
|
||
objectKey,
|
||
contentLength: fileStat.size,
|
||
etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '',
|
||
};
|
||
}
|
||
|
||
async function uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix}) {
|
||
const archivePath = resolvePath(args.uploadArchive);
|
||
if (!existsSync(archivePath)) {
|
||
throw new Error(`待上传备份文件不存在: ${archivePath}`);
|
||
}
|
||
|
||
const manifestPath = resolvePath(args.manifestFile || `${archivePath}.manifest.json`);
|
||
const manifest = existsSync(manifestPath) ? readManifest(manifestPath) : {};
|
||
const dataDir = firstNonEmpty(manifest.dataDir, env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, DEFAULT_PRODUCTION_DATA_DIR);
|
||
const database = firstNonEmpty(args.database, manifest.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
|
||
const objectKey = firstNonEmpty(args.objectKey, manifest.objectKey, buildBackupNames({database, dataDir, objectPrefix}).objectKey);
|
||
|
||
console.log(`[database-backup] 上传已有备份: ${archivePath}`);
|
||
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
|
||
|
||
if (args.dryRun) {
|
||
console.log('[database-backup] dry-run，仅校验上传配置。');
|
||
return;
|
||
}
|
||
|
||
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
|
||
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
|
||
|
||
const uploadedAt = new Date().toISOString();
|
||
writeManifest({
|
||
manifestPath,
|
||
payload: {
|
||
...manifest,
|
||
database,
|
||
bucket: result.bucket,
|
||
objectKey: result.objectKey,
|
||
contentLength: result.contentLength,
|
||
etag: result.etag,
|
||
uploadedAt,
|
||
uploadStatus: 'uploaded',
|
||
},
|
||
});
|
||
|
||
if (args.resultFile) {
|
||
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, ...result, uploadedAt}, null, 2)}\n`, 'utf8');
|
||
}
|
||
|
||
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
|
||
if (!keepLocal) {
|
||
rmSync(archivePath, {force: true});
|
||
rmSync(manifestPath, {force: true});
|
||
console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。');
|
||
} else {
|
||
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
|
||
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
|
||
}
|
||
}
|
||
|
||
async function main() {
|
||
const args = parseArgs(process.argv.slice(2));
|
||
const env = loadEffectiveEnv(args.envFiles);
|
||
const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32';
|
||
const dataDir = resolvePath(firstNonEmpty(
|
||
args.dataDir,
|
||
env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR,
|
||
isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR,
|
||
));
|
||
const workDir = resolvePath(firstNonEmpty(
|
||
args.workDir,
|
||
env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR,
|
||
isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR,
|
||
));
|
||
const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET);
|
||
const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT));
|
||
const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID);
|
||
const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET);
|
||
const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups');
|
||
const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir));
|
||
const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true';
|
||
|
||
for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) {
|
||
if (!value) {
|
||
throw new Error(`缺少 ${label} 配置`);
|
||
}
|
||
}
|
||
|
||
if (args.uploadArchive) {
|
||
await uploadExistingArchive({args, env, bucket, endpoint, accessKeyId, accessKeySecret, objectPrefix});
|
||
return;
|
||
}
|
||
|
||
const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix});
|
||
console.log(`[database-backup] 数据目录: ${dataDir}`);
|
||
console.log(`[database-backup] 本地临时目录: ${workDir}`);
|
||
console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`);
|
||
|
||
if (args.dryRun) {
|
||
console.log('[database-backup] dry-run，仅校验配置，不打包上传。');
|
||
return;
|
||
}
|
||
|
||
let archivePath = '';
|
||
let serviceStopped = false;
|
||
let backupError = null;
|
||
let restoreError = null;
|
||
const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE);
|
||
const restartServicesAfter = collectRestartServicesAfterBackup({args, env});
|
||
try {
|
||
assertSufficientWorkDirSpace({dataDir, workDir, args, env});
|
||
serviceStopped = stopServiceIfNeeded(stopService);
|
||
archivePath = createArchive({dataDir, workDir, fileName});
|
||
} catch (error) {
|
||
backupError = error;
|
||
} finally {
|
||
try {
|
||
if (serviceStopped) {
|
||
restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter});
|
||
} else if (!backupError) {
|
||
restartServicesAfterBackup(restartServicesAfter);
|
||
}
|
||
} catch (error) {
|
||
restoreError = error;
|
||
}
|
||
}
|
||
if (backupError) {
|
||
if (restoreError) {
|
||
throw new AggregateError([backupError, restoreError], `数据库备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`);
|
||
}
|
||
throw backupError;
|
||
}
|
||
if (restoreError) {
|
||
throw restoreError;
|
||
}
|
||
|
||
const manifestPath = `${archivePath}.manifest.json`;
|
||
writeManifest({
|
||
manifestPath,
|
||
payload: {
|
||
createdAt: new Date().toISOString(),
|
||
database,
|
||
dataDir,
|
||
bucket,
|
||
objectKey,
|
||
archivePath,
|
||
uploadStatus: args.deferUpload ? 'deferred' : 'pending',
|
||
},
|
||
});
|
||
|
||
if (args.deferUpload) {
|
||
console.log(`[database-backup] 已生成本地冷备份，延后上传: ${archivePath}`);
|
||
console.log(`[database-backup] 已写入备份清单: ${manifestPath}`);
|
||
if (args.resultFile) {
|
||
writeFileSync(resolvePath(args.resultFile), `${JSON.stringify({archivePath, manifestPath, bucket, objectKey}, null, 2)}\n`, 'utf8');
|
||
}
|
||
return;
|
||
}
|
||
|
||
const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret});
|
||
console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`);
|
||
|
||
writeManifest({
|
||
manifestPath,
|
||
payload: {
|
||
createdAt: new Date().toISOString(),
|
||
database,
|
||
dataDir,
|
||
bucket: result.bucket,
|
||
objectKey: result.objectKey,
|
||
archivePath,
|
||
contentLength: result.contentLength,
|
||
etag: result.etag,
|
||
uploadedAt: new Date().toISOString(),
|
||
uploadStatus: 'uploaded',
|
||
},
|
||
});
|
||
|
||
if (!keepLocal) {
|
||
rmSync(archivePath, {force: true});
|
||
rmSync(manifestPath, {force: true});
|
||
console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。');
|
||
} else {
|
||
console.log(`[database-backup] 已保留本地备份: ${archivePath}`);
|
||
console.log(`[database-backup] 已保留备份清单: ${manifestPath}`);
|
||
}
|
||
}
|
||
|
||
function formatErrorDetails(error) {
|
||
if (!error || typeof error !== 'object') {
|
||
return '';
|
||
}
|
||
return ['code', 'errno', 'syscall', 'hostname', 'host', 'port', 'address']
|
||
.map((field) => {
|
||
const value = error[field];
|
||
return value === undefined || value === null || value === '' ? '' : `${field}=${String(value)}`;
|
||
})
|
||
.filter(Boolean)
|
||
.join(' ');
|
||
}
|
||
|
||
function describeError(error) {
|
||
const lines = [];
|
||
let current = error;
|
||
for (let depth = 0; current && depth < 5; depth += 1) {
|
||
const label = depth === 0 ? 'error' : `cause[${depth}]`;
|
||
if (!(current instanceof Error)) {
|
||
lines.push(`${label}: ${String(current)}`);
|
||
break;
|
||
}
|
||
|
||
lines.push(`${label}: ${current.name}: ${current.message}`);
|
||
const details = formatErrorDetails(current);
|
||
if (details) {
|
||
lines.push(`${label} details: ${details}`);
|
||
}
|
||
if (current instanceof AggregateError) {
|
||
current.errors.slice(0, 3).forEach((item, index) => {
|
||
const itemText = item instanceof Error ? `${item.name}: ${item.message}` : String(item);
|
||
const itemDetails = formatErrorDetails(item);
|
||
lines.push(`${label}.errors[${index}]: ${itemText}${itemDetails ? ` (${itemDetails})` : ''}`);
|
||
});
|
||
}
|
||
current = current.cause;
|
||
}
|
||
return lines;
|
||
}
|
||
|
||
main().catch((error) => {
|
||
for (const line of describeError(error)) {
|
||
console.error(`[database-backup] ${line}`);
|
||
}
|
||
process.exit(1);
|
||
});
|