From 5a8a8562657290fe076e07c74a28c3f84cc7d41f Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Wed, 27 May 2026 19:33:05 +0800 Subject: [PATCH] feat: automate database OSS backups --- .env.example | 11 + deploy/env/api-server.env.example | 12 + .../genarrative-database-backup.service | 18 + .../systemd/genarrative-database-backup.timer | 12 + ...发运维】本地开发验证与生产运维-2026-05-15.md | 30 +- ...Jenkinsfile.production-stdb-module-publish | 7 +- package.json | 3 +- scripts/build-production-release.sh | 3 +- scripts/database-backup-to-oss.mjs | 452 ++++++++++++++++++ scripts/deploy/production-stdb-publish.sh | 28 +- scripts/jenkins-server-provision.sh | 22 +- 11 files changed, 589 insertions(+), 9 deletions(-) create mode 100644 deploy/systemd/genarrative-database-backup.service create mode 100644 deploy/systemd/genarrative-database-backup.timer create mode 100644 scripts/database-backup-to-oss.mjs diff --git a/.env.example b/.env.example index 03d8c2c1..9630fd10 100644 --- a/.env.example +++ b/.env.example @@ -145,6 +145,17 @@ ALIYUN_OSS_POST_EXPIRE_SECONDS="600" ALIYUN_OSS_POST_MAX_SIZE_BYTES="20971520" ALIYUN_OSS_SUCCESS_ACTION_STATUS="200" +# SpacetimeDB 数据目录备份到 OSS。备份 bucket 可与资源 bucket 分离；未设置时脚本回退使用 ALIYUN_OSS_BUCKET。 +GENARRATIVE_DATABASE_BACKUP_DATA_DIR="" +GENARRATIVE_DATABASE_BACKUP_WORK_DIR="" +GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET="" +GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT="" +GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX="database-backups" +GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL="false" +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID="" +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET="" +GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE="" + # Optional model name for custom-world scene image generation. DASHSCOPE_IMAGE_MODEL="wan2.7-image" diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index c7a85bee..19f8b198 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -119,3 +119,15 @@ ALIYUN_OSS_READ_EXPIRE_SECONDS=600 ALIYUN_OSS_POST_EXPIRE_SECONDS=600 ALIYUN_OSS_POST_MAX_SIZE_BYTES=20971520 ALIYUN_OSS_SUCCESS_ACTION_STATUS=200 + +# SpacetimeDB 数据目录 OSS 冷备份配置。可由 cron / Jenkins 调用发布包内 scripts/database-backup-to-oss.mjs。 +GENARRATIVE_DATABASE_BACKUP_DATA_DIR=/stdb +GENARRATIVE_DATABASE_BACKUP_WORK_DIR=/var/lib/genarrative/database-backups +GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= +GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com +GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups +GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +# 可选：定时 / publish 前备份使用独立最小权限 AccessKey；为空时回退 ALIYUN_OSS_ACCESS_KEY_*。 +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= +GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE=spacetimedb.service diff --git a/deploy/systemd/genarrative-database-backup.service b/deploy/systemd/genarrative-database-backup.service new file mode 100644 index 00000000..cde294e2 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup.service @@ -0,0 +1,18 @@ +[Unit] +Description=Genarrative SpacetimeDB OSS Backup +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=root +Group=root +WorkingDirectory=/opt/genarrative/current +EnvironmentFile=/etc/genarrative/api-server.env +ExecStart=/usr/bin/node /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service + +# 备份需要停止 / 启动 spacetimedb.service，并读取 /stdb、写入 /var/lib/genarrative/database-backups。 +PrivateTmp=true +ProtectSystem=full +ReadWritePaths=/stdb /var/lib/genarrative + diff --git a/deploy/systemd/genarrative-database-backup.timer b/deploy/systemd/genarrative-database-backup.timer new file mode 100644 index 00000000..4c222039 --- /dev/null +++ b/deploy/systemd/genarrative-database-backup.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run Genarrative SpacetimeDB OSS Backup Daily + +[Timer] +OnCalendar=*-*-* 03:20:00 +Persistent=true +RandomizedDelaySec=600 +Unit=genarrative-database-backup.service + +[Install] +WantedBy=timers.target + diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 62a24ea8..f7cbd56f 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -1,4 +1,4 @@ -# 本地开发验证与生产运维 +# 本地开发验证与生产运维 更新时间：`2026-05-15` @@ -193,6 +193,31 @@ UI 相关修改要重点验证： 6. Jenkins 数据库导入 / 导出流水线会先加载 `scripts/jenkins-prepare-toolchain-env.sh`，显式补齐 Jenkins 用户的 Node、Cargo、SpacetimeDB 工具链目录；如果目标机器安装路径不同，用 `GENARRATIVE_JENKINS_TOOL_PATHS` 传入额外 `bin` 目录。 7. 本地 `npm run dev` / `npm run dev:api-server` 若没有显式 `GENARRATIVE_SPACETIME_TOKEN`，会在 SpacetimeDB 就绪后调用 `/v1/identity` 创建当前进程专用 Web API identity token，并只注入本次 `api-server` 环境，不写回 `.env.local`。启动日志只打印 identity 前缀，禁止打印 token 明文；若仍出现 `subscribe ... 401 Unauthorized`，先确认是否绕过了项目 dev 脚本或是否连接到非本次启动的 SpacetimeDB server。 +### SpacetimeDB 数据目录 OSS 备份 + +数据库备份不放进 `spacetime-module` reducer / procedure：备份属于文件系统与 OSS 外部副作用，必须由运维脚本在 SpacetimeDB 宿主外执行。当前统一脚本为；生产 provision 还会安装 `genarrative-database-backup.timer`，每天 `03:20` 左右自动执行一次 OSS 冷备份： + +```bash +npm run database:backup:oss -- --data-dir /stdb --stop-service spacetimedb.service +``` + +脚本会将数据目录打包成 `tar.gz`，上传到 `oss://///-.tar.gz`。生产建议做冷备份：传入 `--stop-service spacetimedb.service`，脚本会在打包前停止服务、打包后恢复服务，再上传 OSS。`Genarrative-Stdb-Module-Publish` 默认也会在 `spacetime publish` 前执行同一脚本；备份失败会阻断 publish，只有显式勾选 `SKIP_DATABASE_BACKUP` 或脚本参数 `--skip-backup` 才跳过。若业务不能接受停机窗口，应先规划 SpacetimeDB 原生快照或主备策略，不要直接在写入中的数据目录上做热拷贝并当作强一致备份。 + +生产环境变量模板在 `deploy/env/api-server.env.example`： + +```env +GENARRATIVE_DATABASE_BACKUP_DATA_DIR=/stdb +GENARRATIVE_DATABASE_BACKUP_WORK_DIR=/var/lib/genarrative/database-backups +GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= +GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com +GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups +GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= +GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= +``` + +`GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。 + ## 生产运维 生产部署当前口径： @@ -264,6 +289,7 @@ OpenTelemetry 现阶段默认开启 OTLP traces / metrics / logs，但本地日 - `GENARRATIVE_SPACETIME_SERVER_URL` - `GENARRATIVE_SPACETIME_DATABASE` - `GENARRATIVE_SPACETIME_TOKEN` +- `GENARRATIVE_DATABASE_BACKUP_*` - `GENARRATIVE_LLM_*` - `APIMART_*` - `VECTOR_ENGINE_*` @@ -386,3 +412,5 @@ SELECT * FROM profile_recharge_product_config ORDER BY sort_order ASC; ## 文档维护 当前 `docs/` 只保留少量融合文档。新增稳定知识时优先更新现有文档；只有现有文档无法容纳时才新增带 `【标签名】` 的 Markdown。阶段性流水账、一次性修复记录和已关闭实验不要再新增成长期文档。 + + diff --git a/jenkins/Jenkinsfile.production-stdb-module-publish b/jenkins/Jenkinsfile.production-stdb-module-publish index 13d29880..6b1cecf9 100644 --- a/jenkins/Jenkinsfile.production-stdb-module-publish +++ b/jenkins/Jenkinsfile.production-stdb-module-publish @@ -1,4 +1,4 @@ -pipeline { +pipeline { agent none options { @@ -27,6 +27,7 @@ pipeline { string(name: 'SPACETIME_ROOT_DIR', defaultValue: '/stdb', description: 'spacetime CLI root-dir；需与自托管 spacetimedb.service 一致') string(name: 'SPACETIME_RUN_AS_USER', defaultValue: 'spacetimedb', description: '执行 spacetime publish 的本机用户，默认使用自托管服务用户') booleanParam(name: 'CLEAR_DATABASE', defaultValue: false, description: '是否清空数据库后发布') + booleanParam(name: 'SKIP_DATABASE_BACKUP', defaultValue: false, description: '是否跳过 publish 前 OSS 数据库备份；默认不跳过，备份失败会阻断发布') } stages { @@ -138,6 +139,7 @@ pipeline { steps { script { def clearArg = params.CLEAR_DATABASE ? '--clear-database' : '' + def backupArg = params.SKIP_DATABASE_BACKUP ? '--skip-backup' : '' def rootArg = "--root-dir \"${params.SPACETIME_ROOT_DIR?.trim() ? params.SPACETIME_ROOT_DIR.trim() : '/stdb'}\"" def runAsArg = params.SPACETIME_RUN_AS_USER?.trim() ? "--run-as-user \"${params.SPACETIME_RUN_AS_USER.trim()}\"" @@ -155,7 +157,8 @@ pipeline { ${rootArg} \\ ${runAsArg} \\ ${serverArg} \\ - ${clearArg} + ${clearArg} \\ + ${backupArg} ' """ } diff --git a/package.json b/package.json index 9a6a6da8..853c3c9d 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,8 @@ "codegraph:init": "codegraph init -i .", "codegraph:index": "codegraph index .", "codegraph:sync": "codegraph sync .", - "codegraph:status": "codegraph status ." + "codegraph:status": "codegraph status .", + "database:backup:oss": "node scripts/database-backup-to-oss.mjs" }, "dependencies": { "@tailwindcss/vite": "^4.1.14", diff --git a/scripts/build-production-release.sh b/scripts/build-production-release.sh index 0967523e..803f0762 100644 --- a/scripts/build-production-release.sh +++ b/scripts/build-production-release.sh @@ -461,6 +461,7 @@ copy_required_file "${SCRIPT_DIR}/spacetime-import-migration-json.mjs" "${TARGET copy_required_file "${SCRIPT_DIR}/spacetime-migration-common.mjs" "${TARGET_DIR}/scripts/spacetime-migration-common.mjs" "数据库迁移公共脚本" copy_required_file "${SCRIPT_DIR}/spacetime-authorize-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-authorize-migration-operator.mjs" "数据库迁移授权脚本" copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-revoke-migration-operator.mjs" "数据库迁移撤权脚本" +copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本" copy_required_dir "${REPO_ROOT}/deploy/systemd" "${TARGET_DIR}/deploy/systemd" "systemd 配置" copy_required_dir "${REPO_ROOT}/deploy/nginx" "${TARGET_DIR}/deploy/nginx" "Nginx 配置" @@ -480,7 +481,7 @@ cat >"${TARGET_DIR}/README.md" <] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] + node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] + +说明: + 将 SpacetimeDB 数据目录打包成 .tar.gz，并上传到阿里云 OSS 指定 bucket。 + 默认读取 .env / .env.local / .env.secrets.local；生产服务可传 --env-file /etc/genarrative/api-server.env。 + shell 环境变量优先级最高，不会被 env 文件覆盖。 + +常用环境变量: + GENARRATIVE_DATABASE_BACKUP_DATA_DIR 数据目录；生产建议 /stdb + GENARRATIVE_DATABASE_BACKUP_WORK_DIR 本地临时备份目录；生产建议 /var/lib/genarrative/database-backups + GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET 备份 bucket；未设置时回退 ALIYUN_OSS_BUCKET + GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀，默认 database-backups + GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT + GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz + ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET +`); +} + +function loadEnvFile(filePath, target, protectedKeys) { + if (!existsSync(filePath)) { + return; + } + const rawText = readFileSync(filePath, 'utf8'); + for (const rawLine of rawText.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + const match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u); + if (!match) { + continue; + } + const [, key, rawValue] = match; + if (protectedKeys.has(key)) { + continue; + } + target[key] = rawValue.replace(/^['"]|['"]$/gu, ''); + } +} + +function loadRepoEnv() { + const env = {...process.env}; + const protectedKeys = new Set( + Object.entries(process.env) + .filter(([, value]) => String(value ?? '').trim()) + .map(([key]) => key), + ); + for (const fileName of ['.env', '.env.local', '.env.secrets.local']) { + loadEnvFile(resolve(REPO_ROOT, fileName), env, protectedKeys); + } + return env; +} + +function loadEffectiveEnv(envFiles) { + const env = loadRepoEnv(); + const protectedKeys = new Set( + Object.entries(process.env) + .filter(([, value]) => String(value ?? '').trim()) + .map(([key]) => key), + ); + for (const filePath of envFiles) { + loadEnvFile(resolvePath(filePath), env, protectedKeys); + } + return env; +} + +function parseArgs(argv) { + const options = { + dataDir: '', + workDir: '', + bucket: '', + endpoint: '', + objectPrefix: '', + accessKeyId: '', + accessKeySecret: '', + envFiles: [], + keepLocal: false, + stopService: '', + database: '', + dryRun: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const readValue = () => { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) { + throw new Error(`${arg} 缺少参数值`); + } + index += 1; + return value; + }; + + switch (arg) { + case '--help': + case '-h': + usage(); + process.exit(0); + break; + case '--data-dir': + options.dataDir = readValue(); + break; + case '--work-dir': + options.workDir = readValue(); + break; + case '--bucket': + options.bucket = readValue(); + break; + case '--endpoint': + options.endpoint = readValue(); + break; + case '--object-prefix': + options.objectPrefix = readValue(); + break; + case '--access-key-id': + options.accessKeyId = readValue(); + break; + case '--access-key-secret': + options.accessKeySecret = readValue(); + break; + case '--env-file': + options.envFiles.push(readValue()); + break; + case '--database': + options.database = readValue(); + break; + case '--stop-service': + options.stopService = readValue(); + break; + case '--keep-local': + options.keepLocal = true; + break; + case '--dry-run': + options.dryRun = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + return options; +} + +function firstNonEmpty(...values) { + return values.map((value) => String(value ?? '').trim()).find(Boolean) ?? ''; +} + +function resolvePath(value) { + return isAbsolute(value) ? value : resolve(REPO_ROOT, value); +} + +function normalizeEndpoint(raw) { + return String(raw ?? '') + .trim() + .replace(/^https?:\/\//u, '') + .replace(/\/+$/u, ''); +} + +function sanitizeObjectPart(value, fallback) { + const sanitized = String(value ?? '') + .trim() + .toLowerCase() + .replace(/[^a-z0-9._-]+/gu, '-') + .replace(/-+/gu, '-') + .replace(/^-|-$/gu, ''); + return sanitized || fallback; +} + +function timestampForFile(date = new Date()) { + const pad = (value) => String(value).padStart(2, '0'); + return `${date.getUTCFullYear()}${pad(date.getUTCMonth() + 1)}${pad(date.getUTCDate())}T${pad(date.getUTCHours())}${pad(date.getUTCMinutes())}${pad(date.getUTCSeconds())}Z`; +} + +function buildBackupNames({database, dataDir, objectPrefix}) { + const timestamp = timestampForFile(); + const databasePart = sanitizeObjectPart(database || basename(dataDir), 'spacetimedb'); + const fileName = `${databasePart}-${timestamp}.tar.gz`; + const prefix = String(objectPrefix || 'database-backups') + .trim() + .replace(/^\/+|\/+$/gu, '') + .split('/') + .filter(Boolean) + .map((part) => sanitizeObjectPart(part, 'backup')) + .join('/'); + const objectKey = [prefix, databasePart, fileName].filter(Boolean).join('/'); + return {fileName, objectKey}; +} + +function runCommand(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd ?? REPO_ROOT, + env: options.env ?? process.env, + encoding: 'utf8', + stdio: options.stdio ?? 'pipe', + shell: process.platform === 'win32', + }); + if (result.error) { + throw new Error(`${command} 启动失败: ${result.error.message}`); + } + if (result.status !== 0) { + const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`.trim(); + throw new Error(`${command} 退出码 ${result.status}: ${output}`); + } + return result; +} + +function stopServiceIfNeeded(serviceName) { + if (!serviceName) { + return false; + } + console.log(`[database-backup] 停止服务以获取冷备份: ${serviceName}`); + runCommand('systemctl', ['stop', serviceName], {stdio: 'inherit'}); + return true; +} + +function startServiceIfNeeded(serviceName, wasStopped) { + if (!serviceName || !wasStopped) { + return; + } + console.log(`[database-backup] 恢复服务: ${serviceName}`); + runCommand('systemctl', ['start', serviceName], {stdio: 'inherit'}); +} + +function createArchive({dataDir, workDir, fileName}) { + if (!existsSync(dataDir)) { + throw new Error(`数据库数据目录不存在: ${dataDir}`); + } + const stat = statSync(dataDir); + if (!stat.isDirectory()) { + throw new Error(`数据库数据路径不是目录: ${dataDir}`); + } + mkdirSync(workDir, {recursive: true}); + const archivePath = resolve(workDir, fileName); + const parentDir = dirname(dataDir); + const entryName = basename(dataDir); + console.log(`[database-backup] 打包: ${dataDir} -> ${archivePath}`); + runCommand('tar', ['-czf', archivePath, '-C', parentDir, entryName], {stdio: 'inherit'}); + return archivePath; +} + +function hmac(key, content, encoding) { + return createHmac('sha256', key).update(content).digest(encoding); +} + +function sha256Hex(content) { + return createHash('sha256').update(content).digest('hex'); +} + +function regionFromEndpoint(endpoint) { + const match = /^oss-([a-z0-9-]+)\./u.exec(endpoint); + if (!match) { + throw new Error(`无法从 OSS endpoint 推断 region: ${endpoint}`); + } + return match[1]; +} + +function formatScopeDate(date) { + return timestampForFile(date).slice(0, 8); +} + +function formatOssDate(date) { + return timestampForFile(date).replace(/[-:]/gu, ''); +} + +function encodePath(path) { + return path + .split('/') + .map((segment) => encodeURIComponent(segment).replace(/[!'()*]/gu, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)) + .join('/'); +} + +function canonicalHeaderValue(value) { + return String(value).trim().replace(/\s+/gu, ' '); +} + +function buildAuthorization({method, bucket, endpoint, objectKey, accessKeyId, accessKeySecret, headers, date}) { + const region = regionFromEndpoint(endpoint); + const scopeDate = formatScopeDate(date); + const scope = `${scopeDate}/${region}/${OSS_SERVICE}/${OSS_REQUEST}`; + const canonicalUri = `/${encodeURIComponent(bucket)}/${encodePath(objectKey)}`; + const signedHeaders = Object.fromEntries( + Object.entries(headers).map(([key, value]) => [key.toLowerCase(), canonicalHeaderValue(value)]), + ); + const canonicalHeaders = Object.entries(signedHeaders) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => `${key}:${value}\n`) + .join(''); + const additionalHeaders = 'host'; + const canonicalRequest = [ + method, + canonicalUri, + '', + canonicalHeaders, + additionalHeaders, + UNSIGNED_PAYLOAD, + ].join('\n'); + const stringToSign = [OSS_ALGORITHM, headers['x-oss-date'], scope, sha256Hex(canonicalRequest)].join('\n'); + const signature = hmac(Buffer.from(`aliyun_v4${accessKeySecret}`, 'utf8'), scopeDate); + const regionKey = hmac(signature, region); + const serviceKey = hmac(regionKey, OSS_SERVICE); + const signingKey = hmac(serviceKey, OSS_REQUEST); + const finalSignature = hmac(signingKey, stringToSign, 'hex'); + return `${OSS_ALGORITHM} Credential=${accessKeyId}/${scope},AdditionalHeaders=${additionalHeaders},Signature=${finalSignature}`; +} + +async function uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}) { + const fileStat = statSync(archivePath); + const now = new Date(); + const targetUrl = `https://${bucket}.${endpoint}/${encodePath(objectKey)}`; + const headers = { + host: `${bucket}.${endpoint}`, + 'content-type': 'application/gzip', + 'x-oss-content-sha256': UNSIGNED_PAYLOAD, + 'x-oss-date': formatOssDate(now), + 'x-oss-meta-backup-kind': 'spacetimedb-data-dir', + }; + const authorization = buildAuthorization({ + method: 'PUT', + bucket, + endpoint, + objectKey, + accessKeyId, + accessKeySecret, + headers, + date: now, + }); + + console.log(`[database-backup] 上传 OSS: oss://${bucket}/${objectKey}`); + const response = await fetch(targetUrl, { + method: 'PUT', + headers: { + ...headers, + authorization, + 'content-length': String(fileStat.size), + }, + body: createReadStream(archivePath), + duplex: 'half', + }); + + const responseText = await response.text(); + if (!response.ok) { + throw new Error(`OSS 上传失败 HTTP ${response.status}: ${responseText.slice(0, 500)}`); + } + + return { + bucket, + objectKey, + contentLength: fileStat.size, + etag: response.headers.get('etag')?.replace(/^"|"$/gu, '') ?? '', + }; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const env = loadEffectiveEnv(args.envFiles); + const isProductionLike = existsSync(DEFAULT_PRODUCTION_DATA_DIR) && process.platform !== 'win32'; + const dataDir = resolvePath(firstNonEmpty( + args.dataDir, + env.GENARRATIVE_DATABASE_BACKUP_DATA_DIR, + isProductionLike ? DEFAULT_PRODUCTION_DATA_DIR : DEFAULT_LOCAL_DATA_DIR, + )); + const workDir = resolvePath(firstNonEmpty( + args.workDir, + env.GENARRATIVE_DATABASE_BACKUP_WORK_DIR, + isProductionLike ? DEFAULT_PRODUCTION_WORK_DIR : DEFAULT_LOCAL_WORK_DIR, + )); + const bucket = firstNonEmpty(args.bucket, env.GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET, env.ALIYUN_OSS_BUCKET); + const endpoint = normalizeEndpoint(firstNonEmpty(args.endpoint, env.GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT, env.ALIYUN_OSS_ENDPOINT)); + const accessKeyId = firstNonEmpty(args.accessKeyId, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID, env.ALIYUN_OSS_ACCESS_KEY_ID); + const accessKeySecret = firstNonEmpty(args.accessKeySecret, env.GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET, env.ALIYUN_OSS_ACCESS_KEY_SECRET); + const objectPrefix = firstNonEmpty(args.objectPrefix, env.GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX, 'database-backups'); + const database = firstNonEmpty(args.database, env.GENARRATIVE_SPACETIME_DATABASE, basename(dataDir)); + const keepLocal = args.keepLocal || String(env.GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL ?? '').trim().toLowerCase() === 'true'; + + for (const [label, value] of Object.entries({bucket, endpoint, accessKeyId, accessKeySecret})) { + if (!value) { + throw new Error(`缺少 ${label} 配置`); + } + } + + const {fileName, objectKey} = buildBackupNames({database, dataDir, objectPrefix}); + console.log(`[database-backup] 数据目录: ${dataDir}`); + console.log(`[database-backup] 本地临时目录: ${workDir}`); + console.log(`[database-backup] 目标对象: oss://${bucket}/${objectKey}`); + + if (args.dryRun) { + console.log('[database-backup] dry-run，仅校验配置，不打包上传。'); + return; + } + + let archivePath = ''; + let serviceStopped = false; + try { + serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE)); + archivePath = createArchive({dataDir, workDir, fileName}); + } finally { + startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped); + } + + const result = await uploadArchive({archivePath, bucket, endpoint, objectKey, accessKeyId, accessKeySecret}); + console.log(`[database-backup] 上传完成: ${JSON.stringify(result)}`); + + const manifestPath = `${archivePath}.manifest.json`; + writeFileSync( + manifestPath, + `${JSON.stringify({ + createdAt: new Date().toISOString(), + dataDir, + bucket: result.bucket, + objectKey: result.objectKey, + contentLength: result.contentLength, + etag: result.etag, + }, null, 2)}\n`, + 'utf8', + ); + + if (!keepLocal) { + rmSync(archivePath, {force: true}); + rmSync(manifestPath, {force: true}); + console.log('[database-backup] 已删除本地临时备份文件；如需保留请设置 --keep-local。'); + } else { + console.log(`[database-backup] 已保留本地备份: ${archivePath}`); + console.log(`[database-backup] 已保留备份清单: ${manifestPath}`); + } +} + +main().catch((error) => { + console.error(`[database-backup] ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +}); diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index 38b3e73f..2b7c0c1b 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -5,13 +5,14 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-stdb-publish.sh --source-dir build/ --database [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--clear-database] + ./scripts/deploy/production-stdb-publish.sh --source-dir build/ --database [--server-url http://127.0.0.1:3101] [--server local] [--root-dir /stdb] [--run-as-user spacetimedb] [--clear-database] [--skip-backup] 说明: 进入维护模式，校验 spacetime_module.wasm.sha256，并在生产实例本机执行 spacetime publish。 默认使用 http://127.0.0.1:3101，避免与部署机本机 Git/Web 服务的 3000 端口冲突。 默认使用 /stdb 作为 spacetime CLI root-dir，并以 spacetimedb 用户发布，避免 root CLI 身份污染自托管实例。 发布时固定追加 --no-config，只使用显式参数，避免工作区或用户目录里的 spacetime 配置干扰目标。 + publish 前默认执行一次 OSS 冷备份；备份失败会阻断 publish。仅明确传入 --skip-backup 时跳过。 失败时保留维护模式。 EOF } @@ -43,6 +44,7 @@ SERVER_URL="http://127.0.0.1:3101" SPACETIME_ROOT_DIR="/stdb" RUN_AS_USER="spacetimedb" CLEAR_DATABASE=0 +SKIP_BACKUP=0 DEPLOY_COMPLETED=0 PUBLISH_TMP_DIR="" @@ -81,6 +83,10 @@ while [[ $# -gt 0 ]]; do CLEAR_DATABASE=1 shift ;; + --skip-backup) + SKIP_BACKUP=1 + shift + ;; *) echo "[production-stdb-publish] 未知参数: $1" >&2 usage >&2 @@ -130,6 +136,26 @@ trap on_exit EXIT "${SCRIPT_DIR}/maintenance-on.sh" "spacetime module publish ${DATABASE}" +if [[ "${SKIP_BACKUP}" -ne 1 ]]; then + BACKUP_SCRIPT="${SCRIPT_DIR}/../database-backup-to-oss.mjs" + if [[ ! -f "${BACKUP_SCRIPT}" ]]; then + BACKUP_SCRIPT="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" + fi + if [[ ! -f "${BACKUP_SCRIPT}" ]]; then + echo "[production-stdb-publish] 缺少 publish 前数据库备份脚本: ${BACKUP_SCRIPT}" >&2 + exit 1 + fi + + echo "[production-stdb-publish] publish 前执行 OSS 冷备份" + node "${BACKUP_SCRIPT}" \ + --env-file /etc/genarrative/api-server.env \ + --data-dir "${SPACETIME_ROOT_DIR}" \ + --database "${DATABASE}" \ + --stop-service spacetimedb.service +else + echo "[production-stdb-publish] 已按参数跳过 publish 前数据库备份" +fi + echo "[production-stdb-publish] 校验 wasm" ( cd "${SOURCE_DIR}" diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index e54b42d0..ca9611f9 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -642,8 +642,20 @@ render_api_service() { deploy/systemd/genarrative-api.service } +render_database_backup_service() { + local current_escaped env_escaped + current_escaped="$(escape_sed_replacement "${CURRENT_LINK}")" + env_escaped="$(escape_sed_replacement "${API_ENV_FILE}")" + sed \ + -e "s|/opt/genarrative/current|${current_escaped}|g" \ + -e "s|/etc/genarrative/api-server.env|${env_escaped}|g" \ + deploy/systemd/genarrative-database-backup.service +} + require_path deploy/systemd/spacetimedb.service require_path deploy/systemd/genarrative-api.service +require_path deploy/systemd/genarrative-database-backup.service +require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/otelcol-contrib.service require_path deploy/otelcol/genarrative-debug.yaml require_path deploy/nginx/genarrative.conf @@ -663,7 +675,7 @@ run_cmd id install_build_dependencies install_nginx_brotli_modules install_sccache -run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox +run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups if ! id spacetimedb >/dev/null 2>&1; then run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb @@ -693,11 +705,15 @@ sync_spacetime_install "${SPACETIME_ROOT}" spacetimedb_service="$(mktemp)" api_service="$(mktemp)" +database_backup_service="$(mktemp)" render_spacetimedb_service >"${spacetimedb_service}" render_api_service >"${api_service}" +render_database_backup_service >"${database_backup_service}" install_file "${spacetimedb_service}" /etc/systemd/system/spacetimedb.service 0644 install_file "${api_service}" /etc/systemd/system/genarrative-api.service 0644 -rm -f "${spacetimedb_service}" "${api_service}" +install_file "${database_backup_service}" /etc/systemd/system/genarrative-database-backup.service 0644 +install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644 +rm -f "${spacetimedb_service}" "${api_service}" "${database_backup_service}" if [[ ! -f "${API_ENV_FILE}" ]]; then echo "+ create ${API_ENV_FILE} from example" @@ -732,7 +748,7 @@ if [[ "${ENABLE_SERVICES}" == "true" ]]; then if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then run_cmd systemctl enable otelcol-contrib.service fi - run_cmd systemctl enable spacetimedb.service genarrative-api.service + run_cmd systemctl enable spacetimedb.service genarrative-api.service genarrative-database-backup.timer if [[ "${ENABLE_OTELCOL:-true}" == "true" ]]; then run_cmd systemctl restart otelcol-contrib.service fi