From 4e3c26d5d815957048823de3300d07c63ff71c26 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 8 Jul 2026 11:04:48 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20release=20=E5=8F=91?= =?UTF-8?q?=E5=B8=83=E4=B8=8E=E5=86=B7=E5=A4=87=E4=BB=BD=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 冷备份增加工作目录空间预检并确保失败后恢复依赖服务 API deploy 增加 Pingora 产物硬校验并区分 current 切换前后维护模式处理 Jenkins release 默认构建和部署 Pingora 影子网关并在全量流水线透传参数 补充备份与 API deploy 回归检查和生产运维护栏 更新生产运维文档与项目记忆的事故处理口径 --- deploy/env/api-server.env.example | 2 + docs/project-memory/shared-memory/pitfalls.md | 12 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 7 +- jenkins/Jenkinsfile.production-api-build | 2 +- jenkins/Jenkinsfile.production-api-deploy | 7 +- ...nkinsfile.production-full-build-and-deploy | 4 + package.json | 1 + scripts/check-database-backup-to-oss.mjs | 167 ++++++++++++ scripts/check-production-api-deploy.mjs | 243 ++++++++++-------- scripts/check-production-ops-guardrails.mjs | 51 +++- scripts/database-backup-to-oss.mjs | 184 ++++++++++++- scripts/deploy/production-api-deploy.sh | 45 +++- 12 files changed, 592 insertions(+), 133 deletions(-) create mode 100644 scripts/check-database-backup-to-oss.mjs diff --git a/deploy/env/api-server.env.example b/deploy/env/api-server.env.example index 55e98f44a..7d11c33f4 100644 --- a/deploy/env/api-server.env.example +++ b/deploy/env/api-server.env.example @@ -139,6 +139,8 @@ GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +# 可选：显式要求备份工作目录所在文件系统至少保留的可用空间；为空时按数据目录大小 + 安全余量估算。 +GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES= # 可选：定时 / publish 前备份使用独立最小权限 AccessKey；为空时回退 ALIYUN_OSS_ACCESS_KEY_*。 GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index ab462ce99..eb4bd1d97 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -370,7 +370,7 @@ ## Pingora 直连 80/443 不能只改 env - 现象：`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`，但 `genarrative-pingora-gateway.service` 启动失败，日志出现低端口绑定权限错误。 -- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`pingora-direct-rehearsal-status.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 时，deploy 必须先确认 systemd 最终配置没有 direct-entry `CAP_NET_BIND_SERVICE`、env 仍是 `127.0.0.1:18081` shadow 且未配置 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，再提升 release、切换 current 并 `restart` Pingora shadow；配置不安全时必须在切换 current 前失败并保持维护模式。 +- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`pingora-direct-rehearsal-status.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Jenkins API Build、API Deploy 和 Full Build-And-Deploy 默认要求 Pingora 产物，并用 `--require-pingora-gateway` 在部署阶段硬校验；手工本地 API 包仍需显式 `--include-pingora-gateway` 才会把二进制、checksum 和 manifest artifact 写入发布包。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应在 current 切换前 fail-fast、清理 staging 并退出本次打开的维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 时，deploy 必须先确认 systemd 最终配置没有 direct-entry `CAP_NET_BIND_SERVICE`、env 仍是 `127.0.0.1:18081` shadow 且未配置 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，再提升 release、切换 current 并 `restart` Pingora shadow；配置不安全时必须在切换 current 前失败并退出本次打开的维护模式，current 切换后的 readiness / 服务重启失败仍保留维护模式。 - 处理：确认真实 TLS 证书和 redirect env 已写入 `/etc/genarrative/pingora-gateway.env`、service 模板和 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含这份 env、当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都能读取证书链 / 私钥、current release 的 `pingora-gateway` 已存在且可执行、Nginx 或其它进程已释放 `80/443` 后，先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成只读 JSON runbook，并逐条审阅 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础 readiness、direct enable dry-run、direct enable apply、启用后 `--require-direct` 复核、rollback dry-run、rollback apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；runbook 只用于审阅，不修改系统。正式 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 或回退 smoke 各自验证到不同入口；同时必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检需要 Host 覆盖，再追加 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 把现场巡检入口覆盖成仓库默认值；如需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文。随后先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，再 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`，最后执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，由脚本先跑 direct preflight，再安装 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`、用 `systemctl is-active` 确认服务 active，再以 JSON 模式执行 direct live smoke，验证 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘，并要求 `direct-access-log` 结构化结果 `matchedCount == checked`、`missingCount=0`、`mismatchCount=0`；如果 direct live 退出 0 但缺少该结构化证据，也必须视为启用失败。直连启用后同步调整 `/etc/genarrative/health-patrol.env`：设置 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，本机打 `127.0.0.1` 时设置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`，否则巡检会继续按 Nginx 模式误报。验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''` 或 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''`；回退脚本先跑 `nginx -t`，通过后才移除 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 已移除、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`，随后 reload Nginx、确认 Nginx service 仍为 active，并用 curl smoke URL 证明公网入口已回到 Nginx；回退 smoke URL/body 必须来自切换前真实 Nginx 入口，不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`；回退脚本 `--apply` 不允许省略 `--reload-nginx` 或 `--nginx-smoke-url`，当 smoke URL 指向本机地址时必须同时提供 `--nginx-smoke-host <域名>`，且 host 值不能包含 URL、路径或查询；回退后把 health patrol gateway mode 改回 `nginx`，恢复切换前 public base URL / Host，并用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url <切换前Nginx巡检入口> --require-empty-public-host` 复核；若切换前 Nginx 巡检需要 Host 覆盖，则把 `--require-empty-public-host` 换成 `--expected-public-host <切换前Host>`。若 env 已在回退命令前切回 Nginx，也可给 rollback 脚本追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让它在 Nginx smoke 后自动复核；切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若要同时证明 Pingora shadow 高端口仍活着，追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应为 `gateway=pingora-shadow`。 - 处理补充：不要直接 chmod `/etc/letsencrypt/live` 或 `archive` 来让 Pingora 读取证书；Certbot live 路径通常是 symlink，即使 `stat -L` 看起来是普通文件，父目录权限也会让非 root `genarrative` 用户不可达。先用随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>` 把证书同步到 Pingora 私有目录，再让 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/fullchain.pem` 和 `privkey.pem`。脚本默认 dry-run，`--apply` 才写入，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并拒绝符号链接目标目录或目标文件。 - 处理补充：不要在切换窗口手工编辑 `/etc/genarrative/health-patrol.env` 的三项网关变量；使用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切到直连，回退前用同一脚本传 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>` 并按切换前记录选择 `--clear-public-host` 或 `--public-host <切换前Host>`。脚本只改 gateway mode / public base URL / public Host，并立即复用随包 env 复核脚本，减少空 Host 和旧值残留；生产巡检、env 复核和 env 切换脚本读取的布尔 env 都必须是明确布尔值，非法值直接失败，不能把拼写错误当成 false；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，直接用 `node script.mjs --env-file ...` 或 shebang 执行 `.mjs --env-file ...` 都可能让 Node 抢走业务参数；所有这类命令都必须写成 `node -- script.mjs --env-file ...`，或通过已内置 `node --` 的 npm script 执行。 @@ -411,7 +411,7 @@ - 踩坑补充：证据根目录总审计选择“每类最新证据”后，还必须证明这些证据来自同一次切换时间线。最新证据选择和标准八段时间线证明只接受 `schemaVersion=1` 且带合法、规范 UTC 毫秒格式 `manifest.generatedAt` 的 manifest，命令记录 `startedAt` / `finishedAt` 也必须是 `new Date().toISOString()` 形式；缺失、非法、省略毫秒、本地时区或其它宽松可解析格式都会直接失败，不能用目录 mtime 兜底；证据目录被复制、归档或恢复后，也必须以 manifest 时间为准。同一阶段或同一命令如果出现多个候选共享最新 `manifest.generatedAt`，总审计会以 `AMBIGUOUS_LATEST` 失败并列出重复目录，不能按目录名排序打平；应重新归档该阶段 / 命令证据，或把旧证据移出正式证据根目录后再审计。标准八段证据都被要求时，每段审计状态都必须是 `OK`，`manifest.generatedAt` 必须满足 `pre-cutover -> enable-apply -> post-enable:pingora-health-patrol-direct-env-switch -> post-enable -> rollback-prep:pingora-gateway-shadow-env-switch -> rollback-prep:pingora-health-patrol-nginx-env-switch -> rollback-apply -> post-rollback`，且默认八段跨度不能超过 24 小时；非 OK、倒序或跨度过大都代表可能混入不同切换窗口遗留证据或现场状态未达标，必须失败后重新归档或清理证据根目录。任何证据 manifest 只要显式写入 `cutoverRunId` 字段，就必须是安全非空 ID，不能用空字符串伪装成缺省字段。确需跨更长维护窗口时，只能在生成 runbook 时显式传 `--cutover-evidence-timeline-max-span-ms `，让最终总审计 JSON 记录本次放宽后的 `timeline.maxSpanMs` 与实际 `timeline.spanMs`。 - 踩坑补充：标准八段时间线失败时不要只看顶层 `ok=false` 或 `diagnostics` 文本。`timeline.failedCount` 会按具体失败项累计，`timeline.failureBreakdown` 会把非 OK 证据、缺少时间、cutoverRunId 混入、时间倒序和跨度超限拆开计数；同一次审计可能同时暴露多个证据问题，应逐项修复后重新归档。 - 踩坑补充：同一天多次演练或切换时，只靠“最新证据”和 24 小时窗口仍可能把两轮证据拼在一起。正式 runbook 会生成或接受 `--cutover-run-id `，并把同一 `manifest.cutoverRunId` 写入三阶段证据包、五条真实切换命令证据和最终总审计；最终审计必须带 `--require-cutover-run-id <本次cutoverRunId>`，缺少该字段或 ID 不一致时必须失败。即使人工临时总审计忘记带 `--require-cutover-run-id`，标准八段时间线里只要任一证据声明了 `manifest.cutoverRunId`，八段也必须全部声明同一个值，否则总审计失败。 -- 验证：先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，确认 env、drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和 80/443 已释放；`systemctl cat genarrative-pingora-gateway.service` 必须显示 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env`；启用脚本 apply 必须先通过 current release 自审，失败时不安装 direct-entry drop-in；还必须带 direct HTTPS / HTTP / Host / redirect host / SpacetimeDB database / Pingora access log 参数，并在重启后直接完成 direct live smoke 和 direct-access-log JSON 证据校验；也可用 release readiness `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable` 把 HTTPS、HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id 落盘、env 预检、systemd drop-in、service EnvironmentFile 一致性、当前用户和服务用户证书可读、service 二进制可执行、显式目标库和 WSS 101 一起纳入硬门禁，并拒绝 `--direct-skip-wss`，避免 TLS 证书只按 `127.0.0.1` 误测、HTTP redirect Location 指错域名、service 实际读取另一份 env、root / deploy 用户可读但 systemd 服务用户不可读、current release 缺少可执行 `pingora-gateway`，或 WSS subscribe 隐式打到默认 SpacetimeDB 库。`check-pingora-release-readiness.mjs --help` 的正式直连和只生成 runbook 示例也必须带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，不要让值班人员复制示例后才被 `--require-direct` 拦截。current release 自审、状态快照和证据包的布尔 env 必须是明确布尔值，非法值会失败，不得把拼错的 run / require / fail 开关当成 false。`npm run plan:pingora-direct-cutover -- --require-direct ...` 输出必须包含 Host 与回退巡检入口确认、current release preflight、启用前不带 `--require-direct` 的基础 readiness、direct enable dry-run/apply、启用后带 `--require-direct` 的复核、rollback dry-run/apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；缺少 `--require-direct`、缺少 `--rollback-health-patrol-public-base-url`、缺少 `--direct-pingora-access-log`、redirect Host 漂移或 rollback smoke Host 漂移时必须失败，避免生成缺少正式直连硬门禁或验证不同入口的切换计划。Host 与回退巡检入口确认步骤必须展示回退后要恢复的 health patrol public base URL / Host。直连后 `genarrative-health-patrol.service` 应使用 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，状态 JSON 中 `gatewayMode` 应为 `pingora-direct`，并检查 `genarrative-pingora-gateway.service` 而不是 `nginx.service`；public probe 走 `127.0.0.1` 时应带 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`。回退后 `nginx -t` 必须先通过，`systemctl cat genarrative-pingora-gateway.service` 不应再显示这两条 capability，`systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager` 必须仍指向 current release 的 `pingora-gateway`，`systemctl is-active nginx.service` 应为 `active`，`curl --fail --max-time 5` 访问 `--nginx-smoke-url` 应成功；若 smoke URL 为本机地址必须带 `--nginx-smoke-host <域名>`，证明正式 vhost 已回到 Nginx；随后用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs ...` 复核 health patrol env，必须显示 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 且 public base URL / Host 与切换前记录一致，shadow probe 可选复核必须返回 `gateway=pingora-shadow`。本机提交前还要运行 `npm run check:pingora-direct-enable`、`npm run check:pingora-direct-rollback`、`npm run check:production-health-patrol`、`npm run check:production-api-release`、`npm run check:pingora-production-release-build` 和 `npm run check:production-api-deploy`，确保脚本默认 dry-run 不会安装或删除 drop-in、current release 自审失败时启用脚本不会安装 drop-in、direct live 退出 0 但缺少 `direct-access-log` 结构化证据时启用失败，API release 布局自包含，真实 Pingora release 二进制能构建并进入发布包，API deploy 从发布产物内执行后 current release 自包含；缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、切换命令证据脚本、env 示例目录或 direct live smoke 脚本的发布包都必须部署失败并保持维护模式。正式直连 readiness 必须带 `--direct-health-patrol-env-file /etc/genarrative/health-patrol.env`，并用 `scripts/check-production-health-patrol-env.mjs` 阻断 health patrol 仍停在 Nginx 模式或本机 direct probe 缺少正式 Host；发布包包含 `pingora-gateway` 时，`npm run check:production-api-deploy` 必须覆盖服务 active / inactive 都会在 shadow 配置安全时执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，同时覆盖 direct-entry capability 或公网监听 env 下不会提升 release、不会切 current、不会自动 restart。 +- 验证：先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，确认 env、drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和 80/443 已释放；`systemctl cat genarrative-pingora-gateway.service` 必须显示 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env`；启用脚本 apply 必须先通过 current release 自审，失败时不安装 direct-entry drop-in；还必须带 direct HTTPS / HTTP / Host / redirect host / SpacetimeDB database / Pingora access log 参数，并在重启后直接完成 direct live smoke 和 direct-access-log JSON 证据校验；也可用 release readiness `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable` 把 HTTPS、HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id 落盘、env 预检、systemd drop-in、service EnvironmentFile 一致性、当前用户和服务用户证书可读、service 二进制可执行、显式目标库和 WSS 101 一起纳入硬门禁，并拒绝 `--direct-skip-wss`，避免 TLS 证书只按 `127.0.0.1` 误测、HTTP redirect Location 指错域名、service 实际读取另一份 env、root / deploy 用户可读但 systemd 服务用户不可读、current release 缺少可执行 `pingora-gateway`，或 WSS subscribe 隐式打到默认 SpacetimeDB 库。`check-pingora-release-readiness.mjs --help` 的正式直连和只生成 runbook 示例也必须带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，不要让值班人员复制示例后才被 `--require-direct` 拦截。current release 自审、状态快照和证据包的布尔 env 必须是明确布尔值，非法值会失败，不得把拼错的 run / require / fail 开关当成 false。`npm run plan:pingora-direct-cutover -- --require-direct ...` 输出必须包含 Host 与回退巡检入口确认、current release preflight、启用前不带 `--require-direct` 的基础 readiness、direct enable dry-run/apply、启用后带 `--require-direct` 的复核、rollback dry-run/apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；缺少 `--require-direct`、缺少 `--rollback-health-patrol-public-base-url`、缺少 `--direct-pingora-access-log`、redirect Host 漂移或 rollback smoke Host 漂移时必须失败，避免生成缺少正式直连硬门禁或验证不同入口的切换计划。Host 与回退巡检入口确认步骤必须展示回退后要恢复的 health patrol public base URL / Host。直连后 `genarrative-health-patrol.service` 应使用 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，状态 JSON 中 `gatewayMode` 应为 `pingora-direct`，并检查 `genarrative-pingora-gateway.service` 而不是 `nginx.service`；public probe 走 `127.0.0.1` 时应带 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`。回退后 `nginx -t` 必须先通过，`systemctl cat genarrative-pingora-gateway.service` 不应再显示这两条 capability，`systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager` 必须仍指向 current release 的 `pingora-gateway`，`systemctl is-active nginx.service` 应为 `active`，`curl --fail --max-time 5` 访问 `--nginx-smoke-url` 应成功；若 smoke URL 为本机地址必须带 `--nginx-smoke-host <域名>`，证明正式 vhost 已回到 Nginx；随后用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs ...` 复核 health patrol env，必须显示 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 且 public base URL / Host 与切换前记录一致，shadow probe 可选复核必须返回 `gateway=pingora-shadow`。本机提交前还要运行 `npm run check:pingora-direct-enable`、`npm run check:pingora-direct-rollback`、`npm run check:production-health-patrol`、`npm run check:production-api-release`、`npm run check:pingora-production-release-build` 和 `npm run check:production-api-deploy`，确保脚本默认 dry-run 不会安装或删除 drop-in、current release 自审失败时启用脚本不会安装 drop-in、direct live 退出 0 但缺少 `direct-access-log` 结构化证据时启用失败，API release 布局自包含，真实 Pingora release 二进制能构建并进入发布包，API deploy 从发布产物内执行后 current release 自包含；缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、切换命令证据脚本、env 示例目录或 direct live smoke 脚本的发布包都必须在 current 切换前部署失败并退出本次打开的维护模式。正式直连 readiness 必须带 `--direct-health-patrol-env-file /etc/genarrative/health-patrol.env`，并用 `scripts/check-production-health-patrol-env.mjs` 阻断 health patrol 仍停在 Nginx 模式或本机 direct probe 缺少正式 Host；发布包包含 `pingora-gateway` 时，`npm run check:production-api-deploy` 必须覆盖服务 active / inactive 都会在 shadow 配置安全时执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，同时覆盖 direct-entry capability 或公网监听 env 下不会提升 release、不会切 current、不会自动 restart。 - 顺序补充：正式 runbook 必须先通过 health patrol env 切换脚本预置回 Nginx 和切换前 public base URL / Host，再执行 `rollback apply`；回退脚本内置 env 复核和独立 env 复核都会阻断 public base URL / Host 漂移。 - 顺序补充：正式 runbook 还必须在 `rollback apply` 前预置 Pingora shadow env。启用前和 `--dry-run-cutover` 要求 `80/443` 空闲；启用后 `--require-direct` 复核不再要求端口空闲，因为端口应由 Pingora 占用。回退时要先用 current release 随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs --apply --env-file /etc/genarrative/pingora-gateway.env` 恢复 `GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081` 并清空 TLS / HTTP redirect 低端口监听，再移除 direct-entry drop-in 和重启 Pingora。 - 关联：`deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`、`deploy/env/health-patrol.env.example`、`deploy/env/pingora-direct-live.env.example`、`deploy/env/pingora-canary-live.env.example`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/jenkins-server-provision.sh`、`scripts/build-production-release.sh`、`scripts/deploy/production-api-deploy.sh`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 @@ -466,10 +466,10 @@ ## 生产冷备份后 API 和外部生成 worker 不能只依赖 SpacetimeDB 自恢复 -- 现象：release 机器 `03:20` 冷备份后，`spacetimedb.service` 已恢复，但作品列表、创作入口配置或公开 gallery 继续超时 / 502 / 504，`genarrative-api.service` 保持 stopped；或图片画布生成请求返回队列态后长期显示排队，`external_generation_job` 有 claimable pending，但 `genarrative-external-generation-worker@1.service` / controller 是 inactive。 -- 原因：`genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都配置了 `Requires=spacetimedb.service`，冷备份停止 `spacetimedb.service` 时这些服务会被 systemd 依赖关系一并停止；如果 `genarrative-database-backup.service` 只恢复数据库或只重启 API，外部生成队列就不会被消费。 -- 处理：生产冷备份 unit 和发布脚本必须带 `--restart-service-after genarrative-api.service`、`--restart-service-after genarrative-external-generation-worker@1.service` 和 `--restart-service-after genarrative-external-generation-controller.service`；`genarrative-api.service` 也保留对 controller 的 `Wants` 弱依赖，覆盖“只恢复 API”的现场兜底。仓库用 `npm run check:production-ops` 检查 systemd 模板、API build/deploy 归档和健康巡检链路。现场修复后执行 `systemctl daemon-reload`，但不要为了验证而手动触发冷备份。 -- 验证：`systemctl cat genarrative-database-backup.service` 应包含这些参数；`systemctl is-active spacetimedb.service genarrative-api.service genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service nginx.service` 全为 `active`；`curl -fsS http://127.0.0.1:3101/v1/ping`、`/healthz`、`/readyz` 和代表性 `/api/runtime/puzzle/gallery` 均成功；`get_external_generation_queue_stats_and_return` 不应长期出现 claimable pending。 +- 现象：release 机器 `03:20` 冷备份后，`spacetimedb.service` 已恢复，但作品列表、创作入口配置或公开 gallery 继续超时 / 502 / 504，`genarrative-api.service` 保持 stopped；或图片画布生成请求返回队列态后长期显示排队，`external_generation_job` 有 claimable pending，但 `genarrative-external-generation-worker@1.service` / controller 是 inactive；也可能先看到 `/var/lib/genarrative/database-backups` 把根分区写满，`gzip: stdout: No space left on device`。 +- 原因：`genarrative-api.service`、`genarrative-external-generation-worker@*.service` 和 `genarrative-external-generation-controller.service` 都配置了 `Requires=spacetimedb.service`，冷备份停止 `spacetimedb.service` 时这些服务会被 systemd 依赖关系一并停止；如果备份脚本只在打包成功后重启依赖服务，那么 tar/gzip 因空间不足失败时就只会恢复数据库，外部生成队列和 API 仍无人接管。 +- 处理：生产冷备份 unit 和发布脚本必须带 `--restart-service-after genarrative-api.service`、`--restart-service-after genarrative-external-generation-worker@1.service` 和 `--restart-service-after genarrative-external-generation-controller.service`；备份脚本必须在停止 SpacetimeDB 前做工作目录剩余空间预检，并且一旦已经停过 SpacetimeDB，就算打包失败也要先恢复 SpacetimeDB 与这些依赖服务，再返回原始备份错误。`genarrative-api.service` 也保留对 controller 的 `Wants` 弱依赖，覆盖“只恢复 API”的现场兜底。仓库用 `npm run check:production-ops` 和 `npm run check:database-backup` 检查 systemd 模板、脚本失败路径、API build/deploy 归档和健康巡检链路。现场修复后执行 `systemctl daemon-reload`，但不要为了验证而手动触发冷备份。 +- 验证：`systemctl cat genarrative-database-backup.service` 应包含这些参数；`systemctl is-active spacetimedb.service genarrative-api.service genarrative-external-generation-worker@1.service genarrative-external-generation-controller.service nginx.service` 全为 `active`；`curl -fsS http://127.0.0.1:3101/v1/ping`、`/healthz`、`/readyz` 和代表性 `/api/runtime/puzzle/gallery` 均成功；`npm run check:database-backup` 覆盖空间不足不触碰 systemctl、tar 失败仍恢复依赖服务；`get_external_generation_queue_stats_and_return` 不应长期出现 claimable pending。 - 关联：`deploy/systemd/genarrative-database-backup.service`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 ## Pingora Brotli 不能只看 Content-Encoding diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index c72151731..401703837 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -245,11 +245,12 @@ GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET= GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT=oss-cn-shanghai.aliyuncs.com GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX=database-backups GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL=false +GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID= GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET= ``` -`GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。如果 timer 显示 `enabled` 但 `inactive/dead` 且 `NEXT` / `Trigger` 为空，先写入当前 stamp 避免 `Persistent=true` 在白天立刻补跑冷备份：`touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer && systemctl daemon-reload && systemctl start genarrative-database-backup.timer`，随后确认下一次触发时间约为次日 `03:20`。 +`GENARRATIVE_DATABASE_BACKUP_OSS_BUCKET` 为空时会回退 `ALIYUN_OSS_BUCKET`；AccessKey 默认复用 `ALIYUN_OSS_ACCESS_KEY_ID` / `ALIYUN_OSS_ACCESS_KEY_SECRET`，也可用 `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_ID` / `GENARRATIVE_DATABASE_BACKUP_OSS_ACCESS_KEY_SECRET` 为备份 bucket 单独配置最小权限账号。冷备脚本会在停止 SpacetimeDB 前检查 `GENARRATIVE_DATABASE_BACKUP_WORK_DIR` 所在文件系统剩余空间；未设置 `GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES` 时，按数据目录大小加安全余量估算，空间不足会在停库前失败，避免写满根分区。即使打包或上传前步骤失败，只要脚本已经停过 SpacetimeDB，也会先恢复 SpacetimeDB 并执行 `--restart-service-after` 指定的 API / worker / controller，再带着原始备份错误退出。`Genarrative-Server-Provision` 会创建 `/var/lib/genarrative/database-backups` 并归属 `genarrative:genarrative`，同时安装并启用 `genarrative-database-backup.timer`。手动检查定时器：`systemctl list-timers genarrative-database-backup.timer`；手动触发一次：`systemctl start genarrative-database-backup.service`。如果 timer 显示 `enabled` 但 `inactive/dead` 且 `NEXT` / `Trigger` 为空，先写入当前 stamp 避免 `Persistent=true` 在白天立刻补跑冷备份：`touch /var/lib/systemd/timers/stamp-genarrative-database-backup.timer && systemctl daemon-reload && systemctl start genarrative-database-backup.timer`，随后确认下一次触发时间约为次日 `03:20`。 冷备份后必须做一次只读验收，不要只看 `genarrative-database-backup.service` 是否成功退出： @@ -326,7 +327,7 @@ current release 自审的 `--release-root` 和 `--systemd-service` 不能包含 current release 自审、状态快照、证据包、direct preflight、生产巡检、health patrol env 复核和 env 切换脚本读取的布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，拼写错误会直接失败，避免 `REQUIRE_GATEWAY`、`RUN_HEALTH_PATROL`、`REQUIRE_PINGORA_GATEWAY`、`FAIL_ON_CRITICAL`、`TRUST_X_FORWARDED_FOR` 或巡检切换开关被悄悄当成 false。 -正式直连 runbook 的启用前 release readiness 基础门禁和启用后 `--require-direct` 复核，都必须调用 current release 随包的 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs`。生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和 `production-api-deploy.sh` 都必须携带该聚合门禁脚本；缺失时部署应 fail-fast 并保持维护模式，切换窗口不能回退到源码 checkout 或 Jenkins workspace 的相对路径脚本。 +正式直连 runbook 的启用前 release readiness 基础门禁和启用后 `--require-direct` 复核，都必须调用 current release 随包的 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs`。生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和 `production-api-deploy.sh` 都必须携带该聚合门禁脚本；缺失时部署应在 current 切换前 fail-fast、清理 staging 并退出本次打开的维护模式，切换窗口不能回退到源码 checkout 或 Jenkins workspace 的相对路径脚本。 current release 随包执行的 release readiness 必须追加 `--release-runtime-only`，只运行包内可自包含的 current release 自审、live canary、真实 access log 对账、direct preflight、health patrol env 复核和 direct live smoke；不要在 `/opt/genarrative/current` 上运行默认源码全量门禁。默认不带 `--release-runtime-only` 的聚合门禁仍属于本机 / CI / 构建环境使用，负责覆盖 Cargo、npm、Docker、Nginx 静态 / 真机校验和发布包构建烟测。 @@ -365,7 +366,7 @@ cat /var/lib/genarrative/health-patrol/status.json `Genarrative-Web-Build` 会把 `build//web.tar.gz`、`web.tar.gz.sha256`、`release-manifest.json` 和 `scripts/deploy/production-web-deploy.sh` 直接归档为 Jenkins 构建产物；`Genarrative-Web-Deploy` 只通过 `copyArtifacts` 从指定上游构建复制这些产物和部署脚本，不再在目标机器 checkout Git，再执行随构建归档的 `scripts/deploy/production-web-deploy.sh`。Web 发布不再读取构建机本地缓存目录，也不再通过 release agent `rsync` 回构建机拉取大包；如果 deploy 找不到 `web.tar.gz`，应先检查上游 Web Build 是否按同一 `BUILD_VERSION` 成功归档产物。 -`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会保留维护模式并停止部署；失败会清理 staging 目录且不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会失败且保持维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时拒绝覆盖 / 合并。Pingora 影子网关不是默认 API 归档物；只有显式用 `npm run build:production-release -- --component api-server --include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，发布包才包含 `pingora-gateway` / `pingora-gateway.sha256`，API deploy 会在两者同时存在且 manifest 登记 `pingora-gateway` 时校验并复制到 current release；此时 build 脚本和 Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。发布包包含 Pingora 时，deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载新网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。 +`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会在 current 切换前停止部署，清理 staging 并退出本次打开的维护模式；current 切换后的 Pingora 重启、worker 重启、controller 启动或 readiness 失败仍保留维护模式，避免暴露半发布版本；失败不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、Pingora manifest / 二进制漂移、`--require-pingora-gateway` 缺少 Pingora、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会在 current 切换前失败并退出本次打开的维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现时拒绝覆盖 / 合并，以及 current 切换后的 readiness 失败会保留维护模式。Pingora 影子网关在 `Genarrative-Api-Build`、`Genarrative-Api-Deploy` 和 `Genarrative-Full-Build-And-Deploy` 中默认随 release 构建、归档、复制并用 `--require-pingora-gateway` 硬校验；只有显式取消 `INCLUDE_PINGORA_GATEWAY` 时才允许 API release 不带 `pingora-gateway` / `pingora-gateway.sha256`。本地 CLI 仍保留显式 `npm run build:production-release -- --component api-server --include-pingora-gateway`，用于在需要 Pingora 的手工发布包里登记 manifest 和 checksum；Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。发布包包含 Pingora 时，deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载新网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。 Pingora current release 自审脚本 `scripts/ops/pingora-current-release-audit.mjs`、直连切换状态快照脚本 `scripts/ops/pingora-cutover-status-snapshot.mjs`、证据包脚本 `scripts/ops/pingora-cutover-evidence-bundle.mjs`、命令证据脚本 `scripts/ops/pingora-cutover-command-evidence.mjs`、证据验真脚本 `scripts/ops/pingora-cutover-evidence-verify.mjs`、证据根目录审计脚本 `scripts/ops/pingora-cutover-evidence-audit.mjs` 和 canary access log 对账脚本 `scripts/check-pingora-canary-access-log-parity.mjs` 都属于 API release 的强制随包依赖；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败，避免切换窗口只能靠 Jenkins 工作区或源码 checkout 临时补自审、证据或日志对账脚本。 diff --git a/jenkins/Jenkinsfile.production-api-build b/jenkins/Jenkinsfile.production-api-build index 0a1b0c84a..ea9496ae6 100644 --- a/jenkins/Jenkinsfile.production-api-build +++ b/jenkins/Jenkinsfile.production-api-build @@ -25,7 +25,7 @@ pipeline { string(name: 'COMMIT_HASH', defaultValue: '', description: '可选，指定属于 SOURCE_BRANCH 的 Git commit') string(name: 'BUILD_VERSION', defaultValue: '', description: '发布版本号，留空则使用 Jenkins BUILD_NUMBER') string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱；会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送') - booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '是否额外构建并归档 Pingora 影子网关二进制') + booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: '是否构建并归档 Pingora 影子网关二进制；release 发布默认必须包含') booleanParam(name: 'PUBLISH_AFTER_BUILD', defaultValue: false, description: '构建成功后是否触发 API 发布') string(name: 'DEPLOY_JOB_NAME', defaultValue: 'Genarrative-Api-Deploy', description: 'API 发布流水线作业名') choice(name: 'DEPLOY_TARGET', choices: ['development', 'release'], description: 'PUBLISH_AFTER_BUILD=true 时的逻辑部署目标；development 使用当前 Linux 开发/构建/开发部署 agent') diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index e116474b1..ccd2a32b3 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -16,7 +16,7 @@ pipeline { string(name: 'BUILD_VERSION', defaultValue: '', description: '待发布版本号') string(name: 'BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名') string(name: 'BUILD_NUMBER_TO_DEPLOY', defaultValue: '', description: '要复制归档产物的上游构建号') - booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '上游构建是否包含 Pingora 影子网关产物') + booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: '上游构建是否包含 Pingora 影子网关产物；release 发布默认必须包含') string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: '生产 release 根目录') string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接') string(name: 'SERVICE_NAME', defaultValue: 'genarrative-api.service', description: 'systemd 服务名') @@ -93,12 +93,17 @@ pipeline { bash -lc ' set -euo pipefail chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs" + pingora_deploy_args=() + if [[ "${INCLUDE_PINGORA_GATEWAY:-false}" == "true" ]]; then + pingora_deploy_args+=(--require-pingora-gateway) + fi "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \ --source-dir "build/${BUILD_VERSION}" \ --version "${BUILD_VERSION}" \ --release-root "${RELEASE_ROOT}" \ --current-link "${CURRENT_LINK}" \ --service "${SERVICE_NAME}" \ + "${pingora_deploy_args[@]}" \ --health-url "${HEALTH_URL}" \ --api-env-file "${API_ENV_FILE:-/etc/genarrative/api-server.env}" \ --database "${DATABASE}" \ diff --git a/jenkins/Jenkinsfile.production-full-build-and-deploy b/jenkins/Jenkinsfile.production-full-build-and-deploy index 47904605f..279edb70d 100644 --- a/jenkins/Jenkinsfile.production-full-build-and-deploy +++ b/jenkins/Jenkinsfile.production-full-build-and-deploy @@ -25,6 +25,7 @@ pipeline { booleanParam(name: 'RUN_NPM_CI', defaultValue: true, description: 'Web 构建前是否执行 npm ci') string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱；会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送') string(name: 'MIGRATION_BOOTSTRAP_SECRET_CREDENTIAL_ID', defaultValue: '', description: '可选，透传给 Stdb module 构建的迁移 bootstrap secret 凭据 ID；留空则由 Stdb 构建自动生成') + booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: true, description: 'API release 是否构建、归档并部署 Pingora 影子网关；release 默认必须包含') string(name: 'WEB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Web-Build', description: 'Web 构建流水线作业名') string(name: 'API_BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名') string(name: 'STDB_BUILD_JOB_NAME', defaultValue: 'Genarrative-Stdb-Module-Build', description: 'Stdb 构建流水线作业名') @@ -134,6 +135,7 @@ pipeline { string(name: 'COMMIT_HASH', value: env.SOURCE_COMMIT), string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION), string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''), + booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY), ] env.API_BUILD_NUMBER = apiRun.number.toString() } @@ -196,6 +198,7 @@ pipeline { booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT), string(name: 'BUILD_JOB_NAME', value: params.API_BUILD_JOB_NAME), string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.API_BUILD_NUMBER), + booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY), string(name: 'DATABASE', value: params.DATABASE), string(name: 'SPACETIME_SERVER_URL', value: params.SPACETIME_SERVER_URL ?: ''), ] @@ -234,6 +237,7 @@ pipeline { string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION ?: (params.BUILD_VERSION ?: '')), string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET ?: ''), string(name: 'DATABASE', value: params.DATABASE ?: ''), + string(name: 'INCLUDE_PINGORA_GATEWAY', value: String.valueOf(params.INCLUDE_PINGORA_GATEWAY)), string(name: 'SUMMARY', value: '全量构建发布编排结束'), ] def notificationRecipients = params.NOTIFICATION_EMAILS?.trim() diff --git a/package.json b/package.json index 0136a88a0..69954509c 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "check:encoding": "node scripts/check-encoding.mjs", "check:spacetime-schema": "node scripts/check-spacetime-schema-guard.mjs", "check:production-ops": "node scripts/check-production-ops-guardrails.mjs", + "check:database-backup": "node scripts/check-database-backup-to-oss.mjs", "check:production-health-patrol": "node scripts/check-production-health-patrol.mjs", "check:production-health-patrol-env": "node scripts/check-production-health-patrol-env-check.mjs", "check:production-api-release": "node scripts/check-production-api-release.mjs", diff --git a/scripts/check-database-backup-to-oss.mjs b/scripts/check-database-backup-to-oss.mjs new file mode 100644 index 000000000..af154f6cf --- /dev/null +++ b/scripts/check-database-backup-to-oss.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node + +import {spawnSync} from 'node:child_process'; +import {existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; +import path from 'node:path'; + +const BACKUP_SCRIPT = path.resolve('scripts/database-backup-to-oss.mjs'); +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-database-backup-check-')); +const failures = []; + +try { + main(); +} finally { + rmSync(tmpRoot, {recursive: true, force: true}); +} + +if (failures.length > 0) { + console.error('[check:database-backup] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:database-backup] OK'); + +function main() { + assertInsufficientSpaceStopsBeforeServiceChanges(); + assertArchiveFailureStillRestoresDependentServices(); +} + +function assertInsufficientSpaceStopsBeforeServiceChanges() { + const fixture = createFixture('insufficient-space'); + const result = runBackup(fixture, [ + '--stop-service', + 'spacetimedb.service', + '--restart-service-after', + 'genarrative-api.service', + '--min-free-bytes', + '999999999999999999', + ]); + + assertStatus(result, 1, '空间不足时必须失败。'); + assertIncludes(result.stdout, '备份空间预检', '空间不足失败前应打印空间预检。'); + assertIncludes(result.stderr, '剩余空间不足', '空间不足失败应说明剩余空间不足。'); + assertFileMissing(fixture.systemctlLog, '空间不足时不能调用 systemctl。'); + assertFileMissing(fixture.tarLog, '空间不足时不能调用 tar。'); +} + +function assertArchiveFailureStillRestoresDependentServices() { + const fixture = createFixture('tar-failure'); + const result = runBackup(fixture, [ + '--stop-service', + 'spacetimedb.service', + '--restart-service-after', + 'genarrative-api.service', + '--restart-service-after', + 'genarrative-external-generation-worker@1.service', + '--restart-service-after', + 'genarrative-external-generation-controller.service', + '--min-free-bytes', + '1', + ]); + + assertStatus(result, 1, 'tar 失败时备份脚本必须失败。'); + assertIncludes(result.stderr, 'fake tar failure', 'tar 失败原因应保留在错误输出中。'); + const systemctlLog = readFile(fixture.systemctlLog); + const expectedCommands = [ + 'systemctl stop spacetimedb.service', + 'systemctl start spacetimedb.service', + 'systemctl restart genarrative-api.service', + 'systemctl restart genarrative-external-generation-worker@1.service', + 'systemctl restart genarrative-external-generation-controller.service', + ]; + for (const command of expectedCommands) { + assertIncludes(systemctlLog, command, `tar 失败后必须执行: ${command}`); + } +} + +function createFixture(name) { + const root = path.join(tmpRoot, name); + const binDir = path.join(root, 'bin'); + const dataDir = path.join(root, 'data'); + const workDir = path.join(root, 'work'); + const systemctlLog = path.join(root, 'systemctl.log'); + const tarLog = path.join(root, 'tar.log'); + mkdirSync(binDir, {recursive: true}); + mkdirSync(dataDir, {recursive: true}); + writeFileSync(path.join(dataDir, 'sample.bin'), 'sample backup payload\n', 'utf8'); + writeExecutable( + path.join(binDir, 'systemctl'), + `#!/usr/bin/env bash +printf 'systemctl %s\\n' "$*" >> "${systemctlLog}" +exit 0 +`, + ); + writeExecutable( + path.join(binDir, 'tar'), + `#!/usr/bin/env bash +printf 'tar %s\\n' "$*" >> "${tarLog}" +echo 'fake tar failure' >&2 +exit 2 +`, + ); + return {root, binDir, dataDir, workDir, systemctlLog, tarLog}; +} + +function runBackup(fixture, extraArgs = []) { + return spawnSync( + process.execPath, + [ + BACKUP_SCRIPT, + '--data-dir', + fixture.dataDir, + '--work-dir', + fixture.workDir, + '--bucket', + 'genarrative-test', + '--endpoint', + 'oss-cn-shanghai.aliyuncs.com', + '--access-key-id', + 'test', + '--access-key-secret', + 'test', + ...extraArgs, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + PATH: `${fixture.binDir}${path.delimiter}${process.env.PATH ?? ''}`, + }, + }, + ); +} + +function writeExecutable(filePath, content) { + writeFileSync(filePath, content, 'utf8'); + spawnSync('chmod', ['0755', filePath], {encoding: 'utf8'}); +} + +function readFile(filePath) { + return existsSync(filePath) ? readFileSync(filePath, 'utf8') : ''; +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, expected, reason) { + if (!String(content).includes(expected)) { + failures.push(`${reason} 缺少: ${expected}`); + } +} + +function assertFileMissing(filePath, reason) { + if (existsSync(filePath)) { + failures.push(`${reason} 实际存在: ${filePath}\n${readFile(filePath)}`); + } +} diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 2eacaca7c..db9aa38d0 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -41,6 +41,9 @@ function main() { assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded(); assertDeployRejectsPingoraPublicListenWhenArtifactIncluded(); assertDeployRejectsPingoraArtifactMissingManifestEntry(); + assertDeployRejectsPingoraManifestEntryMissingArtifact(); + assertDeployRequiresPingoraWhenRequested(); + assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch(); assertMissingReleaseManifestFails(); assertReleaseManifestMissingApiArtifactFails(); assertDeployRejectsDotVersion(); @@ -82,6 +85,18 @@ function readOptionalCommandsLog(fixture) { return readFileSync(fixture.commandsLog, 'utf8'); } +function assertMaintenanceCleared(fixture, reason) { + if (existsSync(fixture.maintenanceFile)) { + failures.push(`${reason} 时应退出本次打开的维护模式。`); + } +} + +function assertMaintenanceKept(fixture, reason) { + if (!existsSync(fixture.maintenanceFile)) { + failures.push(`${reason} 时必须保持维护模式。`); + } +} + function assertDeployCopiesPingoraDirectReleaseDependencies() { const fixture = prepareFixture('with-direct-checks'); const result = runDeploy(fixture); @@ -409,9 +424,7 @@ function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() { ) { failures.push('direct-entry capability 存在时不能自动 restart Pingora。'); } - if (!existsSync(fixture.maintenanceFile)) { - failures.push('direct-entry capability 导致部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, 'direct-entry capability 导致部署失败'); assertNoReleasePromoted( fixture, 'direct-entry capability 导致部署失败时不能提升正式 release。', @@ -442,9 +455,7 @@ function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() { ) { failures.push('公网监听 env 存在时不能自动 restart Pingora。'); } - if (!existsSync(fixture.maintenanceFile)) { - failures.push('公网监听 env 导致部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '公网监听 env 导致部署失败'); assertNoReleasePromoted( fixture, '公网监听 env 导致部署失败时不能提升正式 release。', @@ -464,8 +475,67 @@ function assertDeployRejectsPingoraArtifactMissingManifestEntry() { 'release-manifest.json 缺少 pingora-gateway artifact', 'manifest 未登记 Pingora 时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('manifest 未登记 Pingora 导致部署失败时必须保持维护模式。'); + assertMaintenanceCleared(fixture, 'manifest 未登记 Pingora 导致部署失败'); +} + +function assertDeployRejectsPingoraManifestEntryMissingArtifact() { + const fixture = prepareFixture('pingora-manifest-entry-missing-artifact'); + const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + manifest.artifacts.push({ + component: 'pingora-gateway', + path: 'pingora-gateway', + checksum_path: 'pingora-gateway.sha256', + }); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('manifest 登记 Pingora 但发布目录缺少二进制时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'release-manifest.json 登记了 pingora-gateway artifact', + 'manifest 登记 Pingora 但文件缺失时必须给出明确错误。', + ); + assertMaintenanceCleared(fixture, 'manifest 登记 Pingora 但文件缺失导致部署失败'); +} + +function assertDeployRequiresPingoraWhenRequested() { + const fixture = prepareFixture('require-pingora-missing-artifact'); + const result = runDeploy(fixture, { requirePingoraGateway: true }); + + if (result.status === 0) { + failures.push('--require-pingora-gateway 但发布目录缺少 Pingora 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '本次部署要求 Pingora', + '--require-pingora-gateway 缺少 Pingora 文件时必须给出明确错误。', + ); + assertMaintenanceCleared(fixture, '--require-pingora-gateway 缺少 Pingora 文件导致部署失败'); +} + +function assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch() { + const fixture = prepareFixture('readiness-failure'); + const result = runDeploy(fixture, { curlFails: true }); + + if (result.status === 0) { + failures.push('current 切换后的 readiness 失败必须让部署失败。'); + } + assertIncludes( + result.stderr, + 'readiness 检查超时', + 'readiness 失败时必须给出明确错误。', + ); + assertMaintenanceKept(fixture, 'current 切换后的 readiness 失败'); + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + const currentTarget = readlinkSync(fixture.currentLink); + if (currentTarget !== releaseDir) { + failures.push( + `readiness 失败发生在 current 切换后，current link 应指向新 release。实际 ${currentTarget}，预期 ${releaseDir}`, + ); } } @@ -482,9 +552,7 @@ function assertMissingReleaseManifestFails() { '发布产物缺少 release-manifest.json', '缺少 release-manifest.json 时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('缺少 release-manifest.json 导致部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 release-manifest.json 导致部署失败'); } function assertReleaseManifestMissingApiArtifactFails() { @@ -504,9 +572,7 @@ function assertReleaseManifestMissingApiArtifactFails() { 'release-manifest.json 缺少 api-server artifact', 'manifest 缺少 api-server artifact 时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('manifest 缺少 api-server artifact 导致部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, 'manifest 缺少 api-server artifact 导致部署失败'); } function assertDeployRejectsDotVersion() { @@ -679,9 +745,7 @@ function assertDeployCleansStagingReleaseOnFailure() { if (stagingEntries.length > 0) { failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`); } - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, 'staging 构建中失败'); } function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { @@ -714,9 +778,7 @@ function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { if (stagingEntries.length > 0) { failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`); } - if (!existsSync(fixture.maintenanceFile)) { - failures.push('目标 release 竞态失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '目标 release 竞态失败'); } function assertMissingPingoraDirectCheckFails() { @@ -732,9 +794,7 @@ function assertMissingPingoraDirectCheckFails() { '发布产物缺少 Pingora 直连 live smoke 脚本', '缺少 direct live smoke 脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 direct live smoke 脚本导致部署失败'); } function assertMissingPingoraCanaryLiveFails() { @@ -750,9 +810,7 @@ function assertMissingPingoraCanaryLiveFails() { '发布产物缺少 Pingora canary live smoke 脚本', '缺少 canary live smoke 脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 canary live smoke 脚本导致部署失败'); } function assertMissingPingoraCanaryAccessLogParityFails() { @@ -773,9 +831,7 @@ function assertMissingPingoraCanaryAccessLogParityFails() { '发布产物缺少 Pingora canary access log 对账脚本', '缺少 canary access log 对账脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 canary access log 对账脚本导致部署失败'); } function assertMissingBackupScriptFails() { @@ -791,9 +847,7 @@ function assertMissingBackupScriptFails() { '发布产物缺少数据库备份脚本', '缺少数据库备份脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少数据库备份脚本导致部署失败'); } function assertMissingHealthPatrolScriptFails() { @@ -811,9 +865,7 @@ function assertMissingHealthPatrolScriptFails() { '发布产物缺少生产健康巡检脚本', '缺少生产健康巡检脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少生产健康巡检脚本导致部署失败'); } function assertMissingPingoraCurrentReleaseAuditFails() { @@ -834,9 +886,7 @@ function assertMissingPingoraCurrentReleaseAuditFails() { '发布产物缺少 Pingora current release 自审脚本', '缺少 Pingora current release 自审脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora current release 自审脚本导致部署失败'); } function assertMissingPingoraDirectRehearsalStatusFails() { @@ -857,9 +907,7 @@ function assertMissingPingoraDirectRehearsalStatusFails() { '发布产物缺少 Pingora 直连彩排状态脚本', '缺少 Pingora 直连彩排状态脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连彩排状态脚本导致部署失败'); } function assertMissingPingoraCutoverStatusSnapshotFails() { @@ -880,9 +928,7 @@ function assertMissingPingoraCutoverStatusSnapshotFails() { '发布产物缺少 Pingora 直连切换状态快照脚本', '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换状态快照脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceBundleFails() { @@ -903,9 +949,7 @@ function assertMissingPingoraCutoverEvidenceBundleFails() { '发布产物缺少 Pingora 直连切换证据包脚本', '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据包脚本导致部署失败'); } function assertMissingPingoraCutoverCommandEvidenceFails() { @@ -926,9 +970,7 @@ function assertMissingPingoraCutoverCommandEvidenceFails() { '发布产物缺少 Pingora 直连切换命令证据脚本', '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换命令证据脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceVerifyFails() { @@ -949,9 +991,7 @@ function assertMissingPingoraCutoverEvidenceVerifyFails() { '发布产物缺少 Pingora 直连切换证据验真脚本', '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据验真脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceAuditFails() { @@ -974,9 +1014,7 @@ function assertMissingPingoraCutoverEvidenceAuditFails() { '发布产物缺少 Pingora 直连切换证据根目录审计脚本', '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据根目录审计脚本导致部署失败'); } function assertMissingHealthPatrolEnvCheckFails() { @@ -994,9 +1032,7 @@ function assertMissingHealthPatrolEnvCheckFails() { '发布产物缺少生产健康巡检 env 复核脚本', '缺少生产健康巡检 env 复核脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少生产健康巡检 env 复核脚本导致部署失败'); } function assertMissingPingoraReleaseReadinessFails() { @@ -1016,9 +1052,7 @@ function assertMissingPingoraReleaseReadinessFails() { '发布产物缺少 Pingora release readiness 聚合门禁脚本', '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora release readiness 聚合门禁脚本导致部署失败'); } function assertMissingPingoraHealthPatrolEnvSwitchFails() { @@ -1039,9 +1073,7 @@ function assertMissingPingoraHealthPatrolEnvSwitchFails() { '发布产物缺少 Pingora health patrol env 切换脚本', '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora health patrol env 切换脚本导致部署失败'); } function assertMissingPingoraGatewayEnvShadowSwitchFails() { @@ -1064,9 +1096,7 @@ function assertMissingPingoraGatewayEnvShadowSwitchFails() { '发布产物缺少 Pingora gateway env shadow 切换脚本', '缺少 Pingora gateway env shadow 切换脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora gateway env shadow 切换脚本导致部署失败'); } function assertMissingPingoraRealpathCanaryEnableFails() { @@ -1087,9 +1117,7 @@ function assertMissingPingoraRealpathCanaryEnableFails() { '发布产物缺少 Pingora realpath canary 启用脚本', '缺少 Pingora realpath canary 启用脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 启用脚本导致部署失败'); } function assertMissingPingoraRealpathCanaryDisableFails() { @@ -1110,9 +1138,7 @@ function assertMissingPingoraRealpathCanaryDisableFails() { '发布产物缺少 Pingora realpath canary 关闭脚本', '缺少 Pingora realpath canary 关闭脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 关闭脚本导致部署失败'); } function assertMissingPingoraTlsCertSyncFails() { @@ -1133,9 +1159,7 @@ function assertMissingPingoraTlsCertSyncFails() { '发布产物缺少 Pingora TLS 证书同步脚本', '缺少 Pingora TLS 证书同步脚本时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少 Pingora TLS 证书同步脚本导致部署失败'); } function assertMissingEnvExamplesFails() { @@ -1154,9 +1178,7 @@ function assertMissingEnvExamplesFails() { '发布产物缺少环境变量示例目录', '缺少环境变量示例目录时必须给出明确错误。', ); - if (!existsSync(fixture.maintenanceFile)) { - failures.push('部署失败时必须保持维护模式。'); - } + assertMaintenanceCleared(fixture, '缺少环境变量示例目录导致部署失败'); } function prepareFixture(name) { @@ -1432,6 +1454,19 @@ function prepareFixture(name) { [ '#!/usr/bin/env bash', `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'if [[ "${FAKE_CURL_FAIL:-false}" == "true" ]]; then', + ' exit 22', + 'fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'sleep'), + [ + '#!/usr/bin/env bash', + `printf 'sleep %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'exit 0', '', ].join('\n'), @@ -1478,6 +1513,7 @@ function prepareFixture(name) { ); chmodExecutable(path.join(fakeBin, 'systemctl')); chmodExecutable(path.join(fakeBin, 'curl')); + chmodExecutable(path.join(fakeBin, 'sleep')); chmodExecutable(path.join(fakeBin, 'cp')); chmodExecutable(path.join(fakeBin, 'sudo')); @@ -1543,29 +1579,33 @@ function runDeploy(fixture, options = {}) { fixture.sourceDir, 'scripts/deploy/production-api-deploy.sh', ); + const args = [ + deployScript, + '--source-dir', + fixture.sourceDir, + '--version', + options.version ?? fixture.version, + '--release-root', + options.releaseRoot ?? fixture.releaseRoot, + '--current-link', + options.currentLink ?? fixture.currentLink, + '--service', + 'genarrative-api.service', + '--health-url', + 'http://127.0.0.1:18082/readyz', + '--api-env-file', + options.apiEnvFile ?? fixture.apiEnvFile, + '--database', + 'genarrative-prod', + '--spacetime-server-url', + 'http://127.0.0.1:3101', + ]; + if (options.requirePingoraGateway) { + args.push('--require-pingora-gateway'); + } return spawnSync( 'bash', - [ - deployScript, - '--source-dir', - fixture.sourceDir, - '--version', - options.version ?? fixture.version, - '--release-root', - options.releaseRoot ?? fixture.releaseRoot, - '--current-link', - options.currentLink ?? fixture.currentLink, - '--service', - 'genarrative-api.service', - '--health-url', - 'http://127.0.0.1:18082/readyz', - '--api-env-file', - options.apiEnvFile ?? fixture.apiEnvFile, - '--database', - 'genarrative-prod', - '--spacetime-server-url', - 'http://127.0.0.1:3101', - ], + args, { cwd: process.cwd(), encoding: 'utf8', @@ -1578,6 +1618,7 @@ function runDeploy(fixture, options = {}) { options.pingoraDirectEntry === true ? 'true' : 'false', FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile, FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile, + FAKE_CURL_FAIL: options.curlFails === true ? 'true' : 'false', FAKE_CREATE_RELEASE_DURING_COPY: options.createReleaseDuringCopy === true ? 'true' : 'false', FAKE_RELEASE_ROOT: fixture.releaseRoot, diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index d5c0cc487..bd74fa9aa 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -30,6 +30,31 @@ const checks = [ reason: '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', }, + { + file: 'scripts/database-backup-to-oss.mjs', + includes: 'assertSufficientWorkDirSpace({dataDir, workDir, args, env})', + reason: '生产冷备份必须先做工作目录剩余空间预检，避免停库后写满磁盘。', + }, + { + file: 'scripts/database-backup-to-oss.mjs', + includes: 'restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter})', + reason: '生产冷备份打包失败时也必须恢复 SpacetimeDB 及依赖服务。', + }, + { + file: 'package.json', + includes: '"check:database-backup": "node scripts/check-database-backup-to-oss.mjs"', + reason: '数据库冷备份事故路径必须有本地 smoke 检查入口。', + }, + { + file: 'scripts/check-database-backup-to-oss.mjs', + includes: 'assertArchiveFailureStillRestoresDependentServices', + reason: '数据库冷备份 smoke 必须覆盖 tar 失败后恢复依赖服务。', + }, + { + file: 'scripts/check-database-backup-to-oss.mjs', + includes: 'assertInsufficientSpaceStopsBeforeServiceChanges', + reason: '数据库冷备份 smoke 必须覆盖空间不足时不触碰 systemctl。', + }, { file: 'deploy/systemd/genarrative-health-patrol.service', includes: 'scripts/ops/production-health-patrol.mjs', @@ -4756,11 +4781,29 @@ const checks = [ reason: 'API deploy 动态烟测必须覆盖 Pingora 二进制与 release manifest 漂移负例。', }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRequiresPingoraWhenRequested', + reason: + 'API deploy 动态烟测必须覆盖 --require-pingora-gateway 缺失产物负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch', + reason: + 'API deploy 动态烟测必须覆盖 current 切换后的 readiness 失败仍保留维护模式。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '--require-pingora-gateway', + reason: + 'API deploy 必须支持 Jenkins release 对 Pingora 产物做硬要求。', + }, { file: 'scripts/check-production-api-deploy.mjs', includes: 'assertMissingReleaseManifestFails', reason: - 'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护模式。', + 'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护回滚。', }, { file: 'scripts/check-production-api-deploy.mjs', @@ -4831,18 +4874,18 @@ const checks = [ { file: 'scripts/check-production-api-deploy.mjs', includes: 'assertMissingBackupScriptFails', - reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护模式。', + reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护回滚。', }, { file: 'scripts/check-production-api-deploy.mjs', includes: 'assertMissingHealthPatrolScriptFails', - reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护模式。', + reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护回滚。', }, { file: 'scripts/check-production-api-deploy.mjs', includes: 'assertMissingHealthPatrolEnvCheckFails', reason: - 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护模式。', + 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护回滚。', }, { file: 'scripts/check-production-api-deploy.mjs', diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index 79f408ad1..4d4e969a2 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node import {spawnSync} from 'node:child_process'; import {createHash, createHmac} from 'node:crypto'; -import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs'; +import {createReadStream, existsSync, mkdirSync, readFileSync, rmSync, statSync, statfsSync, writeFileSync} from 'node:fs'; import {basename, dirname, isAbsolute, resolve} from 'node:path'; import {fileURLToPath} from 'node:url'; @@ -12,6 +12,8 @@ const DEFAULT_LOCAL_DATA_DIR = resolve(REPO_ROOT, 'server-rs/.spacetimedb/local/ const DEFAULT_LOCAL_WORK_DIR = resolve(REPO_ROOT, 'server-rs/.data/database-backups'); const DEFAULT_PRODUCTION_DATA_DIR = '/stdb'; const DEFAULT_PRODUCTION_WORK_DIR = '/var/lib/genarrative/database-backups'; +const DEFAULT_SPACE_SAFETY_RATIO = 1.1; +const DEFAULT_EXTRA_FREE_BYTES = 512 * 1024 * 1024; const OSS_ALGORITHM = 'OSS4-HMAC-SHA256'; const OSS_SERVICE = 'oss'; const OSS_REQUEST = 'aliyun_v4_request'; @@ -36,6 +38,7 @@ function usage() { GENARRATIVE_DATABASE_BACKUP_OSS_PREFIX 对象前缀，默认 database-backups GENARRATIVE_DATABASE_BACKUP_OSS_ENDPOINT OSS endpoint；未设置时回退 ALIYUN_OSS_ENDPOINT GENARRATIVE_DATABASE_BACKUP_KEEP_LOCAL true 时保留本地 tar.gz + GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES 备份前要求 work dir 所在文件系统至少有这些可用字节；未设置时按数据目录大小估算 ALIYUN_OSS_ACCESS_KEY_ID / ALIYUN_OSS_ACCESS_KEY_SECRET `); } @@ -108,6 +111,7 @@ function parseArgs(argv) { manifestFile: '', objectKey: '', resultFile: '', + minFreeBytes: '', }; for (let index = 0; index < argv.length; index += 1) { @@ -182,6 +186,9 @@ function parseArgs(argv) { case '--result-file': options.resultFile = readValue(); break; + case '--min-free-bytes': + options.minFreeBytes = readValue(); + break; default: throw new Error(`未知参数: ${arg}`); } @@ -253,6 +260,119 @@ function runCommand(command, args, options = {}) { return result; } +function parseByteSize(rawValue, label) { + const value = String(rawValue ?? '').trim(); + if (!value) { + return null; + } + const match = /^(\d+)(?:\s*([KMGTPE]?)(?:I?B?)?)?$/iu.exec(value); + if (!match) { + throw new Error(`${label} 必须是字节数或 K/M/G/T/P/E 后缀大小，实际: ${rawValue}`); + } + const [, amountText, unitText = ''] = match; + const multipliers = { + '': 1n, + K: 1024n, + M: 1024n ** 2n, + G: 1024n ** 3n, + T: 1024n ** 4n, + P: 1024n ** 5n, + E: 1024n ** 6n, + }; + return BigInt(amountText) * multipliers[unitText.toUpperCase()]; +} + +function formatBytes(bytes) { + const value = BigInt(bytes); + const gib = Number(value) / (1024 ** 3); + if (gib >= 1) { + return `${gib.toFixed(1)}GiB`; + } + const mib = Number(value) / (1024 ** 2); + if (mib >= 1) { + return `${mib.toFixed(1)}MiB`; + } + return `${value}B`; +} + +function getDirectorySizeBytes(dataDir) { + const result = runCommand('du', ['-sk', dataDir]); + const [sizeKbText] = String(result.stdout ?? '').trim().split(/\s+/u); + if (!sizeKbText || !/^\d+$/u.test(sizeKbText)) { + throw new Error(`无法解析数据目录大小: ${result.stdout}`); + } + return BigInt(sizeKbText) * 1024n; +} + +function getAvailableBytes(fileSystemPath) { + const stat = statfsSync(fileSystemPath, {bigint: true}); + return stat.bavail * stat.bsize; +} + +function parseSafetyRatio(rawValue) { + const value = String(rawValue ?? '').trim(); + if (!value) { + return DEFAULT_SPACE_SAFETY_RATIO; + } + const ratio = Number(value); + if (!Number.isFinite(ratio) || ratio < 1) { + throw new Error(`GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO 必须是 >= 1 的数字，实际: ${rawValue}`); + } + return ratio; +} + +function calculateRequiredFreeBytes({dataSizeBytes, args, env}) { + const explicitMinFreeBytes = parseByteSize( + firstNonEmpty(args.minFreeBytes, env.GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES), + 'GENARRATIVE_DATABASE_BACKUP_MIN_FREE_BYTES', + ); + if (explicitMinFreeBytes !== null) { + return explicitMinFreeBytes; + } + const ratio = parseSafetyRatio(env.GENARRATIVE_DATABASE_BACKUP_SPACE_SAFETY_RATIO); + const ratioBasisPoints = BigInt(Math.ceil(ratio * 10000)); + const ratioRequirement = (dataSizeBytes * ratioBasisPoints + 9999n) / 10000n; + const extraFreeBytes = parseByteSize( + firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES, String(DEFAULT_EXTRA_FREE_BYTES)), + 'GENARRATIVE_DATABASE_BACKUP_EXTRA_FREE_BYTES', + ); + const extraRequirement = dataSizeBytes + extraFreeBytes; + return ratioRequirement > extraRequirement ? ratioRequirement : extraRequirement; +} + +function assertSufficientWorkDirSpace({dataDir, workDir, args, env}) { + mkdirSync(workDir, {recursive: true}); + const dataSizeBytes = getDirectorySizeBytes(dataDir); + const availableBytes = getAvailableBytes(workDir); + const requiredFreeBytes = calculateRequiredFreeBytes({dataSizeBytes, args, env}); + console.log( + `[database-backup] 备份空间预检: data=${formatBytes(dataSizeBytes)}, available=${formatBytes(availableBytes)}, required=${formatBytes(requiredFreeBytes)}`, + ); + if (availableBytes < requiredFreeBytes) { + throw new Error( + [ + `备份工作目录所在文件系统剩余空间不足: ${workDir}`, + `available=${formatBytes(availableBytes)}`, + `required=${formatBytes(requiredFreeBytes)}`, + `dataDir=${dataDir}`, + `dataSize=${formatBytes(dataSizeBytes)}`, + '为避免停库后写满磁盘，本次备份已在停止服务前中止。', + ].join('；'), + ); + } +} + +function collectRestartServicesAfterBackup({args, env}) { + const serviceNames = [ + ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean), + ...args.restartServicesAfter, + ]; + return [...new Set(serviceNames.filter(Boolean))]; +} + function stopServiceIfNeeded(serviceName) { if (!serviceName) { return false; @@ -271,12 +391,37 @@ function startServiceIfNeeded(serviceName, wasStopped) { } function restartServicesAfterBackup(serviceNames) { + const errors = []; for (const serviceName of serviceNames) { if (!serviceName) { continue; } console.log(`[database-backup] 冷备份后重启依赖服务: ${serviceName}`); - runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'}); + try { + runCommand('systemctl', ['restart', serviceName], {stdio: 'inherit'}); + } catch (error) { + errors.push(error); + } + } + if (errors.length > 0) { + throw new AggregateError(errors, `冷备份后重启依赖服务失败: ${errors.map((error) => error.message).join('; ')}`); + } +} + +function restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}) { + const errors = []; + try { + startServiceIfNeeded(stopService, serviceStopped); + } catch (error) { + errors.push(error); + } + try { + restartServicesAfterBackup(restartServicesAfter); + } catch (error) { + errors.push(error); + } + if (errors.length > 0) { + throw new AggregateError(errors, `恢复冷备份相关服务失败: ${errors.map((error) => error.message).join('; ')}`); } } @@ -523,19 +668,36 @@ async function main() { let archivePath = ''; let serviceStopped = false; + let backupError = null; + let restoreError = null; + const stopService = args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE); + const restartServicesAfter = collectRestartServicesAfterBackup({args, env}); try { - serviceStopped = stopServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE)); + assertSufficientWorkDirSpace({dataDir, workDir, args, env}); + serviceStopped = stopServiceIfNeeded(stopService); archivePath = createArchive({dataDir, workDir, fileName}); + } catch (error) { + backupError = error; } finally { - startServiceIfNeeded(args.stopService || firstNonEmpty(env.GENARRATIVE_DATABASE_BACKUP_STOP_SERVICE), serviceStopped); + try { + if (serviceStopped) { + restoreServicesAfterBackup({stopService, serviceStopped, restartServicesAfter}); + } else if (!backupError) { + restartServicesAfterBackup(restartServicesAfter); + } + } catch (error) { + restoreError = error; + } + } + if (backupError) { + if (restoreError) { + throw new AggregateError([backupError, restoreError], `数据库备份失败，且恢复依赖服务时也失败: ${backupError.message}; ${restoreError.message}`); + } + throw backupError; + } + if (restoreError) { + throw restoreError; } - restartServicesAfterBackup([ - ...String(env.GENARRATIVE_DATABASE_BACKUP_RESTART_SERVICE_AFTER ?? '') - .split(',') - .map((value) => value.trim()) - .filter(Boolean), - ...args.restartServicesAfter, - ]); const manifestPath = `${archivePath}.manifest.json`; writeManifest({ diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 716ce298f..0f3877046 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,14 +5,14 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] 说明: 进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。 默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。 若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。 - 若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，再启动或重启 Pingora 影子服务并复核 active。 - 失败时保留维护模式。 + 若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。 + current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。 EOF } @@ -347,17 +347,20 @@ ensure_pingora_shadow_service() { validate_release_manifest() { local manifest_path="$1" local require_pingora="$2" + local source_dir="$3" if [[ ! -f "${manifest_path}" ]]; then echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2 exit 1 fi - node - "${manifest_path}" "${require_pingora}" <<'NODE' + node - "${manifest_path}" "${require_pingora}" "${source_dir}" <<'NODE' const fs = require('fs'); +const path = require('path'); const manifestPath = process.argv[2]; const requirePingora = process.argv[3] === '1'; +const sourceDir = process.argv[4]; let manifest; try { manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); @@ -380,6 +383,16 @@ if (requirePingora) { process.exit(1); } } + +const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway'); +if (hasPingora) { + const binaryPath = path.join(sourceDir, 'pingora-gateway'); + const checksumPath = path.join(sourceDir, 'pingora-gateway.sha256'); + if (!fs.existsSync(binaryPath) || !fs.existsSync(checksumPath)) { + console.error('[production-api-deploy] release-manifest.json 登记了 pingora-gateway artifact，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。'); + process.exit(1); + } +} NODE } @@ -576,6 +589,9 @@ DATABASE="" SPACETIME_SERVER_URL="" DEPLOY_COMPLETED=0 PINGORA_INCLUDED=0 +REQUIRE_PINGORA_GATEWAY=0 +MAINTENANCE_ENABLED_BY_DEPLOY=0 +CURRENT_LINK_SWITCHED=0 RELEASE_DIR="" STAGING_RELEASE_DIR="" @@ -609,6 +625,10 @@ while [[ $# -gt 0 ]]; do PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}" shift 2 ;; + --require-pingora-gateway) + REQUIRE_PINGORA_GATEWAY=1 + shift + ;; --worker-service-pattern) WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}" shift 2 @@ -709,7 +729,14 @@ on_exit() { local exit_code=$? if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then cleanup_staging_release - echo "[production-api-deploy] 部署失败，保持维护模式。" >&2 + if [[ "${MAINTENANCE_ENABLED_BY_DEPLOY}" -eq 1 && "${CURRENT_LINK_SWITCHED}" -ne 1 ]]; then + echo "[production-api-deploy] 部署失败且尚未切换 current，退出本次打开的维护模式。" >&2 + if ! bash "${SCRIPT_DIR}/maintenance-off.sh"; then + echo "[production-api-deploy] 退出维护模式失败，请人工检查维护文件。" >&2 + fi + else + echo "[production-api-deploy] 部署失败，current 可能已切换或维护模式不是本次打开，保持维护模式。" >&2 + fi fi exit "${exit_code}" } @@ -717,6 +744,7 @@ on_exit() { trap on_exit EXIT bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" +MAINTENANCE_ENABLED_BY_DEPLOY=1 echo "[production-api-deploy] 校验 api-server" ( @@ -730,10 +758,14 @@ echo "[production-api-deploy] 校验 api-server" sha256sum -c pingora-gateway.sha256 fi ) +if [[ "${REQUIRE_PINGORA_GATEWAY}" -eq 1 && ( ! -f "${SOURCE_DIR}/pingora-gateway" || ! -f "${SOURCE_DIR}/pingora-gateway.sha256" ) ]]; then + echo "[production-api-deploy] 本次部署要求 Pingora，但发布目录缺少 pingora-gateway 或 pingora-gateway.sha256。" >&2 + exit 1 +fi if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then PINGORA_INCLUDED=1 fi -validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}" +validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "$(( PINGORA_INCLUDED || REQUIRE_PINGORA_GATEWAY ))" "${SOURCE_DIR}" mkdir -p "${RELEASE_ROOT}" mkdir "${STAGING_RELEASE_DIR}" @@ -960,6 +992,7 @@ fi mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}" STAGING_RELEASE_DIR="" ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}" +CURRENT_LINK_SWITCHED=1 if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}"