From 16ff10111f822a25be85153d9a01d96ba62d1545 Mon Sep 17 00:00:00 2001 From: kdletters <61648117+kdletters@users.noreply.github.com> Date: Tue, 29 Sep 2026 07:19:00 +0800 Subject: [PATCH] =?UTF-8?q?=E9=99=90=E5=88=B6=20dev=20=E5=8F=91=E5=B8=83?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E4=B8=8E=20Jenkins=20=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E6=9A=82=E5=AD=98=E7=9A=84=E5=8E=86=E5=8F=B2=E5=A0=86=E7=A7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - production-api-deploy.sh 新增 --keep-releases（默认 2）：发布成功后保留 current 目标与最近 1 个历史 release，只清理同时含 api-server 或 web 标记的旧目录，清理失败只告警、不改变发布结论 - production-api-deploy.sh 新增 prune_old_releases 与参数校验：非整数或小于 1 的 --keep-releases 在任何动作之前失败 - check-production-api-deploy 增加默认值、显式值和非法值三类夹具，并断言非法参数不会提升 release - check-production-ops 增加 release 保留策略与三个部署 Job 暂存清理的合同 - Api Deploy、Web Deploy、Stdb Publish 三个 Jenkinsfile 在部署或发布成功后只保留最近 2 个 build/ 暂存，失败时保留暂存便于诊断和重跑 - 内联清理片段避开 Groovy 反斜杠转义，并在 set -euo pipefail 下对空 build 目录安全（if [ -d build ] 守卫加兜底） - 开发运维文档补充 release 与 Jenkins 暂存的保留上限口径 - pitfalls 记录 dev 根盘写满的两处根因（发布与暂存无上限、JNLP 幽灵 agent 刷 syslog）以及 Jenkins 内联 shell 的两个坑 --- docs/project-memory/shared-memory/pitfalls.md | 17 +++ ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- jenkins/Jenkinsfile.production-api-deploy | 10 ++ ...Jenkinsfile.production-stdb-module-publish | 10 ++ jenkins/Jenkinsfile.production-web-deploy | 10 ++ scripts/check-production-api-deploy.mjs | 107 ++++++++++++++++++ scripts/check-production-ops-guardrails.mjs | 29 +++++ scripts/deploy/production-api-deploy.sh | 52 ++++++++- 8 files changed, 235 insertions(+), 2 deletions(-) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index b9dc496fc..86995991e 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -1,5 +1,22 @@ # 踩坑与排障记录 +## 2026-09-29 历史 release 与 Jenkins 构建暂存必须显式设置保留上限 + +- **现象**：dev 服务器根盘写满（`/dev/vda2` 47G/50G，可用 960M，99%）。两侧都只写不回收：`Genarrative-Api-Deploy` / `Genarrative-Web-Deploy` / `Genarrative-Stdb-Module-Publish` 三个 Job 工作区的 `build//` 暂存 9.8G，`/opt/genarrative/releases/` 下 68 个历史发布目录 9.2G；另有 journald 归档 700M 和轮转 syslog 437M。 +- **原因**：`production-api-deploy.sh` 用 `cp`（不是 `mv`）把 `${SOURCE_DIR}` 拷进 `${RELEASE_ROOT}/${VERSION}`，所以 Jenkins 工作区里的 `build//` 永远留着；dev 上构建 agent 与发布目标机是同一台机器，同一份产物在盘上存在两份。脚本和 Job 两侧都没有保留上限，按小时级 dev 发布节奏约 1G/天。 +- **处理**：两侧都补了保留上限。服务端：`production-api-deploy.sh` 新增 `--keep-releases`（默认 `2`），发布成功后保留 `current` 目标与最近 1 个历史 release，只删除同时含 `api-server` 或 `web` 标记的旧目录，清理失败只告警、不改变发布结论。CI 侧：`Genarrative-Api-Deploy` / `Genarrative-Web-Deploy` / `Genarrative-Stdb-Module-Publish` 在各自部署 / 发布步骤成功后只保留最近 2 个 `build//`，失败时不清理以便诊断和重跑。`npm run check:production-api-deploy` 增加默认值、显式值和非法值三类夹具，`npm run check:production-ops` 增加对应合同。 +- **不要踩的坑**：直接按 mtime 排序删除会连带删掉发布根目录下不属于发布产物的目录（例如 `dev-mcp-host-*`），必须用 `api-server`/`web` 标记筛选；脚本里的 `mv -T`、`find -printf` 都是 GNU 语义，`npm run check:production-api-deploy` 需要 `sha256sum` 和 `/usr/bin/cp`，Windows 本地跑不了，只在 Linux CI / Linux 检出上有效（本地最低限度用 `bash -n` + `npm run check:production-ops`）。 +- **写 Jenkins 内联 shell 的两个坑**：① Groovy 会处理 `sh '''…'''` / `sh """…"""` 里的反斜杠转义——`\n` 到 shell 手上会变成真实换行（`Jenkinsfile.production-stdb-module-build` 里必须写 `printf "\\r"` 就是同一件事），所以内联片段要么完全不用 `\`，要么写 `\\`；`"""` 是 GString，shell 变量必须写 `\$name`，而 `'''` 不插值、保持 `${name}`。② `set -euo pipefail` 下 `ls build/*/` 在 glob 不匹配时会因 pipefail 把整个部署步骤判失败（实测：`build/` 为空时清理步骤会把一次成功发布判成失败），必须用 `if [ -d build ]` 守卫 + `|| true` 兜底。 +- **关联**：`scripts/deploy/production-api-deploy.sh`、`scripts/check-production-api-deploy.mjs`、`scripts/check-production-ops-guardrails.mjs`、`jenkins/Jenkinsfile.production-api-deploy`、`jenkins/Jenkinsfile.production-web-deploy`、`jenkins/Jenkinsfile.production-stdb-module-publish`。 + +## 2026-09-29 dev 上的 JNLP inbound agent 是历史残留，会在死端口上无限重连刷爆 syslog + +- **现象**：dev 的 `/var/log/syslog` 约 250MB/天，内容是 `jenkins-inbound-agent-start[pid]` 反复输出指向 `http://127.0.0.1:18080/tcpSlaveAgentListener/` 的 `Connection refused` 完整栈（2.6 天 61 万行，其中 `genarrative-release-deploy-01` 占 53 万行）。 +- **原因**：`/etc/jenkins-agent/*.env` 里 `JENKINS_URL=http://127.0.0.1:18080/`，而该机只有 station 反向 ssh 提供的 `18085/18086/18087`，没有 `18080`。真实部署通道是 SSH launcher（`java -jar remoting.jar -workDir /root/jenkins-agent-build`，sshd 会话子进程），Jenkins 工作区、`current` 切换和当日 01:18 的发布都由它完成；两个 JNLP 单元从未连上（`release-deploy-01` 的 workdir 自 2026-05-09 起没有任何 workspace）。 +- **处理**：停止并 disable `jenkins-agent@genarrative-build-01.service` 与 `jenkins-agent@genarrative-release-deploy-01.service`；观测窗口内 `Failed to connect` 新增归零，`genarrative-api` / `spacetimedb` / worker / pingora 保持 active。回滚用 `systemctl enable --now jenkins-agent@.service`。 +- **不要踩的坑**：`ss -t` 在没有状态过滤时不显示 LISTEN，连接被瞬拒时也抓不到 TCP 连接，不能用它判断 agent 是否在线；要看 `journalctl -u jenkins-agent@`、`ss -tlnp` 的端口和 Jenkins 工作区归属。停 JNLP 单元前先确认控制器 `slaveAgentPort=-1` 且 SSH launcher 仍在跑，避免把唯一通道停掉。 +- **关联**：`/etc/systemd/system/jenkins-agent@.service`、`/etc/jenkins-agent/*.env`、`scripts/deploy/install-jenkins-inbound-agent.sh`。 + ## AGC 版本探测必须显式提供隔离用户目录 清空子进程环境后缺少 `USERPROFILE/HOME` 会让 npm 依赖 Windows 后备用户查询，部分机器报 `uv_os_homedir` / `ENOMEM`。正常机器探测成功不能证明环境完整，需同时验证子进程环境。版本探测使用临时用户目录和空 npm 配置，详见 [AGC 实施计划](../../technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md) 的“Web 环境版本探测的用户目录隔离”。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index fba53ca16..097824391 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -808,7 +808,7 @@ Jenkins Copy Artifact 必须保持 `Production` 权限模式；产物生产者 `Genarrative-Web-Build` 会把 `build//web.tar.gz`、`web.tar.gz.sha256`、`release-manifest.json` 和 `scripts/deploy/production-web-deploy.sh` 直接归档为 Jenkins 构建产物；`Genarrative-Web-Deploy` 只通过 `copyArtifacts` 从指定上游构建复制这些产物和部署脚本，不再在目标机器 checkout Git，再执行随构建归档的 `scripts/deploy/production-web-deploy.sh`。Web 发布不再读取构建机本地缓存目录，也不再通过 release agent `rsync` 回构建机拉取大包；如果 deploy 找不到 `web.tar.gz`，应先检查上游 Web Build 是否按同一 `BUILD_VERSION` 成功归档产物。 -`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会在 current 切换前停止部署，清理 staging 并退出本次打开的维护模式；current 切换后的 Pingora 重启、worker 重启、controller 启动或 readiness 失败仍保留维护模式，避免暴露半发布版本；失败不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、Pingora manifest / 二进制漂移、`--require-pingora-gateway` 缺少 Pingora、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会在 current 切换前失败并退出本次打开的维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现时拒绝覆盖 / 合并，以及 current 切换后的 readiness 失败会保留维护模式。Pingora 影子网关在 `Genarrative-Api-Build`、`Genarrative-Api-Deploy` 和 `Genarrative-Full-Build-And-Deploy` 中默认随 release 构建、归档、复制并用 `--require-pingora-gateway` 硬校验；只有显式取消 `INCLUDE_PINGORA_GATEWAY` 时才允许 API release 不带 `pingora-gateway` / `pingora-gateway.sha256`。本地 CLI 仍保留显式 `npm run build:production-release -- --component api-server --include-pingora-gateway`，用于在需要 Pingora 的手工发布包里登记 manifest 和 checksum；Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。发布包包含 Pingora 时，deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载新网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。 +`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会在 current 切换前停止部署，清理 staging 并退出本次打开的维护模式；current 切换后的 Pingora 重启、worker 重启、controller 启动或 readiness 失败仍保留维护模式，避免暴露半发布版本；失败不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。发布成功后 `production-api-deploy.sh` 按 `--keep-releases`（默认 `2`，最小 `1`）只保留 `current` 目标与最近 1 个历史 release，清理更早的旧发布目录；清理范围只包含同时含 `api-server` 或 `web` 标记的目录，隐藏 staging 目录、符号链接和不含这两个标记的目录一律跳过，清理失败只输出告警、不改变本次发布结论。需要更长回滚窗口时显式调大该参数；Jenkins 工作区里的 `build//` 暂存不属于该脚本职责，仍由 Jenkins 侧自行控制。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、Pingora manifest / 二进制漂移、`--require-pingora-gateway` 缺少 Pingora、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会在 current 切换前失败并退出本次打开的维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现时拒绝覆盖 / 合并，以及 current 切换后的 readiness 失败会保留维护模式。Pingora 影子网关在 `Genarrative-Api-Build`、`Genarrative-Api-Deploy` 和 `Genarrative-Full-Build-And-Deploy` 中默认随 release 构建、归档、复制并用 `--require-pingora-gateway` 硬校验；只有显式取消 `INCLUDE_PINGORA_GATEWAY` 时才允许 API release 不带 `pingora-gateway` / `pingora-gateway.sha256`。本地 CLI 仍保留显式 `npm run build:production-release -- --component api-server --include-pingora-gateway`，用于在需要 Pingora 的手工发布包里登记 manifest 和 checksum；Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。发布包包含 Pingora 时，deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载新网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。 Pingora shadow 部署前检查必须按首个权威错误 fail-fast：读取 systemd 最终配置失败或发现 `CAP_NET_BIND_SERVICE` 后，不得继续把空的 EnvironmentFile 解析结果传给 shadow listen 校验，也不得再输出“缺少 EnvironmentFile”或“LISTEN 为空”的连带误报。`npm run check:production-api-deploy` 的 direct-entry fixture 会锁定这一错误顺序，避免值班人员被多条互相矛盾的诊断引向错误配置。 diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index 1c72797a0..323f5abbe 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -123,6 +123,16 @@ pipeline { --bgfilter-worker-env-file "${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}" \ --database "${DATABASE}" \ --spacetime-server-url "${SPACETIME_SERVER_URL:-http://127.0.0.1:3101}" + # 只保留最近 2 个 build/ 暂存：每次部署都用 copyArtifacts 重新落一份， + # 历史暂存不参与发布、只占目标机磁盘；部署失败时不清理，便于诊断和重跑。 + if [ -d build ]; then + stale_list="$(ls -1dt build/*/ 2>/dev/null | tail -n +3 || true)" + while IFS= read -r stale_staging; do + [ -n "${stale_staging}" ] || continue + echo "[staging-cleanup] 清理历史构建暂存: ${stale_staging}" + rm -rf --one-file-system "${stale_staging}" + done <<< "${stale_list}" + fi ' ''' } diff --git a/jenkins/Jenkinsfile.production-stdb-module-publish b/jenkins/Jenkinsfile.production-stdb-module-publish index 3aa58b036..a4f57404b 100644 --- a/jenkins/Jenkinsfile.production-stdb-module-publish +++ b/jenkins/Jenkinsfile.production-stdb-module-publish @@ -148,6 +148,16 @@ pipeline { --worker-env-file "${params.WORKER_ENV_FILE}" \\ ${keepMaintenanceArg} \\ ${backupArg} + # 只保留最近 2 个 build/ 暂存：每次发布都用 copyArtifacts 重新落一份， + # 历史暂存不参与发布、只占目标机磁盘；发布失败时不清理，便于诊断和重跑。 + if [ -d build ]; then + stale_list="$(ls -1dt build/*/ 2>/dev/null | tail -n +3 || true)" + while IFS= read -r stale_staging; do + [ -n "\$stale_staging" ] || continue + echo "[staging-cleanup] 清理历史构建暂存: \$stale_staging" + rm -rf --one-file-system "\$stale_staging" + done <<< "\$stale_list" + fi ' """ } diff --git a/jenkins/Jenkinsfile.production-web-deploy b/jenkins/Jenkinsfile.production-web-deploy index 36bc9571d..1cec39fe6 100644 --- a/jenkins/Jenkinsfile.production-web-deploy +++ b/jenkins/Jenkinsfile.production-web-deploy @@ -84,6 +84,16 @@ pipeline { --release-root "${RELEASE_ROOT}" \ --current-link "${CURRENT_LINK}" \ --web-link "${WEB_LINK}" + # 只保留最近 2 个 build/ 暂存：每次部署都用 copyArtifacts 重新落一份， + # 历史暂存不参与发布、只占目标机磁盘；部署失败时不清理，便于诊断和重跑。 + if [ -d build ]; then + stale_list="$(ls -1dt build/*/ 2>/dev/null | tail -n +3 || true)" + while IFS= read -r stale_staging; do + [ -n "${stale_staging}" ] || continue + echo "[staging-cleanup] 清理历史构建暂存: ${stale_staging}" + rm -rf --one-file-system "${stale_staging}" + done <<< "${stale_list}" + fi ' ''' } diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 0a988c3de..2d0876f64 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -9,6 +9,7 @@ import { readFileSync, readlinkSync, rmSync, + utimesSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; @@ -95,6 +96,9 @@ function main() { assertRealWechatPayUsesLastRefundReconciliationAssignment(); assertDeployCleansStagingReleaseOnFailure(); assertDeployRejectsFinalReleaseRaceAndCleansStaging(); + assertDeployPrunesOldReleases(); + assertDeployHonorsExplicitKeepReleases(); + assertDeployRejectsInvalidKeepReleases(); assertMissingBackupScriptFails(); assertMissingHealthPatrolScriptFails(); assertMissingPingoraCurrentReleaseAuditFails(); @@ -1754,6 +1758,106 @@ function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { assertMaintenanceCleared(fixture, '目标 release 竞态失败'); } +function assertDeployPrunesOldReleases() { + const fixture = prepareFixture('prune-old-releases'); + const nowSeconds = Date.now() / 1000; + const seededReleases = [ + ['20260610-oldest', nowSeconds - 3000], + ['20260611-middle', nowSeconds - 2000], + ['20260613-previous', nowSeconds - 1000], + ]; + for (const [name, mtimeSeconds] of seededReleases) { + const dir = path.join(fixture.releaseRoot, name); + mkdirSync(path.join(dir, 'web'), { recursive: true }); + writeFileSync(path.join(dir, 'web', 'index.html'), `${name}\n`, 'utf8'); + utimesSync(dir, mtimeSeconds, mtimeSeconds); + } + // 发布根目录下不含 api-server/web 的目录不属于发布产物，清理时必须保留。 + const unrelatedDir = path.join(fixture.releaseRoot, 'dev-mcp-host-cache'); + mkdirSync(unrelatedDir, { recursive: true }); + writeFileSync(path.join(unrelatedDir, 'keep.txt'), 'keep\n', 'utf8'); + utimesSync(unrelatedDir, nowSeconds - 4000, nowSeconds - 4000); + + const result = runDeploy(fixture); + + assertStatus(result, 0, '存在历史 release 时完整 fixture 仍应部署成功。'); + if (result.status !== 0) { + return; + } + assertFileExists( + path.join(fixture.releaseRoot, fixture.version), + '本此发布目录必须保留。', + ); + assertFileExists( + path.join(fixture.releaseRoot, '20260613-previous'), + '默认 --keep-releases 2 必须保留最近一个历史 release 以便回滚。', + ); + for (const name of ['20260611-middle', '20260610-oldest']) { + if (existsSync(path.join(fixture.releaseRoot, name))) { + failures.push( + `默认 --keep-releases 2 必须清理更早的历史 release: ${name}`, + ); + } + } + assertFileExists( + path.join(unrelatedDir, 'keep.txt'), + '清理历史 release 不得删除发布根目录下不含 api-server/web 的目录。', + ); + assertIncludes( + result.stdout, + '清理历史 release:', + '清理历史 release 时必须输出被清理的目录。', + ); +} + +function assertDeployHonorsExplicitKeepReleases() { + const fixture = prepareFixture('keep-single-release'); + const nowSeconds = Date.now() / 1000; + const previousReleaseDir = path.join( + fixture.releaseRoot, + '20260613-previous', + ); + mkdirSync(path.join(previousReleaseDir, 'web'), { recursive: true }); + writeFileSync( + path.join(previousReleaseDir, 'web', 'index.html'), + 'previous\n', + 'utf8', + ); + utimesSync(previousReleaseDir, nowSeconds - 1000, nowSeconds - 1000); + + const result = runDeploy(fixture, { keepReleases: 1 }); + + assertStatus(result, 0, '--keep-releases 1 时完整 fixture 仍应部署成功。'); + if (result.status !== 0) { + return; + } + assertFileExists( + path.join(fixture.releaseRoot, fixture.version), + '--keep-releases 1 必须保留本次发布目录。', + ); + if (existsSync(previousReleaseDir)) { + failures.push('--keep-releases 1 必须清理除 current 以外的历史 release。'); + } +} + +function assertDeployRejectsInvalidKeepReleases() { + const fixture = prepareFixture('invalid-keep-releases'); + const result = runDeploy(fixture, { keepReleases: '0' }); + + assertStatus(result, 1, '--keep-releases 0 必须被拒绝。'); + assertIncludes( + result.stderr, + '--keep-releases 必须是大于等于 1 的整数', + '--keep-releases 非法时必须给出明确错误。', + ); + assertNotIncludes( + readOptionalCommandsLog(fixture), + 'systemctl restart genarrative-api.service', + '--keep-releases 非法时不得重启 API 服务。', + ); + assertNoReleasePromoted(fixture, '--keep-releases 非法时不得提升 release'); +} + function assertMissingPingoraDirectCheckFails() { const fixture = prepareFixture('missing-direct-live'); rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs')); @@ -2785,6 +2889,9 @@ function runDeploy(fixture, options = {}) { if (options.keepMaintenance) { args.push('--keep-maintenance-mode'); } + if (options.keepReleases !== undefined) { + args.push('--keep-releases', String(options.keepReleases)); + } return spawnSync('bash', args, { cwd: process.cwd(), encoding: 'utf8', diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 10420911d..07d831a77 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -428,6 +428,24 @@ const checks = [ '--bgfilter-worker-env-file "${BGFILTER_WORKER_ENV_FILE:-/etc/genarrative/bgfilter-worker.env}"', reason: 'API Deploy Job 必须把 BgFilter worker env 路径传给发布脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: '[staging-cleanup]', + reason: + 'API Deploy Job 必须清理历史 build/ 暂存，避免目标机被 copyArtifacts 暂存撑满。', + }, + { + file: 'jenkins/Jenkinsfile.production-web-deploy', + includes: '[staging-cleanup]', + reason: + 'Web Deploy Job 必须清理历史 build/ 暂存，避免目标机被 copyArtifacts 暂存撑满。', + }, + { + file: 'jenkins/Jenkinsfile.production-stdb-module-publish', + includes: '[staging-cleanup]', + reason: + 'Stdb Publish Job 必须清理历史 build/ 暂存，避免目标机被 copyArtifacts 暂存撑满。', + }, { file: 'jenkins/Jenkinsfile.production-full-build-and-deploy', includes: @@ -475,6 +493,17 @@ const checks = [ reason: 'API deploy 必须在 current 切换前复核真实微信支付退款 reconciliation。', }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'prune_old_releases', + reason: + 'API deploy 必须在发布成功后按 --keep-releases 清理历史 release 目录，避免目标机根盘被历史发布撑满。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '--keep-releases 必须是大于等于 1 的整数', + reason: 'API deploy 必须拒绝非法的 --keep-releases 参数。', + }, { file: 'jenkins/Jenkinsfile.production-stdb-module-publish', includes: diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 5919fa220..4eb973d04 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,7 +5,7 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url ] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--require-pingora-gateway] [--bgfilter-worker-service genarrative-bgfilter-worker.service] [--bgfilter-worker-health-url ] [--bgfilter-worker-env-file /etc/genarrative/bgfilter-worker.env] [--no-bgfilter-worker] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--controller-env-file /etc/genarrative/external-generation-controller.env] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--worker-env-file /etc/genarrative/external-generation-worker.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] [--keep-maintenance-mode] [--keep-releases 2] 说明: 进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。 @@ -16,6 +16,7 @@ usage() { 若发布包包含 pingora-gateway，或传入 --require-pingora-gateway，部署脚本会要求 release manifest、二进制与 checksum 一致，再在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，启动或重启 Pingora 影子服务并复核 active。 默认在 readiness 通过后退出维护模式；传入 --keep-maintenance-mode 时保留维护文件，供人工验收后再恢复公网。 current 链接切换前失败时会退出本次打开的维护模式；current 链接切换后失败时保留维护模式，避免暴露半发布版本。 + 发布成功后按 --keep-releases（默认 2）保留 current 目标与最近的历史 release 目录，清理其它含 api-server 或 web 的旧发布目录，避免目标机根盘被历史发布撑满；该清理失败只告警，不改变本次发布结论。 EOF } @@ -1177,6 +1178,7 @@ DEPLOY_COMPLETED=0 PINGORA_INCLUDED=0 REQUIRE_PINGORA_GATEWAY=0 KEEP_MAINTENANCE_MODE=0 +KEEP_RELEASES=2 MAINTENANCE_ENABLED_BY_DEPLOY=0 MAINTENANCE_FILE="${GENARRATIVE_MAINTENANCE_FILE:-/var/lib/genarrative/maintenance/enabled}" CURRENT_LINK_SWITCHED=0 @@ -1222,6 +1224,10 @@ while [[ $# -gt 0 ]]; do KEEP_MAINTENANCE_MODE=1 shift ;; + --keep-releases) + KEEP_RELEASES="${2:?缺少 --keep-releases 的值}" + shift 2 + ;; --worker-service-pattern) WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}" shift 2 @@ -1290,6 +1296,10 @@ done require_argument "${SOURCE_DIR}" "--source-dir" require_absolute_path "${RELEASE_ROOT}" "--release-root" require_absolute_path "${CURRENT_LINK}" "--current-link" +if [[ ! "${KEEP_RELEASES}" =~ ^[0-9]+$ ]] || (( KEEP_RELEASES < 1 )); then + echo "[production-api-deploy] --keep-releases 必须是大于等于 1 的整数: ${KEEP_RELEASES}" >&2 + exit 1 +fi require_absolute_path "${API_ENV_FILE}" "--api-env-file" if [[ -n "${WORKER_ENV_FILE}" ]]; then require_absolute_path "${WORKER_ENV_FILE}" "--worker-env-file" @@ -1359,6 +1369,44 @@ cleanup_rendered_systemd_unit() { RENDERED_SYSTEMD_UNIT_FILE="" } +# 保留 current 目标与最近 keep_total-1 个其它发布目录。 +# 只清理同时不是符号链接、名称不以点开头且含 api-server 或 web 的目录， +# 避免误删发布根目录下不属于发布产物的目录。 +prune_old_releases() { + local keep_total="$1" + local current_target="" + local keep_others=$(( keep_total - 1 )) + local kept_others=0 + local candidate resolved + + if [[ -L "${CURRENT_LINK}" ]]; then + current_target="$(readlink -f "${CURRENT_LINK}" 2>/dev/null || true)" + fi + + while IFS= read -r candidate; do + [[ -n "${candidate}" ]] || continue + case "${candidate}" in + *[[:space:]]*) continue ;; + esac + [[ -d "${candidate}" && ! -L "${candidate}" ]] || continue + [[ -e "${candidate}/api-server" || -e "${candidate}/web" ]] || continue + resolved="$(readlink -f "${candidate}")" + if [[ -n "${current_target}" && "${resolved}" == "${current_target}" ]]; then + continue + fi + if (( kept_others < keep_others )); then + kept_others=$(( kept_others + 1 )) + continue + fi + echo "[production-api-deploy] 清理历史 release: ${candidate}" + rm -rf --one-file-system "${resolved}" + done < <( + find "${RELEASE_ROOT}" -mindepth 1 -maxdepth 1 -type d -name '[!.]*' -printf '%T@ %p\n' 2>/dev/null | + sort -rn | + cut -d' ' -f2- + ) +} + on_exit() { local exit_code=$? cleanup_rendered_systemd_unit @@ -1697,6 +1745,8 @@ for _ in {1..30}; do bash "${SCRIPT_DIR}/maintenance-off.sh" fi DEPLOY_COMPLETED=1 + prune_old_releases "${KEEP_RELEASES}" || + echo "[production-api-deploy] 历史 release 清理失败（仅影响磁盘占用，不影响本次发布结论）" >&2 echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server" exit 0 fi