diff --git a/deploy/env/health-patrol.env.example b/deploy/env/health-patrol.env.example new file mode 100644 index 000000000..b1b9799e5 --- /dev/null +++ b/deploy/env/health-patrol.env.example @@ -0,0 +1,19 @@ +# 复制到 /etc/genarrative/health-patrol.env 后按生产环境调整。 +# 默认不启用 Pingora shadow 巡检;只有同时配置 base URL 与 probe token 才会检查。 + +GENARRATIVE_HEALTH_PATROL_API_BASE_URL=http://127.0.0.1:8082 +GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL=http://127.0.0.1:3101 +GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1 +# 默认公网入口仍按 Nginx 巡检;Pingora 直连切换后改为 pingora-direct。 +GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx +# 本机打 127.0.0.1 时可填正式域名,确保命中正确 vhost / 证书语义。 +GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST= +GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000 +GENARRATIVE_HEALTH_PATROL_SLOW_MS=3000 +GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING=false +GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL=false +GENARRATIVE_HEALTH_PATROL_WEBHOOK_URL= + +# Pingora 影子网关巡检;probe token 应与 /etc/genarrative/pingora-gateway.env 保持一致。 +GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL= +GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN= diff --git a/deploy/env/pingora-canary-live.env.example b/deploy/env/pingora-canary-live.env.example new file mode 100644 index 000000000..9cbff3728 --- /dev/null +++ b/deploy/env/pingora-canary-live.env.example @@ -0,0 +1,35 @@ +# 复制到本机临时 shell 或 CI secret 环境后运行: +# npm run check:pingora-canary-live +# 该检查只读访问已经人工启用的 Nginx -> Pingora canary,不会修改 Nginx。 + +# 例如 http://127.0.0.1 或 https://genarrative.world。 +GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 + +# 默认前缀必须和 deploy/nginx/snippets/genarrative-pingora-canary.conf 保持一致。 +GENARRATIVE_PINGORA_CANARY_PREFIX=/__genarrative_pingora_canary + +# 可选:真实路径 canary 使用 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf。 +# 启用后运行: +# GENARRATIVE_PINGORA_CANARY_MODE=realpath \ +# GENARRATIVE_PINGORA_CANARY_BASE_URL=${GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL} \ +# npm run check:pingora-canary-live +# 或使用 release readiness: +# npm run check:pingora-release-readiness -- --require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> +GENARRATIVE_PINGORA_CANARY_MODE=prefix +GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL=http://127.0.0.1:18083 +GENARRATIVE_PINGORA_REALPATH_CANARY_HOST= + +# 本机直连 127.0.0.1 时必须设置正式域名,避免只命中 Nginx 默认 vhost;release readiness --require-live 必须提供。只填域名或 host:port,不要填 https:// URL。 +GENARRATIVE_PINGORA_CANARY_HOST= + +GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS=5000 + +# live smoke 后必须继续按 request_id 对照 Nginx handoff access log 与 Pingora access log。 +GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG=/var/log/nginx/genarrative.access.log +GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log +GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES=2000 + +# 真实路径 canary 使用独立 Nginx access log,避免和生产真实用户请求混在一起。 +GENARRATIVE_PINGORA_REALPATH_CANARY_NGINX_ACCESS_LOG=/var/log/nginx/genarrative-pingora-realpath-canary.access.log +GENARRATIVE_PINGORA_REALPATH_CANARY_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log +GENARRATIVE_PINGORA_REALPATH_CANARY_ACCESS_LOG_SINCE_LINES=2000 diff --git a/deploy/env/pingora-direct-live.env.example b/deploy/env/pingora-direct-live.env.example new file mode 100644 index 000000000..b3d04c609 --- /dev/null +++ b/deploy/env/pingora-direct-live.env.example @@ -0,0 +1,68 @@ +# 复制到本机临时 shell 或 CI secret 环境后运行: +# npm run check:pingora-direct-live +# 该检查只读访问已经显式配置 TLS_LISTEN / HTTP_REDIRECT_LISTEN 的 pingora-gateway。 +# 切换窗口建议先运行: +# npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free +# release readiness --require-direct 必须确认 TLS/HTTP redirect 端口已释放,复核 health patrol env 已切到 direct 模式,并提供 Pingora access log 路径用于 request_id 落盘校验。 +# 直连启用前: +# /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status +# /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log +# 启用脚本会先执行 current release 自审,确认 pingora-gateway 可执行且 systemd ExecStart 指向 current release;自审失败时不会安装 direct-entry drop-in。 +# 自审和 preflight 都通过后,启用脚本才会安装 drop-in、重启 Pingora、执行 direct live smoke,并强制 WSS subscribe 返回 101,同时用生成的 request_id 反查 Pingora access log。 +# 直连接管后同步更新 /etc/genarrative/health-patrol.env: +# GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct +# GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://127.0.0.1 +# GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名> +# 然后用 node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs 复核。 +# 直连验证失败时: +# /opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> +# 回退脚本会先执行 nginx -t,reload 后确认 nginx.service 仍为 active,并用 smoke URL 证明 Nginx 入口可访问。 +# 回退后把 GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 改回 nginx。 + +# 例如 https://127.0.0.1 或 https://genarrative.world。 +GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL=https://127.0.0.1 + +# 单独 direct live 可选;release readiness --require-direct 必须提供,用于校验 HTTP 入口 301 到 HTTPS,并确认 ACME challenge 没被重定向吞掉。 +GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL=http://127.0.0.1 + +# 本机直连 127.0.0.1 时必须设置正式域名;direct live 会同时用作 Host header 和 TLS SNI。release readiness --require-direct 必须提供。只填域名或 host:port,不要填 https:// URL。 +GENARRATIVE_PINGORA_DIRECT_HOST= + +# 单独 direct live 可选;release readiness --require-direct 必须提供,用于显式校验 HTTP redirect Location 的 host。只填域名或 host:port。 +GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST= + +# 可选:高端口 rehearsal 访问 127.0.0.1:18443、但期望 Location 指向正式域名默认 HTTPS 入口时设置;只能填 https:// base URL,不要带路径、查询或片段。 +GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL= + +# 可选:配置后会额外检查 /__genarrative_pingora/healthz 内部探针。 +GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN= + +# direct live 默认会对 wss:///v1/database//subscribe 做 SpacetimeDB 握手检查;release readiness --require-direct 必须按目标环境显式设置。 +GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE=genarrative-prod + +# release readiness --require-direct 必须显式提供;direct live 会用每个请求生成的 X-Request-Id 反查 Pingora access log,证明直连流量真实进入 Pingora。 +GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log +GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES=2000 + +# direct live 单独运行时可设 true,要求 WSS subscribe 必须返回 101;release readiness --require-direct 会自动强制该口径。 +GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE=true + +# 只允许单独 direct live 临时排障时跳过 WSS;release readiness --require-direct 会直接拒绝,正式切流必须保持 false。 +GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=false + +# 仅本机自签证书 smoke 使用;生产必须保持 false。 +GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=false + +GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS=5000 + +# release readiness --require-direct 会先跑 direct-entry preflight;提供 env 文件后会强制检查 TLS/redirect/cert/key 配置。 +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE=/etc/genarrative/pingora-gateway.env +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true +GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE=true + +# release readiness --require-direct 必须复核健康巡检 env,避免切直连后巡检仍按 Nginx 模式误报。 +GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE=/etc/genarrative/health-patrol.env diff --git a/deploy/logrotate/genarrative-pingora-gateway b/deploy/logrotate/genarrative-pingora-gateway new file mode 100644 index 000000000..660bbd380 --- /dev/null +++ b/deploy/logrotate/genarrative-pingora-gateway @@ -0,0 +1,10 @@ +/var/log/genarrative/pingora-gateway.access.log { + daily + rotate 14 + missingok + notifempty + compress + delaycompress + copytruncate + create 0640 genarrative genarrative +} diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 054734dee..98cddbc0d 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -13,15 +13,69 @@ - `deploy/nginx/genarrative.conf` 与 `deploy/nginx/genarrative-dev-http.conf` 默认开启 gzip。 - 覆盖 `application/json`,用于降低 `/api/runtime/*/gallery` 这类 JSON 列表接口的公网带宽占用。 - 当前推荐等级为 `gzip_comp_level 5`,兼顾 2C/2G 服务器 CPU 与压缩收益。 +- Pingora 影子网关已通过 `GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip`、`GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED=true`、`GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL=5` 与 `GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES=1024` 承接 gzip parity;网关会把压缩算法收敛到 gzip allowlist,并按 `gzip_min_length 1024` 避免压缩小响应,同时沿用 Nginx `gzip_types` 边界不压缩图片资源。改动后必须运行 `npm run check:pingora-gateway-smoke` 验证小响应不压缩、图片资源不压缩,以及 `Accept-Encoding: gzip` 和 `Accept-Encoding: br, gzip` 的大响应都返回 gzip。 ## Brotli - Brotli 只在目标服务器 Nginx 接受 brotli 指令时开启。 +- Pingora 当前正式化口径固定为 gzip-only,不承接 Brotli;做 Nginx -> Pingora canary 时,Brotli 仍由 Nginx 或更前置代理决定是否启用。直连 Pingora 时,不把 Brotli parity 作为切换门禁,避免使用 Pingora 0.8.1 里尚未通过可解压端到端验证的 Brotli compressor 路径。 - Ubuntu / apt 系统的 `Genarrative-Server-Provision` 会安装 `libnginx-mod-http-brotli-filter` 与 `libnginx-mod-http-brotli-static`;非 apt 系统暂不自动安装,仍按下面的能力探测结果决定是否启用。 - Provision 脚本通过临时配置执行 `nginx -t` 做能力探测;探测配置会先 `include /etc/nginx/modules-enabled/*.conf`,避免 Ubuntu 动态模块已安装但测试配置未加载模块导致误判。可用时把模板中的 `# __GENARRATIVE_BROTLI_DIRECTIVES__` 替换为 brotli 指令,不可用时保留注释说明。 - 不要直接在静态模板里无条件写 `brotli on;`,否则没有 brotli 模块的服务器会 `nginx -t` 失败并回滚。 - 不要用 `nginx -V | grep brotli` 判断 brotli 是否可用;Ubuntu apt 安装的 brotli 是动态模块,不会出现在普通编译参数里。应检查包、`/etc/nginx/modules-enabled/` 的 `load_module` 配置,或用包含 `include /etc/nginx/modules-enabled/*.conf` 的临时配置执行 `nginx -t`。 +## Pingora upstream timeouts + +- Pingora 影子网关已显式配置上游连接 / 读 / 写 timeout,避免正式 canary 时依赖 Pingora 默认值。 +- 默认连接超时为 `3000ms`;没有 Nginx 显式长超时的代理路由读取超时为 `60s`。 +- 通用 `/api`、公开列表 / 详情和 SpacetimeDB subscribe 读取超时为 `3600s`,写上游超时为 `3600s`,对齐当前 Nginx `proxy_read_timeout` / `proxy_send_timeout` 口径。 +- 超时会返回 JSON `504 GATEWAY_UPSTREAM_TIMEOUT`,由 `npm run check:pingora-gateway-smoke` 覆盖。 + +## Pingora static cache policy + +- Pingora 直连静态响应会显式写入 `Cache-Control`,避免公网入口切换后 HTML 入口页或 Vite 指纹资源依赖框架默认头。 +- 默认 `GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL=no-cache`,覆盖 `index.html`、`admin/index.html`、目录 index 和 SPA fallback。 +- 默认 `GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable`,只覆盖 `/assets/*` 与 `/admin/assets/*` 中带 Vite 指纹文件名的资源;非指纹静态和 ACME challenge 默认 `GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL=no-cache`。 +- Pingora 静态响应同时写入弱 `ETag`、`Last-Modified` 与 `Accept-Ranges: bytes`,并对 `If-None-Match` / `If-Modified-Since` 返回 `304`;这层只负责浏览器协商缓存,不改变上面的 `Cache-Control` 分档。 +- Pingora 静态文件支持单段 `Range: bytes=`:合法范围返回 `206` 与 `Content-Range`,越界范围返回 `416` 与 `Content-Range: bytes */`,`HEAD + Range` 只返回头。多段 range 暂按完整文件返回,不启用 multipart 响应。`If-Range` 日期匹配时继续返回局部内容,日期旧于文件或弱 ETag 校验器时回完整 `200`。`206` / `304` / `416` 均不做 gzip 压缩,避免局部内容语义和 `Content-Range` 漂移。 +- Pingora 静态路由只允许 `GET` / `HEAD`。非读取方法命中静态候选时返回 `405` 与 `Allow: GET, HEAD`;缺失文件仍返回 `404`,避免错误客户端把静态入口当作可写接口。 +- `npm run check:pingora-gateway-smoke` 会用固定 `X-Request-Id` 对账静态 `304` / `405` / `206` / `416` 的 Pingora access log 行,并同时校验 method / path / status;该 smoke 还会让 direct live 自动发现首页里的普通静态资源和 Vite 指纹资源,确认指纹资源 GET / HEAD / `If-None-Match` 304 / `Range: bytes=0-0` 以及 `public, max-age=31536000, immutable` 缓存头,确保本地静态响应状态和指纹 chunk 长缓存都可进入直连切换证据链。direct live JSON 只给静态 GET / HEAD / 304 / Range 写入白名单 `headers`,包括 `cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`,便于证据包归档后复盘静态缓存与 Range 响应头。 + +## Pingora TLS / redirect + +- Pingora 网关默认仍只监听本机 shadow 端口,不绑定公网 `80/443`。 +- 需要评估直连入口时,显式配置 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` 和 `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`;网关只读取现有证书文件,不负责 Certbot / ACME 自动续期。 +- 需要 HTTP 到 HTTPS 跳转时,再显式配置 `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`;该入口除 `/.well-known/acme-challenge/*` 外统一 301 到 HTTPS,ACME challenge 仍从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取。 +- 同一公网 IP 上还承载 `git.genarrative.world` 时,直连前必须配置 `GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS=git.genarrative.world` 与 `GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM=127.0.0.1:3000`,让命中 Gitea Host 的请求整站转发到 Gitea;该路由不走应用维护页、API body limit 或网关接流保护,避免影响 git clone / push。当前 Pingora TLS listener 只加载一组 cert/key;如果直连 443 同时服务 `dev.genarrative.world` 和 `git.genarrative.world`,证书必须同时覆盖两个域名,不能使用只有单域名 SAN 的证书。 +- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-gateway-env-shadow-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`;current release 自审、直连彩排状态、直连启用、health patrol env 切换、Pingora gateway env shadow 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径,切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- 目标机如果使用 Certbot / Let’s Encrypt 证书,不要直接放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限。先用随包脚本把 root 可读的 live symlink 解析并复制到 Pingora 私有目录,例如 `sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>`;脚本默认 dry-run,`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`,目标目录默认 `root:genarrative 0750`,文件默认 `root:genarrative 0640`,并复核 `genarrative` 服务用户可读。随后把 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/` 下的副本。 +- 默认 systemd service 以 `genarrative` 非 root 用户运行,不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板;切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`,再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`;脚本会先执行随包 current release 自审,确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release,失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora;脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置,用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`,用 `systemctl is-active` 确认 Pingora active,并执行 direct live smoke,强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 +- `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为;正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 +- 目标机直连入口启用前先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`,证明 TLS 文件、service env、current release 二进制和低端口释放都满足切流条件;启用后再运行 `npm run check:pingora-direct-live`,检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe,以及 HTTP 301 / ACME challenge;真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义,direct live 只把它作为路径转发代表,不要求该 GET 创建 identity;release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable` 强制纳入同一条启用后门禁,漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查或 service 二进制可执行检查都会直接失败;启用后 `--require-direct` 复核不再要求端口空闲,因为此时 `80/443` 应由 Pingora 占用。高端口 rehearsal 如果通过 `https://127.0.0.1:18443` 打入、但 Pingora 应返回正式域名默认 HTTPS Location,可额外传 `--direct-redirect-base-url https://<域名>`;`--direct-redirect-host` 仍必须保留,用于直连切换 Host 一致性约束。 +- direct live 在 HTTPS 根路径返回 `200` 且 HTML 中发现 `/assets/` 或 `/admin/assets/` 引用时,会自动请求该静态资源,确认 `Cache-Control`、`ETag`、`Last-Modified`、`Accept-Ranges: bytes`、`HEAD` 头响应、`If-None-Match` / `If-Modified-Since` 的 `304` 响应和 `Range: bytes=0-0` 的 `206 + Content-Range` 响应,并纳入 Pingora access log method/path/status 对账;若首页引用 Vite 指纹资源,还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 以及指纹资源 GET / HEAD / 304 / Range access log 证据。维护模式、非 HTML 或发布包首页没有资产引用时该项标记为 skipped,不阻断维护窗口。 +- 直连启用前先 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`;验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''`,脚本会先运行 `nginx -t`,通过后移除 direct-entry drop-in、重启 Pingora,用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` 已从 systemd 最终配置中移除,用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`,并 reload Nginx、确认 Nginx service 仍为 `active`,最后用 smoke URL 证明 Nginx 入口真实可访问;传入 `--nginx-smoke-expect-body` 时还会要求响应体包含该片段。回退 smoke 的 URL 与 body 必须来自切换前真实 Nginx 入口,不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`;若 smoke URL 打本机地址,再显式追加 `--nginx-smoke-host <域名>` 避免命中默认 vhost。仓库工作区可用 `npm run deploy:pingora-direct-enable -- --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log` 和 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''`。两个脚本默认 dry-run;启用脚本 `--apply` 必须先通过 current release 自审,并带 env、证书可读、service EnvironmentFile 一致性、服务用户证书可读、service 二进制可执行、端口释放预检、direct live 参数和 Pingora access log 参数,回退脚本 `--apply` 必须带 `--reload-nginx` 和 `--nginx-smoke-url`;如果 health patrol env 已预先切回 Nginx,也可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让回退脚本在 Nginx smoke 后复核 `gateway mode=nginx` 且 public base URL / Host 已恢复,若切换前 Nginx 巡检本来需要 Host 覆盖,则把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`;如果还要同时证明 shadow 高端口仍健康,可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `,脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。本机 `npm run check:pingora-direct-enable` / `npm run check:pingora-direct-rollback` 会验证 dry-run 不会安装或删除临时 drop-in、current release 自审失败时启用脚本不会安装 drop-in,并展示启用 / 回退后的 systemd 最终配置核验、ExecStart 指向核验、Pingora active 核验、direct live smoke、Nginx 语法检查、reload 状态核验、Nginx smoke 及响应体片段核验、可选 health patrol env 复核和可选 shadow probe 复核。 +- rollback apply 前必须先执行 `node -- /opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs --apply --env-file /etc/genarrative/pingora-gateway.env`,把 active Pingora env 恢复为 `GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081`,并清空 `TLS_LISTEN`、`HTTP_REDIRECT_LISTEN`、`TLS_CERT_FILE` 和 `TLS_KEY_FILE`;只移除 systemd direct-entry drop-in 而不恢复 env,会让回退后的 Pingora shadow 继续按 direct 低端口或半 TLS 配置启动。 +- `npm run check:pingora-current-release-audit` 会烟测 current release 自审脚本;正式直连 runbook 会先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`,只读确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、`release-manifest.api-server.json` 已登记 `pingora-gateway`、`pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release。 +- `npm run check:pingora-cutover-status-snapshot` 会烟测状态快照脚本,`npm run check:pingora-cutover-evidence-bundle` 会烟测证据包脚本;正式切换窗口与 release readiness 使用 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs` 归档切换前 / 启用后 / 回退后三个阶段的 `manifest.json`、`snapshot.json`、stdout / stderr 和命令记录。直连 runbook 的三个证据包阶段都会向状态快照透传 `--require-pingora-gateway`,在 `checks.current-release-audit` 中保存 Pingora 二进制、sha256、release manifest 和 systemd `ExecStart` 自审结果;启用后证据包还会通过 `--run-direct-live` 保存 `direct-live.json`、direct live stdout / stderr、命令记录、Pingora access log `request_id` 反查结果,以及静态响应头白名单证据。`post-enable` 证据包必须传 `--expected-pingora-env-mode direct`,`post-rollback` 证据包必须传 `--expected-pingora-env-mode shadow`。`manifest.summary.directLiveAccessLog` 会提升 checked / matched / missing / mismatch 对账摘要,`manifest.summary.directLiveStaticHeaders` 会提升普通静态和 Vite 指纹静态的缓存头、校验头、Range `Content-Range` 和 304 状态摘要,`manifest.summary.pingoraEnvShadow` 会提升 `listen`、`tlsListen`、`httpRedirectListen`、`tlsCertFile`、`tlsKeyFile`、`mode` 和 `shadowReady`,便于值班人员不打开 `direct-live.json` 或 `snapshot.json` 也能先判断证据是否完整;如果 direct live 已输出静态资产结果但摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 或 Last-Modified 304 证据,证据包会记为 `CRITICAL`。最终总审计还会带 `--require-phase-direct-live-access-log post-enable --require-phase-direct-live-static-headers post-enable --require-phase-pingora-env-shadow post-rollback`,防止缺少 access log 对账摘要、静态头摘要或回退后 shadow env 摘要的旧证据包混入正式归档。快照脚本只读采集状态,证据包脚本只写指定证据目录,不写 `/etc`、不 reload systemd,也不修改 Nginx 或 Pingora。 +- 最终证据根目录总审计 JSON 先看 `summary.status`、`summary.failedItems[]` 和 `summary.directLiveEvidence[]`:失败时这里会直接指出阶段 / 命令 / 根目录问题,以及 `post-enable` access log 或静态头摘要的短 reason;`phases[]`、`commands[]` 和 `timeline` 继续作为完整机器归档。 +- 最终证据根目录总审计必须同时带五条 `--require-command-executable`:`enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh`、`post-enable:pingora-health-patrol-direct-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs`、`rollback-prep:pingora-gateway-shadow-env-switch:/opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs`、`rollback-prep:pingora-health-patrol-nginx-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs` 和 `rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh`,避免只证明有命令证据,却没有证明真实执行的是 current release 随包切换脚本。 +- `npm run check:production-api-release` 会动态生成 API release,并检查发布包 README 与随包 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover` 输出的最终总审计步骤都保留上述五条 `--require-command-executable` 要求,避免源码文档正确但发布包操作口径漂移。 +- 直连接管后用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切换 health patrol env;该脚本只改 gateway mode、public base URL 和 public Host,并先用临时目标 env 调用随包 `check-production-health-patrol-env.mjs` 复核,复核通过后才原子替换真实 env,复核失败不会落盘。回退到 Nginx 前用同一脚本改回 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>`,若切换前 Nginx 巡检不需要 Host 覆盖则传 `--clear-public-host`,需要 Host 覆盖则传 `--public-host <切换前Host>`;随后 rollback 脚本和独立 env 复核会再次阻断 public base URL / Host 漂移。`plan:pingora-direct-cutover` 会在 JSON runbook 中列出 direct / nginx 两个方向的 env 切换脚本和 env 复核步骤,并先输出 Host 与回退巡检入口确认;正式切换 runbook 会拒绝 `--direct-redirect-host` 或 `--rollback-nginx-smoke-host` 与 `--direct-host` 使用不同 hostname,且必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`,确认步骤会展示回退后要恢复的 public base URL / Host,避免回退计划把现场巡检入口覆盖成仓库默认值。若切换计划希望回退后同时证明 Pingora shadow 高端口仍健康,可给 runbook 追加 `--rollback-pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --rollback-pingora-shadow-probe-token `;JSON 输出会隐藏 token 原文,并把对应参数传给 rollback dry-run / apply 步骤。 + +## Pingora canary + +- `deploy/nginx/snippets/genarrative-pingora-canary.conf` 是默认不启用的人工前缀 canary 入口,只在需要验证 Nginx -> Pingora handoff 时临时 include。它使用 `/__genarrative_pingora_canary/` 前缀改写后转发到 `127.0.0.1:18081`,并返回 `X-Genarrative-Nginx-Handoff: pingora-canary`。 +- `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary。它只能在 Nginx `http` 上下文人工 include,默认监听 `127.0.0.1:18083`,覆盖 `/api/creation-entry/config`、`/v1/identity`、`/v1/database//subscribe`、`/assets/app.js` 和拒绝入口,并写入独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`;不要把它 include 到生产 `443` server 内作为 location 覆盖。 +- 真实路径 canary 片段使用 `access_log ... genarrative_upstream`,如果以独立文件放入 `/etc/nginx/conf.d/`,文件加载顺序必须晚于定义 `log_format genarrative_upstream` 的主站配置。当前目标机可使用 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`;若站点把 `log_format` 移到全局 Nginx 配置,则需确保它仍在所有 `conf.d` server include 之前加载。启用前必须先跑 `nginx -t`,不要用会早于 `genarrative.conf` 加载的文件名。 +- 启用 Nginx canary 前,目标机必须已经有持久 `genarrative-pingora-gateway.service` 运行在 `127.0.0.1:18081`,且带真实 `api-server`、SpacetimeDB、静态目录和 `/var/log/genarrative/pingora-gateway.access.log` 完成本机 shadow 验收;不要从一次性 `/tmp` 网关进程直接切到 Nginx handoff。 +- 启用前必须运行 `npm run check:nginx-pingora-canary`;目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`,同时做两个 snippet 的静态护栏和真实 `nginx -t`。本机或 CI 可运行 `npm run check:pingora-canary-docker`,用 Docker Nginx、真实 Pingora 和 mock 上游复现前缀 canary 与真实路径 canary 两条 handoff 链路。 +- 前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`,再用 `scripts/check-pingora-canary-access-log-parity.mjs` 对账 `/var/log/nginx/genarrative.access.log` 与 `/var/log/genarrative/pingora-gateway.access.log`。 +- 真实路径 canary 不再手工编辑 `/etc/nginx/conf.d/`。目标机从 current release 执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`;脚本会渲染到 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`,保持晚于主站 `log_format genarrative_upstream` 加载,执行 `nginx -t`、reload Nginx,并默认运行 realpath live smoke。`nginx -t`、reload 或 live smoke 任一失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`,脚本会删除该 `zz-...` 文件,执行 `nginx -t` 和 reload,失败时恢复删除前配置。 +- `npm run check:pingora-release-readiness` 默认包含 `npm run check:pingora-realpath-canary-toggle`,用临时 Nginx 目录和 fake `nginx` / `systemctl` / live smoke 验证 realpath canary 启停脚本默认 dry-run、apply 写入、失败回滚和 disable 恢复逻辑;不再把这条脚本护栏留成单独可选项。 +- 真实路径 canary reload 后可再次运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`,再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 +- 正式切换前在源码 checkout / CI / 构建环境跑聚合门禁,并按目标机已启用的 canary 入口选择参数:前缀 canary 已启用时使用 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`;真实路径 canary 已启用时追加或单独使用 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。如果现场只启用了真实路径 canary,不要同时传 `--require-live`。 +- 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁,并同样按现场已启用入口选择:前缀 canary 用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`;真实路径 canary 用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`;验证 Pingora 直连入口时再追加 `--require-direct ...` 参数。未带 `--require-direct` 的 runtime-only 基础门禁会自动执行直连彩排状态复核,确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过;启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态。生成正式直连切换 runbook 时,`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname,只允许端口不同。 + ## 验证 ```bash diff --git a/deploy/nginx/snippets/genarrative-pingora-canary.conf b/deploy/nginx/snippets/genarrative-pingora-canary.conf new file mode 100644 index 000000000..e38e9f162 --- /dev/null +++ b/deploy/nginx/snippets/genarrative-pingora-canary.conf @@ -0,0 +1,64 @@ +# Pingora canary handoff snippet. +# 中文注释:本文件只提供人工 include 后的 Nginx -> Pingora 试跑入口,不默认承接公网流量。 +# 启用前复制到目标机并替换 __GENARRATIVE_PINGORA_PROBE_TOKEN__,确认只允许可信来源访问。 + +location = /__genarrative_pingora_canary/healthz { + allow 127.0.0.1; + allow ::1; + deny all; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__"; + add_header X-Genarrative-Nginx-Handoff pingora-canary always; + proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz; +} + +location ~ ^/__genarrative_pingora_canary/v1/database/[^/]+/subscribe$ { + allow 127.0.0.1; + allow ::1; + deny all; + + rewrite ^/__genarrative_pingora_canary(/.*)$ $1 break; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_read_timeout 3600s; + add_header X-Genarrative-Nginx-Handoff pingora-canary always; + proxy_pass http://127.0.0.1:18081; +} + +location /__genarrative_pingora_canary/ { + allow 127.0.0.1; + allow ::1; + deny all; + + rewrite ^/__genarrative_pingora_canary(/.*)$ $1 break; + proxy_http_version 1.1; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_set_header X-Genarrative-Pingora-Canary "nginx-prefixed"; + add_header X-Accel-Buffering no always; + add_header X-Genarrative-Nginx-Handoff pingora-canary always; + proxy_pass http://127.0.0.1:18081; +} diff --git a/deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf b/deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf new file mode 100644 index 000000000..9ba6f5b2f --- /dev/null +++ b/deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf @@ -0,0 +1,90 @@ +# Pingora real-path canary handoff server. +# 中文注释:本文件只提供人工 include 后的本机真实路径 canary,不默认承接公网流量。 +# 启用前复制到目标机并替换 __GENARRATIVE_PINGORA_PROBE_TOKEN__,只能在 nginx http 上下文 include。 + +server { + listen 127.0.0.1:18083; + server_name genarrative-pingora-realpath-canary.local; + access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream; + error_log /var/log/nginx/genarrative-pingora-realpath-canary.error.log warn; + + allow 127.0.0.1; + allow ::1; + deny all; + + location = /__genarrative_pingora_realpath_canary/healthz { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__"; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz; + } + + location = /api/creation-entry/config { + default_type application/json; + proxy_http_version 1.1; + proxy_buffering off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_set_header X-Genarrative-Pingora-Canary "nginx-realpath"; + add_header X-Accel-Buffering no always; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081; + } + + location ~ ^/v1/database/[^/]+/subscribe$ { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $request_id; + proxy_read_timeout 3600s; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081; + } + + location ^~ /v1/identity { + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $request_id; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081; + } + + location = /assets/app.js { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $request_id; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081; + } + + location = /generated-pingora-canary-smoke { + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Request-Id $request_id; + add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always; + proxy_pass http://127.0.0.1:18081; + } + + location / { + return 404; + } +} diff --git a/deploy/pingora/nginx-route-parity.matrix.json b/deploy/pingora/nginx-route-parity.matrix.json new file mode 100644 index 000000000..8642b8651 --- /dev/null +++ b/deploy/pingora/nginx-route-parity.matrix.json @@ -0,0 +1,375 @@ +{ + "version": 1, + "description": "Pingora shadow gateway routes that must stay aligned with the current Genarrative Nginx production/development templates before any public cutover.", + "routes": [ + { + "id": "acme_challenge", + "samplePath": "/.well-known/acme-challenge/token", + "expect": { + "kind": "static", + "root": "acme", + "mode": "exact" + }, + "nginx": { + "production": [ + "location /.well-known/acme-challenge/", + "root /var/www/html;" + ] + }, + "docs": ["`/.well-known/acme-challenge/*`"] + }, + { + "id": "shadow_probe", + "samplePath": "/__genarrative_pingora/healthz", + "expect": { + "kind": "shadow_probe" + }, + "docs": ["`/__genarrative_pingora/healthz`"] + }, + { + "id": "admin_redirect", + "samplePath": "/admin", + "expect": { + "kind": "redirect_permanent", + "location": "/admin/" + }, + "nginx": { + "production": ["location = /admin", "return 301 /admin/;"], + "development": ["location = /admin", "return 301 /admin/;"] + }, + "docs": ["`/admin`", "301 到 `/admin/`"] + }, + { + "id": "admin_api_proxy", + "samplePath": "/admin/api/users", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": null, + "protectionClass": "admin_api" + }, + "nginx": { + "production": [ + "location ^~ /admin/api/", + "proxy_pass http://genarrative_api/admin/api/;", + "limit_conn genarrative_api_conn 64;", + "limit_req zone=genarrative_admin_rps burst=16 nodelay;" + ], + "development": [ + "location ^~ /admin/api/", + "proxy_pass http://genarrative_api/admin/api/;", + "limit_conn genarrative_api_conn 64;", + "limit_req zone=genarrative_admin_rps burst=16 nodelay;" + ] + }, + "docs": ["`/admin/api/*`"] + }, + { + "id": "admin_assets", + "samplePath": "/admin/assets/index.js", + "expect": { + "kind": "static", + "root": "web", + "mode": "exact" + }, + "nginx": { + "production": ["location ^~ /admin/assets/", "try_files $uri =404;"], + "development": ["location ^~ /admin/assets/", "try_files $uri =404;"] + }, + "docs": ["`/admin/assets/*`"] + }, + { + "id": "admin_spa_fallback", + "samplePath": "/admin/settings", + "expect": { + "kind": "static", + "root": "web", + "mode": "spa_fallback" + }, + "nginx": { + "production": [ + "location ^~ /admin/", + "try_files $uri $uri/ /admin/index.html;" + ], + "development": [ + "location ^~ /admin/", + "try_files $uri $uri/ /admin/index.html;" + ] + }, + "docs": ["`/admin/*`"] + }, + { + "id": "web_assets", + "samplePath": "/assets/app.js", + "expect": { + "kind": "static", + "root": "web", + "mode": "exact" + }, + "nginx": { + "production": ["location ^~ /assets/", "try_files $uri =404;"], + "development": ["location ^~ /assets/", "try_files $uri =404;"] + }, + "docs": ["`/assets/*`"] + }, + { + "id": "puzzle_gallery_list", + "samplePath": "/api/runtime/puzzle/gallery", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": null, + "protectionClass": "gallery_list" + }, + "nginx": { + "production": [ + "location = /api/runtime/puzzle/gallery", + "limit_conn genarrative_api_conn 320;", + "limit_req zone=genarrative_gallery_rps burst=4096 nodelay;", + "add_header X-Accel-Buffering no always;" + ], + "development": [ + "location = /api/runtime/puzzle/gallery", + "limit_conn genarrative_api_conn 320;", + "limit_req zone=genarrative_gallery_rps burst=4096 nodelay;", + "add_header X-Accel-Buffering no always;" + ] + }, + "docs": ["`/api/runtime/puzzle/gallery`"] + }, + { + "id": "custom_world_gallery_list", + "samplePath": "/api/runtime/custom-world-gallery", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": null, + "protectionClass": "gallery_list" + }, + "nginx": { + "production": [ + "location = /api/runtime/custom-world-gallery", + "limit_conn genarrative_api_conn 320;", + "limit_req zone=genarrative_gallery_rps burst=4096 nodelay;", + "add_header X-Accel-Buffering no always;" + ], + "development": [ + "location = /api/runtime/custom-world-gallery", + "limit_conn genarrative_api_conn 320;", + "limit_req zone=genarrative_gallery_rps burst=4096 nodelay;", + "add_header X-Accel-Buffering no always;" + ] + }, + "docs": ["`/api/runtime/custom-world-gallery`"] + }, + { + "id": "puzzle_gallery_detail", + "samplePath": "/api/runtime/puzzle/gallery/work-a", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": null, + "protectionClass": "gallery_detail" + }, + "nginx": { + "production": [ + "location ~ ^/api/runtime/puzzle/gallery/[^/]+$", + "limit_conn genarrative_api_conn 32;", + "limit_req zone=genarrative_api_rps burst=32 nodelay;", + "add_header X-Accel-Buffering no always;" + ], + "development": [ + "location ~ ^/api/runtime/puzzle/gallery/[^/]+$", + "limit_conn genarrative_api_conn 32;", + "limit_req zone=genarrative_api_rps burst=32 nodelay;", + "add_header X-Accel-Buffering no always;" + ] + }, + "docs": ["`/api/runtime/puzzle/gallery/{id}`"] + }, + { + "id": "custom_world_gallery_detail", + "samplePath": "/api/runtime/custom-world-gallery/profile-a/owner-b", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": null, + "protectionClass": "gallery_detail" + }, + "nginx": { + "production": [ + "location ~ ^/api/runtime/custom-world-gallery/[^/]+/[^/]+$", + "limit_conn genarrative_api_conn 32;", + "limit_req zone=genarrative_api_rps burst=32 nodelay;", + "add_header X-Accel-Buffering no always;" + ], + "development": [ + "location ~ ^/api/runtime/custom-world-gallery/[^/]+/[^/]+$", + "limit_conn genarrative_api_conn 32;", + "limit_req zone=genarrative_api_rps burst=32 nodelay;", + "add_header X-Accel-Buffering no always;" + ] + }, + "docs": ["`/api/runtime/custom-world-gallery/{profile}/{owner}`"] + }, + { + "id": "generic_api_proxy", + "samplePath": "/api/creation-entry/config", + "expect": { + "kind": "proxy", + "target": "api", + "bodyLimit": "default", + "protectionClass": "api" + }, + "nginx": { + "production": [ + "location ~ ^/api(?:/|$)", + "client_max_body_size 64m;", + "limit_conn genarrative_api_conn 64;", + "limit_req zone=genarrative_api_rps burst=64 nodelay;", + "add_header X-Accel-Buffering no always;" + ], + "development": [ + "location ~ ^/api(?:/|$)", + "client_max_body_size 64m;", + "limit_conn genarrative_api_conn 64;", + "limit_req zone=genarrative_api_rps burst=64 nodelay;", + "add_header X-Accel-Buffering no always;" + ] + }, + "docs": ["`/api`、`/api/*`"] + }, + { + "id": "spacetime_subscribe", + "samplePath": "/v1/database/genarrative/subscribe", + "expect": { + "kind": "proxy", + "target": "spacetime", + "bodyLimit": null, + "protectionClass": "spacetime" + }, + "nginx": { + "production": [ + "location ~ ^/v1/database/[^/]+/subscribe$", + "proxy_pass http://127.0.0.1:3101;", + "proxy_set_header Upgrade $http_upgrade;", + "proxy_set_header Connection \"Upgrade\";", + "proxy_read_timeout 3600s;" + ], + "development": [ + "location ~ ^/v1/database/[^/]+/subscribe$", + "proxy_pass http://127.0.0.1:3101;", + "proxy_set_header Upgrade $http_upgrade;", + "proxy_set_header Connection \"Upgrade\";", + "proxy_read_timeout 3600s;" + ] + }, + "docs": ["`/v1/database/{db}/subscribe`"] + }, + { + "id": "spacetime_identity", + "samplePath": "/v1/identity", + "expect": { + "kind": "proxy", + "target": "spacetime", + "bodyLimit": null, + "protectionClass": "spacetime" + }, + "nginx": { + "production": [ + "location ^~ /v1/identity", + "proxy_pass http://127.0.0.1:3101;", + "proxy_set_header Upgrade $http_upgrade;", + "proxy_set_header Connection \"Upgrade\";" + ], + "development": [ + "location ^~ /v1/identity", + "proxy_pass http://127.0.0.1:3101;", + "proxy_set_header Upgrade $http_upgrade;", + "proxy_set_header Connection \"Upgrade\";" + ] + }, + "docs": ["`/v1/identity*`"] + }, + { + "id": "v1_forbidden", + "samplePath": "/v1/ping", + "expect": { + "kind": "not_found" + }, + "nginx": { + "production": ["location ^~ /v1/", "return 404;"], + "development": ["location ^~ /v1/", "return 404;"] + }, + "docs": ["`/v1/*`"] + }, + { + "id": "healthz_forbidden", + "samplePath": "/healthz", + "expect": { + "kind": "not_found" + }, + "nginx": { + "production": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ], + "development": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ] + }, + "docs": ["`/healthz*`"] + }, + { + "id": "readyz_forbidden", + "samplePath": "/readyz", + "expect": { + "kind": "not_found" + }, + "nginx": { + "production": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ], + "development": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ] + }, + "docs": ["`/readyz*`"] + }, + { + "id": "generated_assets_forbidden", + "samplePath": "/generated-puzzle-assets/foo.webp", + "expect": { + "kind": "not_found" + }, + "nginx": { + "production": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ], + "development": [ + "location ~ ^/(generated-|healthz|readyz)", + "return 404;" + ] + }, + "docs": ["`/generated-*`"] + }, + { + "id": "web_spa_fallback", + "samplePath": "/some/spa/path", + "expect": { + "kind": "static", + "root": "web", + "mode": "spa_fallback" + }, + "nginx": { + "production": ["location /", "try_files $uri $uri/ /index.html;"], + "development": ["location /", "try_files $uri $uri/ /index.html;"] + }, + "docs": ["其它路径", "失败回退 `/index.html`"] + } + ] +} diff --git a/deploy/pingora/pingora-gateway.env.example b/deploy/pingora/pingora-gateway.env.example new file mode 100644 index 000000000..67b1bc3ff --- /dev/null +++ b/deploy/pingora/pingora-gateway.env.example @@ -0,0 +1,81 @@ +# Pingora 独立网关试点配置示例。 +# 当前只用于本机影子验证,不绑定 80/443,也不替代生产 Nginx。 +# systemd 模板默认读取 /etc/genarrative/pingora-gateway.env。 + +GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081 +# 可选:直连 HTTPS 入口。默认关闭;启用时必须同时配置 cert/key,并把 FORWARDED_PROTO 设为 https。 +# cert/key 必须允许 genarrative 用户读取;不要直接 chmod Let's Encrypt live/archive 原始目录。 +# 可先用随包 scripts/deploy/pingora-tls-cert-sync.mjs 把证书同步到 /etc/genarrative/pingora-tls//。 +# 80/443 必须先从 Nginx 或其它进程释放;高端口 loopback 演练可先使用 127.0.0.1:18443 / 127.0.0.1:18080。 +# 如果要绑定 443/80,必须另行人工启用 systemd direct-entry drop-in 授予 CAP_NET_BIND_SERVICE。 +# GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:18443 +# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/etc/genarrative/pingora-tls/genarrative.world/fullchain.pem +# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/etc/genarrative/pingora-tls/genarrative.world/privkey.pem +# 可选:直连 HTTP 只做 301 到 HTTPS;启用时必须已经配置 TLS_LISTEN,ACME challenge 仍从 ACME_ROOT 静态读取。 +# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:18080 +# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME=https +GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM=127.0.0.1:8082 +GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM=127.0.0.1:3101 +# 可选:同一公网 IP 上还承载 Gitea 时,按 Host 整站转发到 Gitea。 +# 命中这些 Host 的请求不走应用维护页、API body limit 或网关接流保护,避免影响 git clone / push。 +# 直连 443 如果要同时服务这些域名,当前单 listener TLS 配置必须使用覆盖全部 Host 的同一张证书。 +# GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS=git.genarrative.world +# GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM=127.0.0.1:3000 + +GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT=/srv/genarrative/web +GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT=/var/www/html +GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE=/var/lib/genarrative/maintenance/enabled + +GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO=http +GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES=67108864 +# gzip 默认开启;等级和最小响应长度对齐 Nginx gzip_comp_level 5 / gzip_min_length 1024。 +# Pingora 正式化口径固定为 gzip-only;br / zstd 不进入当前网关,Brotli 继续由 Nginx / 前置代理承担。 +GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip +GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED=true +GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL=5 +GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES=1024 + +# 静态响应缓存头:HTML / SPA fallback 保持 no-cache,Vite 指纹资源长缓存,其它静态资源保守 no-cache。 +# 网关会额外写入 ETag / Last-Modified 并支持 304 协商缓存;下面三项只控制 Cache-Control。 +GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL=no-cache +GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable +GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL=no-cache + +# 上游超时默认对齐当前 Nginx 模板:普通无显式长超时路由 60 秒,/api、公开列表/详情和 subscribe 为 3600 秒。 +GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS=3000 +GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS=60 +GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS=3600 +GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS=3600 +GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS=3600 + +# 默认只信任 TCP 对端 IP;只有确认前置 Nginx / LB 会清洗 X-Forwarded-For 后才打开。 +GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false +# 打开 TRUST_X_FORWARDED_FOR 前必须显式确认;否则网关会拒绝启动。 +GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false + +# 单进程内接流保护;默认值对齐当前 Nginx limit_conn / limit_req 口径。 +# INSTANCE_COUNT 默认 1;若横向运行多个 Pingora 实例且仍启用网关接流保护,必须先落地共享保护层并显式确认。 +GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1 +GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false +GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true +GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_MAX_CONCURRENT=64 +GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_RATE_PER_SECOND=30 +GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_BURST=16 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_MAX_CONCURRENT=320 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_RATE_PER_SECOND=5000 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_BURST=4096 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_MAX_CONCURRENT=32 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_RATE_PER_SECOND=300 +GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_BURST=32 +GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT=64 +GENARRATIVE_PINGORA_GATEWAY_API_RATE_PER_SECOND=300 +GENARRATIVE_PINGORA_GATEWAY_API_BURST=64 +GENARRATIVE_PINGORA_GATEWAY_SPACETIME_MAX_CONCURRENT=256 +GENARRATIVE_PINGORA_GATEWAY_SPACETIME_RATE_PER_SECOND=1000 +GENARRATIVE_PINGORA_GATEWAY_SPACETIME_BURST=256 +# 仅供本机影子探针使用;为空时 /__genarrative_pingora/healthz 一律返回 404。 +GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN= + +GENARRATIVE_PINGORA_GATEWAY_LOG=info,pingora=info,pingora_gateway=info +GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE=/var/log/genarrative/pingora-gateway.access.log +GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED=false diff --git a/deploy/systemd/genarrative-database-backup.service b/deploy/systemd/genarrative-database-backup.service index a19481b64..18c5b7e7c 100644 --- a/deploy/systemd/genarrative-database-backup.service +++ b/deploy/systemd/genarrative-database-backup.service @@ -9,7 +9,7 @@ User=root Group=root WorkingDirectory=/opt/genarrative/current EnvironmentFile=/etc/genarrative/api-server.env -ExecStart=/usr/bin/node /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service +ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service # 备份需要停止 / 启动 spacetimedb.service,并读取 /stdb、写入 /var/lib/genarrative/database-backups。 PrivateTmp=true diff --git a/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf b/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf new file mode 100644 index 000000000..6cb0e6d8f --- /dev/null +++ b/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf @@ -0,0 +1,17 @@ +# Pingora 直连公网入口的 systemd drop-in 模板。 +# 默认 provision 只把本文件安装到 /etc/genarrative/pingora/ 作为人工启用材料; +# 需要低端口 80/443 时,才复制到: +# /etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf +# +# 启用前必须同时在 /etc/genarrative/pingora-gateway.env 中显式配置: +# GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:443 +# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:80 +# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=... +# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=... +# 证书链和私钥必须允许 genarrative 用户读取;80/443 必须先从 Nginx 或其它进程释放。 + +[Service] +# 仅授予非 root genarrative 用户绑定 80/443 所需的最小 capability。 +AmbientCapabilities=CAP_NET_BIND_SERVICE +CapabilityBoundingSet=CAP_NET_BIND_SERVICE +NoNewPrivileges=true diff --git a/deploy/systemd/genarrative-pingora-gateway.service b/deploy/systemd/genarrative-pingora-gateway.service new file mode 100644 index 000000000..751130e57 --- /dev/null +++ b/deploy/systemd/genarrative-pingora-gateway.service @@ -0,0 +1,31 @@ +[Unit] +Description=Genarrative Pingora Shadow Gateway +After=network-online.target genarrative-api.service spacetimedb.service +Wants=network-online.target +Requires=genarrative-api.service spacetimedb.service + +[Service] +Type=simple +User=genarrative +Group=genarrative +WorkingDirectory=/opt/genarrative/current +EnvironmentFile=/etc/genarrative/pingora-gateway.env +ExecStartPre=/usr/bin/test -d /srv/genarrative/web +ExecStartPre=/usr/bin/test -d /var/www/html +ExecStart=/opt/genarrative/current/pingora-gateway +Restart=always +RestartSec=5 +KillSignal=SIGINT +TimeoutStopSec=60 +LimitNOFILE=65535 +TasksMax=2048 + +# 影子网关只监听本机高端口,公网入口继续由 Nginx 承接。 +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ReadOnlyPaths=/srv/genarrative/web /var/www/html +ReadWritePaths=/var/lib/genarrative /var/log/genarrative + +[Install] +WantedBy=multi-user.target diff --git a/docs/README.md b/docs/README.md index fc92fc358..acbd05333 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,7 +23,7 @@ 本地通过 SSH alias 管理多台服务器、查看硬件 / systemd / HTTP 健康状态并执行受控服务启停的 egui 桌面工具见 [【开发运维】本地SSH服务器管理面板技术方案-2026-06-11.md](./technical/【开发运维】本地SSH服务器管理面板技术方案-2026-06-11.md)。 -生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md),该文档也是当前生产 Jenkinsfile 的唯一入口。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md);private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md);后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。 +生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md),该文档也是当前生产 Jenkinsfile 的唯一入口。Pingora 只作为独立二进制影子网关试点时,边界、路由口径与替换前验收见 [【开发运维】Pingora独立网关试点-2026-06-11.md](./technical/【开发运维】Pingora独立网关试点-2026-06-11.md)。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md);private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md);后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。 SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index d37951164..41afe3d73 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,89 @@ --- +## 2026-06-13 Pingora 低端口直连只通过显式 systemd drop-in 启用 + +- 背景:`genarrative-pingora-gateway.service` 默认以 `genarrative` 非 root 用户运行,shadow 阶段只监听本机高端口;如果正式评估让 Pingora 直接绑定公网 `80/443`,需要低端口绑定能力,但不能让 Server-Provision 或默认 service 自动改变接流边界。 +- 决策:主 systemd service 保持 shadow 口径,不携带 `CAP_NET_BIND_SERVICE`。仓库提供 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用 drop-in 模板,Server-Provision 只安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 备查和手动覆盖;正式切换窗口从 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 执行时默认读取 current release 随包的 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`,不依赖 `/etc` 参考模板、Jenkins 工作区或源码 checkout。直连切换前先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成 JSON runbook,逐条审阅 Host 与回退巡检入口确认、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply(通过命令证据脚本归档 stdout / stderr / 退出码)、切换后 health patrol 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、rollback apply(通过命令证据脚本归档 stdout / stderr / 退出码)、回退后 health patrol 切回 `nginx` 并恢复切换前 public base URL / Host、回退后 health patrol env Nginx 模式复核;启用前基础门禁不带 `--require-direct`,因为 systemd drop-in 尚未生效,启用后复核必须带 `--require-direct`。正式切换 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname,只允许端口不同,避免 redirect 和回退 smoke 分别验证到不同入口;还必须显式传 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`,切换前 Nginx 巡检需要 Host 覆盖时再传 `--rollback-health-patrol-public-host <切换前Host>`,确认步骤会展示回退后要恢复的 public base URL / Host,避免回退 runbook 覆盖现场原有巡检入口。若需把回退后 Pingora shadow 探针复核纳入 runbook,追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`,JSON 输出会隐藏 token 原文并把参数传给 rollback dry-run / apply。只有切换窗口通过 `pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名>` 先跑 current release 自审,确认发布包自包含、`pingora-gateway` 可执行且 systemd `ExecStart` 指向随包网关,失败时不安装 drop-in;随后跑 direct preflight,确认当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都可读取证书链 / 私钥,确认 service 模板与 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含本次 `/etc/genarrative/pingora-gateway.env`,并确认 service `ExecStart=` 指向的 current release `pingora-gateway` 存在且可执行,再安装到 `/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf`、执行 `systemctl daemon-reload`、重启 Pingora,并用 `systemctl cat` 核验 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl is-active` 确认服务 active、用 direct live smoke 验证 HTTPS / HTTP redirect / ACME / WSS 101 后,才视为授予低端口能力成功。启用前还必须显式配置 TLS / redirect env 和真实证书,并确认 current release 已落盘可执行 `pingora-gateway`、Nginx 或其它进程已释放 `80/443`;启用后仍必须跑 release readiness 门禁;验证失败时统一执行 `pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''` 回到 shadow / Nginx 入口,回退脚本会先运行 `nginx -t`,通过后移除 direct-entry drop-in、重启 Pingora,再用 `systemctl cat` 确认两条低端口 capability 均已从最终 unit 配置中移除,用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`,并 reload Nginx、确认 Nginx service 仍为 `active`,最后用 curl smoke URL 证明 Nginx 入口真实可访问;回退 smoke URL/body 必须来自切换前真实 Nginx 入口,不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`。回退脚本 `--apply` 必须同时带 `--reload-nginx` 和 `--nginx-smoke-url`,避免只撤掉 Pingora 低端口能力却没有证明 Nginx 已重新接流;本机打 `127.0.0.1`、`localhost` 或 `::1` 时,`--apply` 必须带 `--nginx-smoke-host <域名>`,且该值只能是 host 或 `host:port`,避免命中默认 vhost。回退后必须复核 health patrol env 已切回 `nginx` 且 public base URL / Host 恢复为切换前记录值;如果 env 已预先修正,rollback 脚本可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 自动执行这项复核,切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若需证明 Pingora 仍以 shadow 高端口存活,可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `,脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。 +- 决策补充:health patrol env 的直连/回退切换不再靠人工编辑三行变量;正式 runbook 使用 current release 随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply`,只更新 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST` 并立即调用随包 `check-production-health-patrol-env.mjs` 复核。Pingora direct 使用本机 public base URL 时脚本必须带 `--public-host <域名>`;回退到 Nginx 时根据切换前记录传 `--clear-public-host` 或 `--public-host <切换前Host>`。生产巡检、health patrol env 复核和 env 切换脚本读取的布尔 env 必须严格解析,非法值直接失败,不得静默按 false 继续;env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录,也不能包含换行或 NUL;env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数,凡是用 Node 启动项目脚本且要把业务 `--env-file` 传给脚本时,必须写成 `node -- ', + '', + ].join(''), + ); + await writeFile( + path.join(webRoot, 'admin', 'index.html'), + '
admin-shell
', + ); + await writeFile( + path.join(webRoot, 'assets', 'app.js'), + 'console.log("site asset");\n', + ); + await writeFile( + path.join(webRoot, 'assets', 'index-B4dmVw0r.js'), + 'console.log("fingerprinted site asset");\n', + ); + await writeFile( + path.join(webRoot, 'assets', 'large-app.js'), + `console.log("site asset large");\n${'/* asset filler */\n'.repeat(120)}`, + ); + await writeFile( + path.join(webRoot, 'assets', 'large-image.webp'), + Buffer.alloc(4096, 0x52), + ); + await writeFile( + path.join( + webRoot, + 'Icons', + "Admurin's Pixel Items", + "Admurin's Pixel Items", + 'General', + 'Singles', + '499_Iron_Gear.png', + ), + Buffer.concat([ + PNG_MAGIC_BYTES, + Buffer.from([0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52]), + ]), + ); + await writeFile( + path.join(webRoot, 'admin', 'assets', 'admin.js'), + 'console.log("admin asset");', + ); + await writeFile( + path.join(webRoot, 'admin', 'assets', 'admin-Ca8f3012.css'), + 'body{color:#123456;}', + ); + await writeFile( + path.join(webRoot, 'maintenance.html'), + '
maintenance
', + ); + await writeFile( + path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'), + 'acme-token', + ); +} + +async function createSelfSignedCertificate(tempRoot) { + const tlsDir = path.join(tempRoot, 'tls'); + await mkdir(tlsDir, { recursive: true }); + const certFile = path.join(tlsDir, 'cert.pem'); + const keyFile = path.join(tlsDir, 'key.pem'); + runCommand('openssl', [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + keyFile, + '-out', + certFile, + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost,IP:127.0.0.1', + '-days', + '1', + ]); + return { certFile, keyFile }; +} + +async function startApiMock() { + const state = { + requests: [], + releaseHold: undefined, + }; + const server = http.createServer(async (request, response) => { + let body; + try { + body = await readRequestBody(request); + } catch (error) { + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + body: Buffer.alloc(0), + aborted: true, + error: error instanceof Error ? error.message : String(error), + }); + return; + } + + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + body, + }); + + if (request.url?.endsWith('/hold')) { + await new Promise((resolve) => { + state.releaseHold = resolve; + }); + } else if (request.url?.endsWith('/upstream-close')) { + request.socket.destroy(); + return; + } else if (request.url?.endsWith('/slow')) { + await delay(1500); + } + + response.writeHead(200, { + 'Content-Type': 'application/json; charset=utf-8', + 'X-Upstream': 'api', + }); + response.end( + JSON.stringify({ + ok: true, + upstream: 'api', + method: request.method, + url: request.url, + requestId: request.headers['x-request-id'] || '', + forwardedProto: request.headers['x-forwarded-proto'] || '', + forwardedHost: request.headers['x-forwarded-host'] || '', + forwardedFor: request.headers['x-forwarded-for'] || '', + realIp: request.headers['x-real-ip'] || '', + host: request.headers.host || '', + bodyBytes: body.length, + }), + ); + }); + + const port = await listen(server); + return { + port, + state, + }; +} + +async function startSpacetimeMock() { + const state = { + websocketMessages: [], + requests: [], + }; + const server = http.createServer(async (request, response) => { + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + }); + response.writeHead(200, { + 'Content-Type': 'application/json; charset=utf-8', + }); + response.end( + JSON.stringify({ ok: true, upstream: 'spacetime', url: request.url }), + ); + }); + + server.on('upgrade', (request, socket) => { + socket.on('error', () => {}); + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + }); + const key = request.headers['sec-websocket-key']; + if (!key) { + socket.destroy(); + return; + } + + const accept = createHash('sha1') + .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest('base64'); + socket.write( + [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Protocol: ${SPACETIME_WEBSOCKET_PROTOCOL}`, + `Sec-WebSocket-Accept: ${accept}`, + '', + '', + ].join('\r\n'), + ); + socket.write(encodeWebSocketTextFrame('spacetime-upgrade-ok')); + socket.on('data', (chunk) => { + state.websocketMessages.push(chunk); + }); + }); + + const port = await listen(server); + return { + port, + state, + }; +} + +async function startGiteaMock() { + const state = { + requests: [], + }; + const server = http.createServer(async (request, response) => { + const body = await readRequestBody(request); + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + body, + }); + + response.writeHead(200, { + 'Content-Type': 'application/json; charset=utf-8', + 'X-Upstream': 'gitea', + }); + response.end( + JSON.stringify({ + ok: true, + upstream: 'gitea', + method: request.method, + url: request.url, + forwardedProto: request.headers['x-forwarded-proto'] || '', + forwardedHost: request.headers['x-forwarded-host'] || '', + host: request.headers.host || '', + }), + ); + }); + + const port = await listen(server); + return { + port, + state, + }; +} + +async function runSmokeCases( + baseUrl, + tlsBaseUrl, + redirectBaseUrl, + probeToken, + maintenanceFile, + accessLogFile, + api, + spacetime, + gitea, +) { + await expectHttp( + baseUrl, + '/', + 200, + 'site-shell', + '主站根路径返回 index.html', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + baseUrl, + '/some/deep/link', + 200, + 'site-shell', + '主站深链回退 index.html', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp(baseUrl, '/admin', 301, '', '/admin 301 到 /admin/', { + validate: (response) => response.headers.location === '/admin/', + }); + await expectHttp( + baseUrl, + '/admin/settings', + 200, + 'admin-shell', + '后台深链回退 admin/index.html', + ); + const staticAssetResponse = await expectHttp( + baseUrl, + '/assets/app.js', + 200, + 'site asset', + '主站静态资源精确读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + assertStaticValidatorHeaders(staticAssetResponse, '主站静态资源精确读取'); + ensure( + staticAssetResponse.headers['accept-ranges'] === 'bytes', + `主站静态资源精确读取: 缺少 Accept-Ranges=bytes,实际 ${staticAssetResponse.headers['accept-ranges'] || '-'}`, + ); + await expectHttp( + baseUrl, + '/assets/app.js', + 200, + '', + 'HEAD 静态资源只返回响应头', + { + method: 'HEAD', + validate: (response) => + response.body === '' && + response.headers.etag === staticAssetResponse.headers.etag && + response.headers['last-modified'] === + staticAssetResponse.headers['last-modified'] && + Number(response.headers['content-length']) === + Buffer.byteLength('console.log("site asset");\n'), + }, + ); + await expectStaticNotModifiedByEtag( + baseUrl, + '/assets/app.js', + staticAssetResponse, + { + headers: { + 'X-Request-Id': 'static-etag-304-request-id', + }, + }, + ); + await expectStaticNotModifiedByLastModified( + baseUrl, + '/assets/app.js', + staticAssetResponse, + { + headers: { + 'X-Request-Id': 'static-last-modified-304-request-id', + }, + }, + ); + await expectHttp( + baseUrl, + '/assets/app.js', + 405, + '', + '静态资源非读取方法返回 405', + { + method: 'POST', + headers: { + 'X-Request-Id': 'static-method-405-request-id', + }, + body: 'ignored', + validate: (response) => + response.body === '' && + response.headers.allow === 'GET, HEAD' && + response.headers['x-genarrative-gateway'] === 'pingora-shadow', + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + 'console', + 'bytes 0-6/27', + '静态资源 Range 返回 206', + { + headers: { + 'X-Request-Id': 'static-range-206-request-id', + }, + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + 'console', + 'bytes 0-6/27', + '静态资源 If-Range 日期匹配返回 206', + { + headers: { + 'If-Range': staticAssetResponse.headers['last-modified'], + }, + }, + ); + await expectStaticRangeFallback( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 'Tue, 14 Nov 2023 22:13:12 GMT', + '静态资源 If-Range 旧日期回完整文件', + ); + await expectStaticRangeFallback( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + staticAssetResponse.headers.etag, + '静态资源 If-Range 弱 ETag 回完整文件', + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=-8', + 206, + 'sset");\n', + 'bytes 19-26/27', + '静态资源 suffix Range 返回 206', + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=999-1000', + 416, + '', + 'bytes */27', + '静态资源越界 Range 返回 416', + { + headers: { + 'X-Request-Id': 'static-range-416-request-id', + }, + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + '', + 'bytes 0-6/27', + 'HEAD 静态资源 Range 只返回响应头', + { + method: 'HEAD', + expectedContentLength: 7, + headers: { + 'X-Request-Id': 'static-head-range-206-request-id', + }, + }, + ); + await expectHttp( + baseUrl, + '/assets/index-B4dmVw0r.js', + 200, + 'fingerprinted site asset', + '主站指纹静态资源长缓存', + { + validate: (response) => + response.headers['cache-control'] === + 'public, max-age=31536000, immutable', + }, + ); + await expectNotCompressedResponse( + baseUrl, + '/assets/app.js', + 'site asset', + 'gzip 最小长度不压缩小响应', + ); + await expectGzipResponse(baseUrl, '/assets/large-app.js', 'site asset large'); + await expectGzipResponse(baseUrl, '/assets/large-app.js', 'site asset large', { + acceptEncoding: 'br, gzip', + label: 'gzip-only 压缩算法白名单', + }); + await expectNotCompressedResponse( + baseUrl, + '/assets/large-image.webp', + undefined, + 'gzip types 不压缩图片资源', + ); + await expectStaticPng( + baseUrl, + "/Icons/Admurin%27s%20Pixel%20Items/Admurin%27s%20Pixel%20Items/General/Singles/499_Iron_Gear.png", + '百分号编码图标路径返回 PNG', + ); + await expectHttp( + baseUrl, + '/assets%2ffavicon.svg', + 404, + '', + '静态路径拒绝编码斜杠', + ); + await expectRawPathHttp( + baseUrl, + '/assets/%2e%2e/favicon.svg', + 404, + '', + '静态路径拒绝编码上级目录', + ); + await expectHttp( + baseUrl, + '/assets/%GG', + 404, + '', + '静态路径拒绝非法百分号编码', + ); + await expectHttp( + baseUrl, + '/admin/assets/admin.js', + 200, + 'admin asset', + '后台静态资源精确读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + baseUrl, + '/admin/assets/admin-Ca8f3012.css', + 200, + 'body{color', + '后台指纹静态资源长缓存', + { + validate: (response) => + response.headers['cache-control'] === + 'public, max-age=31536000, immutable', + }, + ); + await expectHttp( + baseUrl, + '/.well-known/acme-challenge/token', + 200, + 'acme-token', + 'ACME challenge 读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + tlsBaseUrl, + '/', + 200, + 'site-shell', + 'TLS 直连接口返回 index.html', + { insecureTls: true }, + ); + await expectHttp( + redirectBaseUrl, + '/api/creation-entry/config?from=smoke', + 301, + '', + 'HTTP 入口 301 到 HTTPS', + { + headers: { Host: 'example.test' }, + validate: (response) => + response.headers.location === + 'https://example.test/api/creation-entry/config?from=smoke', + }, + ); + await expectHttp( + redirectBaseUrl, + '/.well-known/acme-challenge/token', + 200, + 'acme-token', + 'HTTP 重定向入口保留 ACME challenge', + ); + await expectHttp( + baseUrl, + '/generated-puzzle-assets/a.png', + 404, + '', + 'generated 私有资产公网拒绝', + ); + await expectHttp(baseUrl, '/healthz', 404, '', '公网 healthz 拒绝'); + await expectHttp( + baseUrl, + '/v1/ping', + 404, + '', + '非白名单 SpacetimeDB 路由拒绝', + ); + + await expectHttp( + baseUrl, + '/__genarrative_pingora/healthz', + 404, + '', + 'shadow probe 无 token 返回 404', + ); + await expectHttp( + baseUrl, + '/__genarrative_pingora/healthz', + 200, + '"gateway":"pingora-shadow"', + 'shadow probe token 通过', + { + headers: { 'X-Genarrative-Pingora-Probe': probeToken }, + }, + ); + + const apiResponse = await expectHttp( + baseUrl, + '/api/creation-entry/config', + 200, + '"upstream":"api"', + '通用 API 转发', + { + headers: { + 'X-Request-Id': 'smoke-request-id', + 'X-Forwarded-For': '203.0.113.10', + Host: 'example.test', + }, + }, + ); + ensure( + apiResponse.headers['x-genarrative-gateway'] === 'pingora-shadow', + 'API 响应缺少网关标记', + ); + ensure( + apiResponse.headers['x-accel-buffering'] === 'no', + 'API 响应缺少 X-Accel-Buffering=no', + ); + const apiPayload = JSON.parse(apiResponse.body); + ensure( + apiPayload.requestId === 'smoke-request-id', + 'API 上游未收到 X-Request-Id', + ); + ensure( + apiPayload.forwardedProto === 'https', + 'API 上游未收到配置的 X-Forwarded-Proto', + ); + ensure(apiPayload.host === 'example.test', 'API 上游 Host 未透传'); + ensure( + apiPayload.forwardedHost === 'example.test', + 'API 上游未收到 X-Forwarded-Host', + ); + ensure( + apiPayload.forwardedFor.endsWith(', 127.0.0.1'), + `API 上游 X-Forwarded-For 未追加 TCP 对端 IP:${apiPayload.forwardedFor}`, + ); + ensure( + apiPayload.realIp === '127.0.0.1', + `API 上游 X-Real-IP 未使用 TCP 对端 IP:${apiPayload.realIp}`, + ); + + await expectHttp( + baseUrl, + '/api/upload', + 413, + 'PAYLOAD_TOO_LARGE', + 'Content-Length body limit', + { + method: 'POST', + body: '0123456789abcdef', + }, + ); + await expectChunkedLimit(baseUrl, '/api/upload'); + + await expectConcurrencyLimit(baseUrl, api); + + const rateLimitHeaders = { 'X-Forwarded-For': '203.0.113.13' }; + const beforeRateLimitRequestCount = api.state.requests.length; + await expectHttp( + baseUrl, + '/api/rate-limited', + 200, + '"upstream":"api"', + 'API RPS 首次请求放行', + { + headers: rateLimitHeaders, + }, + ); + await expectHttp( + baseUrl, + '/api/rate-limited', + 429, + 'GATEWAY_RATE_LIMITED', + 'API RPS 保护', + { + headers: rateLimitHeaders, + }, + ); + ensure( + api.state.requests.length === beforeRateLimitRequestCount + 1, + 'API RPS 保护的 429 请求不应打到上游', + ); + await expectHttp( + baseUrl, + '/api/upstream-close', + 502, + 'GATEWAY_UPSTREAM_ERROR', + 'API 上游断连返回稳定 JSON', + { + headers: { 'X-Forwarded-For': '203.0.113.14' }, + }, + ); + await expectHttp( + baseUrl, + '/api/slow', + 504, + 'GATEWAY_UPSTREAM_TIMEOUT', + 'API 上游读取超时返回稳定 JSON', + { + headers: { 'X-Forwarded-For': '203.0.113.15' }, + timeoutMs: 5000, + }, + ); + + await expectHttp( + baseUrl, + '/v1/identity', + 200, + '"upstream":"spacetime"', + 'SpacetimeDB identity 转发', + ); + await expectWebSocketUpgrade( + baseUrl, + '/v1/database/genarrative/subscribe', + spacetime, + ); + + const giteaBeforeRequests = gitea.state.requests.length; + const giteaResponse = await expectHttp( + baseUrl, + '/api/v1/repos/GenarrativeAI/Genarrative', + 200, + '"upstream":"gitea"', + 'Gitea Host 路由转发', + { + headers: { + Host: 'git.genarrative.world:443', + 'X-Request-Id': 'gitea-host-request-id', + }, + }, + ); + const giteaPayload = JSON.parse(giteaResponse.body || '{}'); + ensure( + giteaPayload.host === 'git.genarrative.world:443', + `Gitea 上游 Host 未透传:${giteaPayload.host || '-'}`, + ); + ensure( + giteaPayload.forwardedHost === 'git.genarrative.world:443', + `Gitea 上游 X-Forwarded-Host 未透传:${giteaPayload.forwardedHost || '-'}`, + ); + ensure( + giteaPayload.forwardedProto === 'https', + `Gitea 上游 X-Forwarded-Proto 未对齐:${giteaPayload.forwardedProto || '-'}`, + ); + await expectHttp( + baseUrl, + '/api/v1/repos/GenarrativeAI/Genarrative', + 200, + '"upstream":"gitea"', + 'Gitea Host 别名路由转发', + { + headers: { + Host: 'git-alias.genarrative.world', + }, + }, + ); + ensure( + gitea.state.requests.length === giteaBeforeRequests + 2, + `Gitea Host 路由没有稳定打到 Gitea mock:${gitea.state.requests.length - giteaBeforeRequests}`, + ); + + await expectAccessLog(accessLogFile); + await expectAccessLogEntries(accessLogFile, [ + { + requestId: 'static-etag-304-request-id', + method: 'GET', + path: '/assets/app.js', + status: 304, + }, + { + requestId: 'static-last-modified-304-request-id', + method: 'GET', + path: '/assets/app.js', + status: 304, + }, + { + requestId: 'static-method-405-request-id', + method: 'POST', + path: '/assets/app.js', + status: 405, + }, + { + requestId: 'static-range-206-request-id', + method: 'GET', + path: '/assets/app.js', + status: 206, + }, + { + requestId: 'static-range-416-request-id', + method: 'GET', + path: '/assets/app.js', + status: 416, + }, + { + requestId: 'static-head-range-206-request-id', + method: 'HEAD', + path: '/assets/app.js', + status: 206, + }, + { + requestId: 'gitea-host-request-id', + method: 'GET', + path: '/api/v1/repos/GenarrativeAI/Genarrative', + status: 200, + proxyTarget: 'Gitea', + }, + ]); + + await mkdir(path.dirname(maintenanceFile), { recursive: true }); + await writeFile(maintenanceFile, 'enabled'); + const giteaRequestsBeforeMaintenance = gitea.state.requests.length; + await expectHttp( + baseUrl, + '/user/login', + 200, + '"upstream":"gitea"', + '维护模式不拦截 Gitea Host', + { + headers: { + Host: 'git.genarrative.world', + }, + }, + ); + ensure( + gitea.state.requests.length === giteaRequestsBeforeMaintenance + 1, + '维护模式 Gitea Host 请求没有打到 Gitea mock', + ); + await expectHttp( + baseUrl, + '/api/creation-entry/config', + 503, + 'MAINTENANCE', + '维护模式 API JSON', + ); + await expectHttp(baseUrl, '/', 503, 'maintenance', '维护模式 Web 页面'); + await expectAccessLogContains(accessLogFile, [ + 'status=503', + 'path=/api/creation-entry/config', + ]); +} + +async function expectAccessLog(accessLogFile) { + console.log('[pingora-gateway-smoke] access log 落盘'); + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + return ( + content.includes('request_id=smoke-request-id') && + content.includes('method=GET') && + content.includes('path=/api/creation-entry/config') && + content.includes('status=200') && + content.includes('proxy_target=Api') && + content.includes('protection_class=api') && + content.includes('status=429') && + content.includes('status=502') && + content.includes('status=504') + ); + } catch { + return false; + } + }); +} + +function assertStaticValidatorHeaders(response, label) { + ensure(Boolean(response.headers.etag), `${label}: 缺少 ETag`); + ensure( + /^W\/"[0-9a-f]+-[0-9a-f]+"$/iu.test(String(response.headers.etag || '')), + `${label}: ETag 格式不符合静态弱校验器口径:${response.headers.etag || '-'}`, + ); + ensure(Boolean(response.headers['last-modified']), `${label}: 缺少 Last-Modified`); + ensure( + Number.isFinite(Date.parse(response.headers['last-modified'] || '')), + `${label}: Last-Modified 不是有效 HTTP 日期:${response.headers['last-modified'] || '-'}`, + ); +} + +async function expectStaticNotModifiedByEtag( + baseUrl, + route, + sourceResponse, + options = {}, +) { + await expectHttp( + baseUrl, + route, + 304, + '', + '静态资源 If-None-Match 返回 304', + { + headers: { + 'If-None-Match': sourceResponse.headers.etag, + ...(options.headers || {}), + }, + validate: (response) => + response.body === '' && + response.headers.etag === sourceResponse.headers.etag && + response.headers['cache-control'] === + sourceResponse.headers['cache-control'], + }, + ); +} + +async function expectStaticNotModifiedByLastModified( + baseUrl, + route, + sourceResponse, + options = {}, +) { + await expectHttp( + baseUrl, + route, + 304, + '', + '静态资源 If-Modified-Since 返回 304', + { + headers: { + 'If-Modified-Since': sourceResponse.headers['last-modified'], + ...(options.headers || {}), + }, + validate: (response) => + response.body === '' && + response.headers['last-modified'] === + sourceResponse.headers['last-modified'] && + response.headers['cache-control'] === + sourceResponse.headers['cache-control'], + }, + ); +} + +async function expectStaticRange( + baseUrl, + route, + range, + status, + bodyNeedle, + contentRange, + label, + options = {}, +) { + const headers = { + Range: range, + 'Accept-Encoding': 'gzip', + ...(options.headers || {}), + }; + await expectHttp(baseUrl, route, status, bodyNeedle, label, { + method: options.method, + headers, + validate: (response) => + response.headers['accept-ranges'] === 'bytes' && + response.headers['content-range'] === contentRange && + !response.headers['content-encoding'] && + (options.method === 'HEAD' ? response.body === '' : true) && + (status === 206 + ? Number(response.headers['content-length']) === + (options.expectedContentLength ?? Buffer.byteLength(bodyNeedle)) + : Number(response.headers['content-length']) === 0), + }); +} + +async function expectStaticRangeFallback(baseUrl, route, range, ifRange, label) { + await expectHttp(baseUrl, route, 200, 'site asset', label, { + headers: { + Range: range, + 'If-Range': ifRange, + 'Accept-Encoding': 'gzip', + }, + validate: (response) => + response.headers['accept-ranges'] === 'bytes' && + !response.headers['content-range'] && + Number(response.headers['content-length']) === + Buffer.byteLength('console.log("site asset");\n'), + }); +} + +async function expectGzipResponse(baseUrl, route, bodyNeedle, options = {}) { + return expectCompressedResponse(baseUrl, route, bodyNeedle, { + acceptEncoding: options.acceptEncoding || 'gzip', + contentEncoding: 'gzip', + decode: (bodyBuffer) => gunzipSync(bodyBuffer).toString('utf8'), + label: options.label || 'gzip 压缩响应', + }); +} + +async function expectNotCompressedResponse(baseUrl, route, bodyNeedle, label) { + console.log(`[pingora-gateway-smoke] ${label}`); + const response = await requestHttp(`${baseUrl}${route}`, { + headers: { + 'Accept-Encoding': 'gzip', + }, + rawBody: true, + }); + + if (response.status !== 200) { + failures.push(`${label}: 期望 HTTP 200,实际 ${response.status}`); + return; + } + if (response.headers['content-encoding']) { + failures.push( + `${label}: 不应返回 Content-Encoding,实际 ${response.headers['content-encoding']}`, + ); + } + const body = response.bodyBuffer.toString('utf8'); + if (bodyNeedle && !body.includes(bodyNeedle)) { + failures.push(`${label}: 响应体缺少 ${bodyNeedle}`); + } +} + +async function expectStaticPng(baseUrl, route, label) { + console.log(`[pingora-gateway-smoke] ${label}`); + const response = await requestHttp(`${baseUrl}${route}`, { + rawBody: true, + }); + + if (response.status !== 200) { + failures.push(`${label}: 期望 HTTP 200,实际 ${response.status}`); + return; + } + if (!String(response.headers['content-type'] || '').startsWith('image/png')) { + failures.push( + `${label}: 期望 Content-Type=image/png,实际 ${response.headers['content-type'] || '-'}`, + ); + } + if (!response.bodyBuffer.subarray(0, 8).equals(PNG_MAGIC_BYTES)) { + failures.push(`${label}: 响应体不是 PNG magic bytes`); + } + if (response.body.includes('')) { + failures.push(`${label}: 命中了 SPA fallback HTML`); + } +} + +async function expectRawPathHttp(baseUrl, route, status, bodyNeedle, label) { + console.log(`[pingora-gateway-smoke] ${label}`); + let response; + try { + response = await rawHttpPathRequest(baseUrl, route); + } catch (error) { + failures.push( + `${label}: ${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + + if (response.status !== status) { + failures.push(`${label}: 期望 HTTP ${status},实际 ${response.status}`); + } + if (bodyNeedle && !response.body.includes(bodyNeedle)) { + failures.push(`${label}: 响应体缺少 ${bodyNeedle}`); + } +} + +async function expectCompressedResponse(baseUrl, route, bodyNeedle, options) { + const label = options.label; + console.log(`[pingora-gateway-smoke] ${label}`); + const response = await requestHttp(`${baseUrl}${route}`, { + headers: { + 'Accept-Encoding': options.acceptEncoding, + }, + rawBody: true, + }); + + if (response.status !== 200) { + failures.push(`${label}: 期望 HTTP 200,实际 ${response.status}`); + return; + } + if (response.headers['content-encoding'] !== options.contentEncoding) { + failures.push( + `${label}: 缺少 Content-Encoding=${options.contentEncoding},实际 ${response.headers['content-encoding'] || '-'}`, + ); + } + if ( + !String(response.headers.vary || '') + .toLowerCase() + .includes('accept-encoding') + ) { + failures.push( + `${label}: 缺少 Vary: Accept-Encoding,实际 ${response.headers.vary || '-'}`, + ); + } + + let decoded = ''; + try { + decoded = options.decode(response.bodyBuffer); + } catch (error) { + failures.push( + `${label}: 解压失败:${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + if (!decoded.includes(bodyNeedle)) { + failures.push(`${label}: 解压后内容缺少 ${bodyNeedle}`); + } +} + +async function expectAccessLogContains(accessLogFile, needles) { + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + return needles.every((needle) => content.includes(needle)); + } catch { + return false; + } + }); +} + +async function expectAccessLogEntries(accessLogFile, expectedEntries) { + console.log('[pingora-gateway-smoke] access log 静态边界状态对账'); + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + const entries = content.split(/\r?\n/u).filter(Boolean).map(parseAccessLogLine); + return expectedEntries.every((expected) => { + const entry = entries.find((item) => item.request_id === expected.requestId); + return ( + entry && + entry.method === expected.method && + entry.path === expected.path && + Number.parseInt(entry.status || '', 10) === expected.status && + entry.proxy_target === (expected.proxyTarget || 'Local') + ); + }); + } catch { + return false; + } + }); +} + +function parseAccessLogLine(line) { + const fields = {}; + for (const part of line.split('\t')) { + const separator = part.indexOf('='); + if (separator <= 0) { + continue; + } + fields[part.slice(0, separator)] = unescapeAccessLogValue( + part.slice(separator + 1), + ); + } + return fields; +} + +function unescapeAccessLogValue(value) { + return value + .replace(/\\\\/gu, '\\') + .replace(/\\t/gu, '\t') + .replace(/\\n/gu, '\n') + .replace(/\\r/gu, '\r'); +} + +async function expectHttp( + baseUrl, + route, + status, + bodyNeedle, + label, + options = {}, +) { + console.log(`[pingora-gateway-smoke] ${label}`); + const pending = options.beforeRequest + ? await options.beforeRequest() + : undefined; + let response; + try { + response = await requestHttp(`${baseUrl}${route}`, { + method: options.method, + headers: options.headers, + body: options.body, + insecureTls: options.insecureTls, + }); + } catch (error) { + failures.push( + `${label}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + if (options.afterRequest) { + try { + await options.afterRequest(pending); + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)); + } + } + } + + if (!response) { + return { status: 0, headers: {}, body: '' }; + } + + if (config.verbose) { + console.log(`[pingora-gateway-smoke] ${label} -> HTTP ${response.status}`); + } + + if (response.status !== status) { + failures.push(`${label}: 期望 HTTP ${status},实际 ${response.status}`); + } + + if (bodyNeedle && !response.body.includes(bodyNeedle)) { + failures.push(`${label}: 响应体缺少 ${bodyNeedle}`); + } + + if (options.validate && !options.validate(response)) { + failures.push(`${label}: 自定义响应校验未通过`); + } + + return response; +} + +async function expectChunkedLimit(baseUrl, route) { + console.log('[pingora-gateway-smoke] chunked body limit'); + const response = await new Promise((resolve, reject) => { + const request = http.request( + `${baseUrl}${route}`, + { + method: 'POST', + headers: { + 'Transfer-Encoding': 'chunked', + 'X-Forwarded-For': '203.0.113.11', + }, + }, + (incoming) => { + collectIncoming(incoming).then(resolve, reject); + }, + ); + request.on('error', reject); + request.write('012345'); + request.write('6789abcdef'); + request.end(); + }); + + if (response.status !== 413 || !response.body.includes('PAYLOAD_TOO_LARGE')) { + failures.push( + `chunked body limit: 期望 413 PAYLOAD_TOO_LARGE,实际 ${response.status} ${response.body}`, + ); + } +} + +async function expectConcurrencyLimit(baseUrl, api) { + console.log('[pingora-gateway-smoke] API 并发保护'); + const hold = await openRawHttpRequest(`${baseUrl}/admin/api/hold`, { + 'X-Forwarded-For': '203.0.113.12', + }); + + try { + await waitForCondition(() => typeof api.state.releaseHold === 'function'); + const beforeLimitedRequestCount = api.state.requests.length; + const response = await rawHttpRequest(`${baseUrl}/admin/api/users`, { + 'X-Forwarded-For': '203.0.113.12', + }); + + if (response.status !== 429) { + failures.push(`API 并发保护: 期望 HTTP 429,实际 ${response.status}`); + } + if (!response.body.includes('GATEWAY_CONCURRENCY_LIMITED')) { + failures.push('API 并发保护: 响应体缺少 GATEWAY_CONCURRENCY_LIMITED'); + } + ensure( + api.state.requests.length === beforeLimitedRequestCount, + `API 并发保护的 429 请求不应打到上游,上游请求为 ${describeRequests(api.state.requests.slice(beforeLimitedRequestCount))}`, + ); + } finally { + api.state.releaseHold?.(); + hold.socket.end(); + const holdResponse = await hold.done.catch((error) => ({ error })); + if (holdResponse?.error) { + failures.push( + `API 并发保护: hold 请求失败:${holdResponse.error.message}`, + ); + } + } +} + +async function expectWebSocketUpgrade(baseUrl, route, spacetime) { + console.log('[pingora-gateway-smoke] SpacetimeDB WebSocket Upgrade'); + const url = new URL(route, baseUrl); + const key = randomBytes(16).toString('base64'); + const response = await new Promise((resolve, reject) => { + const socket = net.connect( + { host: url.hostname, port: Number(url.port) }, + () => { + socket.write( + [ + `GET ${url.pathname} HTTP/1.1`, + `Host: ${url.host}`, + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Key: ${key}`, + 'Sec-WebSocket-Version: 13', + '', + '', + ].join('\r\n'), + ); + }, + ); + + let raw = Buffer.alloc(0); + let settled = false; + const timeout = setTimeout(() => { + settled = true; + socket.destroy(); + reject(new Error('WebSocket Upgrade 超时')); + }, 5000); + + socket.on('data', (chunk) => { + raw = Buffer.concat([raw, chunk]); + const text = raw.toString('latin1'); + if (text.includes('\r\n\r\n') && text.includes('spacetime-upgrade-ok')) { + settled = true; + clearTimeout(timeout); + socket.destroy(); + resolve(raw); + } + }); + socket.on('error', (error) => { + settled = true; + clearTimeout(timeout); + reject(error); + }); + socket.on('close', () => { + clearTimeout(timeout); + if (!settled) { + reject(new Error('WebSocket Upgrade 连接提前关闭')); + } + }); + }); + + const text = response.toString('latin1'); + ensure( + text.startsWith('HTTP/1.1 101'), + `WebSocket Upgrade 未返回 101:${text.split('\r\n')[0] || text}`, + ); + ensure( + text.includes('spacetime-upgrade-ok'), + 'WebSocket tunnel 未收到上游 welcome frame', + ); + const upgradeRequest = spacetime.state.requests.find( + (request) => + request.url === route && request.headers.upgrade === 'websocket', + ); + ensure( + Boolean(upgradeRequest), + 'SpacetimeDB mock 未收到 WebSocket Upgrade 请求', + ); +} + +function requestHttp(url, options = {}) { + return new Promise((resolve, reject) => { + const headers = { ...(options.headers || {}) }; + if ( + options.body !== undefined && + !hasHeader(headers, 'content-length') && + !hasHeader(headers, 'transfer-encoding') + ) { + headers['Content-Length'] = Buffer.byteLength(String(options.body)); + } + const client = url.startsWith('https:') ? https : http; + const request = client.request( + url, + { + method: options.method || 'GET', + headers, + agent: false, + rejectUnauthorized: options.insecureTls ? false : undefined, + }, + (incoming) => { + collectIncoming(incoming).then(resolve, reject); + }, + ); + const timeout = setTimeout(() => { + request.destroy(new Error(`HTTP 请求超时:${url}`)); + }, options.timeoutMs || 5000); + request.on('error', reject); + request.on('close', () => { + clearTimeout(timeout); + }); + if (options.body !== undefined) { + request.write(options.body); + } + request.end(); + }); +} + +function openRawHttpRequest(url, headers = {}) { + return new Promise((resolve, reject) => { + const target = new URL(url); + const socket = net.connect( + { host: target.hostname, port: Number(target.port) }, + () => { + socket.write(buildRawHttpRequest(target, headers)); + resolve({ + socket, + done: collectRawHttpResponse(socket, url, 35000), + }); + }, + ); + socket.on('error', reject); + }); +} + +async function rawHttpRequest(url, headers = {}) { + const request = await openRawHttpRequest(url, headers); + return request.done.finally(() => { + request.socket.destroy(); + }); +} + +function rawHttpPathRequest(baseUrl, route, headers = {}) { + return new Promise((resolve, reject) => { + const target = new URL(baseUrl); + const socket = net.connect( + { host: target.hostname, port: Number(target.port) }, + () => { + socket.write(buildRawHttpPathRequest(target, route, headers)); + collectRawHttpResponse(socket, `${baseUrl}${route}`, 35000) + .then(resolve, reject) + .finally(() => socket.destroy()); + }, + ); + socket.on('error', reject); + }); +} + +function buildRawHttpRequest(target, headers) { + const mergedHeaders = { + Host: target.host, + Connection: 'close', + ...headers, + }; + const headerLines = Object.entries(mergedHeaders).map( + ([name, value]) => `${name}: ${value}`, + ); + return [ + `GET ${target.pathname}${target.search} HTTP/1.1`, + ...headerLines, + '', + '', + ].join('\r\n'); +} + +function buildRawHttpPathRequest(target, route, headers) { + const mergedHeaders = { + Host: target.host, + Connection: 'close', + ...headers, + }; + const headerLines = Object.entries(mergedHeaders).map( + ([name, value]) => `${name}: ${value}`, + ); + return [`GET ${route} HTTP/1.1`, ...headerLines, '', ''].join('\r\n'); +} + +function collectRawHttpResponse(socket, url, timeoutMs) { + return new Promise((resolve, reject) => { + let raw = Buffer.alloc(0); + let settled = false; + const timer = setTimeout(() => { + settled = true; + socket.destroy(); + reject(new Error(`HTTP 请求超时:${url}`)); + }, timeoutMs); + + socket.on('data', (chunk) => { + raw = Buffer.concat([raw, chunk]); + let parsed; + try { + parsed = tryParseRawHttpResponse(raw); + } catch (error) { + settled = true; + clearTimeout(timer); + socket.destroy(); + reject(error); + return; + } + if (parsed) { + settled = true; + clearTimeout(timer); + resolve(parsed); + } + }); + socket.on('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timer); + reject(error); + } + }); + socket.on('close', () => { + if (!settled) { + settled = true; + clearTimeout(timer); + try { + const parsed = tryParseRawHttpResponse(raw, { final: true }); + if (parsed) { + resolve(parsed); + } else { + reject(new Error(`HTTP 响应提前关闭:${url}`)); + } + } catch (error) { + reject(error); + } + } + }); + }); +} + +function tryParseRawHttpResponse(raw, options = {}) { + const headerEnd = raw.indexOf('\r\n\r\n'); + if (headerEnd < 0) { + return null; + } + + const headerText = raw.slice(0, headerEnd).toString('latin1'); + const [statusLine, ...headerLines] = headerText.split('\r\n'); + const status = Number.parseInt(statusLine.split(' ')[1] || '0', 10); + const headers = {}; + for (const line of headerLines) { + const separator = line.indexOf(':'); + if (separator <= 0) { + continue; + } + headers[line.slice(0, separator).trim().toLowerCase()] = line + .slice(separator + 1) + .trim(); + } + + const bodyStart = headerEnd + 4; + const body = raw.slice(bodyStart); + if (hasToken(headers['transfer-encoding'], 'chunked')) { + const decoded = decodeChunkedBody(body); + if (!decoded.complete) { + return null; + } + + return { + status, + headers, + body: decoded.body.toString('utf8'), + }; + } + + const contentLength = Number.parseInt(headers['content-length'] || '', 10); + if (Number.isFinite(contentLength) && body.length < contentLength) { + return null; + } + if (Number.isFinite(contentLength)) { + return { + status, + headers, + body: body.slice(0, contentLength).toString('utf8'), + }; + } + + if (!options.final) { + return null; + } + + return { + status, + headers, + body: body.toString('utf8'), + }; +} + +function hasToken(value, expected) { + if (!value) { + return false; + } + const normalized = expected.toLowerCase(); + return value + .split(',') + .map((token) => token.trim().toLowerCase()) + .includes(normalized); +} + +function decodeChunkedBody(body) { + let offset = 0; + const chunks = []; + + while (offset < body.length) { + const lineEnd = body.indexOf('\r\n', offset); + if (lineEnd < 0) { + return { complete: false, body: Buffer.alloc(0) }; + } + + const sizeLine = body.slice(offset, lineEnd).toString('latin1'); + const sizeText = sizeLine.split(';')[0]?.trim() || ''; + const size = Number.parseInt(sizeText, 16); + if (!Number.isFinite(size)) { + throw new Error(`无法解析 chunked 响应大小:${sizeLine}`); + } + + offset = lineEnd + 2; + if (size === 0) { + const trailerEnd = body.indexOf('\r\n\r\n', offset); + if (trailerEnd >= 0) { + return { complete: true, body: Buffer.concat(chunks) }; + } + if (body.slice(offset, offset + 2).toString('latin1') === '\r\n') { + return { complete: true, body: Buffer.concat(chunks) }; + } + return { complete: false, body: Buffer.alloc(0) }; + } + + const chunkStart = offset; + const chunkEnd = chunkStart + size; + const nextChunkStart = chunkEnd + 2; + if (body.length < nextChunkStart) { + return { complete: false, body: Buffer.alloc(0) }; + } + if (body.slice(chunkEnd, nextChunkStart).toString('latin1') !== '\r\n') { + throw new Error('chunked 响应缺少 chunk 结束换行'); + } + + chunks.push(body.slice(chunkStart, chunkEnd)); + offset = nextChunkStart; + } + + return { complete: false, body: Buffer.alloc(0) }; +} + +function hasHeader(headers, name) { + const expected = name.toLowerCase(); + return Object.keys(headers).some((key) => key.toLowerCase() === expected); +} + +function collectIncoming(incoming, options = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + incoming.on('data', (chunk) => chunks.push(chunk)); + incoming.on('error', reject); + incoming.on('end', () => { + const bodyBuffer = Buffer.concat(chunks); + resolve({ + status: incoming.statusCode || 0, + headers: incoming.headers, + body: options.rawBody ? '' : bodyBuffer.toString('utf8'), + bodyBuffer, + }); + }); + }); +} + +function readRequestBody(request) { + return new Promise((resolve, reject) => { + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('error', reject); + request.on('end', () => resolve(Buffer.concat(chunks))); + }); +} + +function listen(server) { + return new Promise((resolve, reject) => { + server.on('error', reject); + server.on('connection', (socket) => { + sockets.add(socket); + socket.on('close', () => { + sockets.delete(socket); + }); + }); + server.listen(0, '127.0.0.1', () => { + servers.push(server); + const address = server.address(); + if (!address || typeof address === 'string') { + reject(new Error('无法读取 mock server 端口')); + return; + } + resolve(address.port); + }); + }); +} + +function getFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (!address || typeof address === 'string') { + server.close(); + reject(new Error('无法分配临时端口')); + return; + } + const { port } = address; + server.close(() => resolve(port)); + }); + }); +} + +async function waitForHttp(url, expectedStatus) { + const startedAt = Date.now(); + let lastError = ''; + while (Date.now() - startedAt < 10000) { + try { + const response = await requestHttp(url); + if (response.status === expectedStatus) { + return; + } + lastError = `HTTP ${response.status}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + await delay(100); + } + throw new Error(`等待 Pingora 网关就绪超时:${lastError}`); +} + +function runCommand(command, args) { + console.log( + `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`, + ); + const result = spawnSync(command, args, { + cwd: repoRoot, + env: smokeEnv(), + shell: false, + stdio: 'inherit', + }); + + if (result.error) { + throw new Error(`${command} 启动失败:${result.error.message}`); + } + if (result.signal) { + throw new Error(`${command} 被信号终止:${result.signal}`); + } + if ((result.status ?? 0) !== 0) { + throw new Error( + `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${result.status}`, + ); + } +} + +async function runDirectLiveSmoke(tlsBaseUrl, redirectBaseUrl, probeToken, accessLogFile) { + const output = await runCommandAsync('node', [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + tlsBaseUrl, + '--http-base-url', + redirectBaseUrl, + '--host', + new URL(tlsBaseUrl).host, + '--probe-token', + probeToken, + '--pingora-access-log', + accessLogFile, + '--redirect-host', + new URL(tlsBaseUrl).host, + '--spacetime-database', + 'genarrative', + '--require-wss-upgrade', + '--insecure-tls', + '--json', + ]); + assertDirectLiveAccessLogEvidence(output.stdout); +} + +function runCommandAsync(command, args) { + console.log( + `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`, + ); + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + const child = spawn(command, args, { + cwd: repoRoot, + env: smokeEnv(), + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout?.on('data', (chunk) => { + const text = String(chunk); + stdout += text; + process.stdout.write(text); + }); + child.stderr?.on('data', (chunk) => { + const text = String(chunk); + stderr += text; + process.stderr.write(text); + }); + child.on('error', (error) => { + reject(new Error(`${command} 启动失败:${error.message}`)); + }); + child.on('exit', (status, signal) => { + if (signal) { + reject(new Error(`${command} 被信号终止:${signal}`)); + return; + } + if ((status ?? 0) !== 0) { + reject( + new Error( + `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${status}\nstdout:\n${stdout}\nstderr:\n${stderr}`, + ), + ); + return; + } + resolve({ stdout, stderr }); + }); + }); +} + +function assertDirectLiveAccessLogEvidence(stdout) { + const payload = extractJsonObject(stdout, stdout.indexOf('{')); + if (!payload) { + failures.push('direct live JSON 输出缺失。'); + return; + } + let result; + try { + result = JSON.parse(payload); + } catch (error) { + failures.push( + `direct live JSON 输出无法解析:${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + const accessLogCheck = result.results?.find( + (item) => item.name === 'direct-access-log', + ); + const staticAssetCheck = result.results?.find( + (item) => item.name === 'https-static-asset', + ); + ensure( + staticAssetCheck?.fingerprinted?.statusCode === 200, + 'direct live 必须验证首页引用的指纹静态资源 GET 200。', + ); + ensure( + staticAssetCheck?.fingerprinted?.head?.statusCode === 200, + 'direct live 必须验证首页引用的指纹静态资源 HEAD 200。', + ); + ensure( + staticAssetCheck?.fingerprinted?.range?.statusCode === 206, + 'direct live 必须验证首页引用的指纹静态资源 Range 206。', + ); + ensure( + staticAssetCheck?.headers?.['cache-control'] === 'no-cache', + 'direct live JSON 必须保留普通静态资源 Cache-Control 证据。', + ); + ensure( + Boolean(staticAssetCheck?.headers?.etag), + 'direct live JSON 必须保留普通静态资源 ETag 证据。', + ); + ensure( + staticAssetCheck?.range?.headers?.['content-range']?.startsWith('bytes 0-0/'), + 'direct live JSON 必须保留普通静态资源 Content-Range 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.headers?.['cache-control'] === + 'public, max-age=31536000, immutable', + 'direct live JSON 必须保留指纹静态资源 immutable Cache-Control 证据。', + ); + ensure( + Boolean(staticAssetCheck?.fingerprinted?.headers?.etag), + 'direct live JSON 必须保留指纹静态资源 ETag 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.range?.headers?.['content-range']?.startsWith( + 'bytes 0-0/', + ), + 'direct live JSON 必须保留指纹静态资源 Content-Range 证据。', + ); + ensure( + staticAssetCheck?.notModified?.etag?.statusCode === 304, + 'direct live 必须验证首页引用的静态资源 ETag 304。', + ); + ensure( + staticAssetCheck?.notModified?.lastModified?.statusCode === 304, + 'direct live 必须验证首页引用的静态资源 Last-Modified 304。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.etag?.statusCode === 304, + 'direct live 必须验证首页引用的指纹静态资源 ETag 304。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.lastModified?.statusCode === 304, + 'direct live 必须验证首页引用的指纹静态资源 Last-Modified 304。', + ); + ensure( + staticAssetCheck?.notModified?.etag?.headers?.['cache-control'] === + staticAssetCheck?.headers?.['cache-control'], + 'direct live JSON 必须保留普通静态 304 Cache-Control 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.etag?.headers?.['cache-control'] === + staticAssetCheck?.fingerprinted?.headers?.['cache-control'], + 'direct live JSON 必须保留指纹静态 304 Cache-Control 证据。', + ); + if (!accessLogCheck) { + failures.push('direct live JSON 缺少 direct-access-log 检查结果。'); + return; + } + ensure(accessLogCheck.checked >= 10, 'direct live access log 检查请求数不足。'); + ensure( + accessLogCheck.matchedCount === accessLogCheck.checked, + 'direct live access log matchedCount 必须等于 checked。', + ); + ensure( + accessLogCheck.missingCount === 0, + 'direct live access log 不应缺少 request_id。', + ); + ensure( + accessLogCheck.mismatchCount === 0, + 'direct live access log 不应出现 method/path/status 漂移。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'wss-spacetime-subscribe' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 101 && + item.actualStatusCode === 101, + ), + 'direct live access log 必须保留 WSS 101 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-asset-head' && + item.expectedMethod === 'HEAD' && + item.actualMethod === 'HEAD' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留静态 HEAD 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留指纹静态 GET 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-head' && + item.expectedMethod === 'HEAD' && + item.actualMethod === 'HEAD' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留指纹静态 HEAD 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-range' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 206 && + item.actualStatusCode === 206, + ), + 'direct live access log 必须保留指纹静态 Range 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-asset-etag-304' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 304 && + item.actualStatusCode === 304, + ), + 'direct live access log 必须保留静态 ETag 304 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-etag-304' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 304 && + item.actualStatusCode === 304, + ), + 'direct live access log 必须保留指纹静态 ETag 304 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.missing) && + accessLogCheck.missing.length === 0, + 'direct live access log missing 明细必须为空数组。', + ); + ensure( + Array.isArray(accessLogCheck.mismatches) && + accessLogCheck.mismatches.length === 0, + 'direct live access log mismatches 明细必须为空数组。', + ); +} + +function extractJsonObject(text, startIndex) { + if (startIndex < 0) { + return ''; + } + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + + return ''; +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : arg, + ); +} + +function resolveGatewayBinary() { + const explicit = process.env.GENARRATIVE_PINGORA_GATEWAY_BINARY; + if (explicit) { + return explicit; + } + + const candidates = [ + path.join(repoRoot, 'server-rs', 'target', 'debug', 'pingora-gateway'), + path.join(repoRoot, 'target', 'debug', 'pingora-gateway'), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (!found) { + throw new Error( + `未找到 pingora-gateway debug 二进制,请先构建或设置 GENARRATIVE_PINGORA_GATEWAY_BINARY。候选:${candidates.join(', ')}`, + ); + } + return found; +} + +function smokeEnv() { + return { + ...process.env, + PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`, + }; +} + +function collectProcessLogs(child, name) { + child.stdout?.on('data', (chunk) => { + if (config.verbose) { + process.stdout.write(`[${name}] ${chunk}`); + } + }); + child.stderr?.on('data', (chunk) => { + if (config.verbose) { + process.stderr.write(`[${name}] ${chunk}`); + } + }); +} + +function encodeWebSocketTextFrame(text) { + const payload = Buffer.from(text, 'utf8'); + if (payload.length > 125) { + throw new Error('smoke WebSocket frame payload too large'); + } + return Buffer.concat([Buffer.from([0x81, payload.length]), payload]); +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForCondition(predicate, timeoutMs = 2000) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (await predicate()) { + return; + } + await delay(20); + } + throw new Error('等待 smoke 条件达成超时'); +} + +function ensure(condition, message) { + if (!condition) { + failures.push(message); + } +} + +function describeRequests(requests) { + return JSON.stringify( + requests.map((request) => ({ + method: request.method, + url: request.url, + xForwardedFor: request.headers?.['x-forwarded-for'], + aborted: Boolean(request.aborted), + })), + ); +} + +async function cleanup() { + for (const { child, name } of childProcesses.reverse()) { + await stopChild(child, name); + } + + for (const socket of sockets) { + socket.destroy(); + } + sockets.clear(); + + await Promise.all( + servers.reverse().map( + (server) => + new Promise((resolve) => { + const timer = setTimeout(resolve, 1000); + server.close(() => { + clearTimeout(timer); + resolve(); + }); + }), + ), + ); + + await Promise.all( + tempDirs.reverse().map((dir) => rm(dir, { recursive: true, force: true })), + ); +} + +async function stopChild(child, name) { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + + child.kill('SIGTERM'); + const exited = await waitForExit(child, 3000); + if (!exited) { + child.kill('SIGKILL'); + const killed = await waitForExit(child, 3000); + if (!killed) { + failures.push(`${name} 未能退出`); + } + } +} + +function waitForExit(child, timeoutMs) { + return new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(true); + return; + } + + const timer = setTimeout(() => { + child.off('exit', onExit); + resolve(false); + }, timeoutMs); + + function onExit() { + clearTimeout(timer); + resolve(true); + } + + child.once('exit', onExit); + }); +} diff --git a/scripts/check-pingora-health-patrol-env-switch.mjs b/scripts/check-pingora-health-patrol-env-switch.mjs new file mode 100644 index 000000000..922664aeb --- /dev/null +++ b/scripts/check-pingora-health-patrol-env-switch.mjs @@ -0,0 +1,801 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const SWITCH_SCRIPT = 'scripts/deploy/pingora-health-patrol-env-switch.mjs'; +const ENV_CHECK_SCRIPT = path.join( + process.cwd(), + 'scripts/check-production-health-patrol-env.mjs', +); +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-health-patrol-switch-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-health-patrol-env-switch] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-health-patrol-env-switch] OK'); + +function main() { + assertScriptShape(); + assertDryRunDoesNotModifyEnv(); + assertDryRunEnvFileModeIsPrivate(); + assertApplyPreservesEnvFileModeAndOwnership(); + assertApplySwitchesToPingoraDirectAndPreservesOtherKeys(); + assertApplySwitchesBackToNginxAndClearsHost(); + assertApplyDoesNotWriteWhenVerificationFails(); + assertApplyRejectsSymlinkEnvFileBeforeWrite(); + assertMissingManagedKeysAreAppended(); + assertRejectsDirectLoopbackWithoutHost(); + assertRejectsInvalidAndConflictingHostOptions(); + assertRejectsRelativePaths(); + assertRejectsFilesystemRootPaths(); + assertRejectsControlCharacterInputsBeforeWrite(); + assertRejectsInvalidBoolEnv(); + assertReleaseLayoutUsesBundledCheckScript(); +} + +function assertScriptShape() { + const content = readFileSync(SWITCH_SCRIPT, 'utf8'); + assertIncludes(content, '--apply', '切换脚本必须显式要求 --apply 才写 env。'); + assertIncludes( + content, + '当前是 dry-run', + '切换脚本必须在 dry-run 中明确不会写 env。', + ); + assertIncludes( + content, + 'check-production-health-patrol-env.mjs', + '切换脚本必须复用生产 health patrol env 复核脚本。', + ); + assertIncludes( + content, + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host', + '切换脚本必须阻断 direct loopback 缺少正式 Host。', + ); + assertIncludes( + content, + 'health patrol env 中存在重复配置', + '切换脚本必须拒绝重复目标键,避免 env 解析歧义。', + ); + assertIncludes( + content, + 'runHealthPatrolEnvCheck(checkFile);', + '切换脚本必须先用临时目标 env 复核。', + ); + assertIncludes( + content, + 'writeEnvFileAtomically(config.envFile, nextText);', + '切换脚本通过复核后才可原子写入真实 env。', + ); + assertIncludes( + content, + '--env-file 不能是符号链接', + 'apply 写入真实 env 前必须拒绝符号链接目标。', + ); + assertIncludes( + content, + 'DRY_RUN_ENV_FILE_MODE = 0o600', + '临时复核 env 文件权限必须固定为 0600。', + ); + assertIncludes( + content, + 'chownSync(tempFile, currentStat.uid, currentStat.gid);', + '真实 env 原子替换必须保留原文件 owner/group。', + ); +} + +function assertDryRunDoesNotModifyEnv() { + const envFile = writeEnv('dry-run.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ]); + + assertStatus(result, 0, 'dry-run 应成功并通过临时 env 复核。'); + assertIncludes(result.stdout, 'apply=false', 'dry-run 输出必须显示 apply=false。'); + assertIncludes( + result.stdout, + '[check:production-health-patrol-env] OK', + 'dry-run 必须用临时 env 跑复核脚本。', + ); + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('dry-run 不应修改 health patrol env 文件。'); + } +} + +function assertDryRunEnvFileModeIsPrivate() { + const envFile = writeEnv('dry-run-mode.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const modeCheckScript = path.join(tmpRoot, 'check-dry-run-env-mode.mjs'); + writeFileSync( + modeCheckScript, + [ + '#!/usr/bin/env node', + "import { statSync } from 'node:fs';", + "const envFileIndex = process.argv.indexOf('--env-file');", + 'const envFile = process.argv[envFileIndex + 1];', + 'const mode = statSync(envFile).mode & 0o777;', + 'if (mode !== 0o600) {', + ' console.error(`dry-run env mode must be 0600, got ${mode.toString(8)}`);', + ' process.exit(1);', + '}', + 'console.log(`[fake-health-patrol-env-check] dry-run env mode ${mode.toString(8)}`);', + '', + ].join('\n'), + 'utf8', + ); + chmodSync(modeCheckScript, 0o755); + + const result = runSwitch([ + '--env-file', + envFile, + '--check-script', + modeCheckScript, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ]); + + assertStatus(result, 0, 'dry-run 临时复核 env 文件权限应为 0600。'); + assertIncludes( + result.stdout, + 'dry-run env mode 600', + 'dry-run 必须把 0600 临时 env 交给复核脚本。', + ); +} + +function assertApplyPreservesEnvFileModeAndOwnership() { + const envFile = writeEnv('apply-mode-owner.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + chmodSync(envFile, 0o640); + const beforeStat = statSync(envFile); + + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 0, 'apply 原子替换应成功。'); + const afterStat = statSync(envFile); + const beforeMode = beforeStat.mode & 0o777; + const afterMode = afterStat.mode & 0o777; + if (afterMode !== beforeMode) { + failures.push( + `apply 原子替换必须保留 env 文件权限,实际 ${afterMode.toString(8)},预期 ${beforeMode.toString(8)}。`, + ); + } + if (afterStat.uid !== beforeStat.uid || afterStat.gid !== beforeStat.gid) { + failures.push('apply 原子替换必须保留 env 文件 owner/group。'); + } +} + +function assertApplySwitchesToPingoraDirectAndPreservesOtherKeys() { + const envFile = writeEnv('direct.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS: '5000', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 0, 'apply 切到 pingora-direct 应成功。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct', + 'apply 必须写入 pingora-direct gateway mode。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://127.0.0.1', + 'apply 必须写入直连 public base URL。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=example.com', + 'apply 必须写入正式 public Host。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000', + 'apply 必须保留其它 health patrol env 项。', + ); +} + +function assertApplySwitchesBackToNginxAndClearsHost() { + const envFile = writeEnv('nginx.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'example.com', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + + assertStatus(result, 0, 'apply 切回 nginx 并清空 Host 应成功。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + '回退 apply 必须写回 nginx gateway mode。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1', + '回退 apply 必须恢复 Nginx public base URL。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + '回退 apply 必须清空 public Host。', + ); +} + +function assertApplyDoesNotWriteWhenVerificationFails() { + const envFile = writeEnv('apply-verify-fails.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + const failingCheckScript = path.join( + tmpRoot, + 'failing-health-patrol-env-check.mjs', + ); + writeFileSync( + failingCheckScript, + [ + '#!/usr/bin/env node', + 'console.error("[fake-health-patrol-env-check] forced failure");', + 'process.exit(1);', + '', + ].join('\n'), + 'utf8', + ); + + const result = runSwitch([ + '--env-file', + envFile, + '--check-script', + failingCheckScript, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 1, '复核失败时 apply 必须失败。'); + assertIncludes( + result.stderr, + '[fake-health-patrol-env-check] forced failure', + '复核失败时必须透出复核脚本错误。', + ); + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('复核失败时 apply 不应写入真实 health patrol env 文件。'); + } +} + +function assertApplyRejectsSymlinkEnvFileBeforeWrite() { + const targetEnvFile = writeEnv('symlink-target.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const symlinkEnvFile = path.join(tmpRoot, 'symlink-health-patrol.env'); + symlinkSync(targetEnvFile, symlinkEnvFile); + const before = readFileSync(targetEnvFile, 'utf8'); + + const result = runSwitch([ + '--env-file', + symlinkEnvFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 1, 'apply 传入符号链接 env 文件时必须失败。'); + assertIncludes( + result.stderr, + '--env-file 不能是符号链接', + '符号链接 env 文件失败时必须给出明确错误。', + ); + const after = readFileSync(targetEnvFile, 'utf8'); + if (after !== before) { + failures.push('符号链接 env 文件被拒绝时不应写入真实目标文件。'); + } + const linkText = readFileSync(symlinkEnvFile, 'utf8'); + if (linkText !== before) { + failures.push('符号链接 env 文件被拒绝时不应替换或改写链接内容。'); + } +} + +function assertMissingManagedKeysAreAppended() { + const envFile = path.join(tmpRoot, 'append.env'); + writeFileSync( + envFile, + [ + '# existing env', + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000', + '', + ].join('\n'), + 'utf8', + ); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + + assertStatus(result, 0, '缺少目标键时 apply 应追加并通过复核。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + '缺少 gateway mode 时必须追加。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1', + '缺少 public base URL 时必须追加。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + '缺少 public Host 时必须追加。', + ); +} + +function assertRejectsDirectLoopbackWithoutHost() { + const envFile = writeEnv('direct-missing-host.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--clear-public-host', + ]); + + assertStatus(result, 1, 'direct loopback 缺少 Host 必须失败。'); + assertIncludes( + result.stderr, + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host', + 'direct loopback 缺少 Host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidAndConflictingHostOptions() { + const envFile = writeEnv('invalid-host.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const invalidHost = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--public-host', + 'https://example.com', + ]); + assertStatus(invalidHost, 1, '非法 Host 必须失败。'); + assertIncludes( + invalidHost.stderr, + '--public-host 只能是 host 或 host:port', + '非法 Host 必须给出明确错误。', + ); + + const conflictingHost = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--public-host', + 'example.com', + '--clear-public-host', + ]); + assertStatus(conflictingHost, 1, 'Host 与清空 Host 同时传入必须失败。'); + assertIncludes( + conflictingHost.stderr, + '--public-host 和 --clear-public-host 不能同时使用', + 'Host 选项冲突必须给出明确错误。', + ); +} + +function assertRejectsRelativePaths() { + const relativeEnv = runSwitch([ + '--env-file', + 'relative.env', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(relativeEnv, 1, '相对 env 路径必须失败。'); + assertIncludes( + relativeEnv.stderr, + '--env-file 必须是绝对路径', + '相对 env 路径必须给出明确错误。', + ); + + const envFile = writeEnv('relative-check.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const relativeCheck = runSwitch([ + '--env-file', + envFile, + '--check-script', + 'scripts/check-production-health-patrol-env.mjs', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(relativeCheck, 1, '相对 check script 路径必须失败。'); + assertIncludes( + relativeCheck.stderr, + '--check-script 必须是绝对路径', + '相对 check script 路径必须给出明确错误。', + ); +} + +function assertRejectsFilesystemRootPaths() { + const rootEnv = runSwitch([ + '--env-file', + '/', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(rootEnv, 1, '文件系统根目录 env 路径必须失败。'); + assertIncludes( + rootEnv.stderr, + '--env-file 不能是文件系统根目录', + '文件系统根目录 env 路径必须给出明确错误。', + ); + + const envFile = writeEnv('root-check-script.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const rootCheckScript = runSwitch([ + '--env-file', + envFile, + '--check-script', + '/', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(rootCheckScript, 1, '文件系统根目录 check script 路径必须失败。'); + assertIncludes( + rootCheckScript.stderr, + '--check-script 不能是文件系统根目录', + '文件系统根目录 check script 路径必须给出明确错误。', + ); +} + +function assertRejectsControlCharacterInputsBeforeWrite() { + const envFile = writeEnv('control-character.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + + const envFileResult = runSwitch([ + '--env-file', + `${envFile}\nspoofed`, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + assertStatus(envFileResult, 1, '带控制字符的 env 文件路径必须失败。'); + assertIncludes( + envFileResult.stderr, + '--env-file 不能包含换行或 NUL 字符', + '带控制字符的 env 文件路径必须给出明确错误。', + ); + + const checkScriptResult = runSwitch([ + '--env-file', + envFile, + '--check-script', + `${ENV_CHECK_SCRIPT}\nspoofed`, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + assertStatus(checkScriptResult, 1, '带控制字符的 check script 路径必须失败。'); + assertIncludes( + checkScriptResult.stderr, + '--check-script 不能包含换行或 NUL 字符', + '带控制字符的 check script 路径必须给出明确错误。', + ); + + const baseUrlResult = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1\nspoofed', + '--clear-public-host', + '--apply', + ]); + assertStatus(baseUrlResult, 1, '带控制字符的 public base URL 必须失败。'); + assertIncludes( + baseUrlResult.stderr, + '--public-base-url 不能包含换行或 NUL 字符', + '带控制字符的 public base URL 必须给出明确错误。', + ); + + const hostResult = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com\nspoofed', + '--apply', + ]); + assertStatus(hostResult, 1, '带控制字符的 public Host 必须失败。'); + assertIncludes( + hostResult.stderr, + '--public-host 不能包含换行或 NUL 字符', + '带控制字符的 public Host 必须给出明确错误。', + ); + + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('带控制字符的切换参数被拒绝时不应写入真实 env 文件。'); + } +} + +function assertRejectsInvalidBoolEnv() { + const envFile = writeEnv('invalid-bool-env.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runSwitch( + [ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + ], + { + GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST: 'maybe', + }, + ); + assertStatus(result, 1, 'health patrol env 切换脚本必须拒绝非法布尔 env。'); + assertIncludes( + result.stderr, + 'GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST 必须是布尔值', + 'health patrol env 切换脚本非法布尔 env 必须给出明确错误。', + ); +} + +function assertReleaseLayoutUsesBundledCheckScript() { + const releaseRoot = path.join(tmpRoot, 'current-release'); + const releaseScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ); + const releaseCheckScript = path.join( + releaseRoot, + 'scripts/check-production-health-patrol-env.mjs', + ); + const envFile = writeEnv('release-layout.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + + mkdirSync(path.dirname(releaseScript), { recursive: true }); + mkdirSync(path.dirname(releaseCheckScript), { recursive: true }); + copyFileSync(SWITCH_SCRIPT, releaseScript); + copyFileSync('scripts/check-production-health-patrol-env.mjs', releaseCheckScript); + chmodSync(releaseScript, 0o755); + + const result = spawnSync( + 'node', + [ + '--', + releaseScript, + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ], + { cwd: tmpRoot, encoding: 'utf8', env: cleanEnv() }, + ); + + assertStatus( + result, + 0, + 'current release 中的切换脚本 dry-run 应能使用随包 env 复核脚本。', + ); + assertIncludes( + result.stdout, + realpathSync(releaseCheckScript), + 'current release 中的切换脚本必须默认调用随包 env 复核脚本。', + ); + if (!existsSync(releaseCheckScript)) { + failures.push('release layout 测试必须保留随包 env 复核脚本。'); + } +} + +function writeEnv(name, values) { + const filePath = path.join(tmpRoot, name); + const body = Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join('\n'); + writeFileSync(filePath, `${body}\n`, 'utf8'); + return filePath; +} + +function runSwitch(args, extraEnv = {}) { + return spawnSync( + 'node', + ['--', SWITCH_SCRIPT, '--check-script', ENV_CHECK_SCRIPT, ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...cleanEnv(), + ...extraEnv, + }, + }, + ); +} + +function cleanEnv() { + return { + ...process.env, + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: '', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: '', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_ENV_CHECK_SCRIPT: '', + }; +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} diff --git a/scripts/check-pingora-production-release-build.mjs b/scripts/check-pingora-production-release-build.mjs new file mode 100644 index 000000000..73da296c2 --- /dev/null +++ b/scripts/check-pingora-production-release-build.mjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const BUILD_SCRIPT = 'scripts/build-production-release.sh'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-production-release-build-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-production-release-build] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-production-release-build] OK'); + +function main() { + const fixture = prepareFixture(); + const result = runBuild(fixture); + assertStatus(result, 0, 'Pingora production release 真实构建烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'api-server'), + '发布包必须包含 api-server 占位 release binary。', + ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + '发布包必须包含 api-server checksum。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '真实 include Pingora 时发布包必须包含 pingora-gateway。', + ); + assertExecutable( + path.join(releaseDir, 'pingora-gateway'), + 'pingora-gateway 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '真实 include Pingora 时发布包必须包含 pingora-gateway checksum。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + '真实 include Pingora 的发布包仍必须携带 direct live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), + '真实 include Pingora 的发布包仍必须携带 current release 自审脚本。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/nginx-route-parity.matrix.json'), + '真实 include Pingora 的发布包仍必须携带 Nginx/Pingora 路由矩阵。', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (manifest.component_type !== 'api-server') { + failures.push( + `release manifest component_type 应为 api-server,实际 ${manifest.component_type}`, + ); + } + if (!manifest.artifacts?.some((item) => item.path === 'api-server')) { + failures.push('release manifest 必须登记 api-server artifact。'); + } + if (!manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push( + '真实 include Pingora 时 release manifest 必须登记 pingora-gateway artifact。', + ); + } + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function prepareFixture() { + const cargoTargetDir = path.join(tmpRoot, 'cargo-target'); + const binaryDir = path.join( + cargoTargetDir, + 'x86_64-unknown-linux-gnu/release', + ); + const version = `check-pingora-production-release-build-${process.pid}-${Date.now()}`; + mkdirSync(binaryDir, { recursive: true }); + + const apiBinary = path.join(binaryDir, 'api-server'); + writeFileSync(apiBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', apiBinary], { encoding: 'utf8' }); + + return { cargoTargetDir, version }; +} + +function runBuild(fixture) { + return spawnSync( + 'bash', + [ + BUILD_SCRIPT, + '--component', + 'api-server', + '--name', + fixture.version, + '--skip-api-build', + '--include-pingora-gateway', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + CARGO_TARGET_DIR: fixture.cargoTargetDir, + PATH: `${path.join(process.env.HOME || '', '.local', 'bin')}:${process.env.PATH || ''}`, + SOURCE_BRANCH: 'test-branch', + SOURCE_COMMIT: 'test-commit', + }, + }, + ); +} + +function readJson(filePath) { + try { + return JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + failures.push(`${filePath} 不是合法 JSON: ${error.message}`); + return {}; + } +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function assertExecutable(filePath, reason) { + if (!existsSync(filePath)) { + return; + } + if ((statSync(filePath).mode & 0o111) === 0) { + failures.push(`${reason} 文件不可执行: ${filePath}`); + } +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} diff --git a/scripts/check-pingora-realpath-canary-toggle.mjs b/scripts/check-pingora-realpath-canary-toggle.mjs new file mode 100644 index 000000000..22f0e0c95 --- /dev/null +++ b/scripts/check-pingora-realpath-canary-toggle.mjs @@ -0,0 +1,432 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const ENABLE_SCRIPT = 'scripts/deploy/pingora-realpath-canary-enable.sh'; +const DISABLE_SCRIPT = 'scripts/deploy/pingora-realpath-canary-disable.sh'; +const failures = []; +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'genarrative-realpath-canary-')); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-realpath-canary-toggle] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-realpath-canary-toggle] OK'); + +function main() { + assertScriptShape(); + assertDryRunDoesNotWrite(); + assertApplyWritesRenderedConfigAndRunsLive(); + assertEnableRollsBackWhenLiveFails(); + assertDisableRemovesConfig(); + assertDisableRollsBackWhenReloadFails(); + assertRejectsUnsafeInputs(); +} + +function assertScriptShape() { + for (const script of [ENABLE_SCRIPT, DISABLE_SCRIPT]) { + const content = readFileSync(script, 'utf8'); + assertIncludes(content, '--apply', `${script} 必须默认 dry-run 并要求显式 --apply。`); + assertIncludes(content, 'nginx -t', `${script} 帮助文案必须说明会先做 nginx -t。`); + assertIncludes( + content, + 'zz-genarrative-pingora-realpath-canary.conf', + `${script} 必须默认使用晚于主站配置加载的 conf.d 文件名。`, + ); + assertIncludes( + content, + '不能包含换行或 NUL 字符', + `${script} 必须拒绝控制字符参数。`, + ); + assertIncludes(content, '不能是文件系统根目录', `${script} 必须拒绝根路径。`); + } + + const enableContent = readFileSync(ENABLE_SCRIPT, 'utf8'); + assertIncludes( + enableContent, + 'check-pingora-canary-live.mjs', + 'enable 脚本必须默认运行 realpath live smoke。', + ); + assertIncludes( + enableContent, + '恢复写入前配置', + 'enable 脚本失败时必须恢复写入前配置。', + ); + + const disableContent = readFileSync(DISABLE_SCRIPT, 'utf8'); + assertIncludes( + disableContent, + '恢复删除前配置', + 'disable 脚本失败时必须恢复删除前配置。', + ); +} + +function assertDryRunDoesNotWrite() { + const fixture = createFixture('dry-run'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + + const result = runScript(ENABLE_SCRIPT, [ + '--probe-token', + 'dry-run-token', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ]); + + assertStatus(result, 0, 'enable dry-run 应成功。'); + if (existsSync(targetPath)) { + failures.push('enable dry-run 不应写入目标 Nginx 配置。'); + } + assertIncludes(result.stdout, 'dry-run', 'enable dry-run 应输出 dry-run 提示。'); +} + +function assertApplyWritesRenderedConfigAndRunsLive() { + const fixture = createFixture('apply-ok'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + const result = runScript(ENABLE_SCRIPT, [ + '--apply', + '--probe-token', + 'apply-real-token', + '--host', + 'dev.genarrative.world', + '--base-url', + 'http://127.0.0.1:18083', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + '--no-status', + ]); + + assertStatus(result, 0, 'enable apply 应成功。'); + const rendered = readFileSync(targetPath, 'utf8'); + assertIncludes(rendered, '"apply-real-token"', 'enable apply 应替换 probe token。'); + if (rendered.includes('__GENARRATIVE_PINGORA_PROBE_TOKEN__')) { + failures.push('enable apply 后目标配置不应保留 probe token 占位符。'); + } + const calls = readFileSync(fixture.callsPath, 'utf8'); + assertIncludes(calls, 'nginx -t', 'enable apply 必须执行 nginx -t。'); + assertIncludes(calls, 'systemctl reload nginx.service', 'enable apply 必须 reload nginx。'); + assertIncludes( + calls, + 'live --realpath --base-url http://127.0.0.1:18083 --host dev.genarrative.world', + 'enable apply 必须执行 realpath live smoke。', + ); +} + +function assertEnableRollsBackWhenLiveFails() { + const fixture = createFixture('enable-live-fails', { liveExitCode: 7 }); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'previous-config\n', 'utf8'); + + const result = runScript( + ENABLE_SCRIPT, + [ + '--apply', + '--probe-token', + 'rollback-token', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + '--no-status', + ], + fixture.env, + ); + + if (result.status === 0) { + failures.push('realpath live smoke 失败时 enable apply 必须失败。'); + } + const restored = readFileSync(targetPath, 'utf8'); + if (restored !== 'previous-config\n') { + failures.push('realpath live smoke 失败时 enable apply 必须恢复旧配置。'); + } + assertIncludes( + result.stderr, + '恢复写入前配置', + 'enable live 失败时必须说明已恢复写入前配置。', + ); +} + +function assertDisableRemovesConfig() { + const fixture = createFixture('disable-ok'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'enabled-config\n', 'utf8'); + + const result = runScript( + DISABLE_SCRIPT, + [ + '--apply', + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--no-status', + ], + fixture.env, + ); + + assertStatus(result, 0, 'disable apply 应成功。'); + if (existsSync(targetPath)) { + failures.push('disable apply 成功后必须删除 realpath canary 配置。'); + } + const calls = readFileSync(fixture.callsPath, 'utf8'); + assertIncludes(calls, 'nginx -t', 'disable apply 必须执行 nginx -t。'); + assertIncludes(calls, 'systemctl reload nginx.service', 'disable apply 必须 reload nginx。'); +} + +function assertDisableRollsBackWhenReloadFails() { + const fixture = createFixture('disable-reload-fails', { reloadExitCode: 9 }); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'enabled-config\n', 'utf8'); + + const result = runScript(DISABLE_SCRIPT, [ + '--apply', + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--no-status', + ]); + + if (result.status === 0) { + failures.push('reload 失败时 disable apply 必须失败。'); + } + const restored = readFileSync(targetPath, 'utf8'); + if (restored !== 'enabled-config\n') { + failures.push('reload 失败时 disable apply 必须恢复删除前配置。'); + } + assertIncludes( + result.stderr, + '恢复删除前配置', + 'disable reload 失败时必须说明已恢复删除前配置。', + ); +} + +function assertRejectsUnsafeInputs() { + const fixture = createFixture('unsafe-inputs'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + const cases = [ + { + name: 'missing token', + script: ENABLE_SCRIPT, + args: [ + '--apply', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ], + expected: '--apply 必须提供 --probe-token', + }, + { + name: 'url host', + script: ENABLE_SCRIPT, + args: [ + '--probe-token', + 'unsafe-token', + '--host', + 'https://dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ], + expected: '不能是 URL', + }, + { + name: 'relative target', + script: DISABLE_SCRIPT, + args: ['--target-path', 'relative.conf'], + expected: '--target-path 必须是绝对路径', + }, + { + name: 'root target', + script: DISABLE_SCRIPT, + args: ['--target-path', '/'], + expected: '--target-path 不能是文件系统根目录', + }, + ]; + + for (const item of cases) { + const result = runScript(item.script, item.args); + if (result.status === 0) { + failures.push(`${item.name}: 不安全参数必须失败。`); + } + assertIncludes( + `${result.stderr}\n${result.stdout}`, + item.expected, + `${item.name}: 应输出明确错误。`, + ); + } +} + +function createFixture(name, options = {}) { + const root = path.join(tmpRoot, name); + const nginxDir = path.join(root, 'nginx-conf'); + mkdirSync(nginxDir, { recursive: true }); + const callsPath = path.join(root, 'calls.log'); + const templatePath = path.join(root, 'genarrative-pingora-realpath-canary.conf'); + const nginxBinary = path.join(root, 'nginx'); + const systemctlBinary = path.join(root, 'systemctl'); + const liveScript = path.join(root, 'live.mjs'); + + writeFileSync( + templatePath, + [ + 'server {', + ' listen 127.0.0.1:18083;', + ' access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;', + ' location = /__genarrative_pingora_realpath_canary/healthz {', + ' proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__";', + ' proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz;', + ' }', + '}', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + nginxBinary, + [ + '#!/usr/bin/env bash', + `echo "nginx $*" >> ${shellQuote(callsPath)}`, + 'if [[ "${GENARRATIVE_FAKE_NGINX_T_EXIT:-0}" != "0" ]]; then exit "${GENARRATIVE_FAKE_NGINX_T_EXIT}"; fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + systemctlBinary, + [ + '#!/usr/bin/env bash', + `echo "systemctl $*" >> ${shellQuote(callsPath)}`, + `reload_exit="\${GENARRATIVE_FAKE_RELOAD_EXIT:-${Number(options.reloadExitCode || 0)}}"`, + 'if [[ "$1" == "reload" && "${reload_exit}" != "0" ]]; then exit "${reload_exit}"; fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + liveScript, + [ + '#!/usr/bin/env node', + "import { appendFileSync } from 'node:fs';", + `appendFileSync(${JSON.stringify(callsPath)}, 'live ' + process.argv.slice(2).join(' ') + '\\n');`, + `process.exit(Number(process.env.GENARRATIVE_FAKE_LIVE_EXIT || ${Number(options.liveExitCode || 0)}));`, + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(nginxBinary); + chmodExecutable(systemctlBinary); + chmodExecutable(liveScript); + + return { + root, + nginxDir, + callsPath, + templatePath, + nginxBinary, + systemctlBinary, + liveScript, + env: { + GENARRATIVE_FAKE_RELOAD_EXIT: String(options.reloadExitCode || 0), + GENARRATIVE_FAKE_LIVE_EXIT: String(options.liveExitCode || 0), + }, + }; +} + +function runScript(script, args, env = {}) { + return spawnSync('bash', [script, ...args], { + cwd: process.cwd(), + env: { ...process.env, ...env }, + encoding: 'utf8', + }); +} + +function chmodExecutable(file) { + chmodSync(file, 0o755); +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +function assertStatus(result, expected, message) { + if (result.status !== expected) { + failures.push( + `${message} 实际退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, message) { + if (!content.includes(needle)) { + failures.push(`${message} 缺少 ${needle}`); + } +} diff --git a/scripts/check-pingora-release-readiness-plan.mjs b/scripts/check-pingora-release-readiness-plan.mjs new file mode 100644 index 000000000..f2673d276 --- /dev/null +++ b/scripts/check-pingora-release-readiness-plan.mjs @@ -0,0 +1,4336 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; + +const requireDirectBaseArgs = [ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', +]; + +const cutoverRequiredArgs = [ + '--direct-preflight-check-ports-free', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-nginx-smoke-expect-body', + '', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', +]; + +function cutoverPlanArgs(...extraArgs) { + return [ + ...requireDirectBaseArgs, + '--dry-run-cutover', + ...cutoverRequiredArgs, + ...extraArgs, + ]; +} + +const requireLiveBaseArgs = [ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + '/var/log/nginx/genarrative.access.log', + '--live-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--live-access-log-since-lines', + '3000', +]; + +const requireRealpathLiveBaseArgs = [ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--realpath-live-host', + 'example.com', + '--realpath-live-nginx-access-log', + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + '--realpath-live-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--realpath-live-access-log-since-lines', + '3000', +]; + +main(); + +if (failures.length > 0) { + console.error('[check:pingora-release-readiness-plan] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-release-readiness-plan] OK'); + +function main() { + assertDefaultPlanIncludesStaticDirectPreflight(); + assertDefaultPlanIncludesProductionApiReleaseSmoke(); + assertDefaultPlanIncludesProductionApiDeploySmoke(); + assertDefaultPlanIncludesHealthPatrolEnvSmoke(); + assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke(); + assertDefaultPlanIncludesPingoraGatewayEnvShadowSwitchSmoke(); + assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke(); + assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke(); + assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke(); + assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceVerifySmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceAuditSmoke(); + assertDefaultPlanIncludesCanaryAccessLogParitySmoke(); + assertDefaultPlanIncludesRealpathCanaryToggleSmoke(); + assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke(); + assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts(); + assertReleaseRuntimeOnlyPlanIncludesDirectRehearsalStatus(); + assertReleaseRuntimeOnlyAllowsRealpathOnlyCanary(); + assertReleaseRuntimeOnlyKeepsInvokedCurrentSymlinkPath(); + assertReleaseRuntimeOnlyRejectsSourceOnlyFlags(); + assertDryRunCutoverPlanIncludesDirectEnableAndRollbackRunbook(); + assertDryRunCutoverExplicitRollbackBodyIgnoresProcessEnv(); + assertDryRunCutoverPostEnableReadinessRedactsDirectProbeToken(); + assertDryRunCutoverPlanIncludesHostConsistencyConfirmation(); + assertDryRunCutoverAllowsSameHostnameWithDifferentPorts(); + assertDryRunCutoverCanRestoreRollbackHealthPatrolPublicHost(); + assertDryRunCutoverPlanCanIncludeRollbackShadowProbe(); + assertDryRunCutoverPlanCanOverrideEvidenceTimelineMaxSpan(); + assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches(); + assertTechnicalDocCutoverAuditExamplesRequireCommandExecutables(); + assertDryRunCutoverPlanCarriesCutoverRunId(); + assertDryRunCutoverPlanGeneratesDefaultCutoverRunId(); + assertRequireLiveForcesHost(); + assertRequireRealpathLiveForcesHostAndRealpathParity(); + assertRequireLiveRejectsRelativeAccessLog(); + assertRequireLiveRejectsFilesystemRootAccessLog(); + assertRequireLiveRejectsInvalidAccessLogSinceLines(); + assertRequireRealpathLiveRejectsMissingHost(); + assertRequireRealpathLiveRejectsRelativeAccessLog(); + assertLiveSmokeScriptsRejectInvalidTimeout(); + assertDirectLiveRejectsInvalidBoolEnv(); + assertDirectLiveRejectsFilesystemRootAccessLog(); + assertReleaseReadinessRejectsInvalidDirectBoolEnv(); + assertDirectPreflightRejectsInvalidBoolEnv(); + assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection(); + assertCutoverEvidenceScriptsRejectInvalidBoolEnv(); + assertRequireDirectForcesWssUpgrade(); + assertRequireDirectForcesHttpBaseUrl(); + assertRequireDirectForcesHostSni(); + assertRequireDirectForcesRedirectHost(); + assertRequireDirectForcesPingoraAccessLog(); + assertRequireDirectForcesPreflightSystemdCertReadable(); + assertDryRunCutoverForcesPortsFreePreflight(); + assertRequireDirectForcesExplicitSpacetimeDatabase(); + assertRequireDirectForcesHealthPatrolEnvCheck(); + assertDryRunPlanRedactsDirectProbeToken(); + assertRunStepLogRedactsDirectProbeToken(); + assertRequireLiveRejectsMissingHost(); + assertRequireLiveRejectsInvalidHost(); + assertRequireDirectRejectsMissingHttpBaseUrl(); + assertRequireDirectRejectsMissingHost(); + assertRequireDirectRejectsInvalidHost(); + assertRequireDirectRejectsMissingRedirectHost(); + assertRequireDirectRejectsInvalidRedirectHost(); + assertRequireDirectRejectsMissingPingoraAccessLog(); + assertRequireDirectRejectsFilesystemRootPingoraAccessLog(); + assertRequireDirectRejectsMissingSystemdPreflight(); + assertRequireDirectRejectsMissingCertReadablePreflight(); + assertRequireDirectRejectsMissingServiceEnvFilePreflight(); + assertRequireDirectRejectsMissingServiceUserCertReadablePreflight(); + assertRequireDirectRejectsMissingServiceBinaryExecutablePreflight(); + assertRequireDirectAllowsPostEnablePortsOwnedByPingora(); + assertDryRunCutoverRejectsMissingPortsFreePreflight(); + assertRequireDirectRejectsMissingSpacetimeDatabase(); + assertRequireDirectRejectsFilesystemRootPreflightEnvFile(); + assertRequireDirectRejectsMissingHealthPatrolEnvFile(); + assertRequireDirectRejectsRelativeHealthPatrolEnvFile(); + assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile(); + assertRequireDirectRejectsSkipWss(); + assertRequireDirectRejectsInsecureTls(); + assertDryRunCutoverRejectsMissingRequireDirect(); + assertDryRunCutoverRejectsRelativeEvidenceOutputRoot(); + assertDryRunCutoverRejectsFilesystemRootReleaseRoot(); + assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot(); + assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot(); + assertUsageExamplesIncludeDirectPingoraAccessLog(); + assertDryRunCutoverRejectsMissingRollbackSmokeBody(); + assertDryRunCutoverRejectsMissingRollbackHealthPatrolPublicBaseUrl(); + assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicBaseUrl(); + assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicHost(); + assertDryRunCutoverRejectsRedirectHostMismatch(); + assertDryRunCutoverRejectsRollbackHostMismatch(); + assertDryRunCutoverRejectsIncompleteRollbackShadowProbe(); + assertDryRunCutoverRejectsInvalidRollbackShadowProbeUrl(); + assertDryRunCutoverRejectsInvalidEvidenceTimelineMaxSpan(); + assertDryRunCutoverRejectsInvalidCutoverRunId(); +} + +function assertDefaultPlanIncludesStaticDirectPreflight() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora direct entry 静态预检'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 direct-entry 静态预检。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-direct-preflight', + 'direct-entry 静态预检必须调用 check:pingora-direct-preflight。', + ); +} + +function assertDefaultPlanIncludesProductionApiDeploySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'API deploy release 烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 API deploy release 烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-api-deploy', + 'API deploy release 烟测必须调用 check:production-api-deploy。', + ); +} + +function assertDefaultPlanIncludesHealthPatrolEnvSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, '生产健康巡检 env 复核烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含生产健康巡检 env 复核烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-health-patrol-env', + '生产健康巡检 env 复核烟测必须调用 check:production-health-patrol-env。', + ); +} + +function assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora health patrol env 切换烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora health patrol env 切换烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-health-patrol-env-switch', + 'Pingora health patrol env 切换烟测必须调用 check:pingora-health-patrol-env-switch。', + ); +} + +function assertDefaultPlanIncludesPingoraGatewayEnvShadowSwitchSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora gateway env shadow 切换烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora gateway env shadow 切换烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-gateway-env-shadow-switch', + 'Pingora gateway env shadow 切换烟测必须调用 check:pingora-gateway-env-shadow-switch。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 状态快照烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 状态快照烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-status-snapshot', + 'Pingora cutover 状态快照烟测必须调用 check:pingora-cutover-status-snapshot。', + ); +} + +function assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora current release 自审烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora current release 自审烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-current-release-audit', + 'Pingora current release 自审烟测必须调用 check:pingora-current-release-audit。', + ); +} + +function assertDefaultPlanIncludesPingoraDirectRehearsalStatusSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora direct rehearsal 状态烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora direct rehearsal 状态烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-direct-rehearsal-status', + 'Pingora direct rehearsal 状态烟测必须调用 check:pingora-direct-rehearsal-status。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据包烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据包烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-bundle', + 'Pingora cutover 证据包烟测必须调用 check:pingora-cutover-evidence-bundle。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 命令证据烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 命令证据烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-command-evidence', + 'Pingora cutover 命令证据烟测必须调用 check:pingora-cutover-command-evidence。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceVerifySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据 manifest 验真烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据 manifest 验真烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-verify', + 'Pingora cutover 证据 manifest 验真烟测必须调用 check:pingora-cutover-evidence-verify。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceAuditSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据根目录审计烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据根目录审计烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-audit', + 'Pingora cutover 证据根目录审计烟测必须调用 check:pingora-cutover-evidence-audit。', + ); +} + +function assertDefaultPlanIncludesCanaryAccessLogParitySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora canary access log 对账烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora canary access log 对账烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-canary-access-log-parity', + 'Pingora canary access log 对账烟测必须调用 check:pingora-canary-access-log-parity。', + ); +} + +function assertDefaultPlanIncludesRealpathCanaryToggleSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora realpath canary 启停烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora realpath canary 启停烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-realpath-canary-toggle', + 'Pingora realpath canary 启停烟测必须调用 check:pingora-realpath-canary-toggle。', + ); +} + +function assertDefaultPlanIncludesProductionApiReleaseSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'API release build 烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 API release build 烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-api-release', + 'API release build 烟测必须调用 check:production-api-release。', + ); +} + +function assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora production release 真实构建烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora production release 真实构建烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-production-release-build', + 'Pingora production release 真实构建烟测必须调用 check:pingora-production-release-build。', + ); +} + +function assertDryRunCutoverPlanIncludesDirectEnableAndRollbackRunbook() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-release-root', + '/opt/genarrative/current', + )); + const preflight = findStep( + plan, + '切换前 current release 直连 env / 证书 / 端口预检', + ); + const releaseAudit = findStep(plan, '切换前 current release 自包含自审'); + if (!releaseAudit) { + failures.push('dry-run cutover 必须包含切换前 current release 自包含自审。'); + } else { + assertIncludes( + releaseAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + '切换前 current release 自审必须使用 current release 随包脚本。', + ); + assertIncludes( + releaseAudit.args, + '--require-pingora-gateway', + '正式直连 runbook 的 current release 自审必须要求 pingora-gateway 已随包存在。', + ); + assertIncludes( + releaseAudit.args, + '--systemd-show', + '正式直连 runbook 的 current release 自审必须核验 systemd ExecStart。', + ); + } + const preSnapshot = findStep(plan, '切换前状态快照证据包'); + const preVerify = findStep(plan, '切换前证据 manifest 只读验真'); + if (!preSnapshot) { + failures.push('dry-run cutover 必须包含切换前状态快照证据包。'); + } else { + assertIncludes( + preSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '切换前状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + preSnapshot.args, + '--snapshot-script', + '切换前状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + preSnapshot.args, + '--output-root', + '切换前状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + preSnapshot.args, + '/var/log/genarrative/pingora-cutover-evidence', + '切换前状态快照证据包必须默认写入固定证据根目录。', + ); + assertIncludes( + preSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '切换前状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + preSnapshot.args, + '--expected-gateway-mode', + '切换前状态快照证据包必须显式传 expected gateway mode。', + ); + assertIncludes( + preSnapshot.args, + 'nginx', + '切换前状态快照证据包必须确认仍是 Nginx 巡检口径。', + ); + assertIncludes( + preSnapshot.args, + '--run-health-patrol', + '切换前状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + preSnapshot.args, + '--require-pingora-gateway', + '切换前状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + assertIncludes( + preSnapshot.args, + '--fail-on-critical', + '切换前状态快照证据包出现 CRITICAL 必须阻断切换。', + ); + } + if (!preVerify) { + failures.push('dry-run cutover 必须包含切换前证据 manifest 只读验真。'); + } else { + assertIncludes( + preVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '切换前证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + preVerify.args, + '', + '切换前证据验真必须提示替换刚生成的 pre-cutover bundleDir。', + ); + assertIncludes( + preVerify.args, + '--require-summary-ok', + '切换前证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!preflight) { + failures.push('dry-run cutover 必须包含启用前 current release 预检。'); + } else { + assertIncludes( + preflight.args, + '/opt/genarrative/current/scripts/check-pingora-direct-preflight.mjs', + 'dry-run cutover 启用前预检必须使用 current release 随包脚本。', + ); + assertIncludes( + preflight.args, + '--check-service-user-cert-readable', + 'dry-run cutover 启用前预检必须检查 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + preflight.args, + '--check-service-env-file', + 'dry-run cutover 启用前预检必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + preflight.args, + '--check-service-binary-executable', + 'dry-run cutover 启用前预检必须检查 current release Pingora 二进制可执行。', + ); + assertIncludes( + preflight.args, + '--check-ports-free', + 'dry-run cutover 启用前预检必须检查 80/443 端口释放。', + ); + assertIncludes( + preflight.args, + '--systemd-cat', + 'dry-run cutover 启用前预检必须检查当前 systemd direct-entry 状态。', + ); + } + + const baseReadiness = findStep(plan, '切换前 release readiness 基础门禁'); + if (!baseReadiness) { + failures.push( + 'dry-run cutover 必须包含启用前 release readiness 基础门禁。', + ); + } else if (baseReadiness.args.includes('--require-direct')) { + failures.push( + 'dry-run cutover 启用前基础门禁不能带 --require-direct;systemd drop-in 尚未生效。', + ); + } else { + assertIncludes( + baseReadiness.args, + '/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs', + 'dry-run cutover 启用前基础门禁必须使用 current release 随包 release readiness 聚合门禁脚本。', + ); + assertIncludes( + baseReadiness.args, + '--release-runtime-only', + 'dry-run cutover 启用前基础门禁必须使用 current release 包内运行时复核模式。', + ); + } + + const enableDryRun = findStep(plan, 'Pingora direct enable dry-run'); + if (!enableDryRun) { + failures.push('dry-run cutover 必须包含 direct enable dry-run。'); + } else { + assertIncludes( + enableDryRun.command, + '/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'dry-run cutover enable dry-run 必须使用 current release 随包启用脚本。', + ); + if (enableDryRun.args.includes('--apply')) { + failures.push('dry-run cutover enable dry-run 不能带 --apply。'); + } + assertIncludes( + enableDryRun.args, + '--current-release-audit-script', + 'dry-run cutover enable dry-run 必须显式固定 current release 自审脚本。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + 'dry-run cutover enable dry-run 必须使用 current release 随包自审脚本。', + ); + assertIncludes( + enableDryRun.args, + '--current-release-root', + 'dry-run cutover enable dry-run 必须显式固定 current release 根目录。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current', + 'dry-run cutover enable dry-run 必须把 current release 根目录传给启用脚本。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current/scripts/check-pingora-direct-live.mjs', + 'dry-run cutover enable dry-run 必须指定 current release 随包 direct live smoke。', + ); + } + + const enableApply = findStep(plan, 'Pingora direct enable apply'); + const enableApplyVerify = findStep(plan, '启用命令证据 manifest 只读验真'); + if (!enableApply) { + failures.push('dry-run cutover 必须包含 direct enable apply。'); + } else { + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover enable apply 必须通过 current release 随包命令证据脚本归档执行输出。', + ); + assertIncludes( + enableApply.args, + '--command-name', + 'dry-run cutover enable apply 必须显式记录命令证据名称。', + ); + assertIncludes( + enableApply.args, + 'pingora-direct-enable-apply', + 'dry-run cutover enable apply 必须记录固定命令证据名称。', + ); + assertIncludes( + enableApply.args, + '--output-root', + 'dry-run cutover enable apply 必须显式传证据输出根目录。', + ); + assertIncludes( + enableApply.args, + '/var/log/genarrative/pingora-cutover-evidence', + 'dry-run cutover enable apply 必须使用固定证据根目录。', + ); + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'dry-run cutover enable apply 必须在命令证据脚本后执行 current release 随包启用脚本。', + ); + assertIncludes( + enableApply.args, + '--expected-executable', + 'dry-run cutover enable apply 必须把预期真实命令绑定到命令证据。', + ); + assertIncludes( + enableApply.args, + '--require-arg', + 'dry-run cutover enable apply 必须在命令证据脚本中要求真实命令参数。', + ); + assertIncludes( + enableApply.args, + '--apply', + 'dry-run cutover enable apply 必须显式带 --apply。', + ); + assertIncludes( + enableApply.args, + '--current-release-audit-script', + 'dry-run cutover enable apply 必须显式固定 current release 自审脚本。', + ); + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + 'dry-run cutover enable apply 必须使用 current release 随包自审脚本。', + ); + assertIncludes( + enableApply.args, + '--current-release-root', + 'dry-run cutover enable apply 必须显式固定 current release 根目录。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-user-cert-readable', + 'dry-run cutover enable apply 必须验证 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-env-file', + 'dry-run cutover enable apply 必须验证 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-binary-executable', + 'dry-run cutover enable apply 必须验证 current release Pingora 二进制可执行。', + ); + assertIncludes( + enableApply.args, + '--direct-spacetime-database', + 'dry-run cutover enable apply 必须验证显式 SpacetimeDB 数据库名。', + ); + } + if (!enableApplyVerify) { + failures.push('dry-run cutover 必须包含启用命令证据 manifest 只读验真。'); + } else { + assertIncludes( + enableApplyVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '启用命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + enableApplyVerify.args, + '', + '启用命令证据验真必须提示替换刚生成的 enable apply bundleDir。', + ); + assertIncludes( + enableApplyVerify.args, + '--require-summary-ok', + '启用命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + + const postReadiness = findStep(plan, '启用后 release readiness 直连复核'); + const postEnableSnapshot = findStep(plan, '启用后状态快照证据包'); + const postEnableVerify = findStep(plan, '启用后证据 manifest 只读验真'); + if (!postEnableSnapshot) { + failures.push('dry-run cutover 必须包含启用后状态快照证据包。'); + } else { + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '启用后状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '--snapshot-script', + '启用后状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '--output-root', + '启用后状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '启用后状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + 'pingora-direct', + '启用后状态快照证据包必须确认 pingora-direct 巡检口径。', + ); + assertIncludes( + postEnableSnapshot.args, + '--expected-pingora-env-mode', + '启用后状态快照证据包必须显式校验 active Pingora env 姿态。', + ); + assertIncludes( + postEnableSnapshot.args, + 'direct', + '启用后状态快照证据包必须要求 active Pingora env 是 direct 姿态。', + ); + assertIncludes( + postEnableSnapshot.args, + '--expected-public-host', + '启用后状态快照证据包必须确认正式 Host 覆盖。', + ); + assertIncludes( + postEnableSnapshot.args, + '--run-health-patrol', + '启用后状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-live-script', + '启用后状态快照证据包必须显式透传 direct live smoke 脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/check-pingora-direct-live.mjs', + '启用后状态快照证据包必须使用 current release 随包 direct live smoke。', + ); + assertIncludes( + postEnableSnapshot.args, + '--run-direct-live', + '启用后状态快照证据包必须归档 direct live smoke 证据。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-pingora-access-log', + '启用后状态快照证据包必须归档 direct access log request_id 证据。', + ); + assertIncludes( + postEnableSnapshot.args, + '/var/log/genarrative/pingora-gateway.access.log', + '启用后状态快照证据包必须使用显式 Pingora access log 路径。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-spacetime-database', + '启用后状态快照证据包必须传递显式 SpacetimeDB 数据库名。', + ); + assertIncludes( + postEnableSnapshot.args, + '--require-pingora-gateway', + '启用后状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + } + if (!postEnableVerify) { + failures.push('dry-run cutover 必须包含启用后证据 manifest 只读验真。'); + } else { + assertIncludes( + postEnableVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '启用后证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + postEnableVerify.args, + '', + '启用后证据验真必须提示替换刚生成的 post-enable bundleDir。', + ); + assertIncludes( + postEnableVerify.args, + '--require-summary-ok', + '启用后证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!postReadiness) { + failures.push( + 'dry-run cutover 必须包含启用后 release readiness 直连复核。', + ); + } else { + assertIncludes( + postReadiness.args, + '/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs', + '启用后 release readiness 直连复核必须使用 current release 随包 release readiness 聚合门禁脚本。', + ); + assertIncludes( + postReadiness.args, + '--release-runtime-only', + '启用后 release readiness 直连复核必须使用 current release 包内运行时复核模式。', + ); + assertIncludes( + postReadiness.args, + '--require-direct', + 'dry-run cutover 启用后复核必须带 --require-direct。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-user-cert-readable', + 'dry-run cutover 启用后复核必须检查 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-env-file', + 'dry-run cutover 启用后复核必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-binary-executable', + 'dry-run cutover 启用后复核必须检查 Pingora service 二进制可执行。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-systemd', + 'dry-run cutover 启用后复核必须检查 systemd drop-in 生效。', + ); + if (postReadiness.args.includes('--direct-preflight-check-ports-free')) { + failures.push( + 'dry-run cutover 启用后 release readiness 复核不能继续要求 80/443 空闲;此时端口应由 Pingora 直连入口占用。', + ); + } + } + + const healthPatrolEnvCheck = findStep( + plan, + '启用后 health patrol env 直连复核', + ); + if (!healthPatrolEnvCheck) { + failures.push('dry-run cutover 必须包含启用后 health patrol env 复核。'); + } else { + assertIncludes( + healthPatrolEnvCheck.args, + '/opt/genarrative/current/scripts/check-production-health-patrol-env.mjs', + 'dry-run cutover health patrol env 复核必须使用 current release 随包脚本。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover health patrol env 复核必须检查目标 env 文件。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-gateway-mode', + 'dry-run cutover health patrol env 复核必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + 'pingora-direct', + 'dry-run cutover health patrol env 复核必须要求 pingora-direct。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-public-base-url', + 'dry-run cutover health patrol env 复核必须校验 public base URL。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-public-host', + 'dry-run cutover health patrol env 复核必须校验正式 Host。', + ); + } + + const healthPatrolDirect = findStep( + plan, + '切换后 health patrol 切到 Pingora direct', + ); + const healthPatrolDirectVerify = findStep( + plan, + '切换后 health patrol direct 命令证据 manifest 只读验真', + ); + if (!healthPatrolDirect) { + failures.push( + 'dry-run cutover 必须包含切换后 health patrol direct 模式脚本。', + ); + } else { + assertIncludes( + healthPatrolDirect.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover 切换后 health patrol env 切换必须通过命令证据脚本归档。', + ); + assertIncludes( + healthPatrolDirect.args, + '--command-name', + 'dry-run cutover 切换后 health patrol env 切换必须显式记录命令证据名称。', + ); + assertIncludes( + healthPatrolDirect.args, + 'pingora-health-patrol-direct-env-switch', + 'dry-run cutover 切换后 health patrol env 切换必须记录固定命令证据名称。', + ); + assertIncludes( + healthPatrolDirect.args, + '--expected-executable', + 'dry-run cutover 切换后 health patrol env 切换必须绑定真实脚本身份。', + ); + assertIncludes( + healthPatrolDirect.args, + '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'dry-run cutover 切换后必须使用 current release 随包 health patrol env 切换脚本。', + ); + assertIncludes( + healthPatrolDirect.args, + '--apply', + 'dry-run cutover 切换后 health patrol env 切换必须显式带 --apply。', + ); + assertIncludes( + healthPatrolDirect.args, + '--env-file', + 'dry-run cutover 切换后 health patrol env 切换必须显式传 env 文件。', + ); + assertIncludes( + healthPatrolDirect.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover 切换后 health patrol env 切换必须使用显式 env 文件。', + ); + assertIncludes( + healthPatrolDirect.args, + '--gateway-mode', + 'dry-run cutover 切换后 health patrol env 切换必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolDirect.args, + 'pingora-direct', + 'dry-run cutover 切换后 health patrol env 切换必须改为 pingora-direct。', + ); + assertIncludes( + healthPatrolDirect.args, + '--public-base-url', + 'dry-run cutover 切换后 health patrol env 切换必须写 direct public base URL。', + ); + assertIncludes( + healthPatrolDirect.args, + 'https://127.0.0.1', + 'dry-run cutover 切换后 health patrol env 切换必须写 direct HTTPS base URL。', + ); + assertIncludes( + healthPatrolDirect.args, + '--public-host', + 'dry-run cutover 切换后 health patrol env 切换必须写正式 Host。', + ); + assertIncludes( + healthPatrolDirect.args, + 'example.com', + 'dry-run cutover 切换后 health patrol env 切换必须使用正式 Host。', + ); + } + if (!healthPatrolDirectVerify) { + failures.push( + 'dry-run cutover 必须包含切换后 health patrol direct 命令证据 manifest 只读验真。', + ); + } else { + assertIncludes( + healthPatrolDirectVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '切换后 health patrol direct 命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + healthPatrolDirectVerify.args, + '', + '切换后 health patrol direct 命令证据验真必须提示替换刚生成的 bundleDir。', + ); + assertIncludes( + healthPatrolDirectVerify.args, + '--require-summary-ok', + '切换后 health patrol direct 命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + assertStepBefore( + plan, + 'Pingora direct enable apply', + '启用命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 enable apply 命令证据,再验真该命令证据。', + ); + assertStepBefore( + plan, + '启用命令证据 manifest 只读验真', + '切换后 health patrol 切到 Pingora direct', + 'dry-run cutover 必须先验真 enable apply 命令证据,再切换 health patrol 到 pingora-direct。', + ); + assertStepBefore( + plan, + '切换后 health patrol 切到 Pingora direct', + '切换后 health patrol direct 命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 health patrol direct env 命令证据,再验真该命令证据。', + ); + assertStepBefore( + plan, + '切换后 health patrol direct 命令证据 manifest 只读验真', + '启用后 health patrol env 直连复核', + 'dry-run cutover 必须先验真 health patrol direct env 命令证据,再执行直连 env 复核。', + ); + + const rollbackDryRun = findStep(plan, 'Pingora direct rollback dry-run'); + if (!rollbackDryRun) { + failures.push('dry-run cutover 必须包含 rollback dry-run。'); + } else { + assertIncludes( + rollbackDryRun.command, + '/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + 'dry-run cutover rollback dry-run 必须使用 current release 随包回退脚本。', + ); + if (rollbackDryRun.args.includes('--apply')) { + failures.push('dry-run cutover rollback dry-run 不能带 --apply。'); + } + assertIncludes( + rollbackDryRun.args, + '--reload-nginx', + 'dry-run cutover rollback dry-run 必须包含 Nginx reload。', + ); + assertIncludes( + rollbackDryRun.args, + '--nginx-smoke-host', + 'dry-run cutover rollback dry-run 使用 loopback smoke URL 时必须带正式 Host。', + ); + assertIncludes( + rollbackDryRun.args, + '--nginx-smoke-expect-body', + 'dry-run cutover rollback dry-run 必须校验 Nginx smoke 响应体片段。', + ); + assertIncludes( + rollbackDryRun.args, + '', + 'dry-run cutover rollback dry-run 必须使用显式传入的 Nginx smoke body 片段。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-env-file', + 'dry-run cutover rollback dry-run 必须把 health patrol env 复核交给回退脚本。', + ); + assertIncludes( + rollbackDryRun.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover rollback dry-run 必须使用显式 health patrol env 文件。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-expected-public-base-url', + 'dry-run cutover rollback dry-run 必须校验回退 health patrol public base URL。', + ); + assertIncludes( + rollbackDryRun.args, + 'http://127.0.0.1', + 'dry-run cutover rollback dry-run 必须传递显式回退 public base URL。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-require-empty-public-host', + 'dry-run cutover rollback dry-run 默认必须要求回退 public Host 为空。', + ); + } + + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + const rollbackApplyVerify = findStep(plan, '回退命令证据 manifest 只读验真'); + if (!rollbackApply) { + failures.push('dry-run cutover 必须包含 rollback apply。'); + } else { + assertIncludes( + rollbackApply.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover rollback apply 必须通过 current release 随包命令证据脚本归档执行输出。', + ); + assertIncludes( + rollbackApply.args, + '--command-name', + 'dry-run cutover rollback apply 必须显式记录命令证据名称。', + ); + assertIncludes( + rollbackApply.args, + 'pingora-direct-rollback-apply', + 'dry-run cutover rollback apply 必须记录固定命令证据名称。', + ); + assertIncludes( + rollbackApply.args, + '/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + 'dry-run cutover rollback apply 必须在命令证据脚本后执行 current release 随包回退脚本。', + ); + assertIncludes( + rollbackApply.args, + '--expected-executable', + 'dry-run cutover rollback apply 必须把预期真实命令绑定到命令证据。', + ); + assertIncludes( + rollbackApply.args, + '--require-arg', + 'dry-run cutover rollback apply 必须在命令证据脚本中要求真实命令参数。', + ); + assertIncludes( + rollbackApply.args, + '--apply', + 'dry-run cutover rollback apply 必须显式带 --apply。', + ); + assertIncludes( + rollbackApply.args, + 'https://example.com/', + 'dry-run cutover rollback apply 必须包含 Nginx smoke URL。', + ); + assertIncludes( + rollbackApply.args, + 'example.com', + 'dry-run cutover rollback apply 默认必须复用 direct Host 作为 Nginx smoke Host。', + ); + assertIncludes( + rollbackApply.args, + '--nginx-smoke-host', + 'dry-run cutover rollback apply 使用 loopback smoke URL 时必须带正式 Host。', + ); + assertIncludes( + rollbackApply.args, + '--nginx-smoke-expect-body', + 'dry-run cutover rollback apply 必须校验 Nginx smoke 响应体片段。', + ); + assertIncludes( + rollbackApply.args, + '', + 'dry-run cutover rollback apply 必须使用显式传入的 Nginx smoke body 片段。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-env-file', + 'dry-run cutover rollback apply 必须把 health patrol env 复核交给回退脚本。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-expected-public-base-url', + 'dry-run cutover rollback apply 必须校验回退 health patrol public base URL。', + ); + assertIncludes( + rollbackApply.args, + 'http://127.0.0.1', + 'dry-run cutover rollback apply 必须传递显式回退 public base URL。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-require-empty-public-host', + 'dry-run cutover rollback apply 默认必须要求回退 public Host 为空。', + ); + } + if (!rollbackApplyVerify) { + failures.push('dry-run cutover 必须包含回退命令证据 manifest 只读验真。'); + } else { + assertIncludes( + rollbackApplyVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + rollbackApplyVerify.args, + '', + '回退命令证据验真必须提示替换刚生成的 rollback apply bundleDir。', + ); + assertIncludes( + rollbackApplyVerify.args, + '--require-summary-ok', + '回退命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + + const healthPatrolNginx = findStep(plan, '回退前 health patrol 预置回 Nginx'); + const pingoraEnvShadow = findStep(plan, '回退前 Pingora env 预置回 shadow'); + const pingoraEnvShadowVerify = findStep( + plan, + '回退前 Pingora env shadow 命令证据 manifest 只读验真', + ); + const healthPatrolNginxVerify = findStep( + plan, + '回退前 health patrol nginx 命令证据 manifest 只读验真', + ); + if (!pingoraEnvShadow) { + failures.push( + 'dry-run cutover 必须包含回退前 Pingora env shadow 配置预置确认。', + ); + } else { + assertIncludes( + pingoraEnvShadow.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover 回退前 Pingora env shadow 预置必须通过命令证据脚本归档。', + ); + assertIncludes( + pingoraEnvShadow.args, + '--command-name', + 'dry-run cutover 回退前 Pingora env shadow 预置必须显式记录命令证据名称。', + ); + assertIncludes( + pingoraEnvShadow.args, + 'pingora-gateway-shadow-env-switch', + 'dry-run cutover 回退前 Pingora env shadow 预置必须记录固定命令证据名称。', + ); + assertIncludes( + pingoraEnvShadow.args, + '--expected-executable', + 'dry-run cutover 回退前 Pingora env shadow 预置必须绑定真实脚本身份。', + ); + assertIncludes( + pingoraEnvShadow.args, + '/opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + 'dry-run cutover 回退前必须使用 current release 随包 Pingora gateway env shadow 切换脚本。', + ); + assertIncludes( + pingoraEnvShadow.args, + '--apply', + 'dry-run cutover 回退前 Pingora gateway env shadow 切换必须显式带 --apply。', + ); + assertIncludes( + pingoraEnvShadow.args, + '--env-file', + 'dry-run cutover 回退前 Pingora gateway env shadow 切换必须显式传 env 文件。', + ); + assertIncludes( + pingoraEnvShadow.args, + '/tmp/pingora-gateway.env', + 'dry-run cutover 回退前 Pingora gateway env shadow 切换必须使用显式 env 文件。', + ); + if (!pingoraEnvShadow.when.includes('80/443 direct')) { + failures.push( + 'dry-run cutover 回退前 Pingora env shadow 预置说明必须解释从 80/443 direct 配置恢复。', + ); + } + } + if (!pingoraEnvShadowVerify) { + failures.push( + 'dry-run cutover 必须包含回退前 Pingora env shadow 命令证据 manifest 只读验真。', + ); + } else { + assertIncludes( + pingoraEnvShadowVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退前 Pingora env shadow 命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + pingoraEnvShadowVerify.args, + '', + '回退前 Pingora env shadow 命令证据验真必须提示替换刚生成的 bundleDir。', + ); + assertIncludes( + pingoraEnvShadowVerify.args, + '--require-summary-ok', + '回退前 Pingora env shadow 命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!healthPatrolNginx) { + failures.push( + 'dry-run cutover 必须包含回退前 health patrol Nginx 模式预置脚本。', + ); + } else { + assertIncludes( + healthPatrolNginx.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover 回退前 health patrol env 切回 nginx 必须通过命令证据脚本归档。', + ); + assertIncludes( + healthPatrolNginx.args, + '--command-name', + 'dry-run cutover 回退前 health patrol env 切回 nginx 必须显式记录命令证据名称。', + ); + assertIncludes( + healthPatrolNginx.args, + 'pingora-health-patrol-nginx-env-switch', + 'dry-run cutover 回退前 health patrol env 切回 nginx 必须记录固定命令证据名称。', + ); + assertIncludes( + healthPatrolNginx.args, + '--expected-executable', + 'dry-run cutover 回退前 health patrol env 切回 nginx 必须绑定真实脚本身份。', + ); + assertIncludes( + healthPatrolNginx.args, + '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'dry-run cutover 回退前必须使用 current release 随包 health patrol env 切换脚本。', + ); + assertIncludes( + healthPatrolNginx.args, + '--apply', + 'dry-run cutover 回退前 health patrol env 切换必须显式带 --apply。', + ); + assertIncludes( + healthPatrolNginx.args, + '--gateway-mode', + 'dry-run cutover 回退前 health patrol env 切换必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolNginx.args, + 'nginx', + 'dry-run cutover 回退前 health patrol env 切换必须改回 nginx。', + ); + assertIncludes( + healthPatrolNginx.args, + '--public-base-url', + 'dry-run cutover 回退前 health patrol env 切换必须恢复 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginx.args, + 'http://127.0.0.1', + 'dry-run cutover 回退前 health patrol env 切换必须恢复切换前 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginx.args, + '--clear-public-host', + 'dry-run cutover 回退前 health patrol env 切换必须清理 public Host 覆盖。', + ); + } + if (!healthPatrolNginxVerify) { + failures.push( + 'dry-run cutover 必须包含回退前 health patrol nginx 命令证据 manifest 只读验真。', + ); + } else { + assertIncludes( + healthPatrolNginxVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退前 health patrol nginx 命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + healthPatrolNginxVerify.args, + '', + '回退前 health patrol nginx 命令证据验真必须提示替换刚生成的 bundleDir。', + ); + assertIncludes( + healthPatrolNginxVerify.args, + '--require-summary-ok', + '回退前 health patrol nginx 命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + + const healthPatrolIndex = plan.findIndex( + (step) => step.name === '回退前 health patrol 预置回 Nginx', + ); + const rollbackApplyIndex = plan.findIndex( + (step) => step.name === 'Pingora direct rollback apply', + ); + const pingoraEnvShadowIndex = plan.findIndex( + (step) => step.name === '回退前 Pingora env 预置回 shadow', + ); + if ( + pingoraEnvShadowIndex >= 0 && + rollbackApplyIndex >= 0 && + pingoraEnvShadowIndex > rollbackApplyIndex + ) { + failures.push( + 'dry-run cutover 必须先预置 Pingora shadow env,再执行 rollback apply。', + ); + } + if ( + healthPatrolIndex >= 0 && + rollbackApplyIndex >= 0 && + healthPatrolIndex > rollbackApplyIndex + ) { + failures.push( + 'dry-run cutover 必须先预置 health patrol env,再执行 rollback apply。', + ); + } + assertStepBefore( + plan, + '回退前 Pingora env 预置回 shadow', + '回退前 Pingora env shadow 命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 Pingora shadow env 命令证据,再验真该命令证据。', + ); + assertStepBefore( + plan, + '回退前 Pingora env shadow 命令证据 manifest 只读验真', + '回退前 health patrol 预置回 Nginx', + 'dry-run cutover 必须先验真 Pingora shadow env 命令证据,再预置 health patrol nginx env。', + ); + assertStepBefore( + plan, + '回退前 health patrol 预置回 Nginx', + '回退前 health patrol nginx 命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 health patrol nginx env 命令证据,再验真该命令证据。', + ); + assertStepBefore( + plan, + '回退前 health patrol nginx 命令证据 manifest 只读验真', + 'Pingora direct rollback apply', + 'dry-run cutover 必须先验真 health patrol nginx env 命令证据,再执行 rollback apply。', + ); + assertStepBefore( + plan, + 'Pingora direct rollback apply', + '回退命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 rollback apply 命令证据,再验真该命令证据。', + ); + assertStepBefore( + plan, + '回退命令证据 manifest 只读验真', + '回退后 health patrol env Nginx 模式复核', + 'dry-run cutover 必须先验真 rollback apply 命令证据,再执行回退后 env 复核。', + ); + + const healthPatrolNginxCheck = findStep( + plan, + '回退后 health patrol env Nginx 模式复核', + ); + if (!healthPatrolNginxCheck) { + failures.push( + 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核。', + ); + } else { + assertIncludes( + healthPatrolNginxCheck.args, + '/opt/genarrative/current/scripts/check-production-health-patrol-env.mjs', + 'dry-run cutover 回退后 health patrol env 复核必须使用 current release 随包脚本。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-gateway-mode', + 'dry-run cutover 回退后 health patrol env 复核必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'nginx', + 'dry-run cutover 回退后 health patrol env 复核必须要求 nginx。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-public-base-url', + 'dry-run cutover 回退后 health patrol env 复核必须校验 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'http://127.0.0.1', + 'dry-run cutover 回退后 health patrol env 复核必须使用显式回退 public base URL。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--require-empty-public-host', + 'dry-run cutover 回退后 health patrol env 复核必须要求清空 public Host。', + ); + } + + const postRollbackSnapshot = findStep(plan, '回退后状态快照证据包'); + const postRollbackVerify = findStep(plan, '回退后证据 manifest 只读验真'); + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!postRollbackSnapshot) { + failures.push('dry-run cutover 必须包含回退后状态快照证据包。'); + } else { + assertIncludes( + postRollbackSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '回退后状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--snapshot-script', + '回退后状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--output-root', + '回退后状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + postRollbackSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '回退后状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + 'nginx', + '回退后状态快照证据包必须确认 nginx 巡检口径。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--expected-pingora-env-mode', + '回退后状态快照证据包必须显式校验 active Pingora env 姿态。', + ); + assertIncludes( + postRollbackSnapshot.args, + 'shadow', + '回退后状态快照证据包必须要求 active Pingora env 已恢复 shadow 姿态。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--require-empty-public-host', + '默认回退后状态快照证据包必须要求 public Host 为空。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--run-health-patrol', + '回退后状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--require-pingora-gateway', + '回退后状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + } + if (!postRollbackVerify) { + failures.push('dry-run cutover 必须包含回退后证据 manifest 只读验真。'); + } else { + assertIncludes( + postRollbackVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退后证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + postRollbackVerify.args, + '', + '回退后证据验真必须提示替换刚生成的 post-rollback bundleDir。', + ); + assertIncludes( + postRollbackVerify.args, + '--require-summary-ok', + '回退后证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + } else { + assertIncludes( + evidenceRootAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-audit.mjs', + '三阶段总审计必须使用 current release 随包 evidence audit 脚本。', + ); + assertIncludes( + evidenceRootAudit.args, + '--evidence-root', + '三阶段总审计必须显式传证据根目录。', + ); + assertIncludes( + evidenceRootAudit.args, + '/var/log/genarrative/pingora-cutover-evidence', + '三阶段总审计必须使用固定证据根目录。', + ); + assertIncludes( + evidenceRootAudit.args, + '--verify-script', + '三阶段总审计必须显式固定 verifier 脚本。', + ); + assertIncludes( + evidenceRootAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '三阶段总审计必须复用 current release 随包 verifier。', + ); + for (const phase of ['pre-cutover', 'post-enable', 'post-rollback']) { + assertIncludes( + evidenceRootAudit.args, + phase, + `三阶段总审计必须要求 ${phase} 证据。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-phase-direct-live-access-log', + '三阶段总审计必须要求 post-enable 阶段带 direct live access log 对账摘要。', + ); + assertIncludes( + evidenceRootAudit.args, + 'post-enable', + '三阶段总审计必须把 direct live access log 摘要要求绑定到 post-enable 阶段。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-phase-direct-live-static-headers', + '三阶段总审计必须要求 post-enable 阶段带 direct live 静态响应头摘要。', + ); + assertIncludes( + evidenceRootAudit.args, + 'post-enable', + '三阶段总审计必须把 direct live 静态响应头摘要要求绑定到 post-enable 阶段。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-phase-pingora-env-shadow', + '三阶段总审计必须要求 post-rollback 阶段带 Pingora env shadow 摘要。', + ); + assertIncludes( + evidenceRootAudit.args, + 'post-rollback', + '三阶段总审计必须把 Pingora env shadow 摘要要求绑定到 post-rollback 阶段。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-command', + '三阶段总审计必须显式要求真实切换命令证据。', + ); + for (const command of [ + 'enable-apply:pingora-direct-enable-apply', + 'post-enable:pingora-health-patrol-direct-env-switch', + 'rollback-prep:pingora-gateway-shadow-env-switch', + 'rollback-prep:pingora-health-patrol-nginx-env-switch', + 'rollback-apply:pingora-direct-rollback-apply', + ]) { + assertIncludes( + evidenceRootAudit.args, + command, + `三阶段总审计必须要求 ${command} 命令证据。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-command-executable', + '三阶段总审计必须显式要求真实切换脚本身份。', + ); + for (const commandExecutable of [ + 'enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'post-enable:pingora-health-patrol-direct-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'rollback-prep:pingora-gateway-shadow-env-switch:/opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + 'rollback-prep:pingora-health-patrol-nginx-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]) { + assertIncludes( + evidenceRootAudit.args, + commandExecutable, + `三阶段总审计必须要求 ${commandExecutable} 命令脚本身份。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-command-arg', + '三阶段总审计必须显式要求真实切换命令必需参数。', + ); + for (const commandArg of [ + 'enable-apply:pingora-direct-enable-apply:--apply', + 'post-enable:pingora-health-patrol-direct-env-switch:--apply', + 'post-enable:pingora-health-patrol-direct-env-switch:pingora-direct', + 'rollback-prep:pingora-gateway-shadow-env-switch:--apply', + 'rollback-prep:pingora-health-patrol-nginx-env-switch:--apply', + 'rollback-prep:pingora-health-patrol-nginx-env-switch:nginx', + 'rollback-apply:pingora-direct-rollback-apply:--apply', + ]) { + assertIncludes( + evidenceRootAudit.args, + commandArg, + `三阶段总审计必须要求 ${commandArg} 命令参数。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--timeline-max-span-ms', + '三阶段总审计必须显式传标准切换时间线最大跨度。', + ); + assertIncludes( + evidenceRootAudit.args, + '86400000', + '三阶段总审计默认必须把标准切换时间线最大跨度固定为 24 小时。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-cutover-run-id', + '三阶段总审计必须要求同一 cutoverRunId。', + ); + } + +} + +function assertDryRunCutoverPlanCanOverrideEvidenceTimelineMaxSpan() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-evidence-timeline-max-span-ms', + '172800000', + )); + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + '--timeline-max-span-ms', + '三阶段总审计必须传递自定义标准切换时间线最大跨度参数。', + ); + assertIncludes( + evidenceRootAudit.args, + '172800000', + '三阶段总审计必须传递自定义标准切换时间线最大跨度值。', + ); +} + +function assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches() { + const plan = readPlan(cutoverPlanArgs()); + const forbiddenArgs = [ + '--warn-only', + '--allow-extra-files', + '--allow-extra-root-entries', + ]; + + for (const step of plan) { + for (const forbiddenArg of forbiddenArgs) { + assertNotIncludes( + step.args || [], + forbiddenArg, + `正式 dry-run cutover 计划的 ${step.name} 不能携带人工排障放行参数 ${forbiddenArg}。`, + ); + } + } +} + +function assertTechnicalDocCutoverAuditExamplesRequireCommandExecutables() { + const docPath = + 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md'; + const content = readFileSync(docPath, 'utf8'); + const executableArgs = [ + '--require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + '--require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]; + const auditExamples = [ + ...content.matchAll( + /\/opt\/genarrative\/current\/scripts\/ops\/pingora-cutover-evidence-audit\.mjs[\s\S]*?(?=。|` 生成总审计 JSON|`,由脚本|`。)/gu, + ), + ] + .map((match) => match[0]) + .filter( + (example) => + example.includes( + '--require-command enable-apply:pingora-direct-enable-apply', + ) || + example.includes( + '--require-command rollback-apply:pingora-direct-rollback-apply', + ), + ); + + if (auditExamples.length === 0) { + failures.push('Pingora 技术文档必须包含最终证据根目录总审计命令示例。'); + return; + } + for (const example of auditExamples) { + for (const executableArg of executableArgs) { + assertIncludes( + example, + executableArg, + 'Pingora 技术文档里的最终证据根目录总审计命令示例必须要求真实脚本身份。', + ); + } + } +} + +function assertDryRunCutoverPlanIncludesHostConsistencyConfirmation() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-release-root', + '/opt/genarrative/current', + )); + const step = findStep(plan, '切换 Host 一致性确认'); + if (!step) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + return; + } + if ( + !step.args.some((arg) => String(arg).startsWith('cutover-run-id=')) + ) { + failures.push('dry-run cutover Host 一致性确认必须展示 cutoverRunId。'); + } + assertIncludes( + step.args, + 'direct-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 direct Host。', + ); + assertIncludes( + step.args, + 'direct-redirect-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 redirect Host。', + ); + assertIncludes( + step.args, + 'health-patrol-public-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 health patrol Host。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 rollback smoke Host。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-expect-body=', + 'dry-run cutover Host 一致性确认必须展示 rollback smoke body 证据片段。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public base URL。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-host=', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public Host 为空。', + ); +} + +function assertDryRunCutoverPlanCarriesCutoverRunId() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-run-id', + 'cutover-20260617T010000Z', + )); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + } else { + assertIncludes( + hostConsistency.args, + 'cutover-run-id=cutover-20260617T010000Z', + 'Host 一致性确认必须展示显式 cutoverRunId。', + ); + } + + for (const stepName of [ + '切换前状态快照证据包', + 'Pingora direct enable apply', + '启用后状态快照证据包', + 'Pingora direct rollback apply', + '回退后状态快照证据包', + ]) { + const step = findStep(plan, stepName); + if (!step) { + failures.push(`dry-run cutover 必须包含 ${stepName}。`); + continue; + } + assertIncludes( + step.args, + '--cutover-run-id', + `${stepName} 必须传递 cutoverRunId。`, + ); + assertIncludes( + step.args, + 'cutover-20260617T010000Z', + `${stepName} 必须使用同一个 cutoverRunId。`, + ); + } + + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + '--require-cutover-run-id', + '三阶段总审计必须要求同一 cutoverRunId。', + ); + assertIncludes( + evidenceRootAudit.args, + 'cutover-20260617T010000Z', + '三阶段总审计必须使用同一个 cutoverRunId。', + ); +} + +function assertDryRunCutoverPlanGeneratesDefaultCutoverRunId() { + const plan = readPlan(cutoverPlanArgs()); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + return; + } + const runArg = hostConsistency.args.find((arg) => + String(arg).startsWith('cutover-run-id='), + ); + if (!runArg) { + failures.push('dry-run cutover 未显式提供 run id 时必须自动生成 cutoverRunId。'); + return; + } + const runId = runArg.slice('cutover-run-id='.length); + if (!/^pingora-direct-\d{8}T\d{6}Z$/u.test(runId)) { + failures.push(`自动生成的 cutoverRunId 格式不正确: ${runId}`); + } + + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + runId, + '自动生成的 cutoverRunId 必须传给三阶段总审计。', + ); +} + +function assertDryRunCutoverExplicitRollbackBodyIgnoresProcessEnv() { + const previous = + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY; + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY = + 'polluted-body'; + try { + const plan = readPlan(cutoverPlanArgs( + '--cutover-release-root', + '/opt/genarrative/current', + )); + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + if (!rollbackApply) { + failures.push('dry-run cutover 必须包含 rollback apply。'); + return; + } + assertIncludes( + rollbackApply.args, + '', + 'dry-run cutover 必须使用显式 rollback body 断言,且不能被本机 env 污染。', + ); + if (rollbackApply.args.includes('polluted-body')) { + failures.push( + 'readinessPlanEnv 必须清理 GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY,避免 plan 自测受本机 env 污染。', + ); + } + } finally { + if (previous === undefined) { + delete process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY; + } else { + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY = + previous; + } + } +} + +function assertDryRunCutoverPostEnableReadinessRedactsDirectProbeToken() { + const plan = readPlan(cutoverPlanArgs( + '--direct-probe-token', + 'direct-secret-token', + '--cutover-release-root', + '/opt/genarrative/current', + )); + const step = findStep(plan, '启用后 release readiness 直连复核'); + if (!step) { + failures.push('带 direct probe token 的 cutover runbook 必须包含启用后 readiness 复核。'); + return; + } + assertIncludes( + step.args, + '--direct-probe-token', + '启用后 release readiness 直连复核必须继续传递 direct probe token 参数。', + ); + assertIncludes( + step.args, + '', + '启用后 release readiness 直连复核必须隐藏 direct probe token 值。', + ); + if (JSON.stringify(plan).includes('direct-secret-token')) { + failures.push('dry-run cutover JSON 不能泄露 direct probe token 原文。'); + } +} + +function assertDryRunCutoverAllowsSameHostnameWithDifferentPorts() { + const plan = readPlan(cutoverPlanArgs( + '--direct-redirect-host', + 'example.com:443', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-host', + 'example.com:80', + )); + const step = findStep(plan, '切换 Host 一致性确认'); + if (!step) { + failures.push('dry-run cutover 同 hostname 不同端口时仍必须能生成计划。'); + return; + } + assertIncludes( + step.args, + 'direct-redirect-host=example.com:443', + 'dry-run cutover 应允许 redirect Host 使用同 hostname 的显式端口。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-host=example.com:80', + 'dry-run cutover 应允许 rollback smoke Host 使用同 hostname 的显式端口。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover 同 hostname 不同端口时仍必须展示回退 health patrol public base URL。', + ); +} + +function assertDryRunCutoverCanRestoreRollbackHealthPatrolPublicHost() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-health-patrol-public-host', + 'nginx.example.com', + )); + const healthPatrolNginx = findStep(plan, '回退前 health patrol 预置回 Nginx'); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + } else { + assertIncludes( + hostConsistency.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public base URL。', + ); + assertIncludes( + hostConsistency.args, + 'rollback-health-patrol-public-host=nginx.example.com', + 'dry-run cutover Host 一致性确认必须展示显式回退 health patrol public Host。', + ); + } + + if (!healthPatrolNginx) { + failures.push( + 'dry-run cutover 必须包含回退前 health patrol Nginx 模式预置脚本。', + ); + } else { + assertIncludes( + healthPatrolNginx.args, + '--public-host', + 'dry-run cutover 回退前必须能恢复切换前 Nginx public Host。', + ); + assertIncludes( + healthPatrolNginx.args, + 'nginx.example.com', + 'dry-run cutover 回退前必须使用显式回退 public Host。', + ); + } + + const healthPatrolNginxCheck = findStep( + plan, + '回退后 health patrol env Nginx 模式复核', + ); + if (!healthPatrolNginxCheck) { + failures.push( + 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核。', + ); + return; + } + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-public-host', + 'dry-run cutover 回退后 health patrol env 复核必须能校验 Nginx public Host。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'nginx.example.com', + 'dry-run cutover 回退后 health patrol env 复核必须使用显式回退 public Host。', + ); + if (healthPatrolNginxCheck.args.includes('--require-empty-public-host')) { + failures.push( + '显式提供回退 public Host 时,回退后 health patrol env 复核不能同时要求 Host 为空。', + ); + } + + for (const stepName of [ + 'Pingora direct rollback dry-run', + 'Pingora direct rollback apply', + ]) { + const step = findStep(plan, stepName); + if (!step) { + failures.push(`dry-run cutover 必须包含 ${stepName}。`); + continue; + } + assertIncludes( + step.args, + '--health-patrol-expected-public-host', + `dry-run cutover ${stepName} 必须把显式回退 public Host 传给回退脚本。`, + ); + assertIncludes( + step.args, + 'nginx.example.com', + `dry-run cutover ${stepName} 必须使用显式回退 public Host。`, + ); + if (step.args.includes('--health-patrol-require-empty-public-host')) { + failures.push( + `显式提供回退 public Host 时,${stepName} 不能同时要求 Host 为空。`, + ); + } + } +} + +function assertDryRunCutoverPlanCanIncludeRollbackShadowProbe() { + const plan = readPlan(cutoverPlanArgs( + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-pingora-shadow-probe-url', + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + '--rollback-pingora-shadow-probe-token', + 'probe-secret-token', + )); + const rollbackDryRun = findStep(plan, 'Pingora direct rollback dry-run'); + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + for (const step of [rollbackDryRun, rollbackApply]) { + if (!step) { + failures.push('带 shadow probe 的 dry-run cutover 必须包含 rollback 步骤。'); + continue; + } + assertIncludes( + step.args, + '--pingora-shadow-probe-url', + 'dry-run cutover rollback 步骤必须传递 shadow probe URL。', + ); + assertIncludes( + step.args, + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + 'dry-run cutover rollback 步骤必须保留 shadow probe URL。', + ); + assertIncludes( + step.args, + '--pingora-shadow-probe-token', + 'dry-run cutover rollback 步骤必须传递 shadow probe token 参数。', + ); + assertIncludes( + step.args, + '', + 'dry-run cutover rollback 步骤必须隐藏 shadow probe token 值。', + ); + if (step.args.includes('probe-secret-token')) { + failures.push('dry-run cutover JSON 不能泄露 shadow probe token 原文。'); + } + } +} + +function assertRequireLiveForcesHost() { + const plan = readPlan([...requireLiveBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Nginx live canary smoke'); + if (!step) { + failures.push('--require-live 计划必须包含 live canary smoke。'); + return; + } + assertIncludes( + step.args, + '--host', + '--require-live 必须把正式域名 Host 传给 live canary smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-live live canary smoke 必须使用显式传入的正式域名 Host。', + ); + + const parity = findStep(plan, '目标 Nginx live canary access log 对账'); + if (!parity) { + failures.push('--require-live 计划必须在 live smoke 后包含 access log 对账。'); + return; + } + assertIncludes( + parity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + '--require-live 必须调用 canary access log 对账脚本。', + ); + assertIncludes( + parity.args, + '--nginx-log-file', + '--require-live access log 对账必须显式传 Nginx access log。', + ); + assertIncludes( + parity.args, + '/var/log/nginx/genarrative.access.log', + '--require-live access log 对账必须使用显式 Nginx access log 路径。', + ); + assertIncludes( + parity.args, + '--pingora-log-file', + '--require-live access log 对账必须显式传 Pingora access log。', + ); + assertIncludes( + parity.args, + '/var/log/genarrative/pingora-gateway.access.log', + '--require-live access log 对账必须使用显式 Pingora access log 路径。', + ); + assertIncludes( + parity.args, + '--since-lines', + '--require-live access log 对账必须显式传 tail 行数。', + ); + assertIncludes( + parity.args, + '3000', + '--require-live access log 对账必须使用显式 tail 行数。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_canary/healthz', + '--require-live access log 对账必须覆盖 canary healthz。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_canary/api/creation-entry/config', + '--require-live access log 对账必须覆盖代表性 API canary 路径。', + ); +} + +function assertRequireRealpathLiveForcesHostAndRealpathParity() { + const plan = readPlan([...requireRealpathLiveBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Nginx realpath canary smoke'); + if (!step) { + failures.push('--require-realpath-live 计划必须包含 realpath canary smoke。'); + return; + } + assertIncludes( + step.args, + '--realpath', + '--require-realpath-live 必须把 live smoke 切到真实路径 canary 模式。', + ); + assertIncludes( + step.args, + '--host', + '--require-realpath-live 必须把正式域名 Host 传给 realpath live canary smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-realpath-live realpath smoke 必须使用显式传入的正式域名 Host。', + ); + + const parity = findStep(plan, '目标 Nginx realpath canary access log 对账'); + if (!parity) { + failures.push( + '--require-realpath-live 计划必须在 realpath smoke 后包含 access log 对账。', + ); + return; + } + assertIncludes( + parity.args, + '--realpath', + '--require-realpath-live access log 对账必须使用真实路径模式。', + ); + assertIncludes( + parity.args, + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + '--require-realpath-live access log 对账必须使用独立 Nginx realpath canary access log。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_realpath_canary/healthz', + '--require-realpath-live access log 对账必须覆盖 realpath healthz。', + ); + assertIncludes( + parity.args, + '/api/creation-entry/config', + '--require-realpath-live access log 对账必须覆盖真实 API 路径。', + ); + assertIncludes( + parity.args, + '/v1/identity', + '--require-realpath-live access log 对账必须覆盖真实 SpacetimeDB identity 路径。', + ); + assertIncludes( + parity.args, + '/assets/app.js', + '--require-realpath-live access log 对账必须覆盖真实静态资源路径。', + ); +} + +function assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts() { + const plan = readPlan([ + ...requireLiveBaseArgs, + ...requireRealpathLiveBaseArgs, + ...requireDirectBaseArgs, + '--release-runtime-only', + '--dry-run-plan', + ]); + const sourceOnlyNames = new Set([ + 'Rust 路由与保护单测', + 'Pingora mock 上游 smoke', + 'Nginx/Pingora 路由矩阵 parity', + 'Pingora production release 真实构建烟测', + 'API deploy release 烟测', + ]); + for (const name of sourceOnlyNames) { + if (findStep(plan, name)) { + failures.push(`release runtime-only 计划不能包含源码全量门禁: ${name}。`); + } + } + + const audit = findStep(plan, 'current release 自包含自审'); + if (!audit) { + failures.push('release runtime-only 计划必须先包含 current release 自包含自审。'); + } else { + assertAbsoluteScriptArg( + audit.args, + 'scripts/ops/pingora-current-release-audit.mjs', + 'release runtime-only 自审必须使用随当前脚本定位的包内自审脚本。', + ); + assertIncludes( + audit.args, + '--release-root', + 'release runtime-only 自审必须显式传 current release 根目录。', + ); + assertIncludes( + audit.args, + '--require-pingora-gateway', + 'release runtime-only direct 复核必须要求 Pingora 发布物存在。', + ); + assertIncludes( + audit.args, + '--systemd-show', + 'release runtime-only direct 复核必须核验 systemd ExecStart。', + ); + } + + const live = findStep(plan, '目标 Nginx live canary smoke'); + if (!live) { + failures.push('release runtime-only 计划必须包含 live canary smoke。'); + } else { + assertAbsoluteScriptArg( + live.args, + 'scripts/check-pingora-canary-live.mjs', + 'release runtime-only live canary smoke 必须使用随当前脚本定位的包内脚本。', + ); + } + + const liveParity = findStep(plan, '目标 Nginx live canary access log 对账'); + if (!liveParity) { + failures.push('release runtime-only 计划必须包含 live access log 对账。'); + } else { + assertAbsoluteScriptArg( + liveParity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'release runtime-only live access log 对账必须使用随当前脚本定位的包内脚本。', + ); + } + + const realpathLive = findStep(plan, '目标 Nginx realpath canary smoke'); + if (!realpathLive) { + failures.push('release runtime-only 计划必须包含 realpath live canary smoke。'); + } else { + assertAbsoluteScriptArg( + realpathLive.args, + 'scripts/check-pingora-canary-live.mjs', + 'release runtime-only realpath live canary smoke 必须使用随当前脚本定位的包内脚本。', + ); + assertIncludes( + realpathLive.args, + '--realpath', + 'release runtime-only realpath live canary smoke 必须传 --realpath。', + ); + } + + const realpathParity = findStep( + plan, + '目标 Nginx realpath canary access log 对账', + ); + if (!realpathParity) { + failures.push('release runtime-only 计划必须包含 realpath live access log 对账。'); + } else { + assertAbsoluteScriptArg( + realpathParity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'release runtime-only realpath access log 对账必须使用随当前脚本定位的包内脚本。', + ); + assertIncludes( + realpathParity.args, + '--realpath', + 'release runtime-only realpath access log 对账必须传 --realpath。', + ); + } + + const preflight = findStep(plan, '目标 Pingora direct entry preflight'); + if (!preflight) { + failures.push('release runtime-only 计划必须包含 direct preflight。'); + } else { + assertAbsoluteScriptArg( + preflight.args, + 'scripts/check-pingora-direct-preflight.mjs', + 'release runtime-only direct preflight 必须使用随当前脚本定位的包内脚本。', + ); + } + + const healthPatrol = findStep(plan, '目标 health patrol env 直连模式复核'); + if (!healthPatrol) { + failures.push('release runtime-only 计划必须包含 health patrol env 复核。'); + } else { + assertAbsoluteScriptArg( + healthPatrol.args, + 'scripts/check-production-health-patrol-env.mjs', + 'release runtime-only health patrol env 复核必须使用随当前脚本定位的包内脚本。', + ); + } + + const direct = findStep(plan, '目标 Pingora direct live smoke'); + if (!direct) { + failures.push('release runtime-only 计划必须包含 direct live smoke。'); + } else { + assertAbsoluteScriptArg( + direct.args, + 'scripts/check-pingora-direct-live.mjs', + 'release runtime-only direct live smoke 必须使用随当前脚本定位的包内脚本。', + ); + } +} + +function assertReleaseRuntimeOnlyPlanIncludesDirectRehearsalStatus() { + const basePlan = readPlan(['--release-runtime-only', '--dry-run-plan']); + const baseStep = findStep( + basePlan, + '目标 Pingora direct rehearsal 状态复核', + ); + if (!baseStep) { + failures.push( + 'release runtime-only 基础计划必须包含直连彩排状态复核。', + ); + return; + } + assertAbsoluteScriptArg( + baseStep.args, + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + 'release runtime-only 直连彩排状态复核必须使用 current release 随包脚本。', + ); + assertIncludes( + baseStep.args, + '--release-root', + 'release runtime-only 直连彩排状态复核必须显式传 current release 根目录。', + ); + assertIncludes( + baseStep.args, + '--expect-public-gateway', + 'release runtime-only 直连彩排状态复核必须声明公网入口期望。', + ); + assertIncludes( + baseStep.args, + 'nginx', + 'release runtime-only 直连彩排状态复核必须确认 Nginx 仍接公网。', + ); + assertIncludes( + baseStep.args, + '--require-pingora-shadow', + 'release runtime-only 直连彩排状态复核必须要求 Pingora shadow 高端口在线。', + ); + assertIncludes( + baseStep.args, + '--require-realpath-canary', + 'release runtime-only 直连彩排状态复核必须要求 realpath canary 高端口在线。', + ); + assertIncludes( + baseStep.args, + '--require-current-release-gateway', + 'release runtime-only 直连彩排状态复核必须复用 current release Pingora 自审。', + ); + assertIncludes( + baseStep.args, + '--fail-on-critical', + 'release runtime-only 直连彩排状态复核出现 CRITICAL 必须阻断门禁。', + ); + + const directPlan = readPlan([ + ...requireDirectBaseArgs, + '--release-runtime-only', + '--dry-run-plan', + ]); + if (findStep(directPlan, '目标 Pingora direct rehearsal 状态复核')) { + failures.push( + 'release runtime-only --require-direct 阶段不能继续要求 Nginx 接公网的彩排状态。', + ); + } +} + +function assertReleaseRuntimeOnlyAllowsRealpathOnlyCanary() { + const plan = readPlan([ + ...requireRealpathLiveBaseArgs, + '--release-runtime-only', + '--dry-run-plan', + ]); + const prefixLive = findStep(plan, '目标 Nginx live canary smoke'); + if (prefixLive) { + failures.push( + 'release runtime-only 只要求 realpath canary 时不能强制执行前缀 canary smoke。', + ); + } + const prefixParity = findStep(plan, '目标 Nginx live canary access log 对账'); + if (prefixParity) { + failures.push( + 'release runtime-only 只要求 realpath canary 时不能强制执行前缀 canary access log 对账。', + ); + } + + const realpathLive = findStep(plan, '目标 Nginx realpath canary smoke'); + if (!realpathLive) { + failures.push( + 'release runtime-only 只要求 realpath canary 时必须包含 realpath smoke。', + ); + } else { + assertAbsoluteScriptArg( + realpathLive.args, + 'scripts/check-pingora-canary-live.mjs', + 'release runtime-only realpath-only smoke 必须使用 current release 随包脚本。', + ); + assertIncludes( + realpathLive.args, + '--realpath', + 'release runtime-only realpath-only smoke 必须传 --realpath。', + ); + } + + const realpathParity = findStep( + plan, + '目标 Nginx realpath canary access log 对账', + ); + if (!realpathParity) { + failures.push( + 'release runtime-only 只要求 realpath canary 时必须包含 realpath access log 对账。', + ); + } else { + assertAbsoluteScriptArg( + realpathParity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'release runtime-only realpath-only 对账必须使用 current release 随包脚本。', + ); + assertIncludes( + realpathParity.args, + '--realpath', + 'release runtime-only realpath-only 对账必须传 --realpath。', + ); + } +} + +function assertReleaseRuntimeOnlyKeepsInvokedCurrentSymlinkPath() { + const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-release-readiness-current-'), + ); + try { + const releaseDir = path.join(tmpRoot, 'releases', 'test-release'); + const currentLink = path.join(tmpRoot, 'current'); + const scriptDir = path.join(releaseDir, 'scripts'); + mkdirSync(scriptDir, { recursive: true }); + symlinkSync(releaseDir, currentLink, 'dir'); + const invokedScript = path.join( + currentLink, + 'scripts', + 'check-pingora-release-readiness.mjs', + ); + const realScript = path.join( + scriptDir, + 'check-pingora-release-readiness.mjs', + ); + symlinkSync( + path.join(process.cwd(), 'scripts/check-pingora-release-readiness.mjs'), + realScript, + ); + + const result = spawnSync( + 'node', + [invokedScript, '--release-runtime-only', '--dry-run-plan'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `通过 current symlink 读取 runtime-only plan 失败,退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return; + } + const plan = JSON.parse(result.stdout); + const audit = findStep(plan, 'current release 自包含自审'); + const rehearsal = findStep(plan, '目标 Pingora direct rehearsal 状态复核'); + for (const [label, step] of [ + ['current release 自审', audit], + ['直连彩排状态复核', rehearsal], + ]) { + if (!step) { + failures.push( + `通过 current symlink 生成的 runtime-only plan 缺少${label}步骤。`, + ); + continue; + } + assertIncludes( + step.args, + currentLink, + `${label}必须保留调用方 current symlink 作为 release root,而不是展开到真实 release 目录。`, + ); + if (step.args.includes(releaseDir)) { + failures.push( + `${label}不应把 current symlink 展开成真实 release 目录: ${releaseDir}`, + ); + } + } + } catch (error) { + failures.push(`current symlink runtime-only plan 自测异常: ${error.message}`); + } finally { + rmSync(tmpRoot, { recursive: true, force: true }); + } +} + +function assertReleaseRuntimeOnlyRejectsSourceOnlyFlags() { + for (const flag of ['--require-docker', '--pull-docker', '--require-nginx']) { + const result = runReadinessExpectFailure([ + '--release-runtime-only', + flag, + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--release-runtime-only 只执行 current release 包内运行时复核', + `release runtime-only 不能同时接受源码或本机依赖参数 ${flag}。`, + ); + } +} + +function assertRequireDirectForcesWssUpgrade() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--require-wss-upgrade', + '--require-direct 必须自动强制 WSS subscribe 返回 101。', + ); +} + +function assertRequireDirectForcesHttpBaseUrl() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--http-base-url', + '--require-direct 必须把 HTTP redirect / ACME 入口纳入 direct live smoke。', + ); +} + +function assertRequireDirectForcesHostSni() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--host', + '--require-direct 必须把正式域名 Host/SNI 传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct direct live smoke 必须使用显式传入的正式域名 Host/SNI。', + ); +} + +function assertRequireDirectForcesRedirectHost() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--redirect-host', + '--require-direct 必须把显式 redirect Location host 传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct direct live smoke 必须使用显式传入的 redirect Location host。', + ); +} + +function assertRequireDirectForcesPingoraAccessLog() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--pingora-access-log', + '--require-direct 必须把 Pingora access log 落盘校验传给 direct live smoke。', + ); + assertIncludes( + step.args, + '/var/log/genarrative/pingora-gateway.access.log', + '--require-direct direct live smoke 必须使用显式 Pingora access log 路径。', + ); +} + +function assertRequireDirectForcesPreflightSystemdCertReadable() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct entry preflight'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct entry preflight。'); + return; + } + assertIncludes( + step.args, + '--systemd-cat', + '--require-direct 必须检查 systemd drop-in 最终生效配置。', + ); + assertIncludes( + step.args, + '--check-cert-readable', + '--require-direct 必须检查当前用户可读 TLS 证书和私钥。', + ); + assertIncludes( + step.args, + '--check-service-env-file', + '--require-direct 必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + step.args, + '--check-service-user-cert-readable', + '--require-direct 必须检查 systemd 服务用户可读 TLS 证书和私钥。', + ); + assertIncludes( + step.args, + '--check-service-binary-executable', + '--require-direct 必须检查 Pingora service ExecStart 二进制可执行。', + ); + assertNotIncludes( + step.args, + '--check-ports-free', + '--require-direct 启用后复核不能继续要求 TLS/HTTP redirect 端口空闲;此时端口应由 Pingora 占用。', + ); +} + +function assertDryRunCutoverForcesPortsFreePreflight() { + const plan = readPlan(cutoverPlanArgs()); + const step = findStep(plan, '切换前 current release 直连 env / 证书 / 端口预检'); + if (!step) { + failures.push('dry-run cutover 计划必须包含切换前 direct entry preflight。'); + return; + } + assertIncludes( + step.args, + '--check-ports-free', + 'dry-run cutover 启用前 preflight 必须检查 TLS/HTTP redirect 端口已释放。', + ); +} + +function assertRequireDirectForcesExplicitSpacetimeDatabase() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--spacetime-database', + '--require-direct 必须把显式 SpacetimeDB 数据库名传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'genarrative-prod', + '--require-direct direct live smoke 必须使用显式传入的 SpacetimeDB 数据库名。', + ); +} + +function assertRequireDirectForcesHealthPatrolEnvCheck() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 health patrol env 直连模式复核'); + if (!step) { + failures.push('--require-direct 计划必须包含 health patrol env 复核。'); + return; + } + assertIncludes( + step.args, + 'scripts/check-production-health-patrol-env.mjs', + '--require-direct 必须复核 health patrol env。', + ); + assertIncludes( + step.args, + '/etc/genarrative/health-patrol.env', + '--require-direct health patrol env 复核必须使用显式 env 文件。', + ); + assertIncludes( + step.args, + 'pingora-direct', + '--require-direct health patrol env 复核必须要求 pingora-direct。', + ); + assertIncludes( + step.args, + 'https://127.0.0.1', + '--require-direct health patrol env 复核必须校验 direct HTTPS base URL。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct health patrol env 复核必须校验正式 Host。', + ); +} + +function assertDryRunPlanRedactsDirectProbeToken() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--direct-probe-token', + 'direct-secret-token', + '--dry-run-plan', + ]); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('带 direct probe token 的计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--probe-token', + 'dry-run plan direct live smoke 必须传递 probe token 参数。', + ); + assertIncludes( + step.args, + '', + 'dry-run plan direct live smoke 必须隐藏 probe token 值。', + ); + if (step.args.includes('direct-secret-token')) { + failures.push('dry-run plan JSON 不能泄露 direct probe token 原文。'); + } +} + +function assertRunStepLogRedactsDirectProbeToken() { + const output = readFileSync( + 'scripts/check-pingora-release-readiness.mjs', + 'utf8', + ); + assertIncludes( + output, + 'redactSecretArgs(args).join', + '真实执行日志里的命令展示必须使用脱敏参数。', + ); +} + +function assertRequireLiveRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-live 时必须提供 --live-host 或 GENARRATIVE_PINGORA_CANARY_HOST', + '--require-live 缺少 live Host 时必须失败。', + ); +} + +function assertRequireLiveRejectsInvalidHost() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'https://example.com', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-host 只能是 host 或 host:port', + '--require-live 的 live Host 不能接受 URL 或路径。', + ); +} + +function assertRequireLiveRejectsRelativeAccessLog() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + 'nginx.access.log', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-nginx-access-log 必须是绝对路径', + '--require-live 的 Nginx access log 不能接受相对路径。', + ); +} + +function assertRequireLiveRejectsFilesystemRootAccessLog() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + '/', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-nginx-access-log 不能是文件系统根目录', + '--require-live 的 Nginx access log 不能指向文件系统根目录。', + ); +} + +function assertRequireLiveRejectsInvalidAccessLogSinceLines() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-access-log-since-lines', + '0', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-access-log-since-lines 必须是正整数', + '--require-live 的 access log tail 行数必须是正整数。', + ); +} + +function assertRequireRealpathLiveRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-realpath-live 时必须提供 --realpath-live-host 或 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST', + '--require-realpath-live 缺少 realpath live Host 时必须失败。', + ); +} + +function assertRequireRealpathLiveRejectsRelativeAccessLog() { + const result = runReadinessExpectFailure([ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--realpath-live-host', + 'example.com', + '--realpath-live-nginx-access-log', + 'nginx.realpath.access.log', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--realpath-live-nginx-access-log 必须是绝对路径', + '--require-realpath-live 的 Nginx access log 不能接受相对路径。', + ); +} + +function assertRequireDirectRejectsMissingHttpBaseUrl() { + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-release-readiness.mjs', + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('--require-direct 缺少 direct HTTP base URL 时必须失败。'); + return; + } + const output = `${result.stdout}\n${result.stderr}`; + if ( + !output.includes( + '启用 --require-direct 时必须提供 --direct-http-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + ) + ) { + failures.push( + '--require-direct 缺少 direct HTTP base URL 时必须给出明确错误。', + ); + } +} + +function assertRequireDirectRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-host 或 GENARRATIVE_PINGORA_DIRECT_HOST', + '--require-direct 缺少 direct Host/SNI 时必须失败。', + ); +} + +function assertRequireDirectRejectsInvalidHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com/path', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-host 只能是 host 或 host:port', + '--require-direct 的 direct Host/SNI 不能接受 URL 或路径。', + ); +} + +function assertRequireDirectRejectsMissingRedirectHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-redirect-host 或 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + '--require-direct 缺少 direct redirect Location host 时必须失败。', + ); +} + +function assertRequireDirectRejectsInvalidRedirectHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'https://example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-redirect-host 只能是 host 或 host:port', + '--require-direct 的 redirect Location host 不能接受 URL 或路径。', + ); +} + +function assertRequireDirectRejectsMissingPingoraAccessLog() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-pingora-access-log 或 GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + '--require-direct 缺少 direct Pingora access log 时必须失败。', + ); +} + +function assertRequireDirectRejectsFilesystemRootPingoraAccessLog() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-pingora-access-log 不能是文件系统根目录', + '--require-direct 的 Pingora access log 不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsMissingSystemdPreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-systemd 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true', + '--require-direct 缺少 systemd preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingCertReadablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true', + '--require-direct 缺少证书可读 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceEnvFilePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true', + '--require-direct 缺少 service EnvironmentFile preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceUserCertReadablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true', + '--require-direct 缺少服务用户证书可读 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceBinaryExecutablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true', + '--require-direct 缺少 service 二进制可执行 preflight 时必须失败。', + ); +} + +function assertRequireDirectAllowsPostEnablePortsOwnedByPingora() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct entry preflight'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct entry preflight。'); + return; + } + assertNotIncludes( + step.args, + '--check-ports-free', + '--require-direct post-enable 复核不能要求 80/443 空闲。', + ); +} + +function assertDryRunCutoverRejectsMissingPortsFreePreflight() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-nginx-smoke-expect-body', + '', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须提供 --direct-preflight-check-ports-free', + 'dry-run cutover 缺少端口释放 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingSpacetimeDatabase() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-spacetime-database 或 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + '--require-direct 缺少 SpacetimeDB 数据库名时必须失败。', + ); +} + +function assertRequireDirectRejectsFilesystemRootPreflightEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-preflight-env-file 不能是文件系统根目录', + '--require-direct 的 Pingora preflight env 文件不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsMissingHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-health-patrol-env-file 或 GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE', + '--require-direct 缺少 health patrol env 文件时必须失败。', + ); +} + +function assertRequireDirectRejectsRelativeHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + 'health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-health-patrol-env-file 必须是绝对路径', + '--require-direct 的 health patrol env 文件不能接受相对路径。', + ); +} + +function assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-health-patrol-env-file 不能是文件系统根目录', + '--require-direct 的 health patrol env 文件不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsSkipWss() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--direct-skip-wss', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时不能使用 --direct-skip-wss 或 GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=true', + '--require-direct 不能允许跳过 WSS subscribe。', + ); +} + +function assertRequireDirectRejectsInsecureTls() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--direct-insecure-tls', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时不能使用 --direct-insecure-tls 或 GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=true', + '--require-direct 不能允许 insecure TLS。', + ); +} + +function assertDryRunCutoverRejectsMissingRequireDirect() { + const result = runReadinessExpectFailure([ + '--dry-run-cutover', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须同时提供 --require-direct', + 'dry-run cutover 缺少 --require-direct 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRelativeEvidenceOutputRoot() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--cutover-evidence-output-root', + 'relative-evidence', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-output-root 必须是绝对路径', + 'dry-run cutover 证据输出根目录使用相对路径时必须失败。', + ); +} + +function assertDryRunCutoverRejectsFilesystemRootReleaseRoot() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--cutover-release-root', + '/', + ]); + assertFailureIncludes( + result, + '--cutover-release-root 不能是文件系统根目录', + 'dry-run cutover current release 根目录使用 / 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--cutover-evidence-output-root', + '/', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-output-root 不能是文件系统根目录', + 'dry-run cutover 证据输出根目录使用 / 时必须失败。', + ); +} + +function assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot() { + const usage = readUsage(); + assertIncludes( + usage, + '--cutover-evidence-output-root /var/log/genarrative/pingora-cutover-evidence', + '只生成直连切换 runbook 的 usage 示例必须显式传证据输出根目录。', + ); +} + +function assertUsageExamplesIncludeDirectPingoraAccessLog() { + const usage = readUsage(); + assertIncludes( + usage, + 'Pingora 直连入口切换窗口追加:', + 'usage 必须包含正式直连入口示例。', + ); + assertIncludes( + usage, + '只生成直连切换 runbook:', + 'usage 必须包含只生成直连切换 runbook 示例。', + ); + const directExample = usage.slice( + usage.indexOf('Pingora 直连入口切换窗口追加:'), + usage.indexOf('只生成直连切换 runbook:'), + ); + const cutoverExample = usage.slice( + usage.indexOf('只生成直连切换 runbook:'), + ); + assertIncludes( + directExample, + '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + '正式直连入口 usage 示例必须显式传 Pingora access log 路径。', + ); + assertIncludes( + cutoverExample, + '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + '只生成直连切换 runbook 的 usage 示例必须显式传 Pingora access log 路径。', + ); +} + +function assertDryRunCutoverRejectsMissingRollbackHealthPatrolPublicBaseUrl() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--direct-preflight-check-ports-free', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-nginx-smoke-expect-body', + '', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须提供 --rollback-health-patrol-public-base-url', + 'dry-run cutover 缺少回退后 health patrol public base URL 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsMissingRollbackSmokeBody() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--direct-preflight-check-ports-free', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须提供 --rollback-nginx-smoke-expect-body', + 'dry-run cutover 缺少回退 Nginx smoke body 证据片段时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicBaseUrl() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--direct-preflight-check-ports-free', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-nginx-smoke-expect-body', + '', + '--rollback-health-patrol-public-base-url', + 'ftp://127.0.0.1', + ]); + assertFailureIncludes( + result, + '--rollback-health-patrol-public-base-url 必须是 http(s) URL', + 'dry-run cutover 回退 health patrol public base URL 非 http(s) 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicHost() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--rollback-health-patrol-public-host', + 'https://example.com', + ]); + assertFailureIncludes( + result, + '--rollback-health-patrol-public-host 只能是 host 或 host:port', + 'dry-run cutover 回退 health patrol public Host 非 host 格式时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRedirectHostMismatch() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs( + '--direct-redirect-host', + 'redirect.example.com', + ), + ]); + assertFailureIncludes( + result, + '--direct-redirect-host 必须与 --direct-host 使用同一正式 hostname', + 'dry-run cutover redirect Host 与 direct Host 不一致时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRollbackHostMismatch() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--rollback-nginx-smoke-host', + 'nginx.example.com', + ]); + assertFailureIncludes( + result, + '--rollback-nginx-smoke-host 必须与 --direct-host 使用同一正式 hostname', + 'dry-run cutover rollback smoke Host 与 direct Host 不一致时必须失败。', + ); +} + +function assertDryRunCutoverRejectsIncompleteRollbackShadowProbe() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--rollback-pingora-shadow-probe-url', + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + ]); + assertFailureIncludes( + result, + '--rollback-pingora-shadow-probe-url 必须同时提供 --rollback-pingora-shadow-probe-token', + 'dry-run cutover rollback shadow probe URL 缺少 token 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackShadowProbeUrl() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--rollback-pingora-shadow-probe-url', + 'ftp://127.0.0.1/probe', + '--rollback-pingora-shadow-probe-token', + 'probe-secret-token', + ]); + assertFailureIncludes( + result, + '--rollback-pingora-shadow-probe-url 必须是 http(s) URL', + 'dry-run cutover rollback shadow probe URL 非 http(s) 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidEvidenceTimelineMaxSpan() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs(), + '--cutover-evidence-timeline-max-span-ms', + '0', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-timeline-max-span-ms 必须是正整数', + 'dry-run cutover 标准切换时间线最大跨度非正整数时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidCutoverRunId() { + const result = runReadinessExpectFailure([ + ...cutoverPlanArgs( + '--cutover-run-id', + 'cutover 1', + ), + ]); + assertFailureIncludes( + result, + '--cutover-run-id 只能包含 ASCII 字母、数字、点、下划线或短横线', + 'dry-run cutover 必须拒绝不安全 cutoverRunId。', + ); +} + +function runReadinessExpectFailure(args) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('release readiness 缺少硬门禁参数时必须失败。'); + } + return result; +} + +function assertLiveSmokeScriptsRejectInvalidTimeout() { + const cases = [ + { + script: 'scripts/check-pingora-canary-live.mjs', + args: ['--base-url', 'http://127.0.0.1', '--timeout-ms', '0'], + expected: '--timeout-ms 必须是正整数', + reason: 'canary live smoke 必须拒绝非正数 --timeout-ms。', + }, + { + script: 'scripts/check-pingora-canary-live.mjs', + args: ['--base-url', 'http://127.0.0.1'], + env: { + GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS 必须是正整数', + reason: 'canary live smoke 必须拒绝非法 env timeout。', + }, + { + script: 'scripts/check-pingora-direct-live.mjs', + args: ['--https-base-url', 'https://127.0.0.1', '--timeout-ms', '0'], + expected: '--timeout-ms 必须是正整数', + reason: 'direct live smoke 必须拒绝非正数 --timeout-ms。', + }, + { + script: 'scripts/check-pingora-direct-live.mjs', + args: ['--https-base-url', 'https://127.0.0.1'], + env: { + GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS 必须是正整数', + reason: 'direct live smoke 必须拒绝非法 env timeout。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync('node', [testCase.script, ...testCase.args], { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...(testCase.env || {}), + }, + }); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectLiveRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: 'ture', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE 必须是布尔值', + reason: 'direct live smoke 必须拒绝拼写错误的 require WSS env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_SKIP_WSS: 'maybe', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_SKIP_WSS 必须是布尔值', + reason: 'direct live smoke 必须拒绝非法 skip WSS env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS: 'enabled', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS 必须是布尔值', + reason: 'direct live smoke 必须拒绝非法 insecure TLS env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-direct-live.mjs', '--https-base-url', 'https://127.0.0.1'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectLiveRejectsFilesystemRootAccessLog() { + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + 'https://127.0.0.1', + '--pingora-access-log', + '/', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('direct live smoke 必须拒绝 Pingora access log 指向文件系统根目录。'); + } + assertFailureIncludes( + result, + '--pingora-access-log 不能是文件系统根目录', + 'direct live smoke 的 Pingora access log 不能指向文件系统根目录。', + ); +} + +function assertReleaseReadinessRejectsInvalidDirectBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT 必须是布尔值', + reason: 'release readiness 必须拒绝拼写错误的 preflight systemd env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: 'definitely', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE 必须是布尔值', + reason: 'release readiness 必须拒绝非法 require WSS env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', '--dry-run-plan'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectPreflightRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_REQUIRE_LIVE_ENV: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_REQUIRE_LIVE_ENV 必须是布尔值', + reason: 'direct preflight 必须拒绝拼写错误的 require live env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE: 'maybe', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE 必须是布尔值', + reason: 'direct preflight 必须拒绝非法 check ports free env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-direct-preflight.mjs'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection() { + const root = mkdtempSync(path.join(tmpdir(), 'genarrative-direct-preflight-')); + try { + const envFile = path.join(root, 'pingora-gateway.env'); + writeFileSync( + envFile, + [ + 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:18443', + 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:18080', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/tmp/cert.pem', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/tmp/key.pem', + 'GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO=https', + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=2', + 'GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false', + '', + ].join('\n'), + ); + + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-direct-preflight.mjs', + '--env-file', + envFile, + '--require-live-env', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push( + 'direct preflight 必须拒绝未确认共享保护的多实例 Pingora 配置。', + ); + } + assertFailureIncludes( + result, + 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + 'direct preflight 多实例保护失败必须给出明确错误。', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function assertCutoverEvidenceScriptsRejectInvalidBoolEnv() { + const cases = [ + { + script: 'scripts/ops/pingora-current-release-audit.mjs', + env: { + GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY 必须是布尔值', + reason: + 'current release 自审 smoke 必须覆盖非法 require gateway 布尔 env。', + }, + { + script: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + env: { + GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL: 'maybe', + }, + expected: + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL 必须是布尔值', + reason: + 'cutover 状态快照 smoke 必须覆盖非法 run health patrol 布尔 env。', + }, + { + script: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + env: { + GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL: 'enabled', + }, + expected: + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL 必须是布尔值', + reason: + 'cutover 证据包 smoke 必须覆盖非法 fail on critical 布尔 env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync('node', [testCase.script], { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }); + if ((result.status ?? 0) === 0) { + failures.push( + `${testCase.reason} 脚本本体必须失败。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须能在输出中看到明确错误。`, + ); + } +} + +function assertFailureIncludes(result, expected, reason) { + const output = `${result.stdout}\n${result.stderr}`; + if (!output.includes(expected)) { + failures.push(reason); + } +} + +function readPlan(args) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `读取 release readiness dry-run plan 失败,退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return []; + } + try { + return JSON.parse(result.stdout); + } catch (error) { + failures.push( + `release readiness dry-run plan 不是合法 JSON: ${error.message}`, + ); + return []; + } +} + +function readUsage() { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', '--help'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `读取 release readiness usage 失败,退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return ''; + } + return result.stdout; +} + +function readinessPlanEnv() { + return { + ...process.env, + GENARRATIVE_PINGORA_CANARY_HOST: '', + GENARRATIVE_PINGORA_CANARY_BASE_URL: '', + GENARRATIVE_PINGORA_REALPATH_CANARY_HOST: '', + GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL: '', + GENARRATIVE_PINGORA_REALPATH_CANARY_NGINX_ACCESS_LOG: '', + GENARRATIVE_PINGORA_REALPATH_CANARY_PINGORA_ACCESS_LOG: '', + GENARRATIVE_PINGORA_REALPATH_CANARY_ACCESS_LOG_SINCE_LINES: '', + GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL: '', + GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL: '', + GENARRATIVE_PINGORA_DIRECT_HOST: '', + GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE: '', + GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE: '', + GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: '', + GENARRATIVE_PINGORA_DIRECT_SKIP_WSS: '', + GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_HOST: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY: '', + GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN: '', + GENARRATIVE_PINGORA_CUTOVER_RUN_ID: '', + }; +} + +function findStep(plan, name) { + return plan.find((step) => step.name === name); +} + +function assertIncludes(values, expected, reason) { + if (!values.includes(expected)) { + failures.push(`${reason} 缺少: ${expected}`); + } +} + +function assertAbsoluteScriptArg(values, expectedSuffix, reason) { + const normalizedSuffix = expectedSuffix.replace(/\\/g, '/'); + const matched = values.some((value) => { + const text = String(value).replace(/\\/g, '/'); + return text.startsWith('/') && text.endsWith(normalizedSuffix); + }); + if (!matched) { + failures.push(`${reason} 缺少绝对脚本路径后缀: ${expectedSuffix}`); + } +} + +function assertNotIncludes(values, forbidden, reason) { + if (values.includes(forbidden)) { + failures.push(reason); + } +} + +function assertStepBefore(plan, firstName, secondName, reason) { + const firstIndex = plan.findIndex((step) => step.name === firstName); + const secondIndex = plan.findIndex((step) => step.name === secondName); + if (firstIndex < 0 || secondIndex < 0) { + return; + } + if (firstIndex >= secondIndex) { + failures.push(reason); + } +} diff --git a/scripts/check-pingora-release-readiness.mjs b/scripts/check-pingora-release-readiness.mjs new file mode 100644 index 000000000..dd6dd1ff1 --- /dev/null +++ b/scripts/check-pingora-release-readiness.mjs @@ -0,0 +1,2222 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = process.cwd(); +const scriptDir = path.dirname(resolveInvokedScriptPath()); +const releaseRoot = path.resolve(scriptDir, '..'); +const DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; +const DEFAULT_CUTOVER_RUN_ID_PREFIX = 'pingora-direct-'; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); + +const config = parseArgs(process.argv.slice(2)); +const failures = []; + +if (config.dryRunPlan) { + printDryRunPlan(buildSteps(config)); + process.exit(0); +} +if (config.dryRunCutover) { + printDryRunPlan(buildCutoverPlan(config)); + process.exit(0); +} + +for (const step of buildSteps(config)) { + const ok = await runStep(step); + if (!ok && !config.keepGoing) { + break; + } +} + +if (failures.length > 0) { + console.error('\n[pingora-release-readiness] 未通过:'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('\n[pingora-release-readiness] 通过'); + +function resolveInvokedScriptPath() { + const invoked = process.argv[1]; + if (invoked) { + return path.resolve(invoked); + } + return fileURLToPath(import.meta.url); +} + +function parseArgs(argv) { + const result = { + requireDocker: false, + pullDocker: false, + requireNginx: false, + requireLive: false, + requireRealpathLive: false, + requireDirect: false, + liveBaseUrl: process.env.GENARRATIVE_PINGORA_CANARY_BASE_URL || '', + liveHost: process.env.GENARRATIVE_PINGORA_CANARY_HOST || '', + liveAccessLogNginx: + process.env.GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG || + '/var/log/nginx/genarrative.access.log', + liveAccessLogPingora: + process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG || + '/var/log/genarrative/pingora-gateway.access.log', + liveAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES || '2000', + realpathLiveBaseUrl: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL || '', + realpathLiveHost: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_HOST || '', + realpathLiveAccessLogNginx: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_NGINX_ACCESS_LOG || + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + realpathLiveAccessLogPingora: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_PINGORA_ACCESS_LOG || + process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG || + '/var/log/genarrative/pingora-gateway.access.log', + realpathLiveAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_ACCESS_LOG_SINCE_LINES || + process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES || + '2000', + directHttpsBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL || '', + directHttpBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '', + directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', + directRedirectHost: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directRedirectBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '', + directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', + directPingoraAccessLog: + process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '', + directAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES || '2000', + directPreflightEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || '', + directPreflightSystemd: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT', + ), + directPreflightCheckCertReadable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE', + ), + directPreflightCheckServiceEnvFile: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE', + ), + directPreflightCheckServiceUserCertReadable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE', + ), + directPreflightCheckServiceBinaryExecutable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE', + ), + directPreflightCheckPortsFree: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE', + ), + directSpacetimeDatabase: + process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '', + directHealthPatrolEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE || + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '', + directRequireWssUpgrade: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE', + ), + directSkipWss: readBoolEnv('GENARRATIVE_PINGORA_DIRECT_SKIP_WSS'), + directInsecureTls: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS', + ), + cutoverReleaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + cutoverEvidenceOutputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + cutoverEvidenceTimelineMaxSpanMs: parsePositiveIntEnv( + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS', + DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS, + ), + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + rollbackNginxSmokeUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL || '', + rollbackNginxSmokeHost: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_HOST || '', + rollbackNginxSmokeExpectBody: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY || '', + rollbackHealthPatrolPublicBaseUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL || + '', + rollbackHealthPatrolPublicHost: + process.env.GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_HOST || '', + rollbackPingoraShadowProbeUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL || '', + rollbackPingoraShadowProbeToken: + process.env.GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN || '', + dryRunPlan: false, + dryRunCutover: false, + releaseRuntimeOnly: readBoolEnv( + 'GENARRATIVE_PINGORA_RELEASE_RUNTIME_ONLY', + ), + keepGoing: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--require-docker': + result.requireDocker = true; + break; + case '--pull-docker': + result.pullDocker = true; + break; + case '--require-nginx': + result.requireNginx = true; + break; + case '--require-live': + result.requireLive = true; + break; + case '--require-realpath-live': + result.requireRealpathLive = true; + break; + case '--require-direct': + result.requireDirect = true; + break; + case '--live-base-url': + result.liveBaseUrl = requireValue(argv, ++index, arg); + break; + case '--live-host': + result.liveHost = requireValue(argv, ++index, arg); + break; + case '--live-nginx-access-log': + result.liveAccessLogNginx = requireValue(argv, ++index, arg); + break; + case '--live-pingora-access-log': + result.liveAccessLogPingora = requireValue(argv, ++index, arg); + break; + case '--live-access-log-since-lines': + result.liveAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--realpath-live-base-url': + result.realpathLiveBaseUrl = requireValue(argv, ++index, arg); + break; + case '--realpath-live-host': + result.realpathLiveHost = requireValue(argv, ++index, arg); + break; + case '--realpath-live-nginx-access-log': + result.realpathLiveAccessLogNginx = requireValue(argv, ++index, arg); + break; + case '--realpath-live-pingora-access-log': + result.realpathLiveAccessLogPingora = requireValue(argv, ++index, arg); + break; + case '--realpath-live-access-log-since-lines': + result.realpathLiveAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--direct-https-base-url': + result.directHttpsBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-http-base-url': + result.directHttpBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-host': + result.directHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-host': + result.directRedirectHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-base-url': + result.directRedirectBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-probe-token': + result.directProbeToken = requireValue(argv, ++index, arg); + break; + case '--direct-pingora-access-log': + result.directPingoraAccessLog = requireValue(argv, ++index, arg); + break; + case '--direct-access-log-since-lines': + result.directAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--direct-preflight-env-file': + result.directPreflightEnvFile = requireValue(argv, ++index, arg); + break; + case '--direct-preflight-systemd': + result.directPreflightSystemd = true; + break; + case '--direct-preflight-check-cert-readable': + result.directPreflightCheckCertReadable = true; + break; + case '--direct-preflight-check-service-env-file': + result.directPreflightCheckServiceEnvFile = true; + break; + case '--direct-preflight-check-service-user-cert-readable': + result.directPreflightCheckServiceUserCertReadable = true; + break; + case '--direct-preflight-check-service-binary-executable': + result.directPreflightCheckServiceBinaryExecutable = true; + break; + case '--direct-preflight-check-ports-free': + result.directPreflightCheckPortsFree = true; + break; + case '--direct-spacetime-database': + result.directSpacetimeDatabase = requireValue(argv, ++index, arg); + break; + case '--direct-health-patrol-env-file': + result.directHealthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--direct-require-wss-upgrade': + result.directRequireWssUpgrade = true; + break; + case '--direct-skip-wss': + result.directSkipWss = true; + break; + case '--direct-insecure-tls': + result.directInsecureTls = true; + break; + case '--cutover-release-root': + result.cutoverReleaseRoot = requireValue(argv, ++index, arg); + break; + case '--cutover-evidence-output-root': + result.cutoverEvidenceOutputRoot = requireValue(argv, ++index, arg); + break; + case '--cutover-evidence-timeline-max-span-ms': + result.cutoverEvidenceTimelineMaxSpanMs = parsePositiveInt( + requireValue(argv, ++index, arg), + '--cutover-evidence-timeline-max-span-ms', + ); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-url': + result.rollbackNginxSmokeUrl = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-host': + result.rollbackNginxSmokeHost = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-expect-body': + result.rollbackNginxSmokeExpectBody = requireValue(argv, ++index, arg); + break; + case '--rollback-health-patrol-public-base-url': + result.rollbackHealthPatrolPublicBaseUrl = requireValue( + argv, + ++index, + arg, + ); + break; + case '--rollback-health-patrol-public-host': + result.rollbackHealthPatrolPublicHost = requireValue( + argv, + ++index, + arg, + ); + break; + case '--rollback-pingora-shadow-probe-url': + result.rollbackPingoraShadowProbeUrl = requireValue(argv, ++index, arg); + break; + case '--rollback-pingora-shadow-probe-token': + result.rollbackPingoraShadowProbeToken = requireValue( + argv, + ++index, + arg, + ); + break; + case '--dry-run-plan': + result.dryRunPlan = true; + break; + case '--dry-run-cutover': + result.dryRunCutover = true; + break; + case '--release-runtime-only': + result.releaseRuntimeOnly = true; + break; + case '--keep-going': + result.keepGoing = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (result.requireLive && !result.liveBaseUrl) { + throw new Error( + '启用 --require-live 时必须提供 --live-base-url 或 GENARRATIVE_PINGORA_CANARY_BASE_URL', + ); + } + if (result.requireLive && !result.liveHost) { + throw new Error( + '启用 --require-live 时必须提供 --live-host 或 GENARRATIVE_PINGORA_CANARY_HOST', + ); + } + if (result.liveHost) { + validateHostOption(result.liveHost, '--live-host'); + } + if (result.requireLive) { + for (const [label, filePath] of [ + ['--live-nginx-access-log', result.liveAccessLogNginx], + ['--live-pingora-access-log', result.liveAccessLogPingora], + ]) { + validateSafeAbsoluteFilePath(filePath, label); + } + if (!isPositiveIntegerString(result.liveAccessLogSinceLines)) { + throw new Error('--live-access-log-since-lines 必须是正整数。'); + } + } + if (result.requireRealpathLive && !result.realpathLiveBaseUrl) { + throw new Error( + '启用 --require-realpath-live 时必须提供 --realpath-live-base-url 或 GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL', + ); + } + if (result.requireRealpathLive && !result.realpathLiveHost) { + throw new Error( + '启用 --require-realpath-live 时必须提供 --realpath-live-host 或 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST', + ); + } + if (result.realpathLiveHost) { + validateHostOption(result.realpathLiveHost, '--realpath-live-host'); + } + if (result.requireRealpathLive || result.realpathLiveBaseUrl) { + for (const [label, filePath] of [ + ['--realpath-live-nginx-access-log', result.realpathLiveAccessLogNginx], + ['--realpath-live-pingora-access-log', result.realpathLiveAccessLogPingora], + ]) { + validateSafeAbsoluteFilePath(filePath, label); + } + if (!isPositiveIntegerString(result.realpathLiveAccessLogSinceLines)) { + throw new Error('--realpath-live-access-log-since-lines 必须是正整数。'); + } + } + if (result.requireDirect && !result.directHttpsBaseUrl) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-https-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL', + ); + } + if (result.requireDirect && !result.directHttpBaseUrl) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-http-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + ); + } + if (result.requireDirect && !result.directHost) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-host 或 GENARRATIVE_PINGORA_DIRECT_HOST', + ); + } + if (result.directHost) { + validateHostOption(result.directHost, '--direct-host'); + } + if (result.requireDirect && !result.directRedirectHost) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-redirect-host 或 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + ); + } + if (result.directRedirectHost) { + validateHostOption(result.directRedirectHost, '--direct-redirect-host'); + } + if (result.directRedirectBaseUrl) { + validateHttpUrl(result.directRedirectBaseUrl, '--direct-redirect-base-url'); + const parsed = new URL(result.directRedirectBaseUrl); + if (parsed.protocol !== 'https:') { + throw new Error('--direct-redirect-base-url 必须使用 https://。'); + } + if ( + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error( + '--direct-redirect-base-url 只能是 HTTPS base URL,不能包含路径、查询、片段或认证信息。', + ); + } + } + if (result.requireDirect && !result.directPingoraAccessLog) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-pingora-access-log 或 GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + ); + } + if (result.directPingoraAccessLog) { + validateSafeAbsoluteFilePath( + result.directPingoraAccessLog, + '--direct-pingora-access-log', + ); + } + if ( + result.directPingoraAccessLog && + !isPositiveIntegerString(result.directAccessLogSinceLines) + ) { + throw new Error('--direct-access-log-since-lines 必须是正整数。'); + } + if (result.requireDirect && !result.directPreflightEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE', + ); + } + if (result.directPreflightEnvFile) { + validateSafeAbsoluteFilePath( + result.directPreflightEnvFile, + '--direct-preflight-env-file', + ); + } + if (result.requireDirect && !result.directPreflightSystemd) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-systemd 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true', + ); + } + if (result.requireDirect && !result.directPreflightCheckCertReadable) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true', + ); + } + if (result.requireDirect && !result.directPreflightCheckServiceEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true', + ); + } + if ( + result.requireDirect && + !result.directPreflightCheckServiceUserCertReadable + ) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true', + ); + } + if ( + result.requireDirect && + !result.directPreflightCheckServiceBinaryExecutable + ) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true', + ); + } + if (result.dryRunCutover && !result.directPreflightCheckPortsFree) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --direct-preflight-check-ports-free 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE=true,用于切换前证明 Nginx 和其它进程已释放 80/443。', + ); + } + if (result.requireDirect && result.directSkipWss) { + throw new Error( + '启用 --require-direct 时不能使用 --direct-skip-wss 或 GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=true', + ); + } + if (result.requireDirect && !result.directSpacetimeDatabase) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-spacetime-database 或 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + ); + } + if (result.requireDirect && !result.directHealthPatrolEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-health-patrol-env-file 或 GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE', + ); + } + if (result.directHealthPatrolEnvFile) { + validateSafeAbsoluteFilePath( + result.directHealthPatrolEnvFile, + '--direct-health-patrol-env-file', + ); + } + if (result.requireDirect && result.directInsecureTls) { + throw new Error( + '启用 --require-direct 时不能使用 --direct-insecure-tls 或 GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=true', + ); + } + if (result.requireDirect && !result.directSkipWss) { + result.directRequireWssUpgrade = true; + } + if (result.dryRunCutover && !result.requireDirect) { + throw new Error( + '启用 --dry-run-cutover 时必须同时提供 --require-direct,避免生成缺少直连硬门禁的切换计划。', + ); + } + if (result.dryRunCutover && !path.isAbsolute(result.cutoverReleaseRoot)) { + throw new Error('--cutover-release-root 必须是绝对路径。'); + } + if ( + result.dryRunCutover && + isFilesystemRootPath(result.cutoverReleaseRoot) + ) { + throw new Error('--cutover-release-root 不能是文件系统根目录。'); + } + if ( + result.dryRunCutover && + !path.isAbsolute(result.cutoverEvidenceOutputRoot) + ) { + throw new Error('--cutover-evidence-output-root 必须是绝对路径。'); + } + if ( + result.dryRunCutover && + isFilesystemRootPath(result.cutoverEvidenceOutputRoot) + ) { + throw new Error('--cutover-evidence-output-root 不能是文件系统根目录。'); + } + if (result.dryRunCutover && !result.rollbackNginxSmokeUrl) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --rollback-nginx-smoke-url 或 GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL。', + ); + } + if (result.dryRunCutover && !result.rollbackNginxSmokeExpectBody) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --rollback-nginx-smoke-expect-body 或 GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY,且该片段必须来自切换前真实 Nginx 入口。', + ); + } + if (result.dryRunCutover && !result.rollbackNginxSmokeHost) { + result.rollbackNginxSmokeHost = result.directHost || result.liveHost; + } + if (result.rollbackNginxSmokeHost) { + validateHostOption( + result.rollbackNginxSmokeHost, + '--rollback-nginx-smoke-host', + ); + } + if ( + result.dryRunCutover && + !result.rollbackHealthPatrolPublicBaseUrl + ) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --rollback-health-patrol-public-base-url 或 GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL,避免回退 runbook 覆盖现场原有 Nginx 巡检入口。', + ); + } + if (result.rollbackHealthPatrolPublicBaseUrl) { + validateHttpUrl( + result.rollbackHealthPatrolPublicBaseUrl, + '--rollback-health-patrol-public-base-url', + ); + } + if (result.rollbackHealthPatrolPublicHost) { + validateHostOption( + result.rollbackHealthPatrolPublicHost, + '--rollback-health-patrol-public-host', + ); + } + if ( + result.rollbackPingoraShadowProbeUrl && + !result.rollbackPingoraShadowProbeToken + ) { + throw new Error( + '--rollback-pingora-shadow-probe-url 必须同时提供 --rollback-pingora-shadow-probe-token。', + ); + } + if ( + result.rollbackPingoraShadowProbeToken && + !result.rollbackPingoraShadowProbeUrl + ) { + throw new Error( + '--rollback-pingora-shadow-probe-token 必须同时提供 --rollback-pingora-shadow-probe-url。', + ); + } + if (result.rollbackPingoraShadowProbeUrl) { + validateHttpUrl( + result.rollbackPingoraShadowProbeUrl, + '--rollback-pingora-shadow-probe-url', + ); + } + if (result.dryRunCutover) { + assertSameCutoverHostname( + result.directRedirectHost, + result.directHost, + '--direct-redirect-host', + '--direct-host', + ); + assertSameCutoverHostname( + result.rollbackNginxSmokeHost, + result.directHost, + '--rollback-nginx-smoke-host', + '--direct-host', + ); + if (result.liveHost) { + assertSameCutoverHostname( + result.liveHost, + result.directHost, + '--live-host', + '--direct-host', + ); + } + } + if (result.dryRunCutover && !result.cutoverRunId) { + result.cutoverRunId = generateCutoverRunId(); + } + if (result.cutoverRunId) { + validateSafeName(result.cutoverRunId, '--cutover-run-id'); + } + if ( + result.releaseRuntimeOnly && + (result.requireDocker || result.pullDocker || result.requireNginx) + ) { + throw new Error( + '--release-runtime-only 只执行 current release 包内运行时复核,不能同时使用 --require-docker / --pull-docker / --require-nginx;源码全量门禁请在构建环境运行默认模式。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/check-pingora-release-readiness.mjs [options] + +Options: + --require-docker Docker Nginx handoff smoke 必须执行通过,不允许跳过。 + --pull-docker Docker 镜像缺失时允许拉取;通常和 --require-docker 一起用于 CI。 + --require-nginx 本机 Nginx snippet 校验必须执行 nginx -t。 + --require-live 目标 Nginx 已启用 canary 后,强制执行 live canary smoke。 + --require-realpath-live + 目标 Nginx 已启用真实路径 canary 后,强制执行 realpath live canary smoke。 + --require-direct 目标 Pingora 已配置直连 HTTPS / HTTP redirect 后,强制执行 direct preflight 和 direct live smoke。 + --live-base-url live canary base URL;也可用 GENARRATIVE_PINGORA_CANARY_BASE_URL。 + --live-host live canary Host header;也可用 GENARRATIVE_PINGORA_CANARY_HOST。--require-live 时必须显式提供。 + --live-nginx-access-log + live canary 后 Nginx access log 对账路径,默认 /var/log/nginx/genarrative.access.log。 + --live-pingora-access-log + live canary 后 Pingora access log 对账路径,默认 /var/log/genarrative/pingora-gateway.access.log。 + --live-access-log-since-lines + live canary 后只读取日志尾部行数,默认 2000。 + --realpath-live-base-url + 真实路径 canary base URL;也可用 GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL。 + --realpath-live-host + 真实路径 canary Host header;也可用 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST。--require-realpath-live 时必须显式提供。 + --realpath-live-nginx-access-log + 真实路径 canary 独立 Nginx access log,默认 /var/log/nginx/genarrative-pingora-realpath-canary.access.log。 + --realpath-live-pingora-access-log + 真实路径 canary 后 Pingora access log 对账路径,默认 /var/log/genarrative/pingora-gateway.access.log。 + --realpath-live-access-log-since-lines + 真实路径 canary 后只读取日志尾部行数,默认 2000。 + --direct-https-base-url direct HTTPS base URL;也可用 GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL。 + --direct-http-base-url direct HTTP redirect base URL;也可用 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL。 + --direct-host direct live Host header 和 TLS SNI;也可用 GENARRATIVE_PINGORA_DIRECT_HOST。--require-direct 时必须显式提供。 + --direct-redirect-host direct redirect Location host;也可用 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST。--require-direct 时必须显式提供,cutover runbook 中必须和 --direct-host 使用同一 hostname。 + --direct-redirect-base-url + 可选 direct redirect Location HTTPS base URL;用于高端口 rehearsal 访问 127.0.0.1:18443 但期望 Location 指向正式 HTTPS 入口。 + --direct-probe-token direct live shadow probe token;也可用 GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN。 + --direct-pingora-access-log + direct live 后 Pingora access log 落盘校验路径;--require-direct 时必须显式提供。 + --direct-access-log-since-lines + direct live 后只读取 Pingora access log 尾部行数,默认 2000。 + --direct-preflight-env-file + direct entry 预检使用的 pingora-gateway.env;也可用 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE。 + --direct-preflight-systemd direct entry 预检要求 systemctl cat 已包含低端口 capability drop-in。 + --direct-preflight-check-cert-readable + direct entry 预检要求当前用户可读 cert/key。 + --direct-preflight-check-service-env-file + direct entry 预检要求 service EnvironmentFile 包含本次 env 文件;--require-direct 时必须显式提供。 + --direct-preflight-check-service-user-cert-readable + direct entry 预检要求 systemd 服务用户可读 cert/key;--require-direct 时必须显式提供。 + --direct-preflight-check-service-binary-executable + direct entry 预检要求 service ExecStart 指向的 pingora-gateway 已存在且可执行;--require-direct 时必须显式提供。 + --direct-preflight-check-ports-free + direct entry 预检要求 TLS/HTTP redirect 端口当前可绑定;--dry-run-cutover / direct enable apply 前必须显式提供,启用后 --require-direct 复核不要求端口空闲。 + --direct-spacetime-database + direct live WSS subscribe 使用的数据库名;也可用 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE。--require-direct 时必须显式提供。 + --direct-health-patrol-env-file + 目标机 health-patrol env 文件;--require-direct 时必须显式提供,并复核 gateway mode / public base URL / Host。 + --direct-require-wss-upgrade + 要求 direct live WSS subscribe 必须返回 101;--require-direct 时会自动启用。 + --direct-skip-wss 跳过 direct live WSS subscribe 握手;只允许单独 direct live 临时排障,--require-direct 会拒绝。 + --direct-insecure-tls direct live 允许自签证书;仅本机 smoke 使用。 + --cutover-release-root + 生成切换 runbook 时使用的 current release 根目录,默认 /opt/genarrative/current。 + --cutover-evidence-output-root + 生成切换 runbook 时传给证据包脚本的输出根目录,默认 /var/log/genarrative/pingora-cutover-evidence。 + --cutover-evidence-timeline-max-span-ms + 生成切换 runbook 时传给证据根目录总审计的标准五段时间线最大跨度,默认 ${DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS}ms。 + --cutover-run-id 生成切换 runbook 时写入所有证据 manifest 的本次切换批次 ID;不传则自动生成 ${DEFAULT_CUTOVER_RUN_ID_PREFIX}。 + --rollback-nginx-smoke-url + 生成切换 runbook 时传给回退脚本的 Nginx smoke URL;必须显式填写切换前真实 Nginx 入口。 + --rollback-nginx-smoke-host + 生成切换 runbook 时传给回退脚本的 Host header;默认复用 --direct-host,显式传入时必须和 --direct-host 使用同一 hostname。 + --rollback-nginx-smoke-expect-body + 生成切换 runbook 时传给回退脚本的 Nginx smoke 响应体预期片段;必须显式来自切换前真实响应,例如首页 。 + --rollback-health-patrol-public-base-url + 生成切换 runbook 时回退后恢复的 Nginx public base URL;应填切换前 health-patrol env 的原值。 + --rollback-health-patrol-public-host + 生成切换 runbook 时回退后恢复的 public Host;不传则要求清空 Host 覆盖。 + --rollback-pingora-shadow-probe-url + 生成切换 runbook 时传给回退脚本的可选 Pingora shadow 探针 URL;必须和 token 成对出现。 + --rollback-pingora-shadow-probe-token + 生成切换 runbook 时传给回退脚本的可选 Pingora shadow 探针 token;JSON 输出会隐藏 token 值。 + --dry-run-plan 只打印将执行的检查计划,不启动检查命令。 + --dry-run-cutover 只打印直连切换 runbook,不启动检查或修改系统。 + --release-runtime-only + 只执行 current release 包内运行时复核:current release 自审、live canary、access log 对账、direct preflight、health patrol env 和 direct live;不运行 npm/cargo 源码门禁。 + --keep-going 单项失败后继续执行后续检查。 + +默认模式用于本机提交前检查;正式切换窗口建议: + node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> + +Pingora 直连入口切换窗口追加: + node scripts/check-pingora-release-readiness.mjs --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable + +只生成直连切换 runbook: + node scripts/check-pingora-release-readiness.mjs --dry-run-cutover --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --cutover-evidence-output-root /var/log/genarrative/pingora-cutover-evidence --rollback-nginx-smoke-url https://<域名>/ --rollback-nginx-smoke-expect-body '' --rollback-health-patrol-public-base-url http://127.0.0.1 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name, fallback = false) { + const raw = process.env[name]; + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + const normalized = String(raw).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parsePositiveIntEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parsePositiveInt(raw, name); +} + +function parsePositiveInt(value, label) { + const text = String(value || '').trim(); + if (!/^[1-9]\d*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label} 超出 JavaScript 安全整数范围。`); + } + return parsed; +} + +function validateHostOption(value, flag) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${flag} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${flag} 不是合法的 host 或 host:port`); + } +} + +function validateHttpUrl(value, flag) { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${flag} 必须是 http(s) URL: ${value}`); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`${flag} 必须是 http(s) URL: ${value}`); + } +} + +function isPositiveIntegerString(value) { + return /^[1-9]\d*$/u.test(String(value)); +} + +function validateSafeAbsoluteFilePath(value, flag) { + if (!path.isAbsolute(value)) { + throw new Error(`${flag} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${flag} 不能是文件系统根目录。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function hostnameForIdentity(value, flag) { + validateHostOption(value, flag); + const parsed = new URL(`https://${String(value).trim()}`); + return parsed.hostname.replace(/\.$/u, '').toLowerCase(); +} + +function assertSameCutoverHostname(actual, expected, actualFlag, expectedFlag) { + const actualHostname = hostnameForIdentity(actual, actualFlag); + const expectedHostname = hostnameForIdentity(expected, expectedFlag); + if (actualHostname !== expectedHostname) { + throw new Error( + `${actualFlag} 必须与 ${expectedFlag} 使用同一正式 hostname;当前 ${actualHostname} != ${expectedHostname}`, + ); + } +} + +function validateSafeName(value, flag) { + if (!/^[0-9A-Za-z._-]+$/u.test(String(value || ''))) { + throw new Error( + `${flag} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function generateCutoverRunId(now = new Date()) { + const timestamp = now + .toISOString() + .replace(/[-:.]/gu, '') + .replace(/\d{3}Z$/u, 'Z'); + return `${DEFAULT_CUTOVER_RUN_ID_PREFIX}${timestamp}`; +} + +function printDryRunPlan(steps) { + console.log( + JSON.stringify( + steps.map((step) => { + const output = { + name: step.name, + command: step.command, + args: redactSecretArgs(step.args), + }; + if (step.phase) { + output.phase = step.phase; + } + if (step.when) { + output.when = step.when; + } + return output; + }), + null, + 2, + ), + ); +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : arg, + ); +} + +function buildCutoverPlan(config) { + const releaseRoot = config.cutoverReleaseRoot.replace(/\/+$/u, ''); + const enableScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-direct-enable.sh', + ); + const rollbackScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-direct-rollback.sh', + ); + const gatewayEnvShadowSwitchScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + ); + const healthPatrolEnvSwitchScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ); + const statusSnapshotScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ); + const evidenceBundleScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ); + const commandEvidenceScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ); + const evidenceVerifyScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ); + const evidenceAuditScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ); + const currentReleaseAuditScript = path.join( + releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + const releaseReadinessScript = path.join( + releaseRoot, + 'scripts/check-pingora-release-readiness.mjs', + ); + const releasePreflightScript = path.join( + releaseRoot, + 'scripts/check-pingora-direct-preflight.mjs', + ); + const releaseLiveScript = path.join( + releaseRoot, + 'scripts/check-pingora-direct-live.mjs', + ); + const preEnablePreflightArgs = [ + releasePreflightScript, + '--env-file', + config.directPreflightEnvFile, + '--require-live-env', + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + '--check-cert-readable', + '--check-service-env-file', + '--check-service-user-cert-readable', + '--check-service-binary-executable', + '--check-ports-free', + ]; + const enableBaseArgs = [ + '--current-release-audit-script', + currentReleaseAuditScript, + '--current-release-root', + releaseRoot, + '--preflight-script', + releasePreflightScript, + '--preflight-env-file', + config.directPreflightEnvFile, + '--preflight-check-cert-readable', + '--preflight-check-service-env-file', + '--preflight-check-service-user-cert-readable', + '--preflight-check-service-binary-executable', + '--preflight-check-ports-free', + '--direct-live-script', + releaseLiveScript, + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--no-status', + ]; + if (config.directProbeToken) { + enableBaseArgs.push('--direct-probe-token', config.directProbeToken); + } + const rollbackBaseArgs = [ + '--reload-nginx', + '--nginx-smoke-url', + config.rollbackNginxSmokeUrl, + '--nginx-smoke-expect-body', + config.rollbackNginxSmokeExpectBody, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--health-patrol-expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ]; + if (config.rollbackNginxSmokeHost) { + rollbackBaseArgs.push('--nginx-smoke-host', config.rollbackNginxSmokeHost); + } + if (config.rollbackHealthPatrolPublicHost) { + rollbackBaseArgs.push( + '--health-patrol-expected-public-host', + config.rollbackHealthPatrolPublicHost, + ); + } else { + rollbackBaseArgs.push('--health-patrol-require-empty-public-host'); + } + if (config.rollbackPingoraShadowProbeUrl) { + rollbackBaseArgs.push( + '--pingora-shadow-probe-url', + config.rollbackPingoraShadowProbeUrl, + '--pingora-shadow-probe-token', + config.rollbackPingoraShadowProbeToken, + ); + } + const healthPatrolDirectSwitchArgs = [ + '--apply', + '--env-file', + config.directHealthPatrolEnvFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + config.directHttpsBaseUrl, + '--public-host', + config.directHost, + ]; + const pingoraGatewayShadowSwitchArgs = [ + '--apply', + '--env-file', + config.directPreflightEnvFile, + ]; + const healthPatrolNginxSwitchArgs = [ + '--apply', + '--env-file', + config.directHealthPatrolEnvFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--public-host', config.rollbackHealthPatrolPublicHost] + : ['--clear-public-host']), + ]; + + return [ + { + phase: 'preflight', + name: '切换 Host 一致性确认', + command: 'operator-action', + args: [ + 'confirm', + `cutover-run-id=${config.cutoverRunId}`, + `direct-host=${config.directHost}`, + `direct-redirect-host=${config.directRedirectHost}`, + `direct-redirect-base-url=${config.directRedirectBaseUrl || '-'}`, + `health-patrol-public-host=${config.directHost}`, + `rollback-nginx-smoke-host=${config.rollbackNginxSmokeHost}`, + `rollback-nginx-smoke-expect-body=${config.rollbackNginxSmokeExpectBody}`, + `rollback-health-patrol-public-base-url=${config.rollbackHealthPatrolPublicBaseUrl}`, + `rollback-health-patrol-public-host=${config.rollbackHealthPatrolPublicHost}`, + ], + when: 'runbook 生成时已拒绝 hostname 漂移;切换窗口继续人工确认直连、回退 smoke 和回退巡检入口都指向预期目标。', + }, + { + phase: 'preflight', + name: '切换前 current release 自包含自审', + command: 'node', + args: [ + '--', + currentReleaseAuditScript, + '--release-root', + releaseRoot, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--systemd-show', + ], + when: '正式启用前执行,只读确认 current release 已携带 Pingora 切换依赖;直连切换时要求 pingora-gateway 存在且 systemd ExecStart 指向 current release。', + }, + { + phase: 'preflight', + name: '切换前状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'pre-cutover', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'nginx', + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--expected-public-host', config.rollbackHealthPatrolPublicHost] + : ['--require-empty-public-host']), + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--fail-on-critical', + ], + when: '正式启用前执行,只读采集 current release、health patrol env 和 systemd 最终配置,并把 JSON / stdout / stderr / manifest 归档为 Nginx 接流基线证据。', + }, + { + phase: 'preflight', + name: '切换前证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '切换前状态快照证据包生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认 manifest.files 中的 sizeBytes / sha256 与实际归档文件一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'preflight', + name: '切换前 current release 直连 env / 证书 / 端口预检', + command: 'node', + args: ['--', ...preEnablePreflightArgs], + }, + { + phase: 'preflight', + name: '切换前 release readiness 基础门禁', + command: 'node', + args: [ + '--', + releaseReadinessScript, + ...readinessArgs(config, { includeDirect: false }), + ], + }, + { + phase: 'enable-dry-run', + name: 'Pingora direct enable dry-run', + command: enableScript, + args: enableBaseArgs, + }, + { + phase: 'enable-apply', + name: 'Pingora direct enable apply', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'enable-apply', + '--command-name', + 'pingora-direct-enable-apply', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + enableScript, + '--require-arg', + '--apply', + '--', + enableScript, + '--apply', + ...enableBaseArgs, + ], + when: '执行真实 direct enable apply,并把 stdout / stderr / 退出码和脱敏命令记录归档到切换证据目录。', + }, + { + phase: 'enable-apply', + name: '启用命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'enable apply 命令证据生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认命令 stdout / stderr / command-record 与 manifest 元数据一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-enable', + name: '切换后 health patrol 切到 Pingora direct', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'post-enable', + '--command-name', + 'pingora-health-patrol-direct-env-switch', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + healthPatrolEnvSwitchScript, + '--require-arg', + '--apply', + '--require-arg', + 'pingora-direct', + '--', + healthPatrolEnvSwitchScript, + ...healthPatrolDirectSwitchArgs, + ], + when: 'direct enable apply 成功后执行,并用命令证据归档 health patrol env 从 nginx 切到 pingora-direct 的真实 stdout / stderr / 退出码,避免巡检继续要求 nginx.service active。', + }, + { + phase: 'post-enable', + name: '切换后 health patrol direct 命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'health patrol direct env switch 命令证据生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认 env 切换命令 stdout / stderr / command-record 与 manifest 元数据一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-enable', + name: '启用后 health patrol env 直连复核', + command: 'node', + args: [ + '--', + path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + }, + { + phase: 'post-enable', + name: '启用后状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'post-enable', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--direct-live-script', + releaseLiveScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-pingora-env-mode', + 'direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--run-direct-live', + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), + ...(config.directProbeToken + ? ['--direct-probe-token', config.directProbeToken] + : []), + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--fail-on-critical', + ], + when: 'health patrol env 已切到 pingora-direct 且 env 复核通过后执行,归档 direct 接流状态、direct live 和 Pingora access log request_id 证据包。', + }, + { + phase: 'post-enable', + name: '启用后证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '启用后状态快照证据包生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认 snapshot / direct live / stdout / stderr / 命令记录未在归档过程中漂移,且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-enable', + name: '启用后 release readiness 直连复核', + command: 'node', + args: [ + '--', + releaseReadinessScript, + ...readinessArgs(config, { + includeDirect: true, + includeDirectPortsFree: false, + }), + ], + when: 'Pingora 已接管 80/443 后执行;此阶段不再检查端口空闲,因为端口应由 Pingora 直连入口占用。', + }, + { + phase: 'rollback-dry-run', + name: 'Pingora direct rollback dry-run', + command: rollbackScript, + args: rollbackBaseArgs, + when: '启用失败、direct live smoke 失败或业务验证失败时先执行。', + }, + { + phase: 'rollback-prep', + name: '回退前 Pingora env 预置回 shadow', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'rollback-prep', + '--command-name', + 'pingora-gateway-shadow-env-switch', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + gatewayEnvShadowSwitchScript, + '--require-arg', + '--apply', + '--', + gatewayEnvShadowSwitchScript, + ...pingoraGatewayShadowSwitchArgs, + ], + when: 'rollback apply 前执行,并用命令证据归档 Pingora gateway env 从 80/443 direct 配置恢复为 shadow 高端口配置;否则回退脚本移除低端口 capability 后重启 Pingora 可能仍按 80/443 配置启动失败。', + }, + { + phase: 'rollback-prep', + name: '回退前 Pingora env shadow 命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'Pingora gateway shadow env switch 命令证据生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认 env 切换命令 stdout / stderr / command-record 与 manifest 元数据一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'rollback-prep', + name: '回退前 health patrol 预置回 Nginx', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'rollback-prep', + '--command-name', + 'pingora-health-patrol-nginx-env-switch', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + healthPatrolEnvSwitchScript, + '--require-arg', + '--apply', + '--require-arg', + 'nginx', + '--', + healthPatrolEnvSwitchScript, + ...healthPatrolNginxSwitchArgs, + ], + when: 'rollback apply 前执行,并用命令证据归档 health patrol env 从 pingora-direct 预置回 nginx,确保回退脚本内置 health patrol env 复核能校验切换前 public base URL / Host。', + }, + { + phase: 'rollback-prep', + name: '回退前 health patrol nginx 命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'health patrol nginx env switch 命令证据生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认 env 切换命令 stdout / stderr / command-record 与 manifest 元数据一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'rollback-apply', + name: 'Pingora direct rollback apply', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'rollback-apply', + '--command-name', + 'pingora-direct-rollback-apply', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + rollbackScript, + '--require-arg', + '--apply', + '--', + rollbackScript, + '--apply', + ...rollbackBaseArgs, + ], + when: 'health patrol env 预置回 Nginx 且 dry-run 确认后执行,用于回到 Nginx / shadow 入口,并把回退命令 stdout / stderr / 退出码归档。', + }, + { + phase: 'rollback-apply', + name: '回退命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'rollback apply 命令证据生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认回退命令 stdout / stderr / command-record 与 manifest 元数据一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-rollback', + name: '回退后 health patrol env Nginx 模式复核', + command: 'node', + args: [ + '--', + path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'nginx', + ...(config.rollbackHealthPatrolPublicBaseUrl + ? [ + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ] + : []), + ...(config.rollbackHealthPatrolPublicHost + ? [ + '--expected-public-host', + config.rollbackHealthPatrolPublicHost, + ] + : ['--require-empty-public-host']), + ], + when: 'health patrol env 已切回 nginx 后执行,阻断 pingora-direct 模式或回退目标漂移。', + }, + { + phase: 'post-rollback', + name: '回退后状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'post-rollback', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'nginx', + '--expected-pingora-env-mode', + 'shadow', + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--expected-public-host', config.rollbackHealthPatrolPublicHost] + : ['--require-empty-public-host']), + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--fail-on-critical', + ], + when: 'rollback apply 和回退后 env 复核通过后执行,归档 Nginx 接流状态证据包。', + }, + { + phase: 'post-rollback', + name: '回退后证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '回退后状态快照证据包生成后,将上一步 stdout 中的 bundleDir 替换占位符执行,确认回退证据 manifest 和文件内容一致,且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-rollback', + name: '切换证据根目录三阶段总审计', + command: 'node', + args: [ + '--', + evidenceAuditScript, + '--evidence-root', + config.cutoverEvidenceOutputRoot, + '--verify-script', + evidenceVerifyScript, + '--require-phase', + 'pre-cutover', + '--require-phase', + 'post-enable', + '--require-phase', + 'post-rollback', + '--require-phase-direct-live-access-log', + 'post-enable', + '--require-phase-direct-live-static-headers', + 'post-enable', + '--require-phase-pingora-env-shadow', + 'post-rollback', + '--require-command', + 'enable-apply:pingora-direct-enable-apply', + '--require-command', + 'post-enable:pingora-health-patrol-direct-env-switch', + '--require-command', + 'rollback-prep:pingora-gateway-shadow-env-switch', + '--require-command', + 'rollback-prep:pingora-health-patrol-nginx-env-switch', + '--require-command', + 'rollback-apply:pingora-direct-rollback-apply', + '--require-command-executable', + `enable-apply:pingora-direct-enable-apply:${enableScript}`, + '--require-command-executable', + `post-enable:pingora-health-patrol-direct-env-switch:${healthPatrolEnvSwitchScript}`, + '--require-command-executable', + `rollback-prep:pingora-gateway-shadow-env-switch:${gatewayEnvShadowSwitchScript}`, + '--require-command-executable', + `rollback-prep:pingora-health-patrol-nginx-env-switch:${healthPatrolEnvSwitchScript}`, + '--require-command-executable', + `rollback-apply:pingora-direct-rollback-apply:${rollbackScript}`, + '--require-command-arg', + 'enable-apply:pingora-direct-enable-apply:--apply', + '--require-command-arg', + 'post-enable:pingora-health-patrol-direct-env-switch:--apply', + '--require-command-arg', + 'post-enable:pingora-health-patrol-direct-env-switch:pingora-direct', + '--require-command-arg', + 'rollback-prep:pingora-gateway-shadow-env-switch:--apply', + '--require-command-arg', + 'rollback-prep:pingora-health-patrol-nginx-env-switch:--apply', + '--require-command-arg', + 'rollback-prep:pingora-health-patrol-nginx-env-switch:nginx', + '--require-command-arg', + 'rollback-apply:pingora-direct-rollback-apply:--apply', + '--require-cutover-run-id', + config.cutoverRunId, + '--timeline-max-span-ms', + String(config.cutoverEvidenceTimelineMaxSpanMs), + ], + when: '三阶段证据包、enable / rollback apply 命令证据和三条 env 变更命令证据都生成并分别验真后执行,自动找每个 phase 与命令的最新 bundleDir 并再次做只读 manifest 验真,且要求 post-enable manifest.summary.directLiveAccessLog 和 directLiveStaticHeaders 可直接复盘 request_id 对账、静态缓存、校验器、Range 和 304 证据,同时要求 post-rollback manifest.summary.pingoraEnvShadow 证明 Pingora env 已恢复 shadow 高端口。', + }, + ]; +} + +function readinessArgs(config, { includeDirect, includeDirectPortsFree = true }) { + return [ + '--release-runtime-only', + ...(config.requireLive + ? [ + '--require-live', + '--live-base-url', + config.liveBaseUrl, + '--live-host', + config.liveHost, + ] + : []), + ...(config.requireRealpathLive + ? [ + '--require-realpath-live', + '--realpath-live-base-url', + config.realpathLiveBaseUrl, + '--realpath-live-host', + config.realpathLiveHost, + '--realpath-live-nginx-access-log', + config.realpathLiveAccessLogNginx, + '--realpath-live-pingora-access-log', + config.realpathLiveAccessLogPingora, + '--realpath-live-access-log-since-lines', + String(config.realpathLiveAccessLogSinceLines), + ] + : []), + ...(includeDirect + ? [ + '--require-direct', + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), + ...(config.directProbeToken + ? ['--direct-probe-token', config.directProbeToken] + : []), + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--direct-preflight-env-file', + config.directPreflightEnvFile, + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + ...(includeDirectPortsFree + ? ['--direct-preflight-check-ports-free'] + : []), + ] + : []), + ]; +} + +function buildSteps(config) { + if (config.releaseRuntimeOnly) { + return buildReleaseRuntimeSteps(config); + } + + const steps = [ + { + name: 'Rust 路由与保护单测', + command: 'cargo', + args: [ + 'test', + '-p', + 'pingora-gateway', + '--manifest-path', + 'server-rs/Cargo.toml', + ], + }, + { + name: 'Pingora mock 上游 smoke', + command: 'npm', + args: ['run', 'check:pingora-gateway-smoke'], + }, + { + name: 'Nginx/Pingora 路由矩阵 parity', + command: 'npm', + args: ['run', 'check:pingora-route-parity'], + }, + { + name: 'Nginx canary snippet 校验', + command: 'node', + args: [ + 'scripts/check-nginx-pingora-canary.mjs', + ...(config.requireNginx ? ['--require-nginx'] : []), + ], + }, + { + name: 'Docker Nginx handoff smoke', + command: 'node', + args: [ + 'scripts/check-pingora-canary-docker.mjs', + ...(config.requireDocker ? ['--require-docker'] : []), + ...(config.pullDocker ? ['--pull'] : []), + ], + }, + { + name: 'Pingora canary access log 对账烟测', + command: 'npm', + args: ['run', 'check:pingora-canary-access-log-parity'], + }, + { + name: 'Pingora realpath canary 启停烟测', + command: 'npm', + args: ['run', 'check:pingora-realpath-canary-toggle'], + }, + { + name: 'Pingora canary live 参数护栏', + command: 'node', + args: ['scripts/check-pingora-canary-live-guard.mjs'], + }, + { + name: 'Pingora direct entry 静态预检', + command: 'npm', + args: ['run', 'check:pingora-direct-preflight'], + }, + { + name: 'Pingora direct live 参数护栏', + command: 'node', + args: ['scripts/check-pingora-direct-live-guard.mjs'], + }, + { + name: 'Pingora direct rollback dry-run', + command: 'npm', + args: ['run', 'check:pingora-direct-rollback'], + }, + { + name: 'Pingora direct enable dry-run', + command: 'npm', + args: ['run', 'check:pingora-direct-enable'], + }, + { + name: 'Pingora release readiness plan', + command: 'npm', + args: ['run', 'check:pingora-release-readiness-plan'], + }, + { + name: '生产运维护栏', + command: 'npm', + args: ['run', 'check:production-ops'], + }, + { + name: '生产健康巡检烟测', + command: 'npm', + args: ['run', 'check:production-health-patrol'], + }, + { + name: '生产健康巡检 env 复核烟测', + command: 'npm', + args: ['run', 'check:production-health-patrol-env'], + }, + { + name: 'Pingora health patrol env 切换烟测', + command: 'npm', + args: ['run', 'check:pingora-health-patrol-env-switch'], + }, + { + name: 'Pingora gateway env shadow 切换烟测', + command: 'npm', + args: ['run', 'check:pingora-gateway-env-shadow-switch'], + }, + { + name: 'Pingora current release 自审烟测', + command: 'npm', + args: ['run', 'check:pingora-current-release-audit'], + }, + { + name: 'Pingora direct rehearsal 状态烟测', + command: 'npm', + args: ['run', 'check:pingora-direct-rehearsal-status'], + }, + { + name: 'Pingora cutover 状态快照烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-status-snapshot'], + }, + { + name: 'Pingora cutover 证据包烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-bundle'], + }, + { + name: 'Pingora cutover 命令证据烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-command-evidence'], + }, + { + name: 'Pingora cutover 证据 manifest 验真烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-verify'], + }, + { + name: 'Pingora cutover 证据根目录审计烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-audit'], + }, + { + name: 'API release build 烟测', + command: 'npm', + args: ['run', 'check:production-api-release'], + }, + { + name: 'Pingora production release 真实构建烟测', + command: 'npm', + args: ['run', 'check:pingora-production-release-build'], + }, + { + name: 'API deploy release 烟测', + command: 'npm', + args: ['run', 'check:production-api-deploy'], + }, + ]; + + if (config.requireLive || config.liveBaseUrl) { + appendTargetLiveSteps(steps, config, scriptPath); + } + appendTargetRealpathLiveSteps(steps, config, scriptPath); + + if (config.requireDirect || config.directHttpsBaseUrl) { + steps.push({ + name: '目标 Pingora direct entry preflight', + command: 'node', + args: [ + '--', + 'scripts/check-pingora-direct-preflight.mjs', + ...(config.directPreflightEnvFile + ? ['--env-file', config.directPreflightEnvFile, '--require-live-env'] + : []), + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + ...(config.directPreflightCheckCertReadable + ? ['--check-cert-readable'] + : []), + ...(config.directPreflightCheckServiceEnvFile + ? ['--check-service-env-file'] + : []), + ...(config.directPreflightCheckServiceUserCertReadable + ? ['--check-service-user-cert-readable'] + : []), + ...(config.directPreflightCheckServiceBinaryExecutable + ? ['--check-service-binary-executable'] + : []), + ...(config.directPreflightCheckPortsFree ? ['--check-ports-free'] : []), + ], + }); + if (config.requireDirect || config.directHealthPatrolEnvFile) { + steps.push({ + name: '目标 health patrol env 直连模式复核', + command: 'node', + args: [ + '--', + 'scripts/check-production-health-patrol-env.mjs', + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + }); + } + steps.push({ + name: '目标 Pingora direct live smoke', + command: 'node', + args: [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + config.directHttpsBaseUrl, + ...(config.directHttpBaseUrl + ? ['--http-base-url', config.directHttpBaseUrl] + : []), + ...(config.directHost ? ['--host', config.directHost] : []), + ...(config.directRedirectHost + ? ['--redirect-host', config.directRedirectHost] + : []), + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + ...(config.directPingoraAccessLog + ? [ + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + ] + : []), + ...(config.directSpacetimeDatabase + ? ['--spacetime-database', config.directSpacetimeDatabase] + : []), + ...(config.directRequireWssUpgrade ? ['--require-wss-upgrade'] : []), + ...(config.directSkipWss ? ['--skip-wss'] : []), + ...(config.directInsecureTls ? ['--insecure-tls'] : []), + ], + }); + } + + return steps; +} + +function buildReleaseRuntimeSteps(config) { + const steps = [ + { + name: 'current release 自包含自审', + command: 'node', + args: [ + '--', + releaseScriptPath('scripts/ops/pingora-current-release-audit.mjs'), + '--release-root', + releaseRoot, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + ...(config.requireDirect ? ['--systemd-show'] : []), + ], + cwd: releaseRoot, + }, + ]; + + if (!config.requireDirect) { + steps.push({ + name: '目标 Pingora direct rehearsal 状态复核', + command: 'node', + args: [ + '--', + releaseScriptPath('scripts/ops/pingora-direct-rehearsal-status.mjs'), + '--release-root', + releaseRoot, + '--expect-public-gateway', + 'nginx', + '--require-pingora-shadow', + '--require-realpath-canary', + '--require-current-release-gateway', + '--fail-on-critical', + ], + cwd: releaseRoot, + }); + } + + appendTargetLiveSteps(steps, config, releaseScriptPath); + appendTargetRealpathLiveSteps(steps, config, releaseScriptPath); + appendTargetDirectSteps(steps, config, releaseScriptPath); + + return steps; +} + +function appendTargetLiveSteps(steps, config, scriptPath) { + if (!config.requireLive && !config.liveBaseUrl) { + return; + } + steps.push({ + name: '目标 Nginx live canary smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-live.mjs'), + '--base-url', + config.liveBaseUrl, + ...(config.liveHost ? ['--host', config.liveHost] : []), + ], + cwd: releaseRoot, + }); + steps.push({ + name: '目标 Nginx live canary access log 对账', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-access-log-parity.mjs'), + '--nginx-log-file', + config.liveAccessLogNginx, + '--pingora-log-file', + config.liveAccessLogPingora, + '--since-lines', + String(config.liveAccessLogSinceLines), + '--path', + '/__genarrative_pingora_canary/healthz', + '--path', + '/__genarrative_pingora_canary/api/creation-entry/config', + ], + cwd: releaseRoot, + }); +} + +function appendTargetRealpathLiveSteps(steps, config, scriptPath) { + if (!config.requireRealpathLive && !config.realpathLiveBaseUrl) { + return; + } + steps.push({ + name: '目标 Nginx realpath canary smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-live.mjs'), + '--realpath', + '--base-url', + config.realpathLiveBaseUrl, + ...(config.realpathLiveHost ? ['--host', config.realpathLiveHost] : []), + ], + cwd: releaseRoot, + }); + steps.push({ + name: '目标 Nginx realpath canary access log 对账', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-access-log-parity.mjs'), + '--realpath', + '--nginx-log-file', + config.realpathLiveAccessLogNginx, + '--pingora-log-file', + config.realpathLiveAccessLogPingora, + '--since-lines', + String(config.realpathLiveAccessLogSinceLines), + '--path', + '/__genarrative_pingora_realpath_canary/healthz', + '--path', + '/api/creation-entry/config', + '--path', + '/v1/identity', + '--path', + '/assets/app.js', + ], + cwd: releaseRoot, + }); +} + +function appendTargetDirectSteps(steps, config, scriptPath) { + if (!config.requireDirect && !config.directHttpsBaseUrl) { + return; + } + steps.push({ + name: '目标 Pingora direct entry preflight', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-direct-preflight.mjs'), + ...(config.directPreflightEnvFile + ? ['--env-file', config.directPreflightEnvFile, '--require-live-env'] + : []), + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + ...(config.directPreflightCheckCertReadable + ? ['--check-cert-readable'] + : []), + ...(config.directPreflightCheckServiceEnvFile + ? ['--check-service-env-file'] + : []), + ...(config.directPreflightCheckServiceUserCertReadable + ? ['--check-service-user-cert-readable'] + : []), + ...(config.directPreflightCheckServiceBinaryExecutable + ? ['--check-service-binary-executable'] + : []), + ...(config.directPreflightCheckPortsFree ? ['--check-ports-free'] : []), + ], + cwd: releaseRoot, + }); + if (config.requireDirect || config.directHealthPatrolEnvFile) { + steps.push({ + name: '目标 health patrol env 直连模式复核', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + cwd: releaseRoot, + }); + } + steps.push({ + name: '目标 Pingora direct live smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-direct-live.mjs'), + '--https-base-url', + config.directHttpsBaseUrl, + ...(config.directHttpBaseUrl + ? ['--http-base-url', config.directHttpBaseUrl] + : []), + ...(config.directHost ? ['--host', config.directHost] : []), + ...(config.directRedirectHost + ? ['--redirect-host', config.directRedirectHost] + : []), + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + ...(config.directPingoraAccessLog + ? [ + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + ] + : []), + ...(config.directSpacetimeDatabase + ? ['--spacetime-database', config.directSpacetimeDatabase] + : []), + ...(config.directRequireWssUpgrade ? ['--require-wss-upgrade'] : []), + ...(config.directSkipWss ? ['--skip-wss'] : []), + ...(config.directInsecureTls ? ['--insecure-tls'] : []), + ], + cwd: releaseRoot, + }); +} + +function releaseScriptPath(relativePath) { + return path.join(releaseRoot, relativePath); +} + +function scriptPath(relativePath) { + return relativePath; +} + +function runStep(step) { + const args = commandArgsForSpawn(step); + console.log(`\n[pingora-release-readiness] ${step.name}`); + console.log( + `[pingora-release-readiness] ${step.command} ${redactSecretArgs(args).join(' ')}`, + ); + + return new Promise((resolve) => { + const child = spawn(step.command, args, { + cwd: step.cwd || repoRoot, + env: smokeEnv(), + shell: false, + stdio: 'inherit', + }); + + child.on('error', (error) => { + failures.push(`${step.name}: ${step.command} 启动失败:${error.message}`); + resolve(false); + }); + child.on('exit', (status, signal) => { + if (signal) { + failures.push(`${step.name}: 被信号终止:${signal}`); + resolve(false); + return; + } + if ((status ?? 0) !== 0) { + failures.push(`${step.name}: 退出码 ${status}`); + resolve(false); + return; + } + resolve(true); + }); + }); +} + +function commandArgsForSpawn(step) { + if (step.command === 'node' && step.args[0] !== '--') { + return ['--', ...step.args]; + } + return step.args; +} + +function smokeEnv() { + return { + ...process.env, + PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`, + }; +} diff --git a/scripts/check-pingora-route-parity.mjs b/scripts/check-pingora-route-parity.mjs new file mode 100644 index 000000000..78372e43b --- /dev/null +++ b/scripts/check-pingora-route-parity.mjs @@ -0,0 +1,242 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs'; + +const MATRIX_PATH = 'deploy/pingora/nginx-route-parity.matrix.json'; +const PRODUCTION_NGINX_PATH = 'deploy/nginx/genarrative.conf'; +const DEVELOPMENT_NGINX_PATH = 'deploy/nginx/genarrative-dev-http.conf'; +const PINGORA_DOC_PATH = + 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md'; +const PINGORA_GATEWAY_SOURCE = 'server-rs/crates/pingora-gateway/src/main.rs'; + +const VALID_KINDS = new Set([ + 'proxy', + 'static', + 'redirect_permanent', + 'shadow_probe', + 'not_found', +]); +const VALID_PROXY_TARGETS = new Set(['api', 'spacetime']); +const VALID_STATIC_ROOTS = new Set(['web', 'acme']); +const VALID_STATIC_MODES = new Set(['exact', 'spa_fallback']); +const VALID_PROTECTION_CLASSES = new Set([ + 'admin_api', + 'gallery_list', + 'gallery_detail', + 'api', + 'spacetime', +]); +const REQUIRED_ROUTE_IDS = [ + 'acme_challenge', + 'shadow_probe', + 'admin_redirect', + 'admin_api_proxy', + 'admin_assets', + 'admin_spa_fallback', + 'web_assets', + 'puzzle_gallery_list', + 'custom_world_gallery_list', + 'puzzle_gallery_detail', + 'custom_world_gallery_detail', + 'generic_api_proxy', + 'spacetime_subscribe', + 'spacetime_identity', + 'v1_forbidden', + 'healthz_forbidden', + 'readyz_forbidden', + 'generated_assets_forbidden', + 'web_spa_fallback', +]; + +const files = { + production: readFileSync(PRODUCTION_NGINX_PATH, 'utf8'), + development: readFileSync(DEVELOPMENT_NGINX_PATH, 'utf8'), +}; +const docs = readFileSync(PINGORA_DOC_PATH, 'utf8'); +const pingoraGatewaySource = readFileSync(PINGORA_GATEWAY_SOURCE, 'utf8'); +const matrix = JSON.parse(readFileSync(MATRIX_PATH, 'utf8')); +const failures = []; + +function fail(message) { + failures.push(message); +} + +function hasOwn(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +function requireString(value, context) { + if (typeof value !== 'string' || value.trim() === '') { + fail(`${context} 必须是非空字符串。`); + return false; + } + return true; +} + +function validateExpectation(route) { + const context = `${MATRIX_PATH} route ${route.id}`; + const expect = route.expect; + if (!expect || typeof expect !== 'object' || Array.isArray(expect)) { + fail(`${context} 缺少 expect 对象。`); + return; + } + + if (!VALID_KINDS.has(expect.kind)) { + fail(`${context} expect.kind 不支持: ${expect.kind}`); + return; + } + + if (expect.kind === 'proxy') { + if (!VALID_PROXY_TARGETS.has(expect.target)) { + fail(`${context} proxy target 不支持: ${expect.target}`); + } + if ( + hasOwn(expect, 'bodyLimit') && + expect.bodyLimit !== null && + expect.bodyLimit !== 'default' && + (!Number.isInteger(expect.bodyLimit) || expect.bodyLimit < 1) + ) { + fail(`${context} bodyLimit 必须是 null、default 或正整数。`); + } + if (!VALID_PROTECTION_CLASSES.has(expect.protectionClass)) { + fail( + `${context} proxy protectionClass 不支持: ${expect.protectionClass}`, + ); + } + return; + } + + if (hasOwn(expect, 'protectionClass')) { + fail(`${context} 非 proxy 路由不能配置 protectionClass。`); + } + + if (expect.kind === 'static') { + if (!VALID_STATIC_ROOTS.has(expect.root)) { + fail(`${context} static root 不支持: ${expect.root}`); + } + if (!VALID_STATIC_MODES.has(expect.mode)) { + fail(`${context} static mode 不支持: ${expect.mode}`); + } + } + + if ( + expect.kind === 'redirect_permanent' && + !requireString(expect.location, `${context} redirect location`) + ) { + fail(`${context} redirect_permanent 必须配置 location。`); + } +} + +function validateNginxFragments(route) { + for (const environment of ['production', 'development']) { + const fragments = route.nginx?.[environment]; + if (fragments === undefined) { + if (environment === 'production' && route.id !== 'shadow_probe') { + fail(`${MATRIX_PATH} route ${route.id} 缺少 production Nginx 片段。`); + } + continue; + } + if (!Array.isArray(fragments) || fragments.length === 0) { + fail( + `${MATRIX_PATH} route ${route.id} 的 ${environment} Nginx 片段不能为空。`, + ); + continue; + } + + for (const fragment of fragments) { + if (!requireString(fragment, `${route.id} ${environment} Nginx 片段`)) { + continue; + } + if (!files[environment].includes(fragment)) { + fail( + `${environment} Nginx 模板缺少 route ${route.id} 片段: ${fragment}`, + ); + } + } + } +} + +function validateDocFragments(route) { + if (!Array.isArray(route.docs) || route.docs.length === 0) { + fail(`${MATRIX_PATH} route ${route.id} 缺少 docs 片段。`); + return; + } + + for (const fragment of route.docs) { + if (!requireString(fragment, `${route.id} docs 片段`)) { + continue; + } + if (!docs.includes(fragment)) { + fail(`Pingora 试点文档缺少 route ${route.id} 片段: ${fragment}`); + } + } +} + +function validateMatrixShape() { + if (matrix.version !== 1) { + fail(`${MATRIX_PATH} version 必须为 1。`); + } + if (!Array.isArray(matrix.routes) || matrix.routes.length === 0) { + fail(`${MATRIX_PATH} routes 不能为空。`); + return; + } + + const ids = new Set(); + const samplePaths = new Set(); + for (const route of matrix.routes) { + if (!requireString(route.id, `${MATRIX_PATH} route.id`)) { + continue; + } + if (ids.has(route.id)) { + fail(`${MATRIX_PATH} route id 重复: ${route.id}`); + } + ids.add(route.id); + + if (!requireString(route.samplePath, `${route.id} samplePath`)) { + continue; + } + if (!route.samplePath.startsWith('/')) { + fail(`${MATRIX_PATH} route ${route.id} samplePath 必须以 / 开头。`); + } + if (samplePaths.has(route.samplePath)) { + fail(`${MATRIX_PATH} samplePath 重复: ${route.samplePath}`); + } + samplePaths.add(route.samplePath); + + validateExpectation(route); + validateNginxFragments(route); + validateDocFragments(route); + } + + for (const routeId of REQUIRED_ROUTE_IDS) { + if (!ids.has(routeId)) { + fail(`${MATRIX_PATH} 缺少必需 route id: ${routeId}`); + } + } +} + +function validateRustTestUsesMatrix() { + for (const fragment of [ + 'include_str!("../../../../deploy/pingora/nginx-route-parity.matrix.json")', + 'serde_json::from_str(ROUTE_PARITY_MATRIX_JSON)', + 'protection_class_for_route(&route, &case.sample_path)', + 'fn matches_nginx_route_parity_matrix()', + ]) { + if (!pingoraGatewaySource.includes(fragment)) { + fail(`Pingora Rust 路由 parity 测试缺少矩阵接入片段: ${fragment}`); + } + } +} + +validateMatrixShape(); +validateRustTestUsesMatrix(); + +if (failures.length > 0) { + console.error('[check:pingora-route-parity] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log(`[check:pingora-route-parity] OK (${matrix.routes.length} routes)`); diff --git a/scripts/check-pingora-tls-cert-sync.mjs b/scripts/check-pingora-tls-cert-sync.mjs new file mode 100644 index 000000000..a3578fe2c --- /dev/null +++ b/scripts/check-pingora-tls-cert-sync.mjs @@ -0,0 +1,284 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const SCRIPT = 'scripts/deploy/pingora-tls-cert-sync.mjs'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-tls-cert-sync-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-tls-cert-sync] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-tls-cert-sync] OK'); + +function main() { + assertDryRunAcceptsCertbotLiveSymlinks(); + assertApplyCopiesToPrivateTarget(); + assertRejectsRelativeAndRootPaths(); + assertRejectsControlCharactersBeforeApply(); + assertRejectsSymlinkTargetDirectory(); + assertRejectsSymlinkTargetFiles(); + assertRejectsWorldReadableModes(); +} + +function assertDryRunAcceptsCertbotLiveSymlinks() { + const fixture = prepareFixture('dry-run-live-symlink'); + const result = runSync(fixture, []); + assertStatus(result, 0, 'dry-run 应接受 Certbot live symlink 源。'); + if (existsSync(fixture.targetCert) || existsSync(fixture.targetKey)) { + failures.push('dry-run 不应写入目标证书文件。'); + } + const payload = parseJson(result.stdout, 'dry-run JSON 输出'); + assertEqual(payload.mode, 'dry-run', 'dry-run JSON mode 应为 dry-run。'); + assertEqual( + payload.files?.[0]?.source?.sourceIsSymlink, + true, + 'dry-run JSON 应标记源证书来自 symlink。', + ); +} + +function assertApplyCopiesToPrivateTarget() { + const fixture = prepareFixture('apply-copy', { symlinkSources: false }); + const result = runSync(fixture, ['--apply'], { + serviceUser: `#${process.getuid?.() ?? 0}`, + serviceGroup: `#${process.getgid?.() ?? 0}`, + targetUser: `#${process.getuid?.() ?? 0}`, + }); + assertStatus(result, 0, 'apply 应复制证书到目标目录。'); + assertFileContent(fixture.targetCert, 'fixture fullchain\n'); + assertFileContent(fixture.targetKey, 'fixture private key\n'); + assertMode(fixture.targetDir, 0o750, '目标目录权限应为 0750。'); + assertMode(fixture.targetCert, 0o640, '目标证书权限应为 0640。'); + assertMode(fixture.targetKey, 0o640, '目标私钥权限应为 0640。'); +} + +function assertRejectsRelativeAndRootPaths() { + const fixture = prepareFixture('reject-paths'); + const relative = runRaw([ + '--source-cert-file', + 'relative/fullchain.pem', + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + ]); + assertNonZero(relative, '相对源证书路径必须失败。'); + assertIncludes( + `${relative.stdout}\n${relative.stderr}`, + '--source-cert-file 必须是绝对路径', + '相对路径必须给出明确错误。', + ); + + const root = runRaw([ + '--source-cert-file', + fixture.sourceCert, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + '/', + ]); + assertNonZero(root, 'target-dir 指向文件系统根目录必须失败。'); + assertIncludes( + `${root.stdout}\n${root.stderr}`, + '--target-dir 不能是文件系统根目录', + '根目录目标必须给出明确错误。', + ); +} + +function assertRejectsControlCharactersBeforeApply() { + const fixture = prepareFixture('reject-control'); + const result = runRaw([ + '--source-cert-file', + `${fixture.sourceCert}\n--apply`, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + ]); + assertNonZero(result, '带换行的源证书路径必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--source-cert-file 不能包含换行或 NUL 字符', + '控制字符路径必须给出明确错误。', + ); + if (existsSync(fixture.targetCert) || existsSync(fixture.targetKey)) { + failures.push('控制字符参数失败时不应写入目标证书。'); + } +} + +function assertRejectsSymlinkTargetDirectory() { + const fixture = prepareFixture('reject-target-dir-symlink'); + const realTarget = path.join(fixture.root, 'real-target'); + mkdirSync(realTarget, { recursive: true }); + rmSync(fixture.targetDir, { recursive: true, force: true }); + symlinkSync(realTarget, fixture.targetDir); + const result = runSync(fixture, []); + assertNonZero(result, 'target-dir 是 symlink 时必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--target-dir 已存在路径不能包含符号链接', + 'target-dir symlink 必须给出明确错误。', + ); +} + +function assertRejectsSymlinkTargetFiles() { + const fixture = prepareFixture('reject-target-file-symlink'); + mkdirSync(fixture.targetDir, { recursive: true }); + const outside = path.join(fixture.root, 'outside-fullchain.pem'); + writeFileSync(outside, 'outside\n', 'utf8'); + symlinkSync(outside, fixture.targetCert); + const result = runSync(fixture, []); + assertNonZero(result, '目标证书是 symlink 时必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '目标 fullchain.pem 不能是符号链接', + '目标文件 symlink 必须给出明确错误。', + ); +} + +function assertRejectsWorldReadableModes() { + const fixture = prepareFixture('reject-world-readable-mode'); + const result = runSync(fixture, ['--file-mode', '0644']); + assertNonZero(result, 'world-readable file mode 必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--file-mode 不能允许 other 读取证书或私钥', + 'world-readable file mode 必须给出明确错误。', + ); +} + +function prepareFixture(name, options = {}) { + const root = path.join(tmpRoot, name); + const archiveDir = path.join(root, 'archive'); + const liveDir = path.join(root, 'live'); + const targetDir = path.join(root, 'target'); + mkdirSync(archiveDir, { recursive: true }); + mkdirSync(liveDir, { recursive: true }); + writeFileSync(path.join(archiveDir, 'fullchain1.pem'), 'fixture fullchain\n', 'utf8'); + writeFileSync(path.join(archiveDir, 'privkey1.pem'), 'fixture private key\n', 'utf8'); + + const symlinkSources = options.symlinkSources !== false; + const sourceCert = symlinkSources + ? path.join(liveDir, 'fullchain.pem') + : path.join(archiveDir, 'fullchain1.pem'); + const sourceKey = symlinkSources + ? path.join(liveDir, 'privkey.pem') + : path.join(archiveDir, 'privkey1.pem'); + if (symlinkSources) { + symlinkSync('../archive/fullchain1.pem', sourceCert); + symlinkSync('../archive/privkey1.pem', sourceKey); + } + + return { + root, + sourceCert, + sourceKey, + targetDir, + targetCert: path.join(targetDir, 'fullchain.pem'), + targetKey: path.join(targetDir, 'privkey.pem'), + }; +} + +function runSync(fixture, args = [], options = {}) { + return runRaw([ + '--source-cert-file', + fixture.sourceCert, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + '--service-user', + options.serviceUser || `#${process.getuid?.() ?? 0}`, + '--service-group', + options.serviceGroup || `#${process.getgid?.() ?? 0}`, + '--target-user', + options.targetUser || `#${process.getuid?.() ?? 0}`, + '--json', + ...args, + ]); +} + +function runRaw(args) { + return spawnSync('node', [SCRIPT, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function parseJson(text, label) { + try { + return JSON.parse(text); + } catch (error) { + failures.push(`${label} 不是合法 JSON: ${error.message}\n${text}`); + return {}; + } +} + +function assertStatus(result, expected, reason) { + if ((result.status ?? 0) !== expected) { + failures.push( + `${reason} 实际退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertNonZero(result, reason) { + if ((result.status ?? 0) === 0) { + failures.push(`${reason}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } +} + +function assertIncludes(value, expected, reason) { + if (!String(value).includes(expected)) { + failures.push(`${reason} 缺少: ${expected}`); + } +} + +function assertEqual(actual, expected, reason) { + if (actual !== expected) { + failures.push(`${reason} 实际 ${JSON.stringify(actual)},预期 ${JSON.stringify(expected)}`); + } +} + +function assertFileContent(filePath, expected) { + if (!existsSync(filePath)) { + failures.push(`缺少文件: ${filePath}`); + return; + } + const actual = readFileSync(filePath, 'utf8'); + if (actual !== expected) { + failures.push(`${filePath} 内容不符合预期。`); + } +} + +function assertMode(filePath, expected, reason) { + const mode = statSync(filePath).mode & 0o777; + if (mode !== expected) { + failures.push(`${reason} 实际 0${mode.toString(8)},预期 0${expected.toString(8)}。`); + } +} diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs new file mode 100644 index 000000000..3d6d5cc7a --- /dev/null +++ b/scripts/check-production-api-deploy.mjs @@ -0,0 +1,1612 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-production-api-deploy-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-api-deploy] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-api-deploy] OK'); + +function main() { + assertDeployCopiesPingoraDirectReleaseDependencies(); + assertDeployRestartsActivePingoraWhenArtifactIncluded(); + assertDeployStartsInactivePingoraWhenArtifactIncluded(); + assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded(); + assertDeployRejectsPingoraPublicListenWhenArtifactIncluded(); + assertDeployRejectsPingoraArtifactMissingManifestEntry(); + assertMissingReleaseManifestFails(); + assertReleaseManifestMissingApiArtifactFails(); + assertDeployRejectsDotVersion(); + assertDeployRejectsDotDotVersion(); + assertDeployRejectsVersionStartingWithDot(); + assertDeployRejectsExistingReleaseDirectory(); + assertDeployRejectsDirectoryCurrentLink(); + assertDeployRejectsRelativeReleaseRoot(); + assertDeployRejectsRelativeCurrentLink(); + assertDeployRejectsRelativeApiEnvFile(); + assertDeployCleansStagingReleaseOnFailure(); + assertDeployRejectsFinalReleaseRaceAndCleansStaging(); + assertMissingBackupScriptFails(); + assertMissingHealthPatrolScriptFails(); + assertMissingPingoraCurrentReleaseAuditFails(); + assertMissingPingoraDirectRehearsalStatusFails(); + assertMissingPingoraCutoverStatusSnapshotFails(); + assertMissingPingoraCutoverEvidenceBundleFails(); + assertMissingPingoraCutoverCommandEvidenceFails(); + assertMissingPingoraCutoverEvidenceVerifyFails(); + assertMissingPingoraCutoverEvidenceAuditFails(); + assertMissingHealthPatrolEnvCheckFails(); + assertMissingPingoraReleaseReadinessFails(); + assertMissingPingoraRealpathCanaryEnableFails(); + assertMissingPingoraRealpathCanaryDisableFails(); + assertMissingPingoraHealthPatrolEnvSwitchFails(); + assertMissingPingoraGatewayEnvShadowSwitchFails(); + assertMissingPingoraTlsCertSyncFails(); + assertMissingEnvExamplesFails(); + assertMissingPingoraDirectCheckFails(); + assertMissingPingoraCanaryLiveFails(); + assertMissingPingoraCanaryAccessLogParityFails(); +} + +function assertDeployCopiesPingoraDirectReleaseDependencies() { + const fixture = prepareFixture('with-direct-checks'); + const result = runDeploy(fixture); + + assertStatus(result, 0, '完整 fixture 应部署成功。'); + if (result.status !== 0) { + return; + } + assertIncludes( + result.stdout, + '[production-api-deploy] 完成:', + '部署成功时必须输出完成信息。', + ); + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + const currentTarget = readlinkSync(fixture.currentLink); + if (currentTarget !== releaseDir) { + failures.push( + `current link 应指向新 release。实际 ${currentTarget},预期 ${releaseDir}`, + ); + } + + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'current release 必须包含 Pingora 直连启用脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'current release 必须包含 Pingora 直连回退脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'current release 必须包含 Pingora realpath canary 启用脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'current release 必须包含 Pingora realpath canary 关闭脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'current release 必须包含 Pingora health patrol env 切换脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + ), + 'current release 必须包含 Pingora gateway env shadow 切换脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'current release 必须包含 Pingora TLS 证书同步脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'current release 必须包含 Pingora 直连预检脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), + 'current release 必须包含健康巡检 env 复核脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), + 'current release 必须包含 Pingora release readiness 聚合门禁脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), + 'current release 必须包含 Pingora current release 自审脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'), + 'current release 必须包含 Pingora 直连彩排状态脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), + 'current release 必须包含 Pingora 直连切换状态快照脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), + 'current release 必须包含 Pingora 直连切换证据包脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), + 'current release 必须包含 Pingora 直连切换命令证据脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), + 'current release 必须包含 Pingora 直连切换证据验真脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), + 'current release 必须包含 Pingora 直连切换证据根目录审计脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + 'current release 必须包含 Pingora 直连 live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), + 'current release 必须包含 Pingora canary live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), + 'current release 必须包含 Pingora canary access log 对账脚本。', + ); + assertFileExists( + path.join(releaseDir, 'release-manifest.api-server.json'), + 'current release 必须包含 API release manifest 副本。', + ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + 'current release 必须包含 api-server checksum,供随包 current release 自审校验。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/systemd/genarrative-pingora-gateway.service'), + 'current release 必须包含 systemd 主 service 模板。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + 'current release 必须包含 Pingora 直连 drop-in 模板。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'current release 必须包含 Pingora env 示例。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + 'current release 必须包含 Pingora 真实路径 canary Nginx snippet。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/health-patrol.env.example'), + 'current release 必须包含健康巡检 env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), + 'current release 必须包含 Pingora direct live env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), + 'current release 必须包含 Pingora canary live env 示例。', + ); + assertPingoraEnvProductionDefaults( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'current release 内 Pingora env 示例', + ); + + const copiedPreflight = readFileSync( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'utf8', + ); + assertIncludes( + copiedPreflight, + 'deploy/pingora/pingora-gateway.env.example', + '复制后的 preflight 脚本必须仍能从 release root 推导 Pingora 配置。', + ); + + const apiEnv = readFileSync(fixture.apiEnvFile, 'utf8'); + assertIncludes( + apiEnv, + 'GENARRATIVE_SPACETIME_DATABASE=genarrative-prod', + '部署脚本必须写入 SpacetimeDB database。', + ); + assertIncludes( + apiEnv, + 'GENARRATIVE_SPACETIME_SERVER_URL=http://127.0.0.1:3101', + '部署脚本必须写入 SpacetimeDB server URL。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl restart genarrative-api.service', + '部署脚本必须重启 API service。', + ); + assertIncludes( + commandsLog, + 'curl -fsS http://127.0.0.1:18082/readyz', + '部署脚本必须执行 readiness curl。', + ); + + if (existsSync(fixture.maintenanceFile)) { + failures.push('部署成功后应退出维护模式。'); + } +} + +function assertDeployRestartsActivePingoraWhenArtifactIncluded() { + const fixture = prepareFixture('with-active-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture); + + assertStatus( + result, + 0, + '包含 Pingora 且 shadow service active 时应部署成功。', + ); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '包含 Pingora 的发布包部署后 current release 必须包含网关二进制。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '包含 Pingora 的发布包部署后 current release 必须包含网关 checksum。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl cat genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', + ); + assertIncludes( + commandsLog, + 'systemctl restart genarrative-pingora-gateway.service', + 'Pingora shadow service 已 active 时必须随 current release 切换 restart。', + ); + assertIncludes( + result.stdout, + '发布包包含 Pingora,启动或重启 shadow 影子服务', + 'Pingora shadow service 自动重启时必须输出明确提示。', + ); +} + +function assertDeployStartsInactivePingoraWhenArtifactIncluded() { + const fixture = prepareFixture('with-inactive-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture, { pingoraActive: false }); + + assertStatus( + result, + 0, + '包含 Pingora 且 shadow service inactive 但仍是安全 shadow 配置时应部署成功。', + ); + if (result.status !== 0) { + return; + } + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl cat genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', + ); + assertIncludes( + commandsLog, + 'systemctl restart genarrative-pingora-gateway.service', + 'Pingora shadow service inactive 且 shadow 配置安全时必须随 current release 启动。', + ); + assertIncludes( + commandsLog, + 'systemctl is-active --quiet genarrative-pingora-gateway.service', + '启动或重启 Pingora shadow 后必须复核 active。', + ); + assertIncludes( + result.stdout, + '发布包包含 Pingora,启动或重启 shadow 影子服务', + 'Pingora shadow service 自动启动时必须输出明确提示。', + ); +} + +function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() { + const fixture = prepareFixture('with-direct-entry-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture, { pingoraDirectEntry: true }); + + if (result.status === 0) { + failures.push('包含 Pingora 但 systemd 已启用 direct-entry capability 时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'Pingora systemd 已包含 CAP_NET_BIND_SERVICE', + 'direct-entry capability 存在时必须给出明确错误。', + ); + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + if ( + commandsLog.includes( + 'systemctl restart genarrative-pingora-gateway.service', + ) + ) { + failures.push('direct-entry capability 存在时不能自动 restart Pingora。'); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('direct-entry capability 导致部署失败时必须保持维护模式。'); + } + assertNoReleasePromoted( + fixture, + 'direct-entry capability 导致部署失败时不能提升正式 release。', + ); +} + +function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() { + const fixture = prepareFixture('with-public-listen-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + writePingoraEnv(fixture, { + listen: '0.0.0.0:443', + }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('包含 Pingora 但 env 已配置公网监听时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', + '公网监听 env 存在时必须给出明确错误。', + ); + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + if ( + commandsLog.includes( + 'systemctl restart genarrative-pingora-gateway.service', + ) + ) { + failures.push('公网监听 env 存在时不能自动 restart Pingora。'); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('公网监听 env 导致部署失败时必须保持维护模式。'); + } + assertNoReleasePromoted( + fixture, + '公网监听 env 导致部署失败时不能提升正式 release。', + ); +} + +function assertDeployRejectsPingoraArtifactMissingManifestEntry() { + const fixture = prepareFixture('pingora-artifact-missing-manifest-entry'); + addPingoraGatewayArtifact(fixture, { registerInManifest: false }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布包包含 Pingora 但 manifest 未登记时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'release-manifest.json 缺少 pingora-gateway artifact', + 'manifest 未登记 Pingora 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('manifest 未登记 Pingora 导致部署失败时必须保持维护模式。'); + } +} + +function assertMissingReleaseManifestFails() { + const fixture = prepareFixture('missing-release-manifest'); + rmSync(path.join(fixture.sourceDir, 'release-manifest.json')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 release-manifest.json 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 release-manifest.json', + '缺少 release-manifest.json 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('缺少 release-manifest.json 导致部署失败时必须保持维护模式。'); + } +} + +function assertReleaseManifestMissingApiArtifactFails() { + const fixture = prepareFixture('release-manifest-missing-api-artifact'); + writeFileSync( + path.join(fixture.sourceDir, 'release-manifest.json'), + `${JSON.stringify({ version: fixture.version, artifacts: [] }, null, 2)}\n`, + 'utf8', + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('release manifest 缺少 api-server artifact 时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'release-manifest.json 缺少 api-server artifact', + 'manifest 缺少 api-server artifact 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('manifest 缺少 api-server artifact 导致部署失败时必须保持维护模式。'); + } +} + +function assertDeployRejectsDotVersion() { + const fixture = prepareFixture('dot-version'); + const result = runDeploy(fixture, { version: '.' }); + + if (result.status === 0) { + failures.push('--version=. 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version=. 时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version=. 且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsDotDotVersion() { + const fixture = prepareFixture('dot-dot-version'); + const result = runDeploy(fixture, { version: '..' }); + + if (result.status === 0) { + failures.push('--version=.. 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version=.. 时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version=.. 且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsVersionStartingWithDot() { + const fixture = prepareFixture('dot-prefix-version'); + const result = runDeploy(fixture, { version: '.hidden-release' }); + + if (result.status === 0) { + failures.push('--version 以点开头时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version 以点开头时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version 以点开头且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsExistingReleaseDirectory() { + const fixture = prepareFixture('existing-release-directory'); + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + mkdirSync(releaseDir, { recursive: true }); + writeFileSync(path.join(releaseDir, 'old-file'), 'old\n', 'utf8'); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('目标 release 目录已存在时部署必须失败。'); + } + assertIncludes( + result.stderr, + '目标 release 已存在,拒绝覆盖或合并旧文件', + '目标 release 目录已存在时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('目标 release 目录已存在且未进入部署阶段时不应开启维护模式。'); + } + assertFileExists( + path.join(releaseDir, 'old-file'), + '拒绝覆盖既有 release 时必须保留原目录。', + ); +} + +function assertDeployRejectsDirectoryCurrentLink() { + const fixture = prepareFixture('directory-current-link'); + mkdirSync(fixture.currentLink, { recursive: true }); + writeFileSync(path.join(fixture.currentLink, 'old-file'), 'old\n', 'utf8'); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('current 路径已存在但不是符号链接时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'current 链接路径已存在但不是符号链接,拒绝覆盖', + 'current 路径不是符号链接时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('current 路径不是符号链接且未进入部署阶段时不应开启维护模式。'); + } + assertFileExists( + path.join(fixture.currentLink, 'old-file'), + '拒绝覆盖目录型 current 时必须保留原目录内容。', + ); +} + +function assertDeployRejectsRelativeReleaseRoot() { + const fixture = prepareFixture('relative-release-root'); + const result = runDeploy(fixture, { releaseRoot: 'relative/releases' }); + + if (result.status === 0) { + failures.push('--release-root 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--release-root 必须使用绝对路径', + '--release-root 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--release-root 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsRelativeCurrentLink() { + const fixture = prepareFixture('relative-current-link'); + const result = runDeploy(fixture, { currentLink: 'relative/current' }); + + if (result.status === 0) { + failures.push('--current-link 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--current-link 必须使用绝对路径', + '--current-link 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--current-link 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsRelativeApiEnvFile() { + const fixture = prepareFixture('relative-api-env-file'); + const result = runDeploy(fixture, { apiEnvFile: 'relative/api-server.env' }); + + if (result.status === 0) { + failures.push('--api-env-file 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--api-env-file 必须使用绝对路径', + '--api-env-file 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--api-env-file 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployCleansStagingReleaseOnFailure() { + const fixture = prepareFixture('cleans-staging-on-failure'); + rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('缺少数据库备份脚本时部署必须失败。'); + } + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + if (existsSync(releaseDir)) { + failures.push('部署失败时不应留下正式 release 目录。'); + } + const entries = existsSync(fixture.releaseRoot) + ? readDirNames(fixture.releaseRoot) + : []; + const stagingEntries = entries.filter((entry) => + entry.includes(`${fixture.version}.staging`), + ); + if (stagingEntries.length > 0) { + failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { + const fixture = prepareFixture('final-release-race'); + const result = runDeploy(fixture, { createReleaseDuringCopy: true }); + + if (result.status === 0) { + failures.push('最终提升前目标 release 目录被外部创建时部署必须失败。'); + } + assertIncludes( + result.stderr, + '目标 release 在发布过程中出现,拒绝合并 staging', + '最终提升前目标 release 目录被外部创建时必须给出明确错误。', + ); + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + assertFileExists( + path.join(releaseDir, 'raced-file'), + '部署脚本拒绝竞态目标 release 时必须保留外部创建的目录。', + ); + if (existsSync(path.join(releaseDir, 'api-server'))) { + failures.push('目标 release 竞态出现后不应把 staging 内容合并进去。'); + } + const entries = existsSync(fixture.releaseRoot) + ? readDirNames(fixture.releaseRoot) + : []; + const stagingEntries = entries.filter((entry) => + entry.includes(`${fixture.version}.staging`), + ); + if (stagingEntries.length > 0) { + failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('目标 release 竞态失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraDirectCheckFails() { + const fixture = prepareFixture('missing-direct-live'); + rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 direct live smoke 脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连 live smoke 脚本', + '缺少 direct live smoke 脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCanaryLiveFails() { + const fixture = prepareFixture('missing-canary-live'); + rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-canary-live.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 canary live smoke 脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora canary live smoke 脚本', + '缺少 canary live smoke 脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCanaryAccessLogParityFails() { + const fixture = prepareFixture('missing-canary-log-parity'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/check-pingora-canary-access-log-parity.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 canary access log 对账脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora canary access log 对账脚本', + '缺少 canary access log 对账脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingBackupScriptFails() { + const fixture = prepareFixture('missing-backup-script'); + rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少数据库备份脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少数据库备份脚本', + '缺少数据库备份脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingHealthPatrolScriptFails() { + const fixture = prepareFixture('missing-health-patrol-script'); + rmSync( + path.join(fixture.sourceDir, 'scripts/ops/production-health-patrol.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少生产健康巡检脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少生产健康巡检脚本', + '缺少生产健康巡检脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCurrentReleaseAuditFails() { + const fixture = prepareFixture('missing-pingora-current-release-audit'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-current-release-audit.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora current release 自审脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora current release 自审脚本', + '缺少 Pingora current release 自审脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraDirectRehearsalStatusFails() { + const fixture = prepareFixture('missing-pingora-direct-rehearsal-status'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连彩排状态脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连彩排状态脚本', + '缺少 Pingora 直连彩排状态脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverStatusSnapshotFails() { + const fixture = prepareFixture('missing-pingora-cutover-status-snapshot'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换状态快照脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换状态快照脚本', + '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceBundleFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-bundle'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换证据包脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据包脚本', + '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverCommandEvidenceFails() { + const fixture = prepareFixture('missing-pingora-cutover-command-evidence'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换命令证据脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换命令证据脚本', + '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceVerifyFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-verify'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换证据验真脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据验真脚本', + '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceAuditFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-audit'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push( + '发布产物缺少 Pingora 直连切换证据根目录审计脚本时部署必须失败。', + ); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据根目录审计脚本', + '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingHealthPatrolEnvCheckFails() { + const fixture = prepareFixture('missing-health-patrol-env-check'); + rmSync( + path.join(fixture.sourceDir, 'scripts/check-production-health-patrol-env.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少生产健康巡检 env 复核脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少生产健康巡检 env 复核脚本', + '缺少生产健康巡检 env 复核脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraReleaseReadinessFails() { + const fixture = prepareFixture('missing-pingora-release-readiness'); + rmSync( + path.join(fixture.sourceDir, 'scripts/check-pingora-release-readiness.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push( + '发布产物缺少 Pingora release readiness 聚合门禁脚本时部署必须失败。', + ); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora release readiness 聚合门禁脚本', + '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraHealthPatrolEnvSwitchFails() { + const fixture = prepareFixture('missing-pingora-health-patrol-env-switch'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora health patrol env 切换脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora health patrol env 切换脚本', + '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraGatewayEnvShadowSwitchFails() { + const fixture = prepareFixture('missing-pingora-gateway-env-shadow-switch'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push( + '发布产物缺少 Pingora gateway env shadow 切换脚本时部署必须失败。', + ); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora gateway env shadow 切换脚本', + '缺少 Pingora gateway env shadow 切换脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraRealpathCanaryEnableFails() { + const fixture = prepareFixture('missing-pingora-realpath-canary-enable'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-realpath-canary-enable.sh', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora realpath canary 启用脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora realpath canary 启用脚本', + '缺少 Pingora realpath canary 启用脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraRealpathCanaryDisableFails() { + const fixture = prepareFixture('missing-pingora-realpath-canary-disable'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-realpath-canary-disable.sh', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora realpath canary 关闭脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora realpath canary 关闭脚本', + '缺少 Pingora realpath canary 关闭脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraTlsCertSyncFails() { + const fixture = prepareFixture('missing-pingora-tls-cert-sync'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-tls-cert-sync.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora TLS 证书同步脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora TLS 证书同步脚本', + '缺少 Pingora TLS 证书同步脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingEnvExamplesFails() { + const fixture = prepareFixture('missing-env-examples'); + rmSync(path.join(fixture.sourceDir, 'deploy/env'), { + recursive: true, + force: true, + }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少环境变量示例目录时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少环境变量示例目录', + '缺少环境变量示例目录时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function prepareFixture(name) { + const root = path.join(tmpRoot, name); + const sourceDir = path.join(root, 'source'); + const releaseRoot = path.join(root, 'releases'); + const currentLink = path.join(root, 'current'); + const apiEnvFile = path.join(root, 'etc', 'api-server.env'); + const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env'); + const maintenanceFile = path.join(root, 'maintenance', 'enabled'); + const fakeBin = path.join(root, 'bin'); + const commandsLog = path.join(root, 'commands.log'); + const workerStateFile = path.join(root, 'worker-service-enabled'); + const pingoraStateFile = path.join(root, 'pingora-service-active'); + const version = `20260614-${name}`; + + mkdirSync(sourceDir, { recursive: true }); + mkdirSync(fakeBin, { recursive: true }); + mkdirSync(path.dirname(apiEnvFile), { recursive: true }); + mkdirSync(path.join(sourceDir, 'scripts/deploy'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'scripts/ops'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/systemd'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/pingora'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/env'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/nginx/snippets'), { recursive: true }); + + writeFileSync( + path.join(sourceDir, 'api-server'), + '#!/usr/bin/env bash\n', + 'utf8', + ); + writeFileSync( + apiEnvFile, + [ + 'GENARRATIVE_TRACKING_OUTBOX_ENABLED=false', + 'GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS=5000', + '', + ].join('\n'), + 'utf8', + ); + writePingoraEnv({ pingoraEnvFile }); + chmodExecutable(path.join(sourceDir, 'api-server')); + writeSha256(sourceDir, 'api-server'); + writeFileSync( + path.join(sourceDir, 'release-manifest.json'), + `${JSON.stringify( + { + version, + artifacts: [ + { + component: 'api-server', + path: 'api-server', + checksum_path: 'api-server.sha256', + }, + ], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + path.join(sourceDir, 'scripts/database-backup-to-oss.mjs'), + 'console.log("backup");\n', + 'utf8', + ); + writeFileSync( + path.join(sourceDir, 'scripts/ops/production-health-patrol.mjs'), + 'console.log("patrol");\n', + 'utf8', + ); + copyFile( + 'scripts/ops/pingora-current-release-audit.mjs', + path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'), + ); + copyFile( + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + path.join(sourceDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-command-evidence.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), + ); + copyFile( + 'scripts/deploy/production-api-deploy.sh', + path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), + ); + copyFile( + 'scripts/deploy/maintenance-on.sh', + path.join(sourceDir, 'scripts/deploy/maintenance-on.sh'), + ); + copyFile( + 'scripts/deploy/maintenance-off.sh', + path.join(sourceDir, 'scripts/deploy/maintenance-off.sh'), + ); + copyFile( + 'scripts/deploy/pingora-direct-enable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-direct-enable.sh'), + ); + copyFile( + 'scripts/deploy/pingora-direct-rollback.sh', + path.join(sourceDir, 'scripts/deploy/pingora-direct-rollback.sh'), + ); + copyFile( + 'scripts/deploy/pingora-realpath-canary-enable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + ); + copyFile( + 'scripts/deploy/pingora-realpath-canary-disable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + ); + copyFile( + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), + ); + copyFile( + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + path.join(sourceDir, 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs'), + ); + copyFile( + 'scripts/deploy/pingora-tls-cert-sync.mjs', + path.join(sourceDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + ); + copyFile( + 'scripts/check-production-health-patrol-env.mjs', + path.join(sourceDir, 'scripts/check-production-health-patrol-env.mjs'), + ); + copyFile( + 'scripts/check-pingora-release-readiness.mjs', + path.join(sourceDir, 'scripts/check-pingora-release-readiness.mjs'), + ); + copyFile( + 'scripts/check-pingora-direct-preflight.mjs', + path.join(sourceDir, 'scripts/check-pingora-direct-preflight.mjs'), + ); + copyFile( + 'scripts/check-pingora-direct-live.mjs', + path.join(sourceDir, 'scripts/check-pingora-direct-live.mjs'), + ); + copyFile( + 'scripts/check-pingora-canary-live.mjs', + path.join(sourceDir, 'scripts/check-pingora-canary-live.mjs'), + ); + copyFile( + 'scripts/check-pingora-canary-access-log-parity.mjs', + path.join(sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), + ); + copyFile( + 'deploy/systemd/genarrative-pingora-gateway.service', + path.join(sourceDir, 'deploy/systemd/genarrative-pingora-gateway.service'), + ); + copyFile( + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + path.join( + sourceDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + ); + copyFile( + 'deploy/pingora/pingora-gateway.env.example', + path.join(sourceDir, 'deploy/pingora/pingora-gateway.env.example'), + ); + copyFile( + 'deploy/env/health-patrol.env.example', + path.join(sourceDir, 'deploy/env/health-patrol.env.example'), + ); + copyFile( + 'deploy/env/pingora-direct-live.env.example', + path.join(sourceDir, 'deploy/env/pingora-direct-live.env.example'), + ); + copyFile( + 'deploy/env/pingora-canary-live.env.example', + path.join(sourceDir, 'deploy/env/pingora-canary-live.env.example'), + ); + copyFile( + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + path.join( + sourceDir, + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + ), + ); + copyFile( + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + path.join( + sourceDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + ); + chmodExecutable( + path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), + ); + chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-on.sh')); + chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-off.sh')); + + writeFileSync( + path.join(fakeBin, 'systemctl'), + [ + '#!/usr/bin/env bash', + 'set -euo pipefail', + `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'worker_state_file="${FAKE_WORKER_STATE_FILE}"', + 'pingora_state_file="${FAKE_PINGORA_STATE_FILE}"', + 'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', + ' printf "[Service]\\n"', + ' printf "EnvironmentFile=%s\\n" "${FAKE_PINGORA_ENV_FILE}"', + ' printf "ExecStart=/opt/genarrative/current/pingora-gateway\\n"', + ' if [[ "${FAKE_PINGORA_DIRECT_ENTRY:-false}" == "true" ]]; then', + ' printf "AmbientCapabilities=CAP_NET_BIND_SERVICE\\n"', + ' printf "CapabilityBoundingSet=CAP_NET_BIND_SERVICE\\n"', + ' fi', + ' exit 0', + 'fi', + 'if [[ "$1" == "list-units" ]]; then', + ' pattern="${@: -1}"', + ' if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" && -f "${worker_state_file}" ]]; then', + ' printf "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker\\n"', + ' fi', + ' exit 0', + 'fi', + 'if [[ "$1 $2" == "enable --now" && "${3:-}" == "genarrative-external-generation-worker@1.service" ]]; then', + ' printf "enabled\\n" > "${worker_state_file}"', + ' exit 0', + 'fi', + 'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" ]]; then', + ' if [[ "${FAKE_PINGORA_ACTIVE:-true}" == "true" || -f "${pingora_state_file}" ]]; then', + ' exit 0', + ' fi', + ' exit 3', + 'fi', + 'if [[ "$1" == "restart" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', + ' printf "active\\n" > "${pingora_state_file}"', + ' exit 0', + 'fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'curl'), + [ + '#!/usr/bin/env bash', + `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'cp'), + [ + '#!/usr/bin/env bash', + 'set -euo pipefail', + '/usr/bin/cp "$@"', + 'if [[ "${FAKE_CREATE_RELEASE_DURING_COPY:-false}" == "true" ]]; then', + ' marker="${FAKE_RELEASE_ROOT}/.${FAKE_RELEASE_VERSION}.race-created"', + ' if [[ ! -e "${marker}" ]]; then', + ' mkdir -p "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}"', + ' printf "race\\n" > "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}/raced-file"', + ' printf "created\\n" > "${marker}"', + ' fi', + 'fi', + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(path.join(fakeBin, 'systemctl')); + chmodExecutable(path.join(fakeBin, 'curl')); + chmodExecutable(path.join(fakeBin, 'cp')); + + return { + root, + sourceDir, + releaseRoot, + currentLink, + apiEnvFile, + pingoraEnvFile, + maintenanceFile, + fakeBin, + commandsLog, + workerStateFile, + pingoraStateFile, + version, + }; +} + +function writePingoraEnv(fixture, options = {}) { + const filePath = fixture.pingoraEnvFile; + const lines = [ + `GENARRATIVE_PINGORA_GATEWAY_LISTEN=${options.listen ?? '127.0.0.1:18081'}`, + ]; + if (options.tlsListen) { + lines.push(`GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${options.tlsListen}`); + } + if (options.redirectListen) { + lines.push( + `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${options.redirectListen}`, + ); + } + writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); +} + +function addPingoraGatewayArtifact(fixture, options = {}) { + writeFileSync( + path.join(fixture.sourceDir, 'pingora-gateway'), + '#!/usr/bin/env bash\n', + 'utf8', + ); + chmodExecutable(path.join(fixture.sourceDir, 'pingora-gateway')); + writeSha256(fixture.sourceDir, 'pingora-gateway'); + if (options.registerInManifest === false) { + return; + } + const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + manifest.artifacts = Array.isArray(manifest.artifacts) + ? manifest.artifacts + : []; + manifest.artifacts.push({ + component: 'pingora-gateway', + path: 'pingora-gateway', + checksum_path: 'pingora-gateway.sha256', + }); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); +} + +function runDeploy(fixture, options = {}) { + const deployScript = path.join( + fixture.sourceDir, + 'scripts/deploy/production-api-deploy.sh', + ); + return spawnSync( + 'bash', + [ + deployScript, + '--source-dir', + fixture.sourceDir, + '--version', + options.version ?? fixture.version, + '--release-root', + options.releaseRoot ?? fixture.releaseRoot, + '--current-link', + options.currentLink ?? fixture.currentLink, + '--service', + 'genarrative-api.service', + '--health-url', + 'http://127.0.0.1:18082/readyz', + '--api-env-file', + options.apiEnvFile ?? fixture.apiEnvFile, + '--database', + 'genarrative-prod', + '--spacetime-server-url', + 'http://127.0.0.1:3101', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, + GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile, + FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true', + FAKE_PINGORA_DIRECT_ENTRY: + options.pingoraDirectEntry === true ? 'true' : 'false', + FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile, + FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile, + FAKE_CREATE_RELEASE_DURING_COPY: + options.createReleaseDuringCopy === true ? 'true' : 'false', + FAKE_RELEASE_ROOT: fixture.releaseRoot, + FAKE_RELEASE_VERSION: fixture.version, + FAKE_WORKER_STATE_FILE: fixture.workerStateFile, + }, + }, + ); +} + +function copyFile(source, target) { + writeFileSync(target, readFileSync(source, 'utf8'), 'utf8'); +} + +function writeSha256(directory, fileName) { + const result = spawnSync('sha256sum', [fileName], { + cwd: directory, + encoding: 'utf8', + }); + if (result.status !== 0) { + throw new Error(result.stderr || `sha256sum failed for ${fileName}`); + } + writeFileSync( + path.join(directory, `${fileName}.sha256`), + result.stdout, + 'utf8', + ); +} + +function chmodExecutable(filePath) { + spawnSync('chmod', ['0755', filePath], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function assertNoReleasePromoted(fixture, reason) { + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + if (existsSync(releaseDir)) { + failures.push(`${reason} 已存在: ${releaseDir}`); + } + if (existsSync(fixture.currentLink)) { + failures.push(`${reason} 不应切换 current: ${fixture.currentLink}`); + } +} + +function readDirNames(directory) { + return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], { + cwd: process.cwd(), + encoding: 'utf8', + }) + .stdout.split('\n') + .filter(Boolean); +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertPingoraEnvProductionDefaults(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const content = readFileSync(filePath, 'utf8'); + const requiredLines = [ + [ + 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + '压缩算法必须保持 gzip-only,避免 Brotli 在未验证前进入 current release。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', + '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', + '前置代理信任确认开关必须默认关闭。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + '接流保护必须默认开启。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', + '内部探针 token 示例必须保持空值,避免 current release 夹带真实 token。', + ], + ]; + for (const [line, reason] of requiredLines) { + assertIncludes(content, line, `${label} ${reason}`); + } +} diff --git a/scripts/check-production-api-release.mjs b/scripts/check-production-api-release.mjs new file mode 100644 index 000000000..652f09a32 --- /dev/null +++ b/scripts/check-production-api-release.mjs @@ -0,0 +1,591 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const BUILD_SCRIPT = 'scripts/build-production-release.sh'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-production-api-release-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-api-release] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-api-release] OK'); + +function main() { + assertApiReleaseContainsPingoraDirectDependencies(); + assertApiReleaseCanIncludePingoraGatewayArtifact(); +} + +function assertApiReleaseContainsPingoraDirectDependencies() { + const fixture = prepareFixture('api-direct-dependencies'); + const result = runBuild(fixture); + assertStatus(result, 0, 'API release build 烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'api-server'), + 'API release 必须包含 api-server。', + ); + assertExecutable( + path.join(releaseDir, 'api-server'), + 'api-server 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + 'API release 必须包含 api-server checksum。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'), + 'API release 必须包含随产物执行的 API deploy 脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'), + '随产物执行的 API deploy 脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'), + 'API release 必须在 deploy 脚本同目录包含 maintenance-on.sh。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'), + 'deploy 同目录 maintenance-on.sh 必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'), + 'API release 必须在 deploy 脚本同目录包含 maintenance-off.sh。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'), + 'deploy 同目录 maintenance-off.sh 必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'API release 必须包含 Pingora 直连启用脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'Pingora 直连启用脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'API release 必须包含 Pingora 直连回退脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'Pingora 直连回退脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'API release 必须包含 Pingora realpath canary 启用脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'Pingora realpath canary 启用脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'API release 必须包含 Pingora realpath canary 关闭脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'Pingora realpath canary 关闭脚本必须可执行。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'API release 必须包含 Pingora health patrol env 切换脚本。', + ); + assertExecutable( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'Pingora health patrol env 切换脚本必须可执行。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + ), + 'API release 必须包含 Pingora gateway env shadow 切换脚本。', + ); + assertExecutable( + path.join( + releaseDir, + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + ), + 'Pingora gateway env shadow 切换脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'API release 必须包含 Pingora TLS 证书同步脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'Pingora TLS 证书同步脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), + 'API release 必须包含健康巡检 env 复核脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), + 'API release 必须包含 Pingora release readiness 聚合门禁脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-current-release-audit.mjs', + ), + 'API release 必须包含 Pingora current release 自审脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + ), + 'API release 必须包含 Pingora 直连彩排状态脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ), + 'API release 必须包含 Pingora 直连切换状态快照脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据包脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ), + 'API release 必须包含 Pingora 直连切换命令证据脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据验真脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据根目录审计脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'API release 必须包含 Pingora 直连预检脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + 'API release 必须包含 Pingora 直连 live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), + 'API release 必须包含 Pingora canary live smoke 脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/check-pingora-canary-access-log-parity.mjs', + ), + 'API release 必须包含 Pingora canary access log 对账脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway.service', + ), + 'API release 必须包含 Pingora systemd 主 service 模板。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + 'API release 必须包含 Pingora 直连 drop-in 模板。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'API release 必须包含 Pingora env 示例。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + 'API release 必须包含 Pingora 真实路径 canary Nginx snippet。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/health-patrol.env.example'), + 'API release 必须包含健康巡检 env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), + 'API release 必须包含 Pingora direct live env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), + 'API release 必须包含 Pingora canary live env 示例。', + ); + assertPingoraEnvProductionDefaults( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'API release 内 Pingora env 示例', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (manifest.component_type !== 'api-server') { + failures.push( + `release manifest component_type 应为 api-server,实际 ${manifest.component_type}`, + ); + } + if (!manifest.artifacts?.some((item) => item.path === 'api-server')) { + failures.push('release manifest 必须登记 api-server artifact。'); + } + if (manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push('默认 API release 不应登记 pingora-gateway artifact。'); + } + + const releaseReadme = readFileSync( + path.join(releaseDir, 'README.md'), + 'utf8', + ); + assertIncludes( + releaseReadme, + 'Pingora release readiness 聚合门禁、直连启用 / 回退', + 'API release README 必须说明随包携带 Pingora 直连脚本。', + ); + assertIncludes( + releaseReadme, + 'realpath canary 启用 / 关闭', + 'API release README 必须说明随包携带 Pingora realpath canary 启停脚本。', + ); + assertIncludes( + releaseReadme, + 'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态', + 'API release README 必须说明随包携带 Pingora 运行态复核脚本。', + ); + assertIncludes( + releaseReadme, + '状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', + 'API release README 必须说明随包携带 Pingora 切换证据脚本。', + ); + for (const executableArg of requiredCutoverAuditExecutableArgs()) { + assertIncludes( + releaseReadme, + `--require-command-executable ${executableArg}`, + 'API release README 必须记录最终证据根目录总审计绑定 current release 切换脚本。', + ); + } + assertIncludes( + releaseReadme, + 'scripts/deploy/production-api-deploy.sh', + 'API release README 必须说明随包携带 API Deploy 执行入口。', + ); + assertIncludes( + releaseReadme, + '同目录的 `maintenance-on.sh` / `maintenance-off.sh` 必须来自同一发布包', + 'API release README 必须说明 deploy 同目录维护脚本来自同一发布包。', + ); + assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir); + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function assertApiReleaseCanIncludePingoraGatewayArtifact() { + const fixture = prepareFixture('api-with-pingora-artifact', { + includePingoraGateway: true, + }); + const result = runBuild(fixture, { + includePingoraGateway: true, + skipPingoraGatewayBuild: true, + }); + assertStatus(result, 0, '包含 Pingora 的 API release build 烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '显式 include Pingora 时 API release 必须包含 pingora-gateway。', + ); + assertExecutable( + path.join(releaseDir, 'pingora-gateway'), + 'pingora-gateway 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '显式 include Pingora 时 API release 必须包含 pingora-gateway checksum。', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (!manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push( + '显式 include Pingora 时 release manifest 必须登记 pingora-gateway artifact。', + ); + } + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function prepareFixture(name, options = {}) { + const cargoTargetDir = path.join(tmpRoot, name, 'cargo-target'); + const binaryDir = path.join( + cargoTargetDir, + 'x86_64-unknown-linux-gnu/release', + ); + const version = `check-production-api-release-${process.pid}-${Date.now()}`; + mkdirSync(binaryDir, { recursive: true }); + const apiBinary = path.join(binaryDir, 'api-server'); + writeFileSync(apiBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', apiBinary], { encoding: 'utf8' }); + if (options.includePingoraGateway) { + const pingoraBinary = path.join(binaryDir, 'pingora-gateway'); + writeFileSync(pingoraBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', pingoraBinary], { encoding: 'utf8' }); + } + return { cargoTargetDir, version }; +} + +function runBuild(fixture, options = {}) { + const args = [ + BUILD_SCRIPT, + '--component', + 'api-server', + '--name', + fixture.version, + '--skip-api-build', + ]; + if (options.includePingoraGateway) { + args.push('--include-pingora-gateway'); + } + if (options.skipPingoraGatewayBuild) { + args.push('--skip-pingora-gateway-build'); + } + + return spawnSync('bash', args, { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + CARGO_TARGET_DIR: fixture.cargoTargetDir, + SOURCE_BRANCH: 'test-branch', + SOURCE_COMMIT: 'test-commit', + }, + }); +} + +function readJson(filePath) { + try { + return JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + failures.push(`${filePath} 不是合法 JSON: ${error.message}`); + return {}; + } +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function assertExecutable(filePath, reason) { + if (!existsSync(filePath)) { + return; + } + if ((statSync(filePath).mode & 0o111) === 0) { + failures.push(`${reason} 文件不可执行: ${filePath}`); + } +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir) { + const readinessScript = path.join( + releaseDir, + 'scripts/check-pingora-release-readiness.mjs', + ); + const result = spawnSync( + 'node', + [ + readinessScript, + '--dry-run-cutover', + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--direct-preflight-env-file', + '/etc/genarrative/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--cutover-run-id', + 'check-production-api-release', + '--rollback-nginx-smoke-url', + 'https://example.com/', + '--rollback-nginx-smoke-expect-body', + '', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ], + { + cwd: releaseDir, + encoding: 'utf8', + env: { + ...process.env, + GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT: '', + }, + }, + ); + assertStatus( + result, + 0, + 'API release 随包 readiness 脚本必须能生成直连切换 dry-run runbook。', + ); + if (result.status !== 0) { + return; + } + + let plan; + try { + plan = JSON.parse(result.stdout); + } catch (error) { + failures.push( + `API release 随包 readiness dry-run cutover 输出不是合法 JSON: ${error.message}`, + ); + return; + } + + const auditStep = Array.isArray(plan) + ? plan.find((step) => step?.name === '切换证据根目录三阶段总审计') + : null; + if (!auditStep) { + failures.push( + 'API release 随包 readiness dry-run cutover 必须包含切换证据根目录三阶段总审计步骤。', + ); + return; + } + + if (!auditStep.args?.includes('--require-command-executable')) { + failures.push( + 'API release 随包 readiness 总审计步骤必须携带 --require-command-executable。', + ); + } + for (const executableArg of requiredCutoverAuditExecutableArgs()) { + if (!auditStep.args?.includes(executableArg)) { + failures.push( + `API release 随包 readiness 总审计步骤必须绑定 current release 切换脚本,缺少: ${executableArg}`, + ); + } + } +} + +function requiredCutoverAuditExecutableArgs() { + return [ + 'enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]; +} + +function assertPingoraEnvProductionDefaults(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const content = readFileSync(filePath, 'utf8'); + const requiredLines = [ + [ + 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + '压缩算法必须保持 gzip-only,避免 Brotli 在未验证前进入直连发布包。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', + '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', + '前置代理信任确认开关必须默认关闭。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + '接流保护必须默认开启。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', + '内部探针 token 示例必须保持空值,避免发布包夹带真实 token。', + ], + ]; + for (const [line, reason] of requiredLines) { + assertIncludes(content, line, `${label} ${reason}`); + } +} diff --git a/scripts/check-production-health-patrol-env-check.mjs b/scripts/check-production-health-patrol-env-check.mjs new file mode 100644 index 000000000..e8c1c003b --- /dev/null +++ b/scripts/check-production-health-patrol-env-check.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-env-check-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-health-patrol-env] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol-env] OK'); + +function main() { + assertAcceptsPingoraDirectLoopbackWithHost(); + assertRejectsPingoraDirectLoopbackWithoutHost(); + assertRejectsWrongGatewayMode(); + assertAcceptsNginxWithEmptyPublicHost(); + assertRejectsNginxWithStalePublicHost(); + assertRejectsInvalidPublicHost(); + assertRejectsInvalidEnvFilePaths(); + assertRejectsInvalidBoolEnv(); + assertExampleDocumentsDirectMode(); +} + +function assertAcceptsPingoraDirectLoopbackWithHost() { + const envFile = writeEnv('pingora-direct-ok.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + 'https://127.0.0.1', + '--expected-public-host', + 'genarrative.example', + ]); + assertStatus(result, 0, 'Pingora direct loopback 配正式 Host 应通过。'); +} + +function assertRejectsPingoraDirectLoopbackWithoutHost() { + const envFile = writeEnv('pingora-direct-missing-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + 'https://127.0.0.1', + ]); + assertStatus( + result, + 1, + 'Pingora direct loopback 缺 Host 必须失败。', + ); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + 'Pingora direct loopback 缺 Host 必须给出明确错误。', + ); +} + +function assertRejectsWrongGatewayMode() { + const envFile = writeEnv('wrong-mode.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + ]); + assertStatus(result, 1, 'gateway mode 与期望不一致必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 pingora-direct,实际 nginx', + 'gateway mode mismatch 必须给出明确错误。', + ); +} + +function assertAcceptsNginxWithEmptyPublicHost() { + const envFile = writeEnv('nginx-empty-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + '--expected-public-base-url', + 'http://127.0.0.1', + '--require-empty-public-host', + ]); + assertStatus(result, 0, 'Nginx 模式回退且 Host 为空应通过。'); +} + +function assertRejectsNginxWithStalePublicHost() { + const envFile = writeEnv('nginx-stale-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + '--require-empty-public-host', + ]); + assertStatus(result, 1, 'Nginx 回退后残留 Host 覆盖必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空', + 'Nginx 回退后残留 Host 覆盖必须给出明确错误。', + ); +} + +function assertRejectsInvalidPublicHost() { + const envFile = writeEnv('invalid-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'https://genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + ]); + assertStatus(result, 1, '非法 public Host 必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 只能是 host 或 host:port', + '非法 public Host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidEnvFilePaths() { + const relativePath = runCheck([ + '--env-file', + 'health-patrol.env', + '--expected-gateway-mode', + 'nginx', + ]); + assertStatus(relativePath, 1, 'health patrol env 复核必须拒绝相对 env 文件路径。'); + assertIncludes( + `${relativePath.stdout}\n${relativePath.stderr}`, + '--env-file 必须是绝对路径', + '相对 env 文件路径必须给出明确错误。', + ); + + const rootPath = runCheck([ + '--env-file', + '/', + '--expected-gateway-mode', + 'nginx', + ]); + assertStatus(rootPath, 1, 'health patrol env 复核必须拒绝文件系统根目录。'); + assertIncludes( + `${rootPath.stdout}\n${rootPath.stderr}`, + '--env-file 不能是文件系统根目录', + '文件系统根目录 env 文件路径必须给出明确错误。', + ); +} + +function assertRejectsInvalidBoolEnv() { + const envFile = writeEnv('invalid-bool-env.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck( + [ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + ], + { + GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST: 'enabled', + }, + ); + assertStatus(result, 1, 'health patrol env 复核必须拒绝非法布尔 env。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST 必须是布尔值', + 'health patrol env 复核非法布尔 env 必须给出明确错误。', + ); +} + +function assertExampleDocumentsDirectMode() { + const example = readFileSync('deploy/env/health-patrol.env.example', 'utf8'); + assertIncludes( + example, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + 'health patrol env 示例必须保留默认 nginx 模式。', + ); + assertIncludes( + example, + 'Pingora 直连切换后改为 pingora-direct', + 'health patrol env 示例必须说明 Pingora direct 切换口径。', + ); +} + +function writeEnv(name, values) { + const filePath = path.join(tmpRoot, name); + mkdirSync(path.dirname(filePath), { recursive: true }); + const body = Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join('\n'); + writeFileSync(filePath, `${body}\n`, 'utf8'); + return filePath; +} + +function runCheck(args, extraEnv = {}) { + return spawnSync( + process.execPath, + ['--', 'scripts/check-production-health-patrol-env.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST: '', + ...extraEnv, + }, + }, + ); +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} diff --git a/scripts/check-production-health-patrol-env.mjs b/scripts/check-production-health-patrol-env.mjs new file mode 100644 index 000000000..309908df0 --- /dev/null +++ b/scripts/check-production-health-patrol-env.mjs @@ -0,0 +1,313 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); + +const config = parseArgs(process.argv.slice(2)); +const failures = []; +const envValues = readEnvFile(config.envFile); + +checkGatewayMode(); +checkPublicBaseUrl(); +checkPublicHost(); + +if (failures.length > 0) { + console.error('[check:production-health-patrol-env] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol-env] OK'); + +function parseArgs(argv) { + const result = { + envFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: + readBoolEnv('GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST'), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--env-file': + result.envFile = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!result.expectedGatewayMode) { + throw new Error( + '必须提供 --expected-gateway-mode 或 GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE。', + ); + } + if (!path.isAbsolute(result.envFile)) { + throw new Error(`--env-file 必须是绝对路径: ${result.envFile}`); + } + if (isFilesystemRootPath(result.envFile)) { + throw new Error('--env-file 不能是文件系统根目录。'); + } + if (!GATEWAY_MODES.has(result.expectedGatewayMode)) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`, + ); + } + if (result.expectedPublicBaseUrl) { + normalizeBaseUrl(result.expectedPublicBaseUrl, '--expected-public-base-url'); + } + if (result.expectedPublicHost !== null) { + validateHostOption(result.expectedPublicHost, '--expected-public-host'); + } + if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node -- scripts/check-production-health-patrol-env.mjs --expected-gateway-mode [options] + +Options: + --env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 + --expected-gateway-mode nginx 或 pingora-direct。 + --expected-public-base-url 可选,要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 与该 URL 一致。 + --expected-public-host 可选,要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 与该 Host 一致。 + --require-empty-public-host 要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 为空,适合回退到 Nginx 后复核。 +`); +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readEnvFile(filePath) { + let text; + try { + text = readFileSync(filePath, 'utf8'); + } catch (error) { + throw new Error(`无法读取 health patrol env 文件 ${filePath}: ${error.message}`); + } + + const values = new Map(); + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + failures.push(`第 ${index + 1} 行不是 KEY=VALUE 格式: ${rawLine}`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + const value = stripQuotes(line.slice(equalsIndex + 1).trim()); + values.set(key, value); + } + return values; +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function envValue(name) { + return envValues.get(name) ?? ''; +} + +function checkGatewayMode() { + const actual = envValue('GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE'); + if (!actual) { + failures.push( + 'health patrol env 缺少 GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE。', + ); + return; + } + if (actual !== config.expectedGatewayMode) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${config.expectedGatewayMode},实际 ${actual}。`, + ); + } +} + +function checkPublicBaseUrl() { + if (!config.expectedPublicBaseUrl) { + return; + } + + const actual = envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL'); + if (!actual) { + failures.push( + 'health patrol env 缺少 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL。', + ); + return; + } + const expected = normalizeBaseUrl( + config.expectedPublicBaseUrl, + '--expected-public-base-url', + ); + const normalizedActual = normalizeBaseUrl( + actual, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL', + ); + if (normalizedActual !== expected) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${expected},实际 ${normalizedActual}。`, + ); + } +} + +function checkPublicHost() { + const publicHost = envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST'); + const baseUrl = + config.expectedPublicBaseUrl || + envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL'); + + if (publicHost) { + try { + validateHostOption(publicHost, 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST'); + } catch (error) { + failures.push(error.message); + } + } + + if (config.expectedPublicHost !== null && publicHost !== config.expectedPublicHost) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${config.expectedPublicHost},实际 ${publicHost || '(空)'}。`, + ); + } + + if (config.requireEmptyPublicHost && publicHost) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空,实际 ${publicHost}。`, + ); + } + + if ( + config.expectedGatewayMode === 'pingora-direct' && + baseUrl && + isLoopbackUrl(baseUrl) && + !publicHost + ) { + failures.push( + 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST,避免巡检命中默认 vhost。', + ); + } +} + +function normalizeBaseUrl(value, label) { + try { + const parsed = new URL(value); + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new Error('unsupported protocol'); + } + return parsed.toString().replace(/\/+$/u, ''); + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符。`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function isLoopbackUrl(value) { + try { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === 'localhost' || + hostname === '::1' || + hostname === '[::1]' || + hostname.startsWith('127.') + ); + } catch { + return false; + } +} diff --git a/scripts/check-production-health-patrol.mjs b/scripts/check-production-health-patrol.mjs new file mode 100644 index 000000000..b61f12da5 --- /dev/null +++ b/scripts/check-production-health-patrol.mjs @@ -0,0 +1,478 @@ +#!/usr/bin/env node + +import { spawn, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-check-'), +); + +try { + await main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-health-patrol] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol] OK'); + +async function main() { + assertPublicBaseUrlDefaultsToGatewayEntry(); + await assertNginxModeChecksNginxService(); + await assertPingoraDirectModeChecksPingoraServiceAndPublicHost(); + assertRejectsInvalidGatewayMode(); + assertRejectsInvalidPublicHost(); + assertRejectsInvalidTimeoutAndSlowThreshold(); + assertRejectsInvalidBoolEnv(); +} + +function assertPublicBaseUrlDefaultsToGatewayEntry() { + const script = readFileSync( + 'scripts/ops/production-health-patrol.mjs', + 'utf8', + ); + assertIncludes( + script, + "process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n 'http://127.0.0.1'", + 'publicBaseUrl 默认必须指向本机网关入口,不能回落到 API 直连端口。', + ); + if ( + script.includes( + 'process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL', + ) + ) { + failures.push('publicBaseUrl 默认不应回落到 API base URL。'); + } +} + +async function assertNginxModeChecksNginxService() { + const fixture = await prepareFixture('nginx-mode'); + const result = await runPatrol(fixture, ['--gateway-mode', 'nginx']); + + assertStatus(result, 0, 'nginx gateway mode 巡检应成功。'); + if (result.status !== 0) { + return; + } + + const payload = JSON.parse(result.stdout); + assertEqual( + payload.gatewayMode, + 'nginx', + '巡检 JSON 必须记录 gatewayMode=nginx。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl is-active nginx.service', + 'nginx gateway mode 必须检查 nginx.service。', + ); + if (commandsLog.includes('genarrative-pingora-gateway.service')) { + failures.push( + 'nginx gateway mode 不应要求 Pingora gateway service active。', + ); + } +} + +async function assertPingoraDirectModeChecksPingoraServiceAndPublicHost() { + const fixture = await prepareFixture('pingora-direct-mode'); + const result = await runPatrol(fixture, [ + '--gateway-mode', + 'pingora-direct', + '--public-host', + 'genarrative.example', + ]); + + assertStatus(result, 0, 'pingora-direct gateway mode 巡检应成功。'); + if (result.status !== 0) { + return; + } + + const payload = JSON.parse(result.stdout); + assertEqual( + payload.gatewayMode, + 'pingora-direct', + '巡检 JSON 必须记录 gatewayMode=pingora-direct。', + ); + + const commandsLog = readTextFile(fixture.commandsLog, 'systemctl 命令日志'); + assertIncludes( + commandsLog, + 'systemctl is-active genarrative-pingora-gateway.service', + 'pingora-direct gateway mode 必须检查 Pingora gateway service。', + ); + if (commandsLog.includes('systemctl is-active nginx.service')) { + failures.push( + 'pingora-direct gateway mode 不应要求 nginx.service active。', + ); + } + + const requestsLog = readTextFile( + fixture.requestsLog, + 'public probe 请求日志', + ); + assertIncludes( + requestsLog, + 'host=genarrative.example path=/api/creation-entry/config', + 'public probe 必须带正式域名 Host header。', + ); +} + +function assertRejectsInvalidGatewayMode() { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--gateway-mode', + 'direct', + '--skip-journal', + '--json', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push('非法 gateway mode 必须失败。'); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--gateway-mode 只支持 nginx 或 pingora-direct', + '非法 gateway mode 必须给出明确错误。', + ); +} + +function assertRejectsInvalidPublicHost() { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--public-host', + 'https://genarrative.example', + '--skip-journal', + '--json', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push('非法 public host 必须失败。'); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--public-host 只接受域名或 host:port', + '非法 public host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidTimeoutAndSlowThreshold() { + const cases = [ + { + args: ['--timeout-ms', '0'], + env: {}, + expected: '--timeout-ms 必须是正整数', + reason: '非法 CLI timeout 必须失败。', + }, + { + args: ['--slow-ms', 'abc'], + env: {}, + expected: '--slow-ms 必须是正整数', + reason: '非法 CLI slow threshold 必须失败。', + }, + { + args: [], + env: { + GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS 必须是正整数', + reason: '非法 env timeout 必须失败。', + }, + { + args: [], + env: { + GENARRATIVE_HEALTH_PATROL_SLOW_MS: '0', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_SLOW_MS 必须是正整数', + reason: '非法 env slow threshold 必须失败。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--skip-journal', + '--json', + ...testCase.args, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + ...testCase.env, + }, + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING: 'ture', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING 必须是布尔值', + reason: '生产健康巡检必须拒绝拼写错误的 fail-on-warning env。', + }, + { + env: { + GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL: 'maybe', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL 必须是布尔值', + reason: '生产健康巡检必须拒绝非法 skip-journal env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/ops/production-health-patrol.mjs', '--json'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + ...testCase.env, + }, + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +async function prepareFixture(name) { + const root = path.join(tmpRoot, name); + const fakeBin = path.join(root, 'bin'); + const commandsLog = path.join(root, 'commands.log'); + const requestsLog = path.join(root, 'requests.log'); + + mkdirSync(fakeBin, { recursive: true }); + + writeFileSync( + path.join(fakeBin, 'systemctl'), + [ + '#!/usr/bin/env bash', + `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'if [[ "$1" == "list-units" && "$2" == "genarrative-external-generation-worker@*.service" ]]; then', + ' echo "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker 1"', + ' exit 0', + 'fi', + 'echo active', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'journalctl'), + [ + '#!/usr/bin/env bash', + `printf 'journalctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'echo "-- No entries --"', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(path.join(fakeBin, 'systemctl')); + chmodExecutable(path.join(fakeBin, 'journalctl')); + + const server = http.createServer((request, response) => { + writeFileSync( + requestsLog, + `host=${request.headers.host || ''} path=${request.url || ''}\n`, + { + encoding: 'utf8', + flag: 'a', + }, + ); + response.writeHead(200, { + 'Content-Type': 'application/json', + }); + response.end(JSON.stringify({ ok: true })); + }); + await listen(server, '127.0.0.1', 0); + + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('测试 HTTP server 未返回 TCP 端口。'); + } + + return { + root, + fakeBin, + commandsLog, + requestsLog, + server, + baseUrl: `http://127.0.0.1:${address.port}`, + }; +} + +async function runPatrol(fixture, args) { + try { + return await spawnProcess( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--api-base-url', + fixture.baseUrl, + '--spacetime-base-url', + fixture.baseUrl, + '--public-base-url', + fixture.baseUrl, + '--skip-journal', + '--json', + ...args, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, + }, + }, + ); + } finally { + await closeServer(fixture.server); + } +} + +function spawnProcess(command, args, options) { + return new Promise((resolve) => { + const child = spawn(command, args, { + ...options, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.on('error', (error) => { + resolve({ + status: 1, + stdout, + stderr: `${stderr}${error.message}`, + }); + }); + child.on('close', (code, signal) => { + resolve({ + status: code ?? 1, + signal, + stdout, + stderr, + }); + }); + }); +} + +function listen(server, host, port) { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, resolve); + }); +} + +function closeServer(server) { + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +function readTextFile(filePath, label) { + if (!existsSync(filePath)) { + failures.push(`${label} 未生成:${filePath}`); + return ''; + } + return readFileSync(filePath, 'utf8'); +} + +function chmodExecutable(filePath) { + spawnSync('chmod', ['0755', filePath], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertEqual(actual, expected, reason) { + if (actual !== expected) { + failures.push(`${reason} 实际 ${actual},预期 ${expected}。`); + } +} diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 89ce254ec..4ab1e3ff1 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import {readFileSync} from 'node:fs'; +import { readFileSync } from 'node:fs'; const checks = [ { @@ -8,21 +8,955 @@ const checks = [ includes: '--restart-service-after genarrative-api.service', reason: '生产冷备份恢复 SpacetimeDB 后必须显式拉起依赖它的 API 服务。', }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file', + reason: + '生产冷备份 service 必须用 node -- 分隔脚本参数,避免 Node 22 抢占业务 --env-file。', + }, { file: 'deploy/systemd/genarrative-health-patrol.service', includes: 'scripts/ops/production-health-patrol.mjs', reason: '健康巡检 systemd service 必须调用随 API release 发布的巡检脚本。', }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL', + reason: '健康巡检环境示例必须保留 Pingora shadow 可选巡检配置。', + }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + reason: '健康巡检环境示例必须显式保留默认 Nginx 网关模式。', + }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + reason: '健康巡检环境示例必须暴露公网 Host header 配置。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'pingora-direct', + reason: + '健康巡检必须支持 Pingora 直连模式,避免直连切换后误要求 Nginx active。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连巡检模式必须检查 Pingora gateway service。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + reason: '健康巡检必须支持给 public probe 传正式域名 Host header。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'parseRequiredPositiveInt', + reason: + '健康巡检 timeout 和慢请求阈值必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'current release 自包含能力', + reason: 'Pingora current release 自审脚本必须说明它只检查发布包自包含能力。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: 'Pingora current release 自审必须确认 release readiness 聚合门禁脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs', + reason: 'Pingora current release 自审必须确认直连彩排状态脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-direct-rehearsal-status.mjs', + includes: '该脚本只读采集', + reason: 'Pingora 直连彩排状态脚本必须声明只读边界。', + }, + { + file: 'scripts/ops/pingora-direct-rehearsal-status.mjs', + includes: 'ss', + reason: 'Pingora 直连彩排状态脚本必须读取端口监听状态。', + }, + { + file: 'scripts/check-pingora-direct-rehearsal-status.mjs', + includes: 'assertNginxPublicRealpathRehearsalPasses', + reason: 'Pingora 直连彩排状态自测必须覆盖 Nginx 公网 + realpath canary 正例。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: 'Pingora current release 自审必须确认 canary live smoke 脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'Pingora current release 自审必须确认 realpath canary 启用脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'Pingora current release 自审必须确认 realpath canary 关闭脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '--systemd-show', + reason: 'Pingora current release 自审必须能核验 systemd ExecStart 指向 current release。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'pingora-gateway.sha256', + reason: 'Pingora current release 自审必须校验 Pingora 网关 checksum。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'release-manifest.api-server.json', + reason: 'Pingora current release 自审必须读取部署后的 API release manifest。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'release manifest 缺少 pingora-gateway artifact', + reason: 'Pingora current release 自审必须阻断 manifest 未登记 Pingora 产物。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora current release 自审 timeout 参数必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora current release 自审布尔 env 必须严格校验,避免拼写错误静默降级。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora current release 自审必须显式拒绝把 release root 指到 /。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora current release 自审必须在执行 systemctl 前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora current release 自审必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: '完整 release 自审必须确认 canary live smoke 脚本存在', + reason: 'Pingora current release 自审自测必须覆盖 canary live smoke 脚本随包存在。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertChecksumMismatchFails', + reason: 'Pingora current release 自审自测必须覆盖 checksum 不匹配负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertManifestMustRegisterIncludedPingoraGateway', + reason: 'Pingora current release 自审自测必须覆盖 manifest 未登记 Pingora 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora current release 自审自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora current release 自审自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsFilesystemRootReleaseRoot', + reason: 'Pingora current release 自审自测必须覆盖 release root 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsPathArgsWithControlCharacters', + reason: + 'Pingora current release 自审自测必须覆盖带控制字符路径参数在执行 systemctl 前失败。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: 'Pingora 切换状态快照必须把 current release 自审脚本纳入随包依赖采集。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--fail-on-critical', + reason: 'Pingora 切换状态快照必须能作为只读硬门禁运行。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--run-health-patrol', + reason: 'Pingora 切换状态快照必须复用生产健康巡检作为证据。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'current-release-audit', + reason: 'Pingora 切换状态快照必须执行 current release 自审。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'releaseManifest', + reason: 'Pingora 切换状态快照必须收录 manifest 自审结果。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'collectSecretValuesFromEnv', + reason: 'Pingora 切换状态快照必须从 env 中收集敏感值用于后续输出脱敏。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'delete healthPatrolEnv.secretValues', + reason: 'Pingora 切换状态快照 JSON 不能输出 health patrol env 敏感值缓存。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'delete pingoraEnv.secretValues', + reason: 'Pingora 切换状态快照 JSON 不能输出 Pingora env 敏感值缓存。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'redactSecrets', + reason: 'Pingora 切换状态快照必须在收录子命令 stdout/stderr 前脱敏敏感值。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora 切换状态快照 timeout 参数必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora 切换状态快照布尔 env 必须严格校验,避免 run/fail/require 开关拼写错误静默降级。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora 切换状态快照必须显式拒绝把 release root 指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 切换状态快照必须在执行子命令前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 切换状态快照必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--health-patrol-env-file 不能是文件系统根目录', + reason: 'Pingora 切换状态快照必须拒绝把 health patrol env 文件指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--pingora-env-file 不能是文件系统根目录', + reason: 'Pingora 切换状态快照必须拒绝把 Pingora env 文件指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'environmentFileMatchesPingoraEnvFile', + reason: + 'Pingora 切换状态快照必须确认 systemd EnvironmentFile 指向本次 pingora env。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--expected-pingora-env-mode', + reason: + 'Pingora 切换状态快照必须支持按阶段强制校验 active Pingora env 是 direct 或 shadow。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'summarizePingoraEnvPosture', + reason: + 'Pingora 切换状态快照必须输出 active Pingora env 的 shadow/direct 姿态摘要。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'EnvironmentFile 未包含本次 --pingora-env-file', + reason: + 'Pingora 切换状态快照必须在 systemd EnvironmentFile 漂移时给出明确诊断。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertFailOnCriticalRejectsReleaseAuditFailure', + reason: 'Pingora 状态快照自测必须覆盖 current release 自审失败负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertFailOnCriticalRejectsSystemdPingoraEnvDrift', + reason: 'Pingora 状态快照自测必须覆盖 systemd 读取了另一份 Pingora env 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertSnapshotRejectsDirectPingoraEnvModeDrift', + reason: + 'Pingora 状态快照自测必须覆盖启用后 active env 仍是 shadow 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertSnapshotRejectsShadowPingoraEnvModeDrift', + reason: + 'Pingora 状态快照自测必须覆盖回退后 active env 仍残留 direct TLS 配置的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertSnapshotRedactsEnvProbeTokens', + reason: 'Pingora 状态快照自测必须确认 env probe token 不进入快照输出。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora 状态快照自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora 状态快照自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsFilesystemRootReleaseRoot', + reason: 'Pingora 状态快照自测必须覆盖 release root 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsPathArgsWithControlCharacters', + reason: + 'Pingora 状态快照自测必须覆盖带控制字符路径参数在执行 systemctl 前失败。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsFilesystemRootEnvFiles', + reason: 'Pingora 状态快照自测必须覆盖 env 文件参数指到 / 的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'snapshot.stdout.txt', + reason: 'Pingora 切换证据包必须保存状态快照 stdout 便于复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.stdout.txt', + reason: + 'Pingora 切换证据包必须保存 direct live stdout,保留 request_id / access log 复核证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'parseDirectLiveJson', + reason: + 'Pingora 切换证据包必须能从 direct live stdout 中提取 JSON 结果。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'pingoraEnvShadow', + reason: + 'Pingora 切换证据包必须把状态快照中的 Pingora env shadow 摘要提升到 manifest。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--expected-pingora-env-mode', + reason: + 'Pingora 切换证据包必须把 active Pingora env 姿态要求透传给状态快照。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'tlsCertFile 应为空', + reason: + 'Pingora env shadow 摘要必须把 TLS 证书路径残留纳入回退证据诊断。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'snapshotParseError', + reason: + 'Pingora 切换证据包必须把 snapshot 解析失败文件纳入 manifest 和 stdout 索引。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveParseError', + reason: + 'Pingora 切换证据包必须把 direct live 解析失败文件纳入 manifest 和 stdout 索引。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--run-direct-live', + reason: + 'Pingora 切换证据包必须提供显式 direct live 归档开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-pingora-access-log', + reason: + 'Pingora 切换证据包 direct live 归档必须接入 access log request_id 校验路径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'SECRET_VALUE_FLAGS', + reason: 'Pingora 切换证据包命令记录必须集中维护敏感参数脱敏列表。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '', + reason: 'Pingora 切换证据包命令记录不能落盘 probe token 原文。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'args: redactSecretArgs', + reason: 'Pingora 切换证据包命令记录必须保存结构化脱敏参数数组。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateEvidenceCommandArgs', + reason: + 'Pingora 切换证据包必须在执行状态快照或 direct live 前拒绝带控制字符的子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateSafeAbsoluteFilePath', + reason: + 'Pingora 切换证据包 direct live 显式文件路径必须统一拒绝相对路径、根目录和控制字符。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-spacetime-database', + reason: + 'Pingora 切换证据包必须在配置层拒绝带控制字符的 direct SpacetimeDB 数据库名。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-probe-token', + reason: + 'Pingora 切换证据包必须在配置层拒绝带控制字符的 direct live probe token。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 切换证据包必须明确拒绝带换行或 NUL 的状态快照 / direct live 参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: "createHash('sha256')", + reason: 'Pingora 切换证据包 manifest 必须记录证据文件 sha256,便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'sizeBytes', + reason: 'Pingora 切换证据包 manifest 必须记录证据文件大小,便于归档后复核。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'snapshot-command.json 结构化参数数组必须保留快照参数', + reason: 'Pingora 切换证据包自测必须覆盖结构化命令参数记录。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertManifestFileMetadata', + reason: 'Pingora 切换证据包自测必须覆盖 manifest 文件大小与 sha256 元数据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertBundleCanArchiveDirectLiveEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live 证据归档成功路径。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertBundleSummarizesPingoraEnvShadow', + reason: + 'Pingora 切换证据包自测必须覆盖 Pingora env shadow 摘要成功路径。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveFailureStillWritesEvidenceAndFails', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live 失败时保留证据并失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveParseFailureWritesParseErrorEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live stdout 解析失败时保留 parse error 证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertSnapshotParseFailureWritesParseErrorEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 snapshot stdout 解析失败时保留 parse error 证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsSnapshotArgsWithControlCharacters', + reason: + 'Pingora 切换证据包自测必须覆盖带控制字符的状态快照参数在执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsDirectLiveArgsWithControlCharacters', + reason: + 'Pingora 切换证据包自测必须覆盖带控制字符的 direct live 参数在执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: '--direct-pingora-access-log 不能是文件系统根目录', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live access log 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: '--direct-access-log-since-lines 不能包含换行或 NUL 字符', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live access log tail 行数带控制字符的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'BUNDLE_DIR_MODE = 0o750', + reason: 'Pingora 切换证据包目录必须显式限制为 owner/group 可读。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'EVIDENCE_FILE_MODE = 0o640', + reason: 'Pingora 切换证据文件必须显式禁止 world-readable。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: "flag: 'wx'", + reason: 'Pingora 切换证据包写文件必须避免覆盖既有证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validatePhase(config.phase);', + reason: 'Pingora 切换证据包必须拒绝不安全 phase,避免目录名和 manifest 阶段漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateOutputRootForWriting(outputRoot);', + reason: + 'Pingora 切换证据包写入前必须复核 output-root 路径,避免写入软链或非目录。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--output-root 不能是符号链接', + reason: 'Pingora 切换证据包必须拒绝符号链接 output-root。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--output-root 已存在但不是目录', + reason: 'Pingora 切换证据包必须拒绝已存在但不是目录的 output-root。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '不能是文件系统根目录', + reason: 'Pingora 切换证据包必须拒绝把显式路径参数指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora 切换证据包 timeout 参数必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora 切换证据包布尔 env 必须严格校验,避免证据生成开关拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsUnsafePhase', + reason: 'Pingora 切换证据包自测必须覆盖不安全 phase 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertBundleRedactsProbeTokensFromArtifacts', + reason: 'Pingora 切换证据包自测必须确认所有证据文件不落盘 probe token 原文。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertMode', + reason: 'Pingora 切换证据包自测必须断言目录和文件权限。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora 切换证据包自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora 切换证据包自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsSymlinkOutputRootBeforeSnapshot', + reason: + 'Pingora 切换证据包自测必须覆盖符号链接 output-root,且失败时不执行状态快照。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsFileOutputRootBeforeSnapshot', + reason: + 'Pingora 切换证据包自测必须覆盖非目录 output-root,且失败时不执行状态快照。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsFilesystemRootPaths', + reason: + 'Pingora 切换证据包自测必须覆盖显式路径参数指向文件系统根目录的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--require-pingora-gateway', + reason: 'Pingora 切换证据包必须能透传发布物自审要求。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '只写 --output-root 下的新证据目录', + reason: 'Pingora 切换证据包必须明确不会修改 /etc、systemd、Nginx 或 Pingora。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command.stdout.txt', + reason: 'Pingora 切换命令证据脚本必须保存真实命令 stdout。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command.stderr.txt', + reason: 'Pingora 切换命令证据脚本必须保存真实命令 stderr。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command-record.json', + reason: 'Pingora 切换命令证据脚本必须保存结构化命令记录。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "createHash('sha256')", + reason: 'Pingora 切换命令证据 manifest 必须记录证据文件 sha256,便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'sizeBytes', + reason: 'Pingora 切换命令证据 manifest 必须记录证据文件大小,便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'redactSecrets(text', + reason: 'Pingora 切换命令证据脚本必须脱敏 stdout/stderr 中的 probe token。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'validateCommandPath(config.command);', + reason: 'Pingora 切换命令证据脚本必须拒绝非绝对路径真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'path.resolve(command) === path.parse(path.resolve(command)).root', + reason: 'Pingora 切换命令证据脚本必须拒绝文件系统根目录真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '/[\\0\\r\\n]/u.test(command)', + reason: 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令参数不能包含换行或 NUL 字符', + reason: 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的真实命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '--expected-executable', + reason: 'Pingora 切换命令证据脚本必须支持绑定预期真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "validateSafeExecutablePath(config.expectedExecutable, '--expected-executable')", + reason: 'Pingora 切换命令证据生成端必须拒绝不安全 expected executable 路径。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '--require-arg', + reason: 'Pingora 切换命令证据脚本必须支持执行前校验真实命令必需参数。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令与 --expected-executable 不一致', + reason: 'Pingora 切换命令证据脚本必须拒绝挂错真实执行脚本。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令参数缺少 --require-arg 要求', + reason: 'Pingora 切换命令证据脚本必须在执行前拒绝缺少必需参数的真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "validateNoControlCharacters(config.outputRoot, '--output-root')", + reason: + 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的 output-root。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertCommandFailureStillWritesEvidenceAndFails', + reason: 'Pingora 切换命令证据自测必须覆盖真实命令失败时仍保留证据。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsMissingRequiredArgBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖缺少 --apply 时执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsBareCommandName', + reason: 'Pingora 切换命令证据自测必须覆盖 PATH 裸命令名被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsFilesystemRootCommand', + reason: 'Pingora 切换命令证据自测必须覆盖文件系统根目录真实命令被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsOutputRootWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖 output-root 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsCommandWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖带控制字符真实命令被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsCommandArgWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖带控制字符真实命令参数被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsFilesystemRootExpectedExecutable', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsExpectedExecutableWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertCommandEvidenceRedactsProbeTokensFromArtifacts', + reason: 'Pingora 切换命令证据自测必须覆盖 token 不落盘。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertManifestFileMetadata', + reason: 'Pingora 切换命令证据自测必须覆盖 manifest 文件大小与 sha256 元数据。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsSymlinkOutputRootBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖符号链接 output-root 且失败时不执行真实命令。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsMismatchedExpectedExecutableBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 不匹配且失败时不执行真实命令。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertPingoraDirectModeChecksPingoraServiceAndPublicHost', + reason: + '生产健康巡检动态烟测必须覆盖 Pingora 直连模式和 public Host header。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertRejectsInvalidTimeoutAndSlowThreshold', + reason: + '生产健康巡检动态烟测必须覆盖非法 timeout 和慢请求阈值负例。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + '生产健康巡检布尔 env 必须严格校验,避免 fail-on-warning / skip-journal 拼写错误静默降级。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: '生产健康巡检动态烟测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + reason: + '生产健康巡检 env 复核必须阻断直连后本机探测缺少正式 Host 的配置。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + '生产健康巡检 env 复核布尔 env 必须严格校验,避免 require-empty-public-host 拼写错误静默降级。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: 'isFilesystemRootPath', + reason: '生产健康巡检 env 复核必须显式拒绝把 env 文件指到 /。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsPingoraDirectLoopbackWithoutHost', + reason: + '生产健康巡检 env 自测必须覆盖 Pingora direct loopback 缺 Host 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: '生产健康巡检 env 自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsInvalidEnvFilePaths', + reason: '生产健康巡检 env 自测必须覆盖相对路径和 / 路径负例。', + }, + { + file: 'deploy/logrotate/genarrative-pingora-gateway', + includes: '/var/log/genarrative/pingora-gateway.access.log', + reason: + 'Pingora access log 必须有 logrotate 模板,避免 canary 日志无限增长。', + }, { file: 'deploy/systemd/genarrative-health-patrol.timer', includes: 'genarrative-health-patrol.service', reason: '健康巡检 timer 必须绑定巡检 service。', }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + includes: '/var/log/genarrative', + reason: 'Pingora systemd 沙箱必须允许写入 access log 目录。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + includes: '影子网关只监听本机高端口', + reason: + 'Pingora 主 service 必须保持默认 shadow 口径,不能默认声明公网直连。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + excludes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: + 'Pingora 主 service 不能默认带低端口 capability,直连只能通过显式 drop-in 启用。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连 80/443 时必须通过显式 drop-in 授予低端口绑定能力。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连 drop-in 必须把 capability 边界限制为低端口绑定。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: + '/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf', + reason: 'Pingora 直连 drop-in 必须明确人工启用位置,不能被误认为默认生效。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + includes: 'X-Genarrative-Nginx-Handoff', + reason: + 'Pingora canary snippet 必须带明确响应头,方便验证 Nginx 已经切到 Pingora handoff。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + includes: 'deny all', + reason: + 'Pingora canary snippet 默认必须限制访问来源,避免误 include 后公开接流。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'listen 127.0.0.1:18083;', + reason: + 'Pingora 真实路径 canary 必须默认只监听本机端口,避免误 include 后公开接流。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'genarrative-pingora-realpath-canary.access.log', + reason: + 'Pingora 真实路径 canary 必须使用独立 Nginx access log,避免和生产真实路径请求混淆。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'pingora-realpath-canary', + reason: + 'Pingora 真实路径 canary 必须带独立 handoff 标识,方便区分前缀 canary。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'location = /api/creation-entry/config', + reason: + 'Pingora 真实路径 canary 必须覆盖代表性 API 真实路径。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'zz-genarrative-pingora-realpath-canary.conf', + reason: + 'Pingora 真实路径 canary 运维说明必须记录 conf.d 加载顺序,避免早于 log_format 定义加载。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'unknown log format "genarrative_upstream"', + reason: + 'Pingora 试点文档必须记录 realpath canary 的 Nginx log_format 加载顺序踩坑。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'Pingora realpath canary include 要晚于 log_format', + reason: + '团队共享踩坑必须记录 realpath canary 独立 server 在 conf.d 中的加载顺序要求。', + }, { file: 'scripts/jenkins-server-provision.sh', includes: 'genarrative-health-patrol.timer', reason: 'Server-Provision 必须安装并启用健康巡检 timer。', }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-canary.conf', + reason: + 'Server-Provision 必须安装 Pingora canary snippet,供人工启用 Nginx handoff。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-realpath-canary.conf', + reason: + 'Server-Provision 必须安装 Pingora 真实路径 canary snippet,供人工启用直连前中间验证。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-gateway', + reason: 'Server-Provision 必须安装 Pingora access log 的 logrotate 配置。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: + '/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf', + reason: + 'Server-Provision 必须只把 Pingora 直连 drop-in 安装为人工启用模板。', + }, { file: 'scripts/jenkins-server-provision.sh', includes: 'genarrative-external-generation-controller.service', @@ -68,33 +1002,5325 @@ const checks = [ includes: 'production-health-patrol.mjs', reason: '生产 API release 必须携带健康巡检脚本。', }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: '生产 API release 必须携带 Pingora current release 自审脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs', + reason: '生产 API release 必须携带 Pingora 直连彩排状态脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: '生产 API release 必须携带 Pingora 切换状态快照脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: '生产 API release 必须携带 Pingora 切换证据包脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: '生产 API release 必须携带 Pingora 切换命令证据脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-production-health-patrol-env.mjs', + reason: '生产 API release 必须携带健康巡检 env 复核脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-release-readiness.mjs', + reason: + '生产 API release 必须携带 Pingora release readiness 聚合门禁脚本,确保正式 runbook 不依赖源码 checkout。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-canary-live.mjs', + reason: '生产 API release 必须携带 Pingora canary live smoke 脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-canary-access-log-parity.mjs', + reason: '生产 API release 必须携带 Pingora canary access log 对账脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: '--include-pingora-gateway', + reason: 'Pingora 影子网关必须保持显式可选产物打包入口。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_pingora_build_toolchain', + reason: '显式构建 Pingora 影子网关时必须先检查 cmake 和 C/C++ 编译器。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_command cmake', + reason: 'Pingora release 构建必须对 cmake 缺失 fail-fast。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_any_command "C++ 编译器"', + reason: 'Pingora release 构建必须对 C++ 编译器缺失 fail-fast。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: '生产 API release 必须携带随产物执行的 API deploy 脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: + '同目录的 \\`maintenance-on.sh\\` / \\`maintenance-off.sh\\` 必须来自同一发布包', + reason: + '生产 API release README 必须说明 deploy 同目录维护脚本来自同一发布包。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/maintenance-on.sh', + reason: '生产 API release 必须在 deploy 脚本同目录携带 maintenance-on.sh。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/maintenance-off.sh', + reason: + '生产 API release 必须在 deploy 脚本同目录携带 maintenance-off.sh。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: '生产 API release 必须携带 Pingora 直连回退脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: '生产 API release 必须携带 Pingora realpath canary 启用脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: '生产 API release 必须携带 Pingora realpath canary 关闭脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: '生产 API release 必须携带 Pingora health patrol env 切换脚本。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-expected-public-base-url', + reason: + 'Pingora 直连回退脚本必须能复核 health patrol public base URL 已恢复为切换前记录值。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-expected-public-host', + reason: + 'Pingora 直连回退脚本必须能复核切换前存在的 health patrol public Host 覆盖。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: '生产 API release 必须携带 Pingora 直连启用脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: + '生产 API release 必须携带 Pingora 直连预检脚本,确保 current release 可独立执行直连启用。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + '生产 API release 必须携带 Pingora 直连 live smoke 脚本,确保启用脚本 apply 后可独立验收。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'deploy/pingora', + reason: + '生产 API release 必须携带 Pingora 配置目录,供直连预检读取 env 示例和路由矩阵。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'deploy/env', + reason: + '生产 API release 必须携带 env 示例目录,供健康巡检和 Pingora live smoke 复制配置。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ENV_DEPLOY_DIR_SOURCE', + reason: 'API deploy 必须从发布产物复制 env 示例目录到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CANARY_LIVE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora canary live smoke 脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora realpath canary 启用脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora realpath canary 关闭脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 Pingora realpath canary 启用脚本', + reason: 'API deploy 缺少 realpath canary 启用脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 Pingora realpath canary 关闭脚本', + reason: 'API deploy 缺少 realpath canary 关闭脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora current release 自审脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 直连彩排状态脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换状态快照脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换证据包脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换命令证据脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/env/health-patrol.env.example', + reason: 'API release 动态烟测必须断言健康巡检 env 示例随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: 'API release 动态烟测必须断言 Pingora current release 自审脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-direct-rehearsal-status.mjs', + reason: 'API release 动态烟测必须断言 Pingora 直连彩排状态脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换状态快照脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换证据包脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换命令证据脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: 'API release 动态烟测必须断言健康巡检 env 复核脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API release 动态烟测必须断言 Pingora realpath canary 启用脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API release 动态烟测必须断言 Pingora realpath canary 关闭脚本随包发布。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/env/health-patrol.env.example', + reason: + 'API deploy 动态烟测必须断言健康巡检 env 示例复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora current release 自审脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换状态快照脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换证据包脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换命令证据脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: + 'API deploy 动态烟测必须断言 Pingora realpath canary 启用脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: + 'API deploy 动态烟测必须断言 Pingora realpath canary 关闭脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraRealpathCanaryEnableFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora realpath canary 启用脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraRealpathCanaryDisableFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora realpath canary 关闭脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraCutoverCommandEvidenceFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora 切换命令证据脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolEnvCheckFails', + reason: + 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingEnvExamplesFails', + reason: 'API deploy 动态烟测必须覆盖缺少 env 示例目录时 fail-fast。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-direct-enable', + reason: 'Pingora 直连启用必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-realpath-canary-toggle', + reason: 'Pingora realpath canary 启停脚本必须有本地自测入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-realpath-canary-enable', + reason: 'Pingora realpath canary 启用必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-realpath-canary-disable', + reason: 'Pingora realpath canary 关闭必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-direct-rollback', + reason: 'Pingora 直连回退必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-gateway-smoke', + reason: 'Pingora 网关必须保留可重复本地 smoke 入口。', + }, + { + file: 'package.json', + includes: 'check:nginx-pingora-canary', + reason: 'Pingora Nginx canary snippet 必须保留可重复本地校验入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-route-parity', + reason: 'Pingora 与 Nginx 路由矩阵必须保留可重复本地校验入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-live', + reason: 'Pingora Nginx 前缀 canary 必须保留启用后的真实入口 smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-live', + reason: 'Pingora 直连 HTTPS / HTTP redirect 必须保留目标机 live smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-live-guard', + reason: + 'Pingora direct live 参数护栏必须保留单独 npm 入口,方便正式切换前聚合门禁和本地 focused 验证复用。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-preflight', + reason: 'Pingora 直连切换前必须保留目标机 env / systemd / 端口预检入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-enable', + reason: 'Pingora 直连启用必须保留可重复 dry-run 行为检查入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-rollback', + reason: 'Pingora 直连回退必须保留可重复 dry-run 行为检查入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-docker', + reason: 'Pingora Nginx handoff 必须保留本机 Docker 可复现 smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-production-release-build', + reason: 'Pingora release 真实构建烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-release-readiness', + reason: 'Pingora 正式切换前必须保留聚合门禁入口。', + }, + { + file: 'package.json', + includes: 'plan:pingora-direct-cutover', + reason: 'Pingora 直连切换必须保留可审阅 runbook 计划入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-release-readiness-plan', + reason: 'Pingora 聚合门禁计划必须有可重复自检入口。', + }, + { + file: 'package.json', + includes: 'check:production-api-release', + reason: 'API release build 动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:production-health-patrol', + reason: '生产健康巡检动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:production-health-patrol-env', + reason: '生产健康巡检 env 复核动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-health-patrol-env-switch', + reason: 'Pingora health patrol env 切换脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-gateway-env-shadow-switch', + reason: 'Pingora gateway env shadow 切换脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-current-release-audit', + reason: 'Pingora current release 自审必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-rehearsal-status', + reason: 'Pingora 直连彩排状态脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-status-snapshot', + reason: 'Pingora 切换状态快照脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-evidence-bundle', + reason: 'Pingora 切换证据包必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-command-evidence', + reason: 'Pingora 切换命令证据必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-access-log-parity', + reason: 'Pingora canary access log 对账必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-live-guard', + reason: + 'Pingora canary live 参数护栏必须保留单独 npm 入口,方便正式切换前 focused 验证复用。', + }, + { + file: 'package.json', + includes: 'check:production-api-deploy', + reason: 'API deploy 动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-api-release', + reason: 'Pingora 聚合门禁必须执行 API release build 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-production-release-build', + reason: 'Pingora 聚合门禁必须执行真实 Pingora release 构建烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'Pingora production release 真实构建烟测', + reason: 'Pingora 聚合门禁计划自检必须覆盖真实 release 构建烟测。', + }, + { + file: 'scripts/check-pingora-production-release-build.mjs', + includes: '--include-pingora-gateway', + reason: 'Pingora production release 构建烟测必须走真实 include 打包路径。', + }, + { + file: 'scripts/check-pingora-production-release-build.mjs', + includes: 'CARGO_TARGET_DIR', + reason: 'Pingora production release 构建烟测必须使用临时 target,避免污染本地默认 target。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-api-deploy', + reason: 'Pingora 聚合门禁必须执行 API deploy 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'redactSecretArgs(args).join', + reason: 'Pingora 聚合门禁真实执行日志必须隐藏 probe token 参数值。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRunStepLogRedactsDirectProbeToken', + reason: 'Pingora 聚合门禁计划自测必须覆盖真实执行日志 token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-health-patrol', + reason: 'Pingora 聚合门禁必须执行生产健康巡检动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-health-patrol-env', + reason: 'Pingora 聚合门禁必须执行生产健康巡检 env 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-health-patrol-env-switch', + reason: 'Pingora 聚合门禁必须执行 health patrol env 切换脚本烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-gateway-env-shadow-switch', + reason: 'Pingora 聚合门禁必须执行 gateway env shadow 切换脚本烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-current-release-audit', + reason: 'Pingora 聚合门禁必须执行 current release 自审烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-rehearsal-status', + reason: 'Pingora 聚合门禁必须执行直连彩排状态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-status-snapshot', + reason: 'Pingora 聚合门禁必须执行切换状态快照烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-evidence-bundle', + reason: 'Pingora 聚合门禁必须执行切换证据包烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-command-evidence', + reason: 'Pingora 聚合门禁必须执行切换命令证据烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'Pingora cutover runbook apply 步骤必须使用 current release 随包命令证据脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--expected-executable', + reason: 'Pingora cutover runbook apply 步骤必须把命令证据绑定到预期 current release 脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-arg', + reason: 'Pingora cutover runbook apply 步骤必须让命令证据脚本提前校验 --apply 参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'pingora-direct-rollback-apply', + reason: 'Pingora runbook 自检必须确认 rollback apply 被命令证据脚本归档。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '必须在命令证据脚本中要求真实命令参数', + reason: 'Pingora runbook 自检必须锁住命令证据脚本的 --apply 早失败参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'pingora-direct-enable-apply', + reason: 'Pingora runbook 自检必须确认 enable apply 被命令证据脚本归档。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用命令证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在 enable apply 命令证据生成后立即验真 manifest。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 必须提示用 enable apply 命令证据 bundleDir 执行即时验真。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-summary-ok', + reason: 'Pingora cutover runbook 的即时证据验真必须要求 manifest.summary.status 为 OK。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退命令证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在 rollback apply 命令证据生成后立即验真 manifest。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 必须提示用 rollback apply 命令证据 bundleDir 执行即时验真。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用命令证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 enable apply 命令证据即时验真脚本来源。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '回退命令证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 rollback apply 命令证据即时验真脚本来源。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须先验真 enable apply 命令证据,再切换 health patrol 到 pingora-direct', + reason: 'Pingora runbook 自检必须锁住 enable apply 命令证据即时验真的执行顺序。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须先验真 rollback apply 命令证据,再执行回退后 env 复核', + reason: 'Pingora runbook 自检必须锁住 rollback apply 命令证据即时验真的执行顺序。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-canary-access-log-parity', + reason: 'Pingora 聚合门禁必须执行 canary access log 对账烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-canary-live-guard.mjs', + reason: + 'Pingora 聚合门禁必须执行 canary live 参数护栏,避免正式切换前漏掉脚本输入污染负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'Pingora canary access log 对账烟测', + reason: 'Pingora 聚合门禁计划自检必须覆盖 canary access log 对账烟测。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: '复核失败时 apply 不应写入真实 health patrol env 文件', + reason: 'Pingora health patrol env 切换脚本自测必须覆盖复核失败不落盘。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora health patrol env 切换脚本布尔 env 必须严格校验,避免 clear-public-host 拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'DRY_RUN_ENV_FILE_MODE = 0o600', + reason: 'Pingora health patrol env 切换脚本的临时复核 env 必须显式限制为 0600。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertDryRunEnvFileModeIsPrivate', + reason: 'Pingora health patrol env 切换脚本自测必须动态验证临时复核 env 权限。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'runHealthPatrolEnvCheck(checkFile);', + reason: 'Pingora health patrol env 切换脚本必须先复核临时目标 env。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'writeEnvFileAtomically(config.envFile, nextText);', + reason: 'Pingora health patrol env 切换脚本只能在复核通过后原子写入真实 env。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'chownSync(tempFile, currentStat.uid, currentStat.gid);', + reason: + 'Pingora health patrol env 切换脚本原子替换真实 env 时必须保留 owner/group。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertApplyPreservesEnvFileModeAndOwnership', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖 apply 保留 mode 与 owner/group。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '--env-file 不能是符号链接', + reason: + 'Pingora health patrol env 切换脚本 apply 必须拒绝符号链接 env,避免替换链接或写入非预期目标。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora health patrol env 切换脚本必须显式拒绝把路径参数指到 /。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '--check-script 不能是文件系统根目录', + reason: 'Pingora health patrol env 切换脚本必须拒绝把复核脚本路径指到 /。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'function validateNoControlCharacters(value, label)', + reason: + 'Pingora health patrol env 切换脚本必须拒绝 env 路径、check script、public URL 和 Host 中的换行 / NUL。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertApplyRejectsSymlinkEnvFileBeforeWrite', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖符号链接 env 被拒绝且不写目标文件。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsFilesystemRootPaths', + reason: 'Pingora health patrol env 切换脚本自测必须覆盖路径参数指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeWrite', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖控制字符参数在 apply 写入前失败。', + }, + { + file: 'scripts/check-pingora-gateway-env-shadow-switch.mjs', + includes: 'assertApplyRestoresShadowAndPreservesOtherKeys', + reason: 'Pingora gateway env shadow 切换脚本自测必须覆盖恢复 shadow 并保留其它键。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_LISTEN: \'127.0.0.1:18081\'', + reason: 'Pingora gateway env shadow 切换脚本必须固定恢复 shadow 高端口监听。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN: \'\'', + reason: 'Pingora gateway env shadow 切换脚本必须清空 TLS 低端口监听。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN: \'\'', + reason: 'Pingora gateway env shadow 切换脚本必须清空 HTTP redirect 低端口监听。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE: \'\'', + reason: + 'Pingora gateway env shadow 切换脚本必须清空 TLS cert,避免回退后形成不完整 TLS 配置。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE: \'\'', + reason: + 'Pingora gateway env shadow 切换脚本必须清空 TLS key,避免回退后形成不完整 TLS 配置。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'assertShadowEnv(checkFile);', + reason: 'Pingora gateway env shadow 切换脚本必须先复核临时目标 env。', + }, + { + file: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + includes: 'writeEnvFileAtomically(config.envFile, nextText);', + reason: 'Pingora gateway env shadow 切换脚本只能在复核通过后原子写入真实 env。', + }, + { + file: 'scripts/check-pingora-gateway-env-shadow-switch.mjs', + includes: 'assertRejectsSymlinkEnvFileBeforeWrite', + reason: 'Pingora gateway env shadow 切换脚本自测必须覆盖符号链接 env 被拒绝。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--release-runtime-only', + reason: + 'Pingora release readiness 必须支持 current release 包内运行时复核模式,避免目标机依赖源码 checkout。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'buildReleaseRuntimeSteps', + reason: 'Pingora release runtime-only 模式必须单独构建包内自包含检查步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'releaseScriptPath', + reason: 'Pingora release runtime-only 模式必须从 current release 解析随包脚本绝对路径。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-live', + reason: 'Pingora 正式切换聚合门禁必须支持目标机 live canary 强制验收。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-live 时必须提供 --live-host', + reason: 'Pingora 强制 live canary 门禁不能跳过正式域名 Host header 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '目标 Nginx live canary access log 对账', + reason: 'Pingora 强制 live canary 门禁必须在 live smoke 后执行真实 access log 对账。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--live-nginx-access-log', + reason: 'Pingora 强制 live canary 门禁必须允许显式指定 Nginx access log。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--live-pingora-access-log', + reason: 'Pingora 强制 live canary 门禁必须允许显式指定 Pingora access log。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'validateSafeAbsoluteFilePath(filePath, label);', + reason: + 'Pingora release readiness 的 live/direct 文件路径必须统一拒绝相对路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts', + reason: 'Pingora release readiness plan 自测必须覆盖 runtime-only 使用 current release 随包脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseRuntimeOnlyRejectsSourceOnlyFlags', + reason: 'Pingora release readiness plan 自测必须覆盖 runtime-only 拒绝源码/本机依赖参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 计划必须在 live smoke 后包含 access log 对账', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live smoke 后真实日志对账。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 的 Nginx access log 不能接受相对路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live 日志路径负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireLiveRejectsFilesystemRootAccessLog', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 live 日志路径指向文件系统根目录的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-direct', + reason: 'Pingora 正式切换聚合门禁必须支持目标机直连入口强制验收。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--dry-run-cutover', + reason: 'Pingora 正式直连切换必须支持只打印可审阅 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换 Host 一致性确认', + reason: 'Pingora cutover runbook 必须输出 Host 一致性确认步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'assertSameCutoverHostname', + reason: 'Pingora cutover runbook 必须在生成阶段阻断 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--rollback-pingora-shadow-probe-url', + reason: 'Pingora cutover runbook 必须支持把可选 shadow probe 参数传给 rollback。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--rollback-health-patrol-public-base-url', + reason: + 'Pingora cutover runbook 必须显式声明回退后恢复的 Nginx public base URL。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '避免回退 runbook 覆盖现场原有 Nginx 巡检入口', + reason: + 'Pingora cutover runbook 不能把回退后的 health patrol public base URL 隐式写成仓库默认值。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 输出不能泄露 probe token 原文。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'Pingora direct enable apply', + reason: 'Pingora cutover runbook 必须显式列出 direct enable apply 步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'Pingora direct rollback apply', + reason: 'Pingora cutover runbook 必须显式列出失败回退 apply 步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换后 health patrol 切到 Pingora direct', + reason: + 'Pingora cutover runbook 必须提醒直连启用后把健康巡检切到 pingora-direct。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退前 health patrol 预置回 Nginx', + reason: + 'Pingora cutover runbook 必须在 rollback apply 前提醒把健康巡检预置回 nginx。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退前 Pingora env 预置回 shadow', + reason: + 'Pingora cutover runbook 必须在 rollback apply 前提醒把 Pingora env 预置回 shadow。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'Pingora cutover runbook 必须用随包脚本恢复 Pingora shadow env。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 回退前 Pingora gateway env shadow 切换必须显式带 --apply。', + reason: + 'Pingora runbook 自检必须锁住 rollback apply 前 Pingora shadow env 脚本化预置步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'pingora-health-patrol-env-switch.mjs', + reason: + 'Pingora cutover runbook 必须使用随包脚本切换 health patrol env,不能只给手工编辑提示。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前 current release 自包含自审', + reason: 'Pingora cutover runbook 必须先审 current release 自包含能力。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前状态快照', + reason: + 'Pingora cutover runbook 必须在启用前采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前状态快照证据包', + reason: + 'Pingora cutover runbook 必须在启用前归档状态快照证据包。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-evidence-output-root', + reason: 'Pingora cutover runbook 必须显式固定证据包输出根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用后状态快照', + reason: + 'Pingora cutover runbook 必须在启用后采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--run-direct-live', + reason: + 'Pingora cutover runbook 启用后证据包必须归档 direct live 和 access log request_id 证据。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用后状态快照证据包必须归档 direct live smoke 证据', + reason: + 'Pingora runbook 自检必须锁住 post-enable 证据包的 direct live 归档。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在每个证据包阶段后提示只读验真。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 current release 随包证据验真脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '证据验真必须要求 manifest.summary.status=OK', + reason: 'Pingora runbook 自检必须锁住即时证据验真的 summary OK 要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches', + reason: 'Pingora runbook 自检必须确认正式切换计划不携带人工证据放行参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--allow-extra-root-entries', + reason: 'Pingora runbook 自检必须显式覆盖证据根目录额外条目放行参数不得进入正式计划。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'findExtraEvidenceEntries', + reason: 'Pingora 证据验真脚本必须扫描 manifest.files 未登记的额外条目。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '--require-summary-ok', + reason: 'Pingora 证据验真脚本必须支持正式 runbook 的 summary OK 强制模式。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'manifest.summary.status is not OK', + reason: 'Pingora 证据验真脚本必须在 summary 非 OK 时给出结构化失败原因。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra file not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记普通文件。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra directory not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记目录。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra symlink not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记符号链接。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '--allow-extra-files', + reason: 'Pingora 证据验真脚本允许额外条目时必须显式使用例外开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '正式切换归档不应使用', + reason: 'Pingora 证据验真脚本必须声明额外条目例外不属于正式归档口径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 证据验真脚本必须拒绝带换行或 NUL 的入口路径和 manifest 文件名。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 证据验真脚本必须对控制字符路径给出明确错误。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraFileFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记普通文件默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraDirectoryFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记目录默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraSymlinkFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记符号链接默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertAllowExtraFilesOverrideSucceeds', + reason: 'Pingora 证据验真自测必须覆盖显式允许额外条目的人工例外。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRequireSummaryOkFailsForCriticalManifest', + reason: 'Pingora 证据验真自测必须覆盖 summary 非 OK 时 --require-summary-ok 失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRequireSummaryOkFailsWhenSummaryMissing', + reason: 'Pingora 证据验真自测必须覆盖缺少 summary 时 --require-summary-ok 失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRejectsEntryPathControlCharacters', + reason: 'Pingora 证据验真自测必须覆盖入口路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRejectsManifestFilePathControlCharacters', + reason: 'Pingora 证据验真自测必须覆盖 manifest 文件 path 带控制字符的负例。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'manifest.files` 视为闭集', + reason: 'Pingora 试点文档必须说明证据验真默认闭集归档口径。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: 'manifest.files` 视为闭集', + reason: '生产运维文档必须说明证据验真默认闭集归档口径。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: 'manifest.files` 视为闭集', + reason: '团队共享决策必须记录 Pingora 证据验真闭集归档口径。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'manifest.files` 视为闭集', + reason: '团队共享踩坑必须记录 Pingora 证据目录不能夹带未登记条目。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'pingora-gateway-env-shadow-switch.mjs --apply', + reason: 'Pingora 试点文档必须记录 rollback apply 前用随包脚本恢复 Pingora shadow env。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: 'pingora-gateway-env-shadow-switch.mjs --apply', + reason: '生产运维文档必须记录 rollback apply 前用随包脚本恢复 Pingora shadow env。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: 'pingora-gateway-env-shadow-switch.mjs --apply', + reason: '团队共享决策必须记录 rollback apply 前用随包脚本恢复 Pingora shadow env。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'pingora-gateway-env-shadow-switch.mjs --apply', + reason: '团队共享踩坑必须记录 rollback apply 前用随包脚本恢复 Pingora shadow env。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'Nginx README 的 API release 清单必须包含 Pingora gateway env shadow 切换脚本。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'rollback apply 前必须先执行', + reason: 'Nginx README 必须说明 rollback apply 前先恢复 Pingora gateway shadow env。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'TLS_CERT_FILE', + reason: 'Nginx README 必须说明 Pingora gateway shadow 回退会清空 TLS cert/key 路径。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--expected-pingora-env-mode direct', + reason: 'Nginx README 必须说明 post-enable 证据包会强制校验 Pingora env direct 姿态。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--expected-pingora-env-mode shadow', + reason: 'Nginx README 必须说明 post-rollback 证据包会强制校验 Pingora env shadow 姿态。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'manifest.summary.pingoraEnvShadow', + reason: 'Nginx README 必须说明证据 manifest 会提升 Pingora env shadow 摘要。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'tlsCertFile', + reason: 'Nginx README 必须说明 Pingora env shadow 摘要包含 TLS cert/key 残留字段。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'tlsCertFile', + reason: 'Pingora 试点文档必须说明证据摘要包含 TLS cert/key 残留字段。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换证据根目录三阶段总审计', + reason: 'Pingora cutover runbook 必须在三阶段证据后执行根目录总审计。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'enable-apply:pingora-direct-enable-apply', + reason: + 'Pingora cutover runbook 总审计必须要求 direct enable apply 命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollback-apply:pingora-direct-rollback-apply', + reason: + 'Pingora cutover runbook 总审计必须要求 direct rollback apply 命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'post-enable:pingora-health-patrol-direct-env-switch', + reason: + 'Pingora cutover runbook 总审计必须要求 health patrol 切到 direct 的命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollback-prep:pingora-gateway-shadow-env-switch', + reason: + 'Pingora cutover runbook 总审计必须要求回退前 Pingora gateway env shadow 切换命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollback-prep:pingora-health-patrol-nginx-env-switch', + reason: + 'Pingora cutover runbook 总审计必须要求回退前 health patrol 切回 Nginx 的命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-phase-pingora-env-shadow', + reason: + 'Pingora cutover runbook 总审计必须要求 post-rollback Pingora env shadow 证据。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须使用 current release 随包 evidence audit 脚本', + reason: 'Pingora runbook 自检必须锁住 current release 随包证据根目录审计脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须要求 post-rollback 阶段带 Pingora env shadow 摘要。', + reason: + 'Pingora runbook 自检必须锁住总审计里的 post-rollback shadow env 证据要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换命令证据', + reason: 'Pingora runbook 自检必须锁住总审计里的 enable / rollback 和 env 切换命令证据要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'post-enable:pingora-health-patrol-direct-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: + 'Pingora runbook 自检必须锁住 health patrol direct env 切换命令的真实脚本身份。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'rollback-prep:pingora-gateway-shadow-env-switch:/opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: + 'Pingora runbook 自检必须锁住 Pingora gateway shadow env 切换命令的真实脚本身份。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'rollback-prep:pingora-health-patrol-nginx-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: + 'Pingora runbook 自检必须锁住 health patrol nginx env 切换命令的真实脚本身份。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-command-executable', + reason: 'Pingora cutover runbook 总审计必须要求真实切换命令证据的真实脚本身份。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换脚本身份', + reason: 'Pingora runbook 自检必须锁住总审计里的真实切换命令脚本身份要求。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-command-arg', + reason: 'Pingora cutover runbook 总审计必须要求真实切换命令证据包含关键参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换命令必需参数', + reason: 'Pingora runbook 自检必须锁住总审计里的真实切换命令参数要求。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "commandName: 'pingora-health-patrol-direct-env-switch'", + reason: + 'Pingora 证据根目录审计标准时间线必须包含 health patrol direct env 切换命令。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "commandName: 'pingora-gateway-shadow-env-switch'", + reason: + 'Pingora 证据根目录审计标准时间线必须包含 Pingora gateway shadow env 切换命令。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "commandName: 'pingora-health-patrol-nginx-env-switch'", + reason: + 'Pingora 证据根目录审计标准时间线必须包含 health patrol nginx env 切换命令。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '完整切换时间线总审计必须检查三阶段和五条命令', + reason: + 'Pingora 证据根目录审计自测必须把完整时间线固定为三阶段和五条命令。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-run-id', + reason: 'Pingora cutover runbook 必须公开并传递切换批次 ID。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-cutover-run-id', + reason: 'Pingora cutover runbook 最终总审计必须要求同一切换批次 ID。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverPlanCarriesCutoverRunId', + reason: 'Pingora runbook 自检必须覆盖切换批次 ID 贯穿所有证据步骤。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '不会修改证据目录', + reason: 'Pingora 证据根目录审计脚本必须声明只读边界。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command :', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command-executable ::', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令真实脚本身份。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: + '--require-command-executable executable 不能包含换行或 NUL 字符', + reason: + 'Pingora 证据根目录审计脚本必须拒绝带控制字符的命令脚本身份要求。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 证据根目录审计脚本必须拒绝带换行或 NUL 的根目录和 verifier 路径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '!/[\\0\\r\\n]/u.test(value)', + reason: + 'Pingora 证据根目录审计脚本必须拒绝 manifest 文件名里的控制字符。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'command executable 带控制字符必须失败', + reason: + 'Pingora 证据根目录审计自测必须覆盖命令脚本身份要求带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根目录带控制字符必须失败', + reason: 'Pingora 证据根目录审计自测必须覆盖 evidence-root 控制字符负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'verifier 路径带控制字符必须失败', + reason: 'Pingora 证据根目录审计自测必须覆盖 verify-script 控制字符负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertManifestFileNameControlCharacterFails', + reason: 'Pingora 证据根目录审计自测必须覆盖 manifest 文件名控制字符负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command-arg ::', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令必需参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'mergeRequiredCommands', + reason: 'Pingora 命令脚本身份要求必须隐式纳入命令证据审计,不能被静默忽略。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.expectedExecutable 必须是', + reason: 'Pingora 证据根目录审计脚本必须阻断命令证据 expectedExecutable 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'command.executable 必须是', + reason: 'Pingora 证据根目录审计脚本必须阻断命令证据真实 executable 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'readCommandRecord', + reason: 'Pingora 证据根目录审计脚本必须读取独立 command-record.json 做语义复核。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'compareCommandRecords', + reason: 'Pingora 证据根目录审计脚本必须阻断 manifest.command 与 command-record.json 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'stdoutPath', + reason: 'Pingora 证据根目录审计脚本必须复核命令 stdout 文件引用一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'stderrPath', + reason: 'Pingora 证据根目录审计脚本必须复核命令 stderr 文件引用一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'jsonValuesEqual', + reason: 'Pingora 证据根目录审计脚本必须复核命令参数数组一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'args 不能包含换行或 NUL 字符', + reason: + 'Pingora 证据根目录审计脚本必须拒绝被控制字符污染的命令参数证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'scalarFields', + reason: 'Pingora 证据根目录审计脚本必须复核脱敏命令字符串一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "'command'", + reason: 'Pingora 证据根目录审计脚本必须把脱敏命令字符串纳入一致性比较。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'checkCommandRecordTimes', + reason: 'Pingora 证据根目录审计脚本必须校验命令记录 startedAt / finishedAt / durationMs。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "'--require-summary-ok'", + reason: 'Pingora 证据根目录审计脚本复用 verifier 时必须启用 strict summary 模式。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.summary.status 必须是 OK', + reason: 'Pingora 证据根目录审计脚本必须阻断 summary 非 OK 的阶段或命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'candidate.commandName === null', + reason: 'Pingora 证据根目录审计脚本必须防止命令证据冒充阶段证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.summary.exitCode 必须是 0', + reason: 'Pingora 证据根目录审计脚本必须阻断退出码非 0 的命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'CUTOVER_TIMELINE_ORDER', + reason: 'Pingora 证据根目录审计脚本必须内置标准直连切换时间线顺序。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '疑似混入不同切换窗口证据', + reason: 'Pingora 证据根目录审计脚本必须阻断标准切换时间线倒序证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000', + reason: 'Pingora 证据根目录审计脚本必须默认限制标准切换证据窗口为 24 小时。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--timeline-max-span-ms', + reason: 'Pingora 证据根目录审计脚本必须允许切换窗口显式覆盖最大时间跨度。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-cutover-run-id', + reason: 'Pingora 证据根目录审计脚本必须能按切换批次 ID 过滤证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.cutoverRunId', + reason: 'Pingora 证据根目录审计脚本必须读取并校验 manifest.cutoverRunId。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--allow-extra-root-entries', + reason: 'Pingora 证据根目录审计脚本允许额外根目录条目时必须显式使用例外开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '正式切换归档不应使用', + reason: 'Pingora 证据根目录审计脚本必须声明额外根目录条目例外不属于正式归档口径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '证据根目录只能包含证据目录,发现普通文件', + reason: 'Pingora 证据根目录审计脚本必须默认拒绝根目录普通文件。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '证据根目录只能包含带 manifest.json 的证据目录', + reason: 'Pingora 证据根目录审计脚本必须默认拒绝无 manifest 的根目录子目录。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootFileFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖根目录普通文件默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootDirectoryFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖无 manifest 根目录子目录默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootSymlinkFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖根目录符号链接默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertAllowExtraRootEntriesSucceeds', + reason: 'Pingora 证据根目录审计自测必须覆盖显式允许额外根目录条目的人工例外。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandExecutableImplicitlyRequiresCommand', + reason: 'Pingora 证据根目录审计自测必须覆盖命令脚本身份参数隐式要求命令证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandExecutableMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令脚本身份漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandArgImplicitlyRequiresCommand', + reason: 'Pingora 证据根目录审计自测必须覆盖命令必需参数隐式要求命令证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandArgMissingFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据缺少 --apply 参数负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.command 漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordTimelineMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令记录时间线漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordOutputPathMismatchFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.files 的 stdout/stderr 引用漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordArgsMismatchFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.command 的参数漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordArgsControlCharacterFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖命令参数证据带控制字符的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '时间跨度', + reason: 'Pingora 证据根目录审计脚本必须阻断标准切换时间线跨度过大的证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCriticalPhaseSummaryFails', + reason: 'Pingora 证据根目录审计自测必须覆盖阶段证据 summary 非 OK 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandBundleCannotSatisfyPhaseRequirement', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据不能满足阶段证据要求。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '必须让 verifier 启用 --require-summary-ok', + reason: 'Pingora 证据根目录审计自测必须覆盖总审计调用 verifier 的 strict summary 模式。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertFailedCommandSummaryFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据退出失败负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertOutOfOrderCutoverTimelineFails', + reason: 'Pingora 证据根目录审计自测必须覆盖标准切换时间线倒序负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertLongCutoverTimelineFails', + reason: 'Pingora 证据根目录审计自测必须覆盖标准切换时间线跨度过大负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCutoverRunIdMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖切换批次 ID 不匹配负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-evidence-timeline-max-span-ms', + reason: 'Pingora cutover runbook 必须公开证据时间线最大跨度参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式传标准切换时间线最大跨度', + reason: 'Pingora runbook 自检必须锁住总审计的时间线最大跨度参数。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退后状态快照', + reason: + 'Pingora cutover runbook 必须在回退后采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退后 health patrol env Nginx 模式复核', + reason: + 'Pingora cutover runbook 必须在回退后复核 health patrol env 已回到 Nginx 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包自审脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包 health patrol env 切换脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包切换状态快照脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包切换证据包脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsRelativeEvidenceOutputRoot', + reason: 'Pingora runbook 自检必须覆盖证据包输出根目录相对路径负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsFilesystemRootReleaseRoot', + reason: 'Pingora runbook 自检必须覆盖 current release 根目录指向 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot', + reason: 'Pingora runbook 自检必须覆盖证据包输出根目录指向 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora runbook 参数校验必须显式拒绝文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot', + reason: + 'Pingora runbook usage 示例必须显式展示证据包输出根目录,避免值班复制命令时依赖隐藏默认值。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertUsageExamplesIncludeDirectPingoraAccessLog', + reason: + 'Pingora release readiness usage 示例必须显式展示 direct access log 参数,避免值班复制命令后被 --require-direct 拦住。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--clear-public-host', + reason: 'Pingora runbook 自检必须覆盖回退后清空 health patrol public Host。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核', + reason: 'Pingora runbook 自检必须覆盖回退后的健康巡检 env 机器复核步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'cutoverReleaseRoot', + reason: 'Pingora cutover runbook 必须允许固定 current release 根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-health-patrol-env-file', + reason: 'Pingora 强制直连门禁必须允许指定 health patrol env 文件做复核。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '目标 health patrol env 直连模式复核', + reason: 'Pingora 强制直连门禁必须复核 health patrol 已切到 direct 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-health-patrol-env-file', + reason: 'Pingora 强制直连门禁不能跳过 health patrol env 复核。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'Pingora 直连入口门禁必须先执行 direct-entry 预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-env-file', + reason: 'Pingora 直连入口门禁必须允许指定目标机 env 文件做预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-preflight-env-file', + reason: 'Pingora 强制直连门禁不能跳过目标机 env 预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-preflight-systemd', + reason: 'Pingora 强制直连门禁不能跳过 systemd drop-in 生效核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable', + reason: 'Pingora 强制直连门禁不能跳过 TLS 证书和私钥可读核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file', + reason: + 'Pingora 强制直连门禁不能跳过 service EnvironmentFile 与 preflight env 一致性核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable', + reason: + 'Pingora 强制直连门禁不能跳过 systemd 服务用户读取 TLS 证书和私钥核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable', + reason: + 'Pingora 强制直连门禁不能跳过 current release service 二进制可执行核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --dry-run-cutover 时必须提供 --direct-preflight-check-ports-free', + reason: + 'Pingora 切换前 runbook 门禁不能跳过 80/443 端口释放核验;启用后直连复核不再要求端口空闲。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-http-base-url', + reason: 'Pingora 强制直连门禁不能跳过 HTTP redirect / ACME live 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-host', + reason: + 'Pingora 强制直连门禁不能跳过正式域名 Host header 和 TLS SNI 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-redirect-host', + reason: 'Pingora 强制直连门禁不能跳过 HTTP redirect Location host 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-pingora-access-log', + reason: 'Pingora 强制直连门禁不能跳过 direct access log 落盘校验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: "validateSafeAbsoluteFilePath(\n result.directPingoraAccessLog,\n '--direct-pingora-access-log'", + reason: + 'Pingora 强制直连门禁必须拒绝相对 access log 路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--http-base-url', + reason: + 'Pingora 直连入口门禁必须把 HTTP redirect / ACME base URL 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--host', + reason: + 'Pingora 直连入口门禁必须把正式域名 Host/SNI 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--redirect-host', + reason: + 'Pingora 直连入口门禁必须把 redirect Location host 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--pingora-access-log', + reason: + 'Pingora 直连入口门禁必须把 access log 路径传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-systemd', + reason: 'Pingora 直连入口门禁必须支持检查 systemd drop-in 是否生效。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-user-cert-readable', + reason: 'Pingora 直连入口门禁必须支持检查 systemd 服务用户可读证书。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-env-file', + reason: + 'Pingora 直连入口门禁必须支持检查 service EnvironmentFile 与 preflight env 一致。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-binary-executable', + reason: 'Pingora 直连入口门禁必须支持检查 service ExecStart 二进制可执行。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-ports-free', + reason: 'Pingora 直连入口门禁必须支持检查 TLS/HTTP redirect 端口已释放。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-spacetime-database', + reason: + 'Pingora 直连入口门禁必须允许指定 WSS subscribe 的目标 SpacetimeDB 库。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-spacetime-database', + reason: 'Pingora 强制直连门禁不能让 WSS subscribe 隐式使用默认数据库名。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时不能使用 --direct-skip-wss', + reason: 'Pingora 强制直连门禁不能允许正式直连跳过 WSS subscribe。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时不能使用 --direct-insecure-tls', + reason: 'Pingora 强制直连门禁不能允许生产直连跳过 TLS 信任校验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-require-wss-upgrade', + reason: 'Pingora 直连入口门禁必须支持强制要求 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'result.directRequireWssUpgrade = true', + reason: + 'Pingora --require-direct 硬门禁默认必须强制 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-canary-docker.mjs', + reason: 'Pingora 正式切换聚合门禁必须包含 Docker Nginx handoff smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-route-parity', + reason: 'Pingora 正式切换聚合门禁必须包含 Nginx / Pingora 路由 parity。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-rollback', + reason: 'Pingora 正式切换聚合门禁必须包含直连回退 dry-run 行为检查。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-preflight', + reason: 'Pingora 正式切换聚合门禁必须包含直连入口静态预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-enable', + reason: 'Pingora 正式切换聚合门禁必须包含直连启用 dry-run 行为检查。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-release-readiness-plan', + reason: 'Pingora 正式切换聚合门禁必须自检关键计划语义。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把 HTTP redirect / ACME 入口纳入 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 HTTP redirect / ACME 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 必须把正式域名 Host 传给 live canary smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-live 的正式域名 Host 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 direct enable apply', + reason: 'Pingora 聚合门禁计划自检必须覆盖直连启用 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 rollback apply', + reason: 'Pingora 聚合门禁计划自检必须覆盖失败回退 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 缺少 --require-direct 时必须失败', + reason: 'Pingora cutover runbook 不能在缺少直连硬门禁参数时生成。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 Host 一致性确认步骤', + reason: 'Pingora cutover runbook 必须显式展示 direct / health patrol / rollback Host 一致性。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover redirect Host 与 direct Host 不一致时必须失败', + reason: 'Pingora cutover runbook 必须阻断 redirect Host 与正式入口 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback smoke Host 与 direct Host 不一致时必须失败', + reason: 'Pingora cutover runbook 必须阻断 rollback smoke Host 与正式入口 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback 步骤必须传递 shadow probe URL', + reason: 'Pingora cutover runbook 自检必须覆盖可选 rollback shadow probe 传递。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover JSON 不能泄露 shadow probe token 原文', + reason: 'Pingora cutover runbook 自检必须覆盖 shadow probe token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run plan JSON 不能泄露 direct probe token 原文', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct probe token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '启用后 release readiness 直连复核必须继续传递 direct probe token 参数', + reason: + 'Pingora cutover runbook 自检必须覆盖启用后 readiness 复核继续检查 direct probe。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 启用前基础门禁必须使用 current release 包内运行时复核模式', + reason: 'Pingora cutover runbook 自检必须覆盖启用前 readiness 使用 runtime-only 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用后 release readiness 直连复核必须使用 current release 包内运行时复核模式', + reason: 'Pingora cutover runbook 自检必须覆盖启用后 readiness 使用 runtime-only 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '启用后 release readiness 直连复核必须使用 current release 随包 release readiness 聚合门禁脚本', + reason: + 'Pingora cutover runbook 自检必须覆盖启用后 readiness 从 current release 随包脚本执行。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + 'dry-run cutover 启用前基础门禁必须使用 current release 随包 release readiness 聚合门禁脚本', + reason: + 'Pingora cutover runbook 自检必须覆盖启用前 readiness 从 current release 随包脚本执行。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover JSON 不能泄露 direct probe token 原文', + reason: + 'Pingora cutover runbook 自检必须覆盖 direct probe token 在切换计划中脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback shadow probe URL 缺少 token 时必须失败', + reason: 'Pingora cutover runbook 自检必须覆盖 shadow probe 参数成对出现。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须把正式域名 Host/SNI 传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的正式域名 Host/SNI 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把显式 redirect Location host 传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 redirect Location host 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 systemd drop-in 最终生效配置', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 systemd drop-in 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查当前用户可读 TLS 证书和私钥', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的证书可读硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 service EnvironmentFile 包含本次 env', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 service env 一致性硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 systemd 服务用户可读 TLS 证书和私钥', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的服务用户证书可读硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须检查 Pingora service ExecStart 二进制可执行', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 service 二进制可执行硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 启用前预检必须检查 80/443 端口释放', + reason: + 'Pingora 聚合门禁计划自检必须覆盖切换前端口释放硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把显式 SpacetimeDB 数据库名传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的显式 WSS 数据库名。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把 Pingora access log 落盘校验传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 Pingora access log 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct HTTP base URL 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 direct HTTP base URL 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 缺少 live Host 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 live Host 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 的 live Host 不能接受 URL 或路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live Host 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct Host/SNI 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少正式域名 Host/SNI 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 的 direct Host/SNI 不能接受 URL 或路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct Host/SNI 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct redirect Location host 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 redirect Location host 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 的 redirect Location host 不能接受 URL 或路径', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 redirect Location host 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 systemd preflight 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 systemd preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少证书可读 preflight 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 service EnvironmentFile preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 service env 一致性 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少服务用户证书可读 preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少服务用户证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 service 二进制可执行 preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 service 二进制可执行 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct post-enable 复核不能要求 80/443 空闲', + reason: + 'Pingora 聚合门禁计划自检必须覆盖启用后直连复核不再要求端口空闲。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 SpacetimeDB 数据库名时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少显式 WSS 数据库名的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct Pingora access log 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 direct access log 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 不能允许跳过 WSS subscribe', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct skip WSS 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 不能允许 insecure TLS', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct insecure TLS 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须自动强制 WSS subscribe 返回 101', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 WSS 101 硬要求。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--require-docker', + reason: + 'Pingora Docker canary smoke 必须支持在 CI / 目标 agent 上强制执行。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'Pingora Docker canary smoke 必须复用 live canary 断言,避免双套验收漂移。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--realpath', + reason: + 'Pingora Docker canary smoke 必须覆盖真实路径 canary handoff。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'genarrative-pingora-realpath-canary.conf', + reason: + 'Pingora Docker canary smoke 必须加载真实路径 canary snippet。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--network', + reason: + 'Pingora Docker canary smoke 必须真实启动 Nginx 容器验证 handoff 链路。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'log_format genarrative_upstream', + reason: + 'Pingora Docker canary smoke 必须按生产 Nginx access log 口径写 request_id。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'scripts/check-pingora-canary-access-log-parity.mjs', + reason: + 'Pingora Docker canary smoke 必须复用真实 access log 对账脚本验证 handoff。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/api/creation-entry/config', + reason: + 'Pingora Docker canary access log 对账必须覆盖代表性 API canary 路径。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/v1/identity', + reason: + 'Pingora Docker canary access log 对账必须覆盖 SpacetimeDB identity 路径。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/assets/app.js', + reason: 'Pingora Docker canary access log 对账必须覆盖静态资源路径。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: '该脚本只读比较 Nginx canary handoff access log 和 Pingora access log', + reason: 'Pingora canary access log 对账脚本必须说明只读边界。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'request_id', + reason: 'Pingora canary access log 对账必须按 request_id 串联 Nginx 与 Pingora 日志。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'realpath', + reason: 'Pingora canary access log 对账必须支持真实路径 canary 模式。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: '/__genarrative_pingora/healthz', + reason: 'Pingora canary access log 对账必须覆盖 canary healthz 到 shadow healthz 的真实映射。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora canary access log 对账 since-lines 必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'validateSafeAbsoluteFilePath(file, label);', + reason: 'Pingora canary access log 对账必须拒绝相对日志路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'function validateNoControlCharacters(value, label)', + reason: + 'Pingora canary access log 对账必须拒绝日志路径、前缀、必需路径和 since-lines 中的换行 / NUL。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsFilesystemRootLogPaths', + reason: 'Pingora canary access log 对账自测必须覆盖日志路径指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsLogPathControlCharacters', + reason: 'Pingora canary access log 对账自测必须覆盖日志路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsPrefixAndPathControlCharacters', + reason: + 'Pingora canary access log 对账自测必须覆盖 canary prefix 和必需路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsParsedLogPathControlCharacters', + reason: + 'Pingora canary access log 对账自测必须覆盖日志行解析出的 path 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: '缺少对应 Pingora access log', + reason: 'Pingora canary access log 对账自测必须覆盖缺少 Pingora 日志的失败场景。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsInvalidSinceLines', + reason: + 'Pingora canary access log 对账自测必须覆盖非法 since-lines 参数和 env 负例。', + }, + { + file: 'deploy/env/pingora-canary-live.env.example', + includes: 'GENARRATIVE_PINGORA_CANARY_BASE_URL', + reason: 'Pingora canary live smoke 必须有可复制的环境变量示例。', + }, + { + file: 'deploy/env/pingora-canary-live.env.example', + includes: '不要填 https:// URL', + reason: 'Pingora canary live smoke 环境示例必须提示 Host 只填 host。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL', + reason: 'Pingora direct live smoke 必须有可复制的环境变量示例。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + reason: + 'Pingora direct live smoke 环境示例必须暴露 HTTP redirect base URL。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + reason: + 'Pingora direct live smoke 环境示例必须暴露 HTTP redirect Location host。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL', + reason: + 'Pingora direct live smoke 环境示例必须暴露高端口 rehearsal 的 redirect Location base URL 覆盖。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '只填域名或 host:port', + reason: 'Pingora direct live smoke 环境示例必须提示 Host 参数格式。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + reason: + 'Pingora direct live smoke 环境示例必须暴露 WSS subscribe 使用的数据库名。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE', + reason: 'Pingora direct live smoke 环境示例必须暴露 WSS 101 强制开关。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct', + reason: 'Pingora direct live env 示例必须提醒直连后切换健康巡检模式。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'current release 自审', + reason: + 'Pingora direct live env 示例必须提醒直连启用 apply 前先做 current release 自审。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-env-file', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含 service EnvironmentFile 一致性预检。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-user-cert-readable', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含服务用户证书可读预检。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-binary-executable', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含 service 二进制可执行预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: + 'plan:pingora-direct-cutover` 会在 JSON runbook 中列出 direct / nginx 两个方向的 env 切换脚本和 env 复核步骤', + reason: + 'Nginx / Pingora 运维文档必须说明 cutover runbook 包含健康巡检切换脚本与复核步骤。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--rollback-health-patrol-public-base-url', + reason: + 'Nginx / Pingora 运维文档必须提醒 runbook 显式声明回退后的 Nginx 巡检入口。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-env-file', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含 service EnvironmentFile 一致性预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-user-cert-readable', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含服务用户证书可读预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-binary-executable', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含 service 二进制可执行预检。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'X-Genarrative-Nginx-Handoff', + reason: 'Pingora canary live smoke 必须校验 Nginx handoff 响应头。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '/__genarrative_pingora_canary', + reason: 'Pingora canary live smoke 必须默认检查前缀 canary 入口。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '/__genarrative_pingora_realpath_canary/healthz', + reason: 'Pingora canary live smoke 必须支持真实路径 canary healthz 探针。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-realpath-live', + reason: 'Pingora 聚合门禁必须支持把真实路径 canary 纳入目标机 live 复核。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '目标 Nginx realpath canary access log 对账', + reason: 'Pingora 聚合门禁计划自检必须覆盖真实路径 canary access log 对账。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'validateHostOption(result.host,', + reason: 'Pingora canary live smoke 必须提前拒绝非法 Host header 参数。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora canary live smoke timeout 参数必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora canary live smoke 必须在发起 canary 请求前拒绝带控制字符的 base URL、prefix、Host、path 和 timeout。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora canary live smoke 必须对带换行或 NUL 的现场参数给出明确错误。', + }, + { + file: 'scripts/check-pingora-canary-live-guard.mjs', + includes: 'assertRejectsControlCharacter', + reason: + 'Pingora canary live guard 必须动态覆盖控制字符参数在发起 canary 请求前失败。', + }, + { + file: 'scripts/check-pingora-canary-live-guard.mjs', + includes: '--prefix', + reason: 'Pingora canary live guard 必须覆盖 canary prefix 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'X-Genarrative-Gateway', + reason: 'Pingora direct live smoke 必须确认响应来自 Pingora 网关。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'http-api-redirect', + reason: 'Pingora direct live smoke 必须覆盖 HTTP 到 HTTPS 重定向。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-http2-alpn', + reason: 'Pingora direct live smoke 必须覆盖 HTTPS HTTP/2 ALPN。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'v2.bsatn.spacetimedb', + reason: + 'Pingora direct live smoke 必须覆盖 SpacetimeDB SDK 默认 WSS 子协议透传。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'wss-spacetime-subscribe', + reason: 'Pingora direct live smoke 必须覆盖 WSS subscribe 握手。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '--require-wss-upgrade', + reason: + 'Pingora direct live smoke 必须支持强制要求 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateHostOption(result.redirectHost,', + reason: + 'Pingora direct live smoke 必须提前拒绝非法 redirect Location host 参数。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateHttpsBaseUrl(result.redirectBaseUrl,', + reason: + 'Pingora direct live smoke 必须提前拒绝非法 redirect Location base URL 参数。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora direct live smoke timeout 参数必须严格校验,不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora direct live smoke 必须在发起 HTTPS / HTTP / WSS 请求前拒绝带控制字符的 URL、Host、token、路径和日志参数。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora direct live smoke 必须对带换行或 NUL 的现场参数给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateSafeAbsoluteFilePath', + reason: + 'Pingora direct live smoke 的 access log 路径必须统一复核绝对路径、根目录和控制字符。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora direct live smoke 布尔 env 必须严格校验,避免拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '不能是文件系统根目录', + reason: 'Pingora direct live smoke 必须拒绝 Pingora access log 指向文件系统根目录。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: 'assertRejectsControlCharacter', + reason: + 'Pingora direct live guard 必须动态覆盖控制字符参数在发起 live 请求前失败。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: '--probe-token', + reason: 'Pingora direct live guard 必须覆盖 probe token 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: '--pingora-access-log', + reason: 'Pingora direct live guard 必须覆盖 access log 路径控制字符负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-direct-live-guard.mjs', + reason: + 'Pingora 聚合 readiness 必须执行 direct live 参数护栏,避免正式切换前漏掉脚本输入污染负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'readBoolEnv(', + reason: + 'Pingora release readiness 直连布尔 env 必须统一严格解析,避免拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectLiveRejectsInvalidBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct live 非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseReadinessRejectsInvalidDirectBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 release readiness 非法直连布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootPingoraAccessLog', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 Pingora access log 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootPreflightEnvFile', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 preflight env 文件指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 health patrol env 文件指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectLiveRejectsFilesystemRootAccessLog', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct live 单脚本日志路径指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectPreflightRejectsInvalidBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct preflight 非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection', + reason: + 'Pingora release readiness plan 自测必须覆盖未确认共享保护的多实例 direct preflight 负例。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora direct preflight 必须检查低端口 capability drop-in。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora direct preflight 布尔 env 必须严格校验,避免检查项拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + reason: 'Pingora direct preflight 必须检查直连 TLS env。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-ports-free', + reason: 'Pingora direct preflight 必须支持检查 80/443 是否已释放。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-user-cert-readable', + reason: 'Pingora direct preflight 必须支持检查 systemd 服务用户可读证书。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-env-file', + reason: + 'Pingora direct preflight 必须支持检查 service EnvironmentFile 包含本次 env。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-binary-executable', + reason: + 'Pingora direct preflight 必须支持检查 service ExecStart 二进制可执行。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'assertProtectionInstanceBoundary', + reason: + 'Pingora direct preflight 必须检查多实例与进程内接流保护的边界。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: + 'Pingora direct preflight 多实例保护失败必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'ExecStart=', + reason: + 'Pingora direct preflight 必须从 systemd service ExecStart 推导 current release 二进制路径。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'sudo', + reason: + 'Pingora direct preflight 必须能以 systemd 服务用户身份验证 cert/key 可读性。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'fileURLToPath(import.meta.url)', + reason: + 'Pingora direct preflight 默认路径必须从脚本所在 release root 推导,不能依赖 Jenkins 工作区。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora direct preflight 必须在执行 systemctl 或 sudo 前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora direct preflight 必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'function runCommand(command, args)', + reason: + 'Pingora direct preflight 的 systemctl / sudo 子命令必须经统一执行前参数校验。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: "validateSafeAbsoluteFilePath(result.envFile, '--env-file')", + reason: 'Pingora direct preflight 必须拒绝 --env-file 指向文件系统根目录。', + }, + { + file: 'scripts/check-pingora-direct-preflight-guard.mjs', + includes: 'assertRejectsEnvFileWithControlCharacters', + reason: 'Pingora direct preflight 自测必须覆盖 --env-file 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-preflight-guard.mjs', + includes: 'assertRejectsSystemdServiceWithControlCharactersBeforeSystemctl', + reason: + 'Pingora direct preflight 自测必须覆盖 systemd service 控制字符在执行 systemctl 前失败。', + }, + { + file: 'package.json', + includes: 'node scripts/check-pingora-direct-preflight-guard.mjs', + reason: 'Pingora direct preflight npm 自测入口必须执行控制字符负例护栏。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: + 'Pingora direct preflight 必须能从 release root 读取 deploy/pingora 配置。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: + '公网直连 Pingora 时 GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR 必须保持 false', + reason: + 'Pingora direct preflight 必须阻断公网直连时误信任 X-Forwarded-For。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertDirectPreflightRejectsPublicForwardedForTrust', + reason: + 'Pingora direct preflight 自测必须覆盖公网直连误信任 X-Forwarded-For 的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply', + reason: 'Pingora 直连回退脚本默认必须是 dry-run,只能显式 apply 修改系统。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply 必须同时提供 --reload-nginx', + reason: 'Pingora 直连回退 apply 前必须强制 reload Nginx 并核验 active。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '${NGINX_BINARY} -t', + reason: 'Pingora 直连回退 reload Nginx 前必须先执行 nginx -t。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目录不能是符号链接', + reason: + 'Pingora 直连回退脚本 apply 前必须拒绝符号链接 drop-in 目录,避免删除非预期 systemd 位置。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目标不能是符号链接', + reason: + 'Pingora 直连回退脚本 apply 前必须拒绝符号链接 drop-in 目标,避免误以为已回退真实 drop-in。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目标已存在但不是普通文件', + reason: 'Pingora 直连回退脚本 apply 前必须拒绝非普通文件 drop-in 目标。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply 必须同时提供 --nginx-smoke-url', + reason: 'Pingora 直连回退 apply 前必须强制 Nginx 入口 smoke。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host', + reason: + 'Pingora 直连回退本机 smoke 必须显式使用正式 Host,避免命中默认 vhost。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--nginx-smoke-expect-body', + reason: + 'Pingora 直连回退必须支持 Nginx smoke 响应体证据片段,避免 HTTP 200 命中错误入口。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'nginx smoke evidence matched expected body fragment', + reason: 'Pingora 直连回退必须在 Nginx smoke 响应体匹配后输出证据摘要。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'Nginx smoke 响应缺少预期片段', + reason: 'Pingora 直连回退必须拒绝 Nginx smoke 响应体漂移。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '只能是 host 或 host:port', + reason: 'Pingora 直连回退必须拒绝非法 Nginx smoke Host。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--curl-binary', + reason: + 'Pingora 直连回退必须支持覆盖 curl 可执行文件,便于测试和特殊环境。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-env-file', + reason: 'Pingora 直连回退必须支持可选复核 health patrol env 已切回 nginx。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-env-check-script', + reason: + 'Pingora 直连回退 health patrol env 复核必须能固定 current release 随包脚本。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'node -- "${health_patrol_args[@]}"', + reason: + 'Pingora 直连回退 health patrol env 复核必须用 node -- 分隔脚本参数,避免 Node 22 抢占业务 --env-file。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--require-empty-public-host', + reason: 'Pingora 直连回退 health patrol env 复核必须阻断 direct Host 覆盖残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--pingora-shadow-probe-url', + reason: 'Pingora 直连回退必须支持可选复核 shadow 高端口探针。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--pingora-shadow-probe-token', + reason: 'Pingora 直连回退 shadow 探针复核必须使用 probe token。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'X-Genarrative-Pingora-Probe: ', + reason: 'Pingora 直连回退 shadow 探针命令日志必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run Pingora shadow probe 命令不应输出 probe token 原文', + reason: 'Pingora 直连回退自测必须覆盖 shadow probe token 不进命令日志。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'gateway=pingora-shadow', + reason: 'Pingora 直连回退 shadow 探针复核必须检查响应身份。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '${CURL_BINARY} --fail', + reason: 'Pingora 直连回退 reload Nginx 后必须执行 smoke URL 验证。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl daemon-reload', + reason: 'Pingora 直连回退移除 drop-in 后必须 reload systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连回退脚本必须默认重启 Pingora 网关服务。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'direct-entry.conf', + reason: 'Pingora 直连回退脚本必须移除 direct-entry drop-in。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl cat ${SERVICE_NAME}', + reason: 'Pingora 直连回退 apply 后必须核验 systemd drop-in 最终已移除。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连回退 apply 后必须拒绝 AmbientCapabilities 残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连回退 apply 后必须拒绝 CapabilityBoundingSet 残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl is-active ${NGINX_SERVICE}', + reason: 'Pingora 直连回退 reload Nginx 后必须核验 Nginx active 状态。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'Nginx reload 后状态不是 active', + reason: 'Pingora 直连回退必须拒绝 Nginx reload 后非 active 状态。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply', + reason: 'Pingora 直连启用脚本默认必须是 dry-run,只能显式 apply 修改系统。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl daemon-reload', + reason: 'Pingora 直连启用安装 drop-in 后必须 reload systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连启用脚本必须默认重启 Pingora 网关服务。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + reason: + 'Pingora 直连启用脚本默认必须读取 current release 随包 direct-entry 模板,不能依赖 /etc 参考模板。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_SCRIPT', + reason: + 'Pingora 直连启用脚本必须支持固定 current release 自审脚本,避免切换窗口依赖源码工作区。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'run_current_release_audit', + reason: + 'Pingora 直连启用 apply 前必须先跑 current release 自包含自审,再修改 systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--require-pingora-gateway', + reason: + 'Pingora 直连启用 apply 前 current release 自审必须要求 Pingora 网关产物存在。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--systemd-show', + reason: + 'Pingora 直连启用 apply 前 current release 自审必须核验 systemd ExecStart。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--service-unit-path', + reason: + 'Pingora 直连启用脚本必须能固定随包主 service 模板作为 ExecStart 真相源。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager', + reason: + 'Pingora 直连启用脚本 apply 后必须核验 systemd ExecStart 指向 current release 网关二进制。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenCurrentReleaseAuditFailsBeforeInstall', + reason: + 'Pingora 直连启用脚本自测必须覆盖 current release 自审失败时不安装 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenSystemdExecStartDiffers', + reason: + 'Pingora 直连启用脚本自测必须覆盖 systemd ExecStart 指向旧 release 的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '该脚本不写入 TLS env', + reason: 'Pingora 直连启用脚本不能隐式修改 TLS/env/证书边界。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-env-file', + reason: 'Pingora 直连启用 apply 前必须强制通过 env 预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-cert-readable', + reason: 'Pingora 直连启用 apply 前必须强制通过证书可读预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-service-env-file', + reason: + 'Pingora 直连启用 apply 前必须强制通过 service EnvironmentFile 一致性预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemd_env_file_matches', + reason: + 'Pingora 直连启用 apply 后必须精确核验 systemctl cat 中的 EnvironmentFile。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: + '--apply 必须同时提供 --preflight-check-service-user-cert-readable', + reason: 'Pingora 直连启用 apply 前必须强制通过服务用户证书可读预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: + '--apply 必须同时提供 --preflight-check-service-binary-executable', + reason: 'Pingora 直连启用 apply 前必须强制通过 service 二进制可执行预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-ports-free', + reason: 'Pingora 直连启用 apply 前必须强制通过端口释放预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-https-base-url', + reason: 'Pingora 直连启用 apply 后必须强制验证 HTTPS 直连入口。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-http-base-url', + reason: 'Pingora 直连启用 apply 后必须强制验证 HTTP redirect / ACME 入口。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-host', + reason: 'Pingora 直连启用 apply 后必须用正式域名 Host/SNI 验证。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-redirect-host', + reason: 'Pingora 直连启用 apply 后必须验证 redirect Location host。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--direct-redirect-base-url', + reason: + 'Pingora 直连启用脚本必须支持把高端口 rehearsal 的 redirect base URL 透传给 direct live smoke。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-spacetime-database', + reason: 'Pingora 直连启用 apply 后必须验证目标库 WSS subscribe。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl cat ${SERVICE_NAME}', + reason: 'Pingora 直连启用 apply 后必须核验 systemd drop-in 最终生效。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl is-active ${SERVICE_NAME}', + reason: 'Pingora 直连启用 apply 后必须核验服务已 active。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连启用 apply 后必须核验 AmbientCapabilities。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连启用 apply 后必须核验 CapabilityBoundingSet。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--require-wss-upgrade', + reason: 'Pingora 直连启用 apply 后 direct live smoke 必须强制 WSS 101。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'print_redacted_command node "${direct_live_args[@]}"', + reason: 'Pingora 直连启用脚本展示 direct live 命令时必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 不应安装临时 drop-in 文件', + reason: 'Pingora 直连启用检查必须证明默认 dry-run 不修改 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--probe-token ', + reason: 'Pingora 直连启用检查必须覆盖 direct live 命令 token 脱敏。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示并执行 direct preflight', + reason: 'Pingora 直连启用检查必须证明可在启用前执行 direct preflight。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + 'release current 目录中的 enable dry-run 必须默认读取随包携带的 direct-entry 模板', + reason: + 'Pingora 直连启用检查必须证明从 /opt/genarrative/current 执行时默认使用发布包内 systemd 模板。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-env-file 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 apply 跳过 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-check-cert-readable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + '--apply 缺少 --preflight-check-service-user-cert-readable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过服务用户证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + '--apply 缺少 --preflight-check-service-binary-executable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过 service 二进制可执行 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-check-ports-free 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过端口释放 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示启用后 systemd drop-in 生效核验', + reason: 'Pingora 直连启用检查必须覆盖启用后的 systemd 生效核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示启用后 service active 核验', + reason: 'Pingora 直连启用检查必须覆盖启用后的 service active 核验。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-https-base-url 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct HTTPS 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-http-base-url 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct HTTP 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-host 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct Host/SNI 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-redirect-host 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 redirect host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-spacetime-database 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 WSS 数据库名的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct live smoke 失败时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 失败负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct preflight 脚本缺失时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct preflight 脚本缺失负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct live smoke 脚本缺失时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 脚本缺失负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--template-path 必须是绝对路径', + reason: 'Pingora 直连启用检查必须覆盖模板相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--dropin-path 必须是绝对路径', + reason: 'Pingora 直连启用检查必须覆盖 drop-in 相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'drop-in 目录不能是符号链接', + reason: + 'Pingora 直连启用脚本 apply 前必须拒绝符号链接 drop-in 目录,避免写到非预期 systemd 位置。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'drop-in 目标不能是符号链接', + reason: + 'Pingora 直连启用脚本 apply 前必须拒绝符号链接 drop-in 目标,避免覆盖非预期文件。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'reject_control_characters', + reason: + 'Pingora 直连启用脚本必须在修改 systemd 前拒绝带换行或 NUL 的切换参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'reject_filesystem_root_path', + reason: + 'Pingora 直连启用脚本必须在修改 systemd 前拒绝文件系统根目录路径参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 直连启用脚本拒绝控制字符参数时必须给出明确错误。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '不能是文件系统根目录', + reason: + 'Pingora 直连启用脚本拒绝文件系统根目录路径时必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyRejectsSymlinkDropinDirectoryBeforeInstall', + reason: 'Pingora 直连启用检查必须覆盖符号链接 drop-in 目录负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyRejectsSymlinkDropinFileBeforeInstall', + reason: 'Pingora 直连启用检查必须覆盖符号链接 drop-in 目标负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeApply', + reason: + 'Pingora 直连启用检查必须覆盖控制字符参数会在 apply 写 systemd 前失败。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertFilesystemRootPathsRejected', + reason: + 'Pingora 直连启用检查必须覆盖文件系统根目录路径会在 apply 写 systemd 前失败。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--preflight-script 必须是绝对路径', + reason: + 'Pingora 直连启用脚本的 preflight 脚本路径必须固定为绝对路径,避免切换窗口受 cwd 影响。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--direct-live-script 必须是绝对路径', + reason: + 'Pingora 直连启用脚本的 direct live smoke 脚本路径必须固定为绝对路径,避免切换窗口受 cwd 影响。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '相对路径 direct preflight 脚本必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 direct preflight 脚本相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '相对路径 direct live smoke 脚本必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 脚本相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 不应删除临时 drop-in 文件', + reason: 'Pingora 直连回退检查必须证明默认 dry-run 不修改 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示回退后 systemd drop-in 移除核验', + reason: 'Pingora 直连回退检查必须覆盖回退后的 systemd 移除核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示回退后 ExecStart 指向核验', + reason: 'Pingora 直连回退检查必须覆盖回退后的 current release ExecStart 核验。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '回退脚本 ExecStart 核验必须以随包主 service 模板为真相源', + reason: 'Pingora 直连回退脚本必须以随包主 service 模板为 ExecStart 真相源。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 reload Nginx 后 active 核验', + reason: 'Pingora 直连回退检查必须覆盖 Nginx reload 后状态核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 Nginx 入口 smoke', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke dry-run 命令。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--apply 缺少 --reload-nginx 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 apply 跳过 Nginx reload 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'nginx -t 失败时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx 配置语法错误负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--apply 缺少 --nginx-smoke-url 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖缺少 Nginx smoke URL 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '缺少 --nginx-smoke-host 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 loopback smoke 缺少 Host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '非法 --nginx-smoke-host 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖非法 smoke Host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Nginx smoke 失败时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke 失败负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Nginx smoke 响应体不匹配时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke 响应体漂移负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback apply 必须使用显式传入的 Nginx smoke body 片段', + reason: + 'Pingora cutover runbook 自检必须覆盖 rollback Nginx smoke body 显式配置门禁。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollbackNginxSmokeExpectBody', + reason: 'Pingora cutover runbook 必须把 rollback Nginx smoke body 片段作为显式配置。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 health patrol env 回退后复核命令', + reason: 'Pingora 直连回退检查必须覆盖 health patrol env 复核 dry-run 命令。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --health-patrol-env-file 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 health patrol env 相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'health patrol env 仍停留在 pingora-direct 时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖巡检 env 未切回 nginx 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 Pingora shadow probe 命令且隐藏 token', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe dry-run 命令并避免泄露 token。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'shadow probe URL 缺少 token 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe 参数成对出现。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Pingora shadow probe body 错误时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe 响应身份错误负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'systemctl show ExecStart 指向旧 release 时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 systemd ExecStart 漂到旧 release 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--service-unit-path 必须是绝对路径', + reason: 'Pingora 直连回退检查必须覆盖主 service 模板相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--dropin-path 必须是绝对路径', + reason: 'Pingora 直连回退检查必须覆盖相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'require_absolute_path_if_path_like "--nginx-binary"', + reason: + 'Pingora 直连回退脚本覆盖 Nginx 二进制时,路径形式必须固定为绝对路径。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'require_absolute_path_if_path_like "--curl-binary"', + reason: + 'Pingora 直连回退脚本覆盖 curl 二进制时,路径形式必须固定为绝对路径。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --nginx-binary 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 Nginx 二进制相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --curl-binary 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 curl 二进制相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'validate_http_url "--nginx-smoke-url"', + reason: 'Pingora 直连回退脚本必须在改 systemd 前拒绝非法 Nginx smoke URL。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'reject_control_characters', + reason: + 'Pingora 直连回退脚本必须在 nginx -t、删除 drop-in 或 reload 前拒绝带换行或 NUL 的回退参数。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'reject_filesystem_root_path', + reason: + 'Pingora 直连回退脚本必须在 nginx -t、删除 drop-in 或 reload 前拒绝文件系统根目录路径参数。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 直连回退脚本拒绝控制字符参数时必须给出明确错误。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '不能是文件系统根目录', + reason: + 'Pingora 直连回退脚本拒绝文件系统根目录路径时必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '非法 --nginx-smoke-url 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖非法 Nginx smoke URL 负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeApply', + reason: + 'Pingora 直连回退检查必须覆盖控制字符参数会在 nginx -t 和删除 drop-in 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertRejectsFilesystemRootPathsBeforeApply', + reason: + 'Pingora 直连回退检查必须覆盖文件系统根目录路径会在 nginx -t 和删除 drop-in 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsSymlinkDropinDirectoryBeforeNginxTest', + reason: 'Pingora 直连回退检查必须覆盖符号链接 drop-in 目录会在 nginx -t 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsSymlinkDropinFileBeforeRemoval', + reason: 'Pingora 直连回退检查必须覆盖符号链接 drop-in 目标不会删除真实目标文件。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsDirectoryDropinTargetBeforeNginxTest', + reason: 'Pingora 直连回退检查必须覆盖目录型 drop-in 目标会在 nginx -t 前失败。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API deploy 必须把 Pingora 直连回退脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API deploy 必须把 Pingora 直连启用脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API deploy 必须把 Pingora 直连预检脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: 'API deploy 必须把 Pingora canary live smoke 脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API deploy 必须把 Pingora 直连 live smoke 脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API deploy 必须把 Pingora release readiness 聚合门禁脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'deploy/pingora', + reason: 'API deploy 必须把 Pingora 配置目录复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'deploy/systemd', + reason: + 'API deploy 必须把 systemd 配置目录复制到 current release,供直连预检读取主 service 和 drop-in 模板。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ENV_DEPLOY_DIR_SOURCE', + reason: 'API deploy 必须从发布包内复制 deploy/env 示例目录。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: 'API release build 动态烟测必须覆盖随产物执行的 deploy 脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'API release README 必须说明随包携带 API Deploy 执行入口', + reason: + 'API release build 动态烟测必须覆盖发布包 README 中的 deploy 执行入口说明。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/maintenance-on.sh', + reason: 'API release build 动态烟测必须覆盖 deploy 同目录维护开启脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/maintenance-off.sh', + reason: 'API release build 动态烟测必须覆盖 deploy 同目录维护关闭脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API release build 动态烟测必须覆盖直连启用脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API release build 动态烟测必须覆盖直连回退脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API release build 动态烟测必须覆盖 health patrol env 切换脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'API release build 动态烟测必须覆盖 gateway env shadow 切换脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API release build 动态烟测必须覆盖直连预检脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: + 'API release build 动态烟测必须覆盖健康巡检 env 复核脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API release build 动态烟测必须覆盖 Pingora release readiness 聚合门禁脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'API release build 动态烟测必须覆盖 canary live smoke 脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API release build 动态烟测必须覆盖直连 live smoke 脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/systemd/genarrative-pingora-gateway.service', + reason: 'API release build 动态烟测必须覆盖 systemd 模板进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: 'API release build 动态烟测必须覆盖 Pingora 配置进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: '--skip-api-build', + reason: 'API release build 动态烟测必须避免真实构建 Rust。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: 'API deploy 动态烟测必须从发布产物内执行 deploy 脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/maintenance-on.sh', + reason: 'API deploy 动态烟测必须覆盖 deploy 同目录维护开启脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/maintenance-off.sh', + reason: 'API deploy 动态烟测必须覆盖 deploy 同目录维护关闭脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'API deploy 动态烟测必须覆盖 Pingora canary live smoke 脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API deploy 动态烟测必须覆盖 Pingora release readiness 聚合门禁脚本进入 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'release-manifest.json 缺少 pingora-gateway artifact', + reason: + 'API deploy 必须阻断带 Pingora 二进制但 manifest 未登记 Pingora artifact 的发布包。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 release-manifest.json', + reason: + 'API deploy 必须强制要求 release manifest,避免 current release 缺少产物证据根。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${RELEASE_ROOT}" "--release-root"', + reason: + 'API deploy 发布根目录必须是绝对路径,避免 Jenkins 工作目录影响 release 位置。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${CURRENT_LINK}" "--current-link"', + reason: + 'API deploy current 链接必须是绝对路径,避免相对路径切错入口。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${API_ENV_FILE}" "--api-env-file"', + reason: + 'API deploy 环境文件必须是绝对路径,避免相对路径写入工作区文件。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '^[0-9A-Za-z][0-9A-Za-z._-]*$', + reason: + 'API deploy release version 必须以数字或字母开头,避免点目录或隐藏 staging 名称污染 release 根目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '--version 不能是点目录', + reason: + 'API deploy 必须显式拒绝 . / .. 这类点目录版本名。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '.staging.$$', + reason: + 'API deploy 必须先写 staging release,避免失败时污染正式 release 目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '目标 release 已存在,拒绝覆盖或合并旧文件', + reason: + 'API deploy 必须拒绝同版本 release 目录复用,避免旧文件混入 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '目标 release 在发布过程中出现,拒绝合并 staging', + reason: + 'API deploy 在最终提升前必须再次拒绝同版本 release 竞态出现,避免把 staging 合并进旧目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'current 链接路径已存在但不是符号链接,拒绝覆盖', + reason: + 'API deploy 必须拒绝目录型 current 路径,避免 ln 把新链接写进旧目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"', + reason: + 'API deploy 切换 current 必须用 -T 固定符号链接替换语义。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"', + reason: + 'API deploy 必须用非合并语义把 staging release 提升为正式 release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraArtifactMissingManifestEntry', + reason: + 'API deploy 动态烟测必须覆盖 Pingora 二进制与 release manifest 漂移负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingReleaseManifestFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertReleaseManifestMissingApiArtifactFails', + reason: + 'API deploy 动态烟测必须覆盖 release manifest 缺少 api-server artifact 的负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDotVersion', + reason: + 'API deploy 动态烟测必须覆盖 . 版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDotDotVersion', + reason: + 'API deploy 动态烟测必须覆盖 .. 版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsVersionStartingWithDot', + reason: + 'API deploy 动态烟测必须覆盖以点开头版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsExistingReleaseDirectory', + reason: + 'API deploy 动态烟测必须覆盖同版本 release 目录已存在时拒绝覆盖。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDirectoryCurrentLink', + reason: + 'API deploy 动态烟测必须覆盖 current 路径不是符号链接时拒绝覆盖。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeReleaseRoot', + reason: + 'API deploy 动态烟测必须覆盖 release root 相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeCurrentLink', + reason: + 'API deploy 动态烟测必须覆盖 current link 相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeApiEnvFile', + reason: + 'API deploy 动态烟测必须覆盖 api env 文件相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployCleansStagingReleaseOnFailure', + reason: + 'API deploy 动态烟测必须覆盖失败时清理 staging release 且不留下正式 release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsFinalReleaseRaceAndCleansStaging', + reason: + 'API deploy 动态烟测必须覆盖最终提升前 release 目录竞态出现时拒绝合并并清理 staging。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingBackupScriptFails', + reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolScriptFails', + reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolEnvCheckFails', + reason: + 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API deploy 动态烟测必须覆盖直连预检脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: + 'API deploy 动态烟测必须覆盖健康巡检 env 复核脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API deploy 动态烟测必须覆盖 health patrol env 切换脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'API deploy 动态烟测必须覆盖 gateway env shadow 切换脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API deploy 动态烟测必须覆盖直连 live smoke 脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/systemd/genarrative-pingora-gateway.service', + reason: 'API deploy 动态烟测必须覆盖 systemd 模板进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: 'API deploy 动态烟测必须覆盖 Pingora 配置进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'fakeBin', + reason: + 'API deploy 动态烟测必须使用 fake systemctl/curl,禁止触碰真实服务。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API Build 归档必须包含 Pingora 直连回退脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API Build 归档必须包含 Pingora 直连启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API Build 归档必须包含 Pingora 直连预检脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: 'API Build 归档必须包含 Pingora 直连 live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'deploy/pingora/**', + reason: 'API Build 归档必须包含 Pingora 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'deploy/systemd/**', + reason: 'API Build 归档必须包含 systemd 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连回退脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连预检脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连 live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'deploy/pingora/**', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'deploy/systemd/**', + reason: 'API Deploy 复制上游产物时必须包含 systemd 配置目录。', + }, + { + file: 'deploy/pingora/nginx-route-parity.matrix.json', + includes: 'spacetime_subscribe', + reason: 'Pingora 路由矩阵必须覆盖 SpacetimeDB WebSocket subscribe。', + }, + { + file: 'scripts/check-pingora-route-parity.mjs', + includes: 'deploy/nginx/genarrative.conf', + reason: 'Pingora 路由矩阵检查必须覆盖生产 Nginx 模板。', + }, + { + file: 'scripts/check-pingora-route-parity.mjs', + includes: 'deploy/nginx/genarrative-dev-http.conf', + reason: 'Pingora 路由矩阵检查必须覆盖开发服 Nginx 模板。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'ROUTE_PARITY_MATRIX_JSON', + reason: 'Pingora Rust 路由单测必须读取共享路由矩阵。', + }, + { + file: 'scripts/check-nginx-pingora-canary.mjs', + includes: '--require-nginx', + reason: + 'Pingora canary 校验脚本必须支持在有 Nginx 的环境强制执行 nginx -t。', + }, + { + file: 'scripts/check-nginx-pingora-canary.mjs', + includes: 'proxy_set_header Upgrade $http_upgrade;', + reason: + 'Pingora canary 校验脚本必须覆盖 SpacetimeDB WebSocket Upgrade 设置。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'SpacetimeDB WebSocket Upgrade', + reason: 'Pingora 网关 smoke 必须覆盖 SpacetimeDB WebSocket 代理。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'v2.bsatn.spacetimedb', + reason: 'Pingora 网关 smoke 必须覆盖 SpacetimeDB WSS 子协议透传。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip 压缩响应', + reason: 'Pingora 网关 smoke 必须覆盖 gzip 响应压缩。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip-only 压缩算法白名单', + reason: 'Pingora 网关 smoke 必须覆盖压缩算法白名单。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip 最小长度不压缩小响应', + reason: 'Pingora 网关 smoke 必须覆盖 Nginx gzip_min_length parity。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip types 不压缩图片资源', + reason: 'Pingora 网关 smoke 必须覆盖 Nginx gzip_types parity。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '主站指纹静态资源长缓存', + reason: 'Pingora 网关 smoke 必须覆盖 Vite 指纹资源 immutable 缓存头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: "response.headers['cache-control'] === 'no-cache'", + reason: 'Pingora 网关 smoke 必须覆盖 HTML 和普通静态资源 no-cache 口径。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-None-Match 返回 304', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 ETag 协商缓存。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Modified-Since 返回 304', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 Last-Modified 协商缓存。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源非读取方法返回 405', + reason: 'Pingora 网关 smoke 必须覆盖静态资源只允许 GET / HEAD。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GET, HEAD', + reason: 'Pingora 静态响应必须对非读取方法返回 Allow: GET, HEAD。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 Range 返回 206', + reason: 'Pingora 网关 smoke 必须覆盖静态资源单 Range 局部读取。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Range 日期匹配返回 206', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 If-Range 匹配时继续返回局部内容。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Range 旧日期回完整文件', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 If-Range 不匹配时回完整内容。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源越界 Range 返回 416', + reason: 'Pingora 网关 smoke 必须覆盖静态资源越界 Range 响应。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'access log 静态边界状态对账', + reason: + 'Pingora 网关 smoke 必须按 request_id 对账静态 304 / 405 / 206 / 416 access log,避免直连证据链漏掉本地响应状态。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'entry.method === expected.method', + reason: + 'Pingora 网关 smoke 必须按 request_id 对账静态 access log 的 method,避免 HEAD / POST 边界被状态码巧合掩盖。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Accept-Ranges', + reason: 'Pingora 静态响应必须声明 bytes Range 能力。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Content-Range', + reason: 'Pingora 静态 Range 响应必须写入 Content-Range。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Last-Modified', + reason: 'Pingora 静态响应必须写入 Last-Modified,避免直连后协商缓存能力退化。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'if-none-match', + reason: 'Pingora 静态响应必须支持 ETag 条件请求返回 304。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL', + reason: 'Pingora 网关必须暴露指纹静态资源 Cache-Control 配置。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'has_fingerprinted_file_name', + reason: 'Pingora 网关必须按 Vite 指纹文件名区分长期缓存静态资源。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: '不能包含换行或 NUL 字符', + reason: 'Pingora 静态缓存头配置必须拒绝响应头注入字符。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL', + reason: 'Pingora 网关 smoke 必须覆盖 gzip 配置校验。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS', + reason: 'Pingora 网关 smoke 必须覆盖压缩算法配置校验。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_CONCURRENCY_LIMITED', + reason: 'Pingora 网关 smoke 必须覆盖接流并发保护。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'PAYLOAD_TOO_LARGE', + reason: 'Pingora 网关 smoke 必须覆盖请求体大小限制。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'access log 落盘', + reason: 'Pingora 网关 smoke 必须覆盖 access log 文件写入。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '配置错误启动失败', + reason: 'Pingora 网关 smoke 必须覆盖不安全配置 fail-fast。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'SECRET_VALUE_FLAGS', + reason: 'Pingora 网关 smoke 必须集中维护敏感参数脱敏列表。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'redactSecretArgs(args).join', + reason: 'Pingora 网关 smoke 打印 direct live 命令时必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_UPSTREAM_ERROR', + reason: 'Pingora 网关 smoke 必须覆盖上游断连时的稳定 JSON 错误。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_UPSTREAM_TIMEOUT', + reason: 'Pingora 网关 smoke 必须覆盖上游超时时的稳定 JSON 错误。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'TLS 直连接口返回 index.html', + reason: 'Pingora 网关 smoke 必须覆盖可选 TLS 直连接口。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'HTTP 入口 301 到 HTTPS', + reason: 'Pingora 网关 smoke 必须覆盖可选 HTTP 到 HTTPS 重定向入口。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'direct-access-log', + reason: 'Pingora direct live smoke 必须覆盖 Pingora access log 落盘校验。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset', + reason: + 'Pingora direct live smoke 必须自动发现并探测首页引用的静态资源,避免直连只验证 API / WSS 路径。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-range', + reason: + 'Pingora direct live smoke 必须验证首页静态资产 Range: bytes=0-0,避免直连后媒体/浏览器探测能力退化。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-head', + reason: + 'Pingora direct live smoke 必须验证首页静态资产 HEAD 行为,覆盖 curl -I、监控探针和 CDN 预检。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset', + reason: + 'Pingora direct live smoke 必须额外探测首页引用的 Vite 指纹静态资源。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'public, max-age=31536000, immutable', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源长期缓存头,避免直连后旧 tab chunk 缓存口径退化。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'pickEvidenceHeaders', + reason: + 'Pingora direct live smoke 必须把静态响应头白名单写入 JSON 证据,便于切换证据包复盘。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: "'content-range'", + reason: + 'Pingora direct live smoke 的静态响应头 JSON 证据必须包含 Content-Range。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-head', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 HEAD 行为。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-range', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 Range 行为。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-etag-304', + reason: + 'Pingora direct live smoke 必须验证首页静态资源 ETag 条件请求返回 304。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-etag-304', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 ETag 条件请求返回 304。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '', + reason: + 'Pingora 本地 smoke fixture 必须让 direct live 自动发现指纹静态资源。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live 必须验证首页引用的指纹静态资源 GET 200。', + reason: + 'Pingora 本地 smoke 必须从 direct live JSON 结构化确认指纹资源 GET 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live JSON 必须保留指纹静态资源 immutable Cache-Control 证据。', + reason: + 'Pingora 本地 smoke 必须确认 direct live JSON 可复盘指纹资源长期缓存头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live JSON 必须保留指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 本地 smoke 必须确认 direct live JSON 可复盘指纹资源 Range 响应头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live access log 必须保留指纹静态 Range 的 method/path/status 明细。', + reason: + 'Pingora 本地 smoke 必须确认指纹资源 Range 进入 access log method/path/status 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live 必须验证首页引用的指纹静态资源 ETag 304。', + reason: + 'Pingora 本地 smoke 必须从 direct live JSON 结构化确认指纹资源 304 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live access log 必须保留指纹静态 ETag 304 的 method/path/status 明细。', + reason: + 'Pingora 本地 smoke 必须确认指纹资源 304 进入 access log method/path/status 证据。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'Content-Range 不符合 bytes 0-0/', + reason: + 'Pingora direct live smoke 必须校验静态资产 Range 响应的 Content-Range。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'request_id', + reason: 'Pingora direct live smoke 必须按 request_id 反查 access log。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'matchedCount', + reason: + 'Pingora direct live smoke 的 access log 证据必须记录匹配数量,便于证据包复盘。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'mismatches', + reason: + 'Pingora direct live smoke 的 access log 证据必须保留 method/path/status 漂移明细。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'expectedMethod', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的预期 HTTP 方法。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'actualMethod', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的实际 HTTP 方法。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'expectedStatusCode', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的预期状态码。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'actualStatusCode', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的实际状态码。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-access-log 检查必须保留 mismatches 明细数组。', + reason: + 'Pingora 切流证据包自测必须防止 direct live access log JSON 明细退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-access-log 检查必须保留已匹配 request_id 的预期方法。', + reason: + 'Pingora 切流证据包自测必须防止 direct live access log method 明细退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.json 必须保留指纹静态资源 Cache-Control 证据。', + reason: + 'Pingora 切流证据包自测必须防止 direct live 静态响应头证据退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.json 必须保留指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 切流证据包自测必须防止 direct live Range 响应头证据退化。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveStaticHeaders', + reason: + 'Pingora 切流证据包 manifest 必须提升 direct live 静态响应头摘要,便于快速复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveStaticHeaders?.ok', + reason: + 'Pingora 切流证据包必须把静态响应头摘要完整性纳入 direct live 成功判定。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'ETag 304 statusCode 应为 304', + reason: + 'Pingora 切流证据包必须诊断静态 ETag 304 证据缺失。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'Range statusCode 应为 206', + reason: + 'Pingora 切流证据包必须诊断静态 Range 证据缺失。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'manifest 必须提升指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 切流证据包自测必须防止 manifest 静态响应头摘要退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveMissingStaticHeadersSummaryFails', + reason: + 'Pingora 切流证据包自测必须覆盖 direct live JSON 缺少静态响应头证据的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveStaticHeaderDiagnosticsFails', + reason: + 'Pingora 切流证据包自测必须覆盖静态 Range / 304 摘要不完整时失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'manifest 必须提升普通静态资源 ETag 304 证据。', + reason: + 'Pingora 切流证据包自测必须防止 manifest 普通静态 304 摘要退化。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'summaryDirectLiveStaticHeaders', + reason: + 'Pingora 证据根目录审计必须读取阶段 manifest 的 direct live 静态响应头摘要。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-phase-pingora-env-shadow', + reason: + 'Pingora 证据根目录审计必须支持强制校验 post-rollback Pingora env shadow 摘要。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'summaryPingoraEnvShadow', + reason: + 'Pingora 证据根目录审计必须读取阶段 manifest 的 Pingora env shadow 摘要。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'tlsCertFile 必须为空', + reason: + 'Pingora 证据根目录审计必须拒绝 post-rollback TLS 证书路径残留。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'shadowReady 必须为 true', + reason: + 'Pingora 证据根目录审计必须拒绝 post-rollback 快照姿态不是 shadow 的证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'directLiveStaticHeaders: latest.summaryDirectLiveStaticHeaders', + reason: + 'Pingora 证据根目录审计输出必须提升阶段 direct live 静态响应头摘要。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequirePhasePingoraEnvShadowFailsWhenDirectLowPortsRemain', + reason: + 'Pingora 证据根目录审计自测必须覆盖 post-rollback 低端口 env 残留失败路径。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根审计输出必须提升 post-enable direct live 指纹静态 Cache-Control 摘要。', + reason: + 'Pingora 证据根目录审计自测必须覆盖 direct live 静态响应头摘要输出。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根审计输出必须提升 post-enable direct live access log 摘要。', + reason: + 'Pingora 证据根目录审计自测必须覆盖 direct live access log 摘要输出。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '静态 GET / HEAD / 304 / Range 结果还必须在 `direct-live.json` 中保留白名单响应头', + reason: + 'Pingora 试点文档必须说明 direct live JSON 会归档静态响应头白名单证据。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '`cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`', + reason: + 'Pingora 试点文档必须固定 direct live 静态响应头证据白名单。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '静态摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 / Last-Modified 304 证据时,证据包都会直接记为 `CRITICAL`', + reason: + 'Pingora 试点文档必须说明静态响应头证据不完整会让证据包失败。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '总审计输出会把阶段 manifest 中的 `directLiveAccessLog` 与 `directLiveStaticHeaders` 一并带到对应 `phases[]` 项', + reason: + 'Pingora 试点文档必须说明最终证据根审计会输出 direct live 摘要。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'direct live JSON 只给静态 GET / HEAD / 304 / Range 写入白名单 `headers`', + reason: + 'Nginx README 必须说明 Pingora direct live 静态头部证据归档口径。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'manifest.summary.directLiveStaticHeaders', + reason: + 'Nginx README 必须说明切流 manifest 会提升 direct live 静态响应头摘要。', + }, + { + file: 'deploy/nginx/README.md', + includes: '证据包会记为 `CRITICAL`', + reason: + 'Nginx README 必须说明 direct live 静态响应头摘要缺关键证据时会阻断证据包。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: 'API / WSS 检查不落原始响应头', + reason: + '团队共享决策必须记录 direct live 只归档静态白名单响应头。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: 'directLiveStaticHeaders', + reason: + '团队共享决策必须记录切流 manifest 提升 direct live 静态响应头摘要。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: '静态头摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 / Last-Modified 304 证据时整包记为 `CRITICAL`', + reason: + '团队共享决策必须记录静态响应头摘要不完整会阻断证据包。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '不要把 `direct-live.json` 当成只有状态码的摘要', + reason: + '团队共享踩坑必须提醒 direct live JSON 需要可复盘静态响应头证据。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'manifest.summary.directLiveStaticHeaders', + reason: + '团队共享踩坑必须提醒切流 manifest 需要可快速复盘静态响应头摘要。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 或 Last-Modified 304 证据,证据包会直接记为 `CRITICAL`', + reason: + '团队共享踩坑必须提醒静态响应头摘要不完整会阻断证据包。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveAccessLog', + reason: + 'Pingora 切流证据包 manifest 必须提升 direct live access log 摘要,便于快速复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'direct live JSON missing direct-access-log result', + reason: + 'Pingora 切流证据包必须在 direct live JSON 缺少 access log 结果时标记 CRITICAL。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveMissingAccessLogSummaryFails', + reason: + 'Pingora 切流证据包自测必须覆盖 direct live JSON 缺少 access log 结果的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-pingora-access-log', + reason: 'Pingora 直连启用 apply 必须强制 access log 落盘校验参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct_live_args+=(--json)', + reason: + 'Pingora 直连启用 apply 后必须用 JSON 模式运行 direct live,才能校验结构化接流证据。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct live JSON 缺少 direct-access-log 结构化结果', + reason: + 'Pingora 直连启用 apply 必须拒绝缺少 direct-access-log 结构化结果的 direct live 输出。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct-access-log evidence checked=', + reason: + 'Pingora 直连启用 apply 必须在 direct access log 结构化证据通过后输出摘要。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-pingora-access-log 必须被拒绝', + reason: 'Pingora 直连启用自测必须覆盖缺少 access log 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenDirectLiveAccessLogJsonMissing', + reason: + 'Pingora 直连启用自测必须覆盖 direct live 退出 0 但缺少 access log JSON 证据的负例。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '--pingora-access-log', + reason: 'Pingora 本地网关 smoke 必须把临时 access log 传给 direct live smoke。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS', + reason: + 'Pingora 网关必须暴露上游读取超时配置,避免正式切换时和 Nginx 超时口径漂移。', + }, { file: 'scripts/deploy/production-api-deploy.sh', includes: 'production-health-patrol.mjs', reason: 'API deploy 必须把健康巡检脚本复制到 current release。', }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少数据库备份脚本', + reason: 'API deploy 缺少数据库备份脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少生产健康巡检脚本', + reason: 'API deploy 缺少健康巡检脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: 'WORKSPACE_BACKUP_SCRIPT_SOURCE', + reason: 'API deploy 不应再从部署工作区兜底复制数据库备份脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: 'WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE', + reason: 'API deploy 不应再从部署工作区兜底复制健康巡检脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: '回退使用部署工作区脚本', + reason: 'API deploy 不应再用部署工作区脚本掩盖发布产物缺失。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'pingora-gateway.sha256', + reason: 'API deploy 看到 Pingora 影子网关产物时必须校验 checksum 后复制。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ensure_pingora_shadow_service', + reason: + 'API deploy 看到 Pingora 影子网关产物时必须先进入受控 shadow service 启动流程。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'systemctl cat "${service_name}"', + reason: + 'API deploy 自动启动或重启 Pingora 前必须读取 systemd 最终配置。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'CAP_NET_BIND_SERVICE', + reason: + 'API deploy 自动启动或重启 Pingora 前必须拒绝疑似 direct-entry 低端口 capability。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', + reason: + 'API deploy 自动启动 Pingora 时必须限制为本机 shadow 高端口,避免误拉起公网直连入口。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'systemctl restart "${service_name}"', + reason: + 'API deploy 看到 Pingora 影子网关产物且 shadow 配置安全时必须随 current release 启动或重启。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRestartsActivePingoraWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 Pingora 产物发布后 active shadow service 自动重启。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployStartsInactivePingoraWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 但配置安全时会被主动拉起。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 direct-entry capability 下不会自动拉起 Pingora。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraPublicListenWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖公网监听 env 下不会自动拉起 Pingora。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'assertApiReleaseCanIncludePingoraGatewayArtifact', + reason: 'API release 动态烟测必须覆盖显式 include Pingora 影子网关产物。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: '--skip-pingora-gateway-build', + reason: 'API release 动态烟测应能用假二进制快速覆盖 Pingora 产物布局。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'assertPingoraEnvProductionDefaults', + reason: 'API release 动态烟测必须校验 Pingora env 的生产安全默认值。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertPingoraEnvProductionDefaults', + reason: 'API deploy 动态烟测必须校验 current release 内 Pingora env 的生产安全默认值。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED', + reason: 'Pingora 网关配置示例必须保留接流保护总开关。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1', + reason: 'Pingora 网关配置示例必须显式保留默认单实例接流保护口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false', + reason: 'Pingora 网关配置示例必须默认不确认共享保护层。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED', + reason: 'Pingora 网关配置示例必须暴露 gzip 总开关。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + reason: 'Pingora 网关配置示例必须暴露可选 TLS 直连入口。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN', + reason: 'Pingora 网关配置示例必须暴露可选 HTTP 到 HTTPS 重定向入口。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + reason: 'Pingora 网关配置示例必须暴露压缩算法白名单。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'Pingora 正式化口径固定为 gzip-only', + reason: 'Pingora 网关配置示例必须明确当前正式化压缩口径。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Brotli', + reason: 'Pingora 试点文档必须说明 Brotli 尚未进入 Pingora 的边界。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Pingora 正式化口径固定为 gzip-only', + reason: 'Pingora 试点文档必须把 Brotli 取舍落成明确结论。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + excludes: '补齐 Brotli 取舍', + reason: 'Pingora 试点文档不能继续把 Brotli 取舍留成待办。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'Pingora Brotli 不能只看 Content-Encoding', + reason: '团队共享踩坑必须记录 Pingora Brotli 端到端解压风险。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL=5', + reason: 'Pingora 网关配置示例必须对齐 Nginx gzip 压缩等级。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES=1024', + reason: 'Pingora 网关配置示例必须对齐 Nginx gzip_min_length。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL=no-cache', + reason: 'Pingora 网关配置示例必须暴露 HTML / SPA fallback 缓存头。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable', + reason: 'Pingora 网关配置示例必须暴露指纹静态资源长期缓存头。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Vite 指纹静态资源', + reason: 'Pingora 试点文档必须说明直连静态缓存头口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: + 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS=3600', + reason: 'Pingora 网关配置示例必须保留上游 API 读取超时口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: + 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS=3600', + reason: 'Pingora 网关配置示例必须保留长连接读取超时口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT', + reason: 'Pingora 网关配置示例必须保留通用 API 并发保护参数。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_RATE_PER_SECOND', + reason: 'Pingora 网关配置示例必须保留公开列表 RPS 保护参数。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE', + reason: 'Pingora 网关配置示例必须暴露 access log 文件路径。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'API 上游未收到 X-Forwarded-Host', + reason: 'Pingora gateway smoke 必须断言 API 上游收到 Nginx parity 的 X-Forwarded-Host。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'API 上游 X-Forwarded-For 未追加 TCP 对端 IP', + reason: + 'Pingora gateway smoke 必须断言 X-Forwarded-For 追加真实 TCP 对端 IP。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'API 代理头', + reason: 'Pingora 试点文档必须说明 API 代理头 parity 门禁。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'X-Real-IP', + reason: 'Pingora 试点文档必须说明 X-Real-IP 的 TCP 对端 IP 口径。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + reason: 'Pingora direct live env 示例必须暴露 direct access log 复核路径。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES=2000', + reason: 'Pingora direct live env 示例必须暴露 direct access log tail 行数。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: 'Nginx README 的 Pingora 直连 runbook 必须包含 access log 落盘校验参数。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: 'Pingora 试点文档必须展示 direct access log 落盘校验参数。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: '生产运维文档必须展示 Pingora 直连 access log 落盘校验参数。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: '直连日志证据补充', + reason: '团队共享决策必须记录 Pingora 直连 access log 证据门禁。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: '团队共享踩坑必须记录 Pingora 直连 access log 参数。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: '启用后 `--require-direct --release-runtime-only` 复核不再要求端口空闲', + reason: + '生产运维文档必须明确 Pingora 启用后直连复核不再要求 80/443 空闲。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: '不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`', + reason: + '生产运维文档必须阻止回退 smoke 继续沿用旧的固定 healthz 响应片段。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: "--rollback-nginx-smoke-expect-body ''", + reason: '生产运维文档必须展示来自真实 Nginx 首页的回退响应体片段示例。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '`--require-direct` 用于 Pingora 已接管后的直连复核,不再要求 `--direct-preflight-check-ports-free`', + reason: + 'Pingora 试点文档必须明确端口释放检查只属于切流前和启用 apply 阶段。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: "--rollback-nginx-smoke-expect-body ''", + reason: 'Pingora 试点文档必须展示真实 Nginx 回退 smoke 响应体片段。', + }, + { + file: 'deploy/nginx/README.md', + includes: '启用后 `--require-direct` 复核不再要求端口空闲', + reason: 'Nginx README 必须记录 Pingora 启用后 direct 复核的端口语义。', + }, + { + file: 'deploy/nginx/README.md', + includes: "--nginx-smoke-expect-body ''", + reason: 'Nginx README 必须展示真实 Nginx 回退 smoke 响应体片段。', + }, + { + file: 'deploy/nginx/README.md', + includes: '不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`', + reason: 'Nginx README 必须防止回退 smoke 示例回到旧 healthz 口径。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: '不再默认假定 `/healthz` 会返回 `"ok":true`', + reason: '团队共享决策必须记录回退 smoke 的真实 Nginx 入口要求。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: '启用后复核应看到端口由 Pingora 占用而不是空闲', + reason: + '团队共享决策必须记录启用后 --require-direct 不再检查端口空闲。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '不要继续用固定 `/healthz` 与 `"ok":true`', + reason: '团队共享踩坑必须提醒回退 smoke 不能沿用旧 healthz 片段。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '启用后 `--require-direct` 复核不再要求端口空闲', + reason: '团队共享踩坑必须提醒端口空闲只属于切流前门禁。', + }, + { + file: 'deploy/nginx/README.md', + excludes: + '--nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body \'"ok":true\'', + reason: 'Nginx README 不应再给出固定 healthz + ok:true 的回退命令。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + excludes: + '--nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body \'"ok":true\'', + reason: '生产运维文档不应再给出固定 healthz + ok:true 的回退命令。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + excludes: 'rollback-nginx-smoke-expect-body="ok":true', + reason: '生产运维 runbook 不应再默认展示 ok:true 作为回退响应体片段。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + excludes: 'rollback-nginx-smoke-expect-body="ok":true', + reason: 'Pingora 试点 runbook 不应再默认展示 ok:true 作为回退响应体片段。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + excludes: + '--nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>', + reason: '团队共享踩坑不应再把固定本机 healthz 当成可执行回退示例。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED', + reason: 'Pingora 网关配置示例必须暴露 X-Forwarded-For 信任确认开关。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Brotli', + reason: 'Pingora 试点文档必须明确 gzip 以外的压缩边界。', + }, + { + file: 'deploy/nginx/README.md', + includes: '直连 Pingora 时,不把 Brotli parity 作为切换门禁', + reason: 'Nginx 压缩文档必须明确 Pingora Brotli 不作为直连切换门禁。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: + 'Genarrative-Api-Deploy` 只能从上游构建归档复制并执行 `build//scripts/deploy/production-api-deploy.sh`', + reason: 'Pingora 试点文档必须明确 API Deploy 执行入口来自 build 产物。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: 'Pingora 试点文档必须明确多实例接流保护确认边界。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: '生产运维文档必须明确多实例 Pingora 与进程内保护的启动 / preflight 边界。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: '团队共享决策必须记录 Pingora 多实例接流保护边界。', + }, + { + file: 'docs/project-memory/shared-memory/decision-log.md', + includes: '禁止继续执行部署工作区根部脚本', + reason: '团队共享决策必须记录 API Deploy 不再执行 workspace 根部脚本。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'API deploy 脚本本身也不能继续用部署工作区根部', + reason: + '团队共享踩坑必须记录 workspace 根部 deploy 脚本会掩盖发布包布局问题。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: '备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底', + reason: '团队共享踩坑必须记录备份/巡检脚本也不能使用 workspace fallback。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'Pingora 影子网关已通过', + reason: 'Nginx 压缩文档必须说明 Pingora gzip parity 口径。', + }, { file: 'jenkins/Jenkinsfile.production-api-build', includes: 'scripts/ops/production-health-patrol.mjs', reason: 'API Build 归档必须包含健康巡检脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs', + reason: 'API Build 归档必须包含 Pingora current release 自审脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API Build 归档必须包含 Pingora 切换状态快照脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据包脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API Build 归档必须包含 Pingora 切换命令证据脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据验真脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据根目录审计脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs', + reason: 'API Build 归档必须包含健康巡检 env 复核脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs', + reason: 'API Build 归档必须包含 Pingora release readiness 聚合门禁脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs', + reason: 'API Build 归档必须包含 Pingora canary live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs', + reason: 'API Build 归档必须包含 Pingora canary access log 对账脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh', + reason: 'API Build 归档必须包含 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh', + reason: 'API Build 归档必须包含 build 产物内的维护开启脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh', + reason: 'API Build 归档必须包含 build 产物内的维护关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API Build 归档必须包含 Pingora health patrol env 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'API Build 归档必须包含 Pingora gateway env shadow 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API Build 归档必须包含 Pingora realpath canary 启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API Build 归档必须包含 Pingora realpath canary 关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**', + reason: 'API Build 归档必须包含 env 示例目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**', + reason: 'API Build 归档必须包含 Nginx/Pingora canary snippet。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + excludes: ',scripts/deploy/production-api-deploy.sh', + reason: 'API Build 不应再归档工作区根部 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'INCLUDE_PINGORA_GATEWAY', + reason: 'API Build 必须保留 Pingora 影子网关可选构建参数。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'command -v cmake', + reason: 'API Build 勾选 Pingora 时必须前置检查 cmake。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: '构建 Pingora 影子网关缺少 C++ 编译器', + reason: 'API Build 勾选 Pingora 时必须前置检查 C++ 编译器。', + }, { file: 'jenkins/Jenkinsfile.production-api-deploy', includes: 'scripts/ops/production-health-patrol.mjs', reason: 'API Deploy 复制上游产物时必须包含健康巡检脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora current release 自审脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换状态快照脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据包脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换命令证据脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据验真脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据根目录审计脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs', + reason: 'API Deploy 复制上游产物时必须包含健康巡检 env 复核脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora release readiness 聚合门禁脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora canary live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora canary access log 对账脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora realpath canary 启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora realpath canary 关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh', + reason: 'API Deploy 必须复制并执行 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + '"build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh"', + reason: 'API Deploy 执行入口必须是 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh', + reason: 'API Deploy 必须复制 build 产物内的维护开启脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API Deploy 必须复制 build 产物内的 Pingora health patrol env 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + reason: 'API Deploy 必须复制 build 产物内的 Pingora gateway env shadow 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh', + reason: 'API Deploy 必须复制 build 产物内的维护关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/deploy/env/**', + reason: 'API Deploy 必须从上游 build 产物复制 env 示例目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/deploy/nginx/**', + reason: 'API Deploy 必须从上游 build 产物复制 Nginx/Pingora canary snippet。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'NGINX_DEPLOY_DIR_SOURCE', + reason: 'API Deploy 必须把发布产物内 deploy/nginx 复制到 current release。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + excludes: ',scripts/deploy/production-api-deploy.sh', + reason: 'API Deploy 不应再复制工作区根部 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'pingora-gateway.sha256', + reason: + 'API Deploy 复制上游产物时必须保留 Pingora 影子网关可选 checksum 产物。', + }, +]; + +const nodeEnvFileCommandFiles = [ + 'package.json', + 'deploy/systemd/genarrative-database-backup.service', + 'scripts/deploy/production-stdb-publish.sh', + 'scripts/deploy/pingora-direct-enable.sh', + 'scripts/deploy/pingora-direct-rollback.sh', ]; let failed = false; for (const check of checks) { const content = readFileSync(check.file, 'utf8'); - if (!content.includes(check.includes)) { + if (check.includes && !content.includes(check.includes)) { failed = true; console.error( `[check:production-ops] ${check.file} 缺少 ${check.includes}。${check.reason}`, ); } + if (check.excludes && content.includes(check.excludes)) { + failed = true; + console.error( + `[check:production-ops] ${check.file} 不应包含 ${check.excludes}。${check.reason}`, + ); + } +} + +for (const file of nodeEnvFileCommandFiles) { + const content = readFileSync(file, 'utf8'); + const commandText = content.replace(/\\\r?\n\s*/g, ' '); + const unsafeCommands = commandText.match( + /\bnode\b(?!\s+--)(?=[^;&|\n]*\.mjs)(?=[^;&|\n]*--env-file)[^;&|\n]*/g, + ); + if (!unsafeCommands) { + continue; + } + + failed = true; + for (const command of unsafeCommands) { + console.error( + `[check:production-ops] ${file} 中的 Node 命令必须用 node -- 分隔业务 --env-file 参数: ${command.trim()}`, + ); + } } if (failed) { diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index b01cf5571..4c5587b3d 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -20,8 +20,8 @@ const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD'; function usage() { console.log(`用法: npm run database:backup:oss -- [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] - node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] - node scripts/database-backup-to-oss.mjs --upload-archive + node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] + node -- scripts/database-backup-to-oss.mjs --upload-archive 说明: 将 SpacetimeDB 数据目录打包成 .tar.gz,并上传到阿里云 OSS 指定 bucket。 diff --git a/scripts/deploy/pingora-direct-enable.sh b/scripts/deploy/pingora-direct-enable.sh new file mode 100644 index 000000000..55bfa52ff --- /dev/null +++ b/scripts/deploy/pingora-direct-enable.sh @@ -0,0 +1,922 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" +SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}" +DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service" +SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_DIRECT_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}" +DEFAULT_TEMPLATE_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf" +TEMPLATE_PATH="${GENARRATIVE_PINGORA_DIRECT_TEMPLATE_PATH:-${DEFAULT_TEMPLATE_PATH}}" +DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}" +PREFLIGHT_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-direct-preflight.mjs}" +PREFLIGHT_ENV_FILE="${GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE:-}" +CURRENT_RELEASE_AUDIT_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_SCRIPT:-${REPO_ROOT}/scripts/ops/pingora-current-release-audit.mjs}" +CURRENT_RELEASE_ROOT="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_ROOT:-${REPO_ROOT}}" +CURRENT_RELEASE_AUDIT_TIMEOUT_MS="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_TIMEOUT_MS:-}" +PREFLIGHT_SYSTEMD="false" +PREFLIGHT_CHECK_CERT_READABLE="false" +PREFLIGHT_CHECK_SERVICE_ENV_FILE="false" +PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE="false" +PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE="false" +PREFLIGHT_CHECK_PORTS_FREE="false" +DIRECT_LIVE_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_LIVE_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-direct-live.mjs}" +DIRECT_HTTPS_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL:-}" +DIRECT_HTTP_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL:-}" +DIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_HOST:-}" +DIRECT_REDIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST:-}" +DIRECT_REDIRECT_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL:-}" +DIRECT_PROBE_TOKEN="${GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN:-}" +DIRECT_PINGORA_ACCESS_LOG="${GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG:-}" +DIRECT_ACCESS_LOG_SINCE_LINES="${GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES:-}" +DIRECT_SPACETIME_DATABASE="${GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE:-}" +DIRECT_TIMEOUT_MS="${GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS:-}" +APPLY="false" +STATUS_AFTER="true" +VERIFY_SYSTEMD_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-direct-enable.sh [--apply] [--preflight-env-file ] [--preflight-systemd] [--preflight-check-cert-readable] [--preflight-check-service-env-file] [--preflight-check-service-user-cert-readable] [--preflight-check-service-binary-executable] [--preflight-check-ports-free] + [--direct-https-base-url ] [--direct-http-base-url ] [--direct-host ] [--direct-redirect-host ] [--direct-redirect-base-url ] + [--direct-spacetime-database ] [--direct-probe-token ] [--direct-pingora-access-log ] [--direct-access-log-since-lines ] [--direct-timeout-ms ] [--direct-live-script ] [--no-postcheck] + [--current-release-audit-script ] [--current-release-root ] [--current-release-audit-timeout-ms ] + [--service ] [--service-unit-path ] [--template-path ] [--dropin-path ] + +说明: + 启用 Pingora 直连低端口入口:把已准备好的 direct-entry systemd drop-in 模板安装到 + genarrative-pingora-gateway.service.d/direct-entry.conf,执行 daemon-reload, + 重启 genarrative-pingora-gateway.service,让服务具备绑定 80/443 的最小 capability。 + +默认是 dry-run,只打印将执行的命令;必须显式传 --apply 才会修改系统状态。 +该脚本不写入 TLS env、不复制证书、不停止 Nginx;--apply 会先执行 current release 自包含自审, +确认发布包已携带 Pingora 网关且 systemd ExecStart 指向随包二进制;--apply 时必须提供 --preflight-env-file、 +--preflight-check-cert-readable、--preflight-check-service-env-file、--preflight-check-service-user-cert-readable、--preflight-check-service-binary-executable、--preflight-check-ports-free 和 direct live smoke 参数, +先通过 direct preflight,安装并重启 Pingora,再通过 direct live smoke 后才视为启用成功。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + --no-postcheck) + VERIFY_SYSTEMD_AFTER="false" + shift + ;; + --service) + SERVICE_NAME="${2:-}" + if [[ -z "${SERVICE_NAME}" ]]; then + echo "[pingora-direct-enable] --service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --service-unit-path) + SERVICE_UNIT_PATH="${2:-}" + if [[ -z "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-enable] --service-unit-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --template-path) + TEMPLATE_PATH="${2:-}" + if [[ -z "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] --template-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --dropin-path) + DROPIN_PATH="${2:-}" + if [[ -z "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] --dropin-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-script) + PREFLIGHT_SCRIPT="${2:-}" + if [[ -z "${PREFLIGHT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --preflight-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-env-file) + PREFLIGHT_ENV_FILE="${2:-}" + if [[ -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] --preflight-env-file 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-audit-script) + CURRENT_RELEASE_AUDIT_SCRIPT="${2:-}" + if [[ -z "${CURRENT_RELEASE_AUDIT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --current-release-audit-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-root) + CURRENT_RELEASE_ROOT="${2:-}" + if [[ -z "${CURRENT_RELEASE_ROOT}" ]]; then + echo "[pingora-direct-enable] --current-release-root 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-audit-timeout-ms) + CURRENT_RELEASE_AUDIT_TIMEOUT_MS="${2:-}" + if [[ -z "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" ]]; then + echo "[pingora-direct-enable] --current-release-audit-timeout-ms 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-systemd) + PREFLIGHT_SYSTEMD="true" + shift + ;; + --preflight-check-cert-readable) + PREFLIGHT_CHECK_CERT_READABLE="true" + shift + ;; + --preflight-check-service-env-file) + PREFLIGHT_CHECK_SERVICE_ENV_FILE="true" + shift + ;; + --preflight-check-service-user-cert-readable) + PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE="true" + shift + ;; + --preflight-check-service-binary-executable) + PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE="true" + shift + ;; + --preflight-check-ports-free) + PREFLIGHT_CHECK_PORTS_FREE="true" + shift + ;; + --direct-live-script) + DIRECT_LIVE_SCRIPT="${2:-}" + if [[ -z "${DIRECT_LIVE_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --direct-live-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-https-base-url) + DIRECT_HTTPS_BASE_URL="${2:-}" + if [[ -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-https-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-http-base-url) + DIRECT_HTTP_BASE_URL="${2:-}" + if [[ -z "${DIRECT_HTTP_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-http-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-host) + DIRECT_HOST="${2:-}" + if [[ -z "${DIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --direct-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-redirect-host) + DIRECT_REDIRECT_HOST="${2:-}" + if [[ -z "${DIRECT_REDIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --direct-redirect-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-redirect-base-url) + DIRECT_REDIRECT_BASE_URL="${2:-}" + if [[ -z "${DIRECT_REDIRECT_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-redirect-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-probe-token) + DIRECT_PROBE_TOKEN="${2:-}" + if [[ -z "${DIRECT_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-enable] --direct-probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-pingora-access-log) + DIRECT_PINGORA_ACCESS_LOG="${2:-}" + if [[ -z "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + echo "[pingora-direct-enable] --direct-pingora-access-log 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-access-log-since-lines) + DIRECT_ACCESS_LOG_SINCE_LINES="${2:-}" + if [[ -z "${DIRECT_ACCESS_LOG_SINCE_LINES}" ]]; then + echo "[pingora-direct-enable] --direct-access-log-since-lines 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-spacetime-database) + DIRECT_SPACETIME_DATABASE="${2:-}" + if [[ -z "${DIRECT_SPACETIME_DATABASE}" ]]; then + echo "[pingora-direct-enable] --direct-spacetime-database 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-timeout-ms) + DIRECT_TIMEOUT_MS="${2:-}" + if [[ -z "${DIRECT_TIMEOUT_MS}" ]]; then + echo "[pingora-direct-enable] --direct-timeout-ms 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + *) + echo "[pingora-direct-enable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-direct-enable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +reject_filesystem_root_path() { + local label="$1" + local value="$2" + if is_filesystem_root_path "${value}"; then + echo "[pingora-direct-enable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +reject_control_characters "--service" "${SERVICE_NAME}" +reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_control_characters "--template-path" "${TEMPLATE_PATH}" +reject_control_characters "--dropin-path" "${DROPIN_PATH}" +reject_control_characters "--preflight-script" "${PREFLIGHT_SCRIPT}" +reject_control_characters "--preflight-env-file" "${PREFLIGHT_ENV_FILE}" +reject_control_characters "--current-release-audit-script" "${CURRENT_RELEASE_AUDIT_SCRIPT}" +reject_control_characters "--current-release-root" "${CURRENT_RELEASE_ROOT}" +reject_control_characters "--current-release-audit-timeout-ms" "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" +reject_control_characters "--direct-live-script" "${DIRECT_LIVE_SCRIPT}" +reject_control_characters "--direct-https-base-url" "${DIRECT_HTTPS_BASE_URL}" +reject_control_characters "--direct-http-base-url" "${DIRECT_HTTP_BASE_URL}" +reject_control_characters "--direct-host" "${DIRECT_HOST}" +reject_control_characters "--direct-redirect-host" "${DIRECT_REDIRECT_HOST}" +reject_control_characters "--direct-redirect-base-url" "${DIRECT_REDIRECT_BASE_URL}" +reject_control_characters "--direct-probe-token" "${DIRECT_PROBE_TOKEN}" +reject_control_characters "--direct-pingora-access-log" "${DIRECT_PINGORA_ACCESS_LOG}" +reject_control_characters "--direct-access-log-since-lines" "${DIRECT_ACCESS_LOG_SINCE_LINES}" +reject_control_characters "--direct-spacetime-database" "${DIRECT_SPACETIME_DATABASE}" +reject_control_characters "--direct-timeout-ms" "${DIRECT_TIMEOUT_MS}" + +reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_filesystem_root_path "--template-path" "${TEMPLATE_PATH}" +reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}" +reject_filesystem_root_path "--preflight-env-file" "${PREFLIGHT_ENV_FILE}" +reject_filesystem_root_path "--preflight-script" "${PREFLIGHT_SCRIPT}" +reject_filesystem_root_path "--current-release-audit-script" "${CURRENT_RELEASE_AUDIT_SCRIPT}" +reject_filesystem_root_path "--current-release-root" "${CURRENT_RELEASE_ROOT}" +reject_filesystem_root_path "--direct-live-script" "${DIRECT_LIVE_SCRIPT}" +reject_filesystem_root_path "--direct-pingora-access-log" "${DIRECT_PINGORA_ACCESS_LOG}" + +if [[ "${TEMPLATE_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --template-path 必须是绝对路径: ${TEMPLATE_PATH}" >&2 + exit 1 +fi + +if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ "${DROPIN_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2 + exit 1 +fi + +if [[ -n "${PREFLIGHT_ENV_FILE}" && "${PREFLIGHT_ENV_FILE}" != /* ]]; then + echo "[pingora-direct-enable] --preflight-env-file 必须是绝对路径: ${PREFLIGHT_ENV_FILE}" >&2 + exit 1 +fi + +if [[ "${PREFLIGHT_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --preflight-script 必须是绝对路径: ${PREFLIGHT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${CURRENT_RELEASE_AUDIT_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --current-release-audit-script 必须是绝对路径: ${CURRENT_RELEASE_AUDIT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${CURRENT_RELEASE_ROOT}" != /* ]]; then + echo "[pingora-direct-enable] --current-release-root 必须是绝对路径: ${CURRENT_RELEASE_ROOT}" >&2 + exit 1 +fi + +if [[ "${DIRECT_LIVE_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --direct-live-script 必须是绝对路径: ${DIRECT_LIVE_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-env-file,避免跳过直连 env / 证书预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_CERT_READABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-cert-readable,避免跳过 TLS 证书和私钥可读预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-env-file,避免检查的 env 文件和 systemd service 实际读取的 EnvironmentFile 不一致。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-user-cert-readable,避免跳过 systemd 服务用户读取 TLS 证书和私钥预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-binary-executable,避免 current release 缺少可执行 pingora-gateway 时才在重启阶段失败。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_PORTS_FREE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-ports-free,避免 Nginx 或其它进程仍占用 80/443。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-https-base-url,启用后验证 Pingora HTTPS 直连入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HTTP_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-http-base-url,启用后验证 HTTP redirect / ACME 入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-host,启用后用正式域名 Host/SNI 验证直连入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_REDIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-redirect-host,启用后验证 HTTP redirect Location host。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-pingora-access-log,启用后验证直连请求已写入 Pingora access log。" >&2 + exit 1 +fi + +if [[ -n "${DIRECT_PINGORA_ACCESS_LOG}" && "${DIRECT_PINGORA_ACCESS_LOG}" != /* ]]; then + echo "[pingora-direct-enable] --direct-pingora-access-log 必须是绝对路径: ${DIRECT_PINGORA_ACCESS_LOG}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_SPACETIME_DATABASE}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-spacetime-database,启用后验证目标库 WSS subscribe。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] direct-entry 模板不存在: ${TEMPLATE_PATH}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-enable] 主 service 模板不存在: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${CURRENT_RELEASE_AUDIT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] current release 自审脚本不存在: ${CURRENT_RELEASE_AUDIT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${PREFLIGHT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] direct preflight 脚本不存在: ${PREFLIGHT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${DIRECT_LIVE_SCRIPT}" ]]; then + echo "[pingora-direct-enable] direct live smoke 脚本不存在: ${DIRECT_LIVE_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" ]]; then + DROPIN_DIR="$(dirname "${DROPIN_PATH}")" + if [[ -L "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-enable] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-enable] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -L "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2 + exit 1 + fi +fi + +run_cmd() { + echo "+ $*" + if [[ "${APPLY}" == "true" ]]; then + "$@" + fi +} + +print_redacted_command() { + local previous="" output=() + for arg in "$@"; do + if [[ "${previous}" == "--probe-token" || "${previous}" == "--direct-probe-token" ]]; then + output+=("") + else + output+=("${arg}") + fi + previous="${arg}" + done + echo "+ ${output[*]}" +} + +run_systemctl() { + if [[ "${APPLY}" != "true" ]]; then + run_cmd systemctl "$@" + return + fi + if command -v systemctl >/dev/null 2>&1; then + run_cmd systemctl "$@" + else + echo "[pingora-direct-enable] 未找到 systemctl,无法执行: systemctl $*" >&2 + exit 1 + fi +} + +run_preflight() { + if [[ -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] 未提供 --preflight-env-file,仅执行脚本 dry-run;--apply 会强制要求预检。" + return + fi + + preflight_args=( + "${PREFLIGHT_SCRIPT}" + --env-file "${PREFLIGHT_ENV_FILE}" + --require-live-env + ) + if [[ "${PREFLIGHT_SYSTEMD}" == "true" ]]; then + preflight_args+=(--systemd-cat) + fi + if [[ "${PREFLIGHT_CHECK_CERT_READABLE}" == "true" ]]; then + preflight_args+=(--check-cert-readable) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" == "true" ]]; then + preflight_args+=(--check-service-env-file) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE}" == "true" ]]; then + preflight_args+=(--check-service-user-cert-readable) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE}" == "true" ]]; then + preflight_args+=(--check-service-binary-executable) + fi + if [[ "${PREFLIGHT_CHECK_PORTS_FREE}" == "true" ]]; then + preflight_args+=(--check-ports-free) + fi + + echo "+ node -- ${preflight_args[*]}" + node -- "${preflight_args[@]}" +} + +run_current_release_audit() { + audit_args=( + "${CURRENT_RELEASE_AUDIT_SCRIPT}" + --release-root "${CURRENT_RELEASE_ROOT}" + --require-pingora-gateway + --systemd-show + --systemd-service "${SERVICE_NAME}" + ) + if [[ -n "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" ]]; then + audit_args+=(--timeout-ms "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}") + fi + + echo "+ node -- ${audit_args[*]}" + if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] dry-run:--apply 前会先执行 current release 自包含自审。" + return + fi + + node -- "${audit_args[@]}" +} + +systemd_env_file_matches() { + local expected="$1" + local line value word normalized + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == EnvironmentFile=* ]] || continue + value="${line#EnvironmentFile=}" + [[ -n "${value}" ]] || continue + for word in ${value}; do + normalized="${word%\"}" + normalized="${normalized#\"}" + normalized="${normalized%\'}" + normalized="${normalized#\'}" + normalized="${normalized#-}" + if [[ "${normalized}" == "${expected}" ]]; then + return 0 + fi + done + done <<<"${unit_content}" + return 1 +} + +read_expected_exec_start() { + local line value + if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then + return 1 + fi + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == ExecStart=* ]] || continue + value="${line#ExecStart=}" + [[ -n "${value}" ]] || continue + printf '%s\n' "${value}" + return 0 + done <"${SERVICE_UNIT_PATH}" + return 1 +} + +first_exec_word() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ "${value}" == \"* ]]; then + value="${value#\"}" + printf '%s\n' "${value%%\"*}" + return + fi + printf '%s\n' "${value%%[[:space:]]*}" +} + +systemd_show_exec_start_matches() { + local expected_exec="$1" + local expected_binary actual_show actual_binary + expected_binary="$(first_exec_word "${expected_exec}")" + [[ -n "${expected_binary}" ]] || return 1 + actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)" + actual_binary="${actual_show}" + if [[ "${actual_show}" == *"path="* ]]; then + actual_binary="${actual_show#*path=}" + actual_binary="${actual_binary%% ;*}" + elif [[ "${actual_show}" == ExecStart=* ]]; then + actual_binary="${actual_show#ExecStart=}" + actual_binary="$(first_exec_word "${actual_binary}")" + else + actual_binary="$(first_exec_word "${actual_binary}")" + fi + [[ "${actual_binary}" == "${expected_binary}" ]] +} + +verify_systemd_exec_start() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 Pingora ExecStart 指向核验。" + return + fi + + expected_exec_start="$(read_expected_exec_start || true)" + if [[ -z "${expected_exec_start}" ]]; then + echo "[pingora-direct-enable] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2 + exit 1 + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + echo "[pingora-direct-enable] dry-run:--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl,无法核验 Pingora ExecStart。" >&2 + exit 1 + fi + + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + if ! systemd_show_exec_start_matches "${expected_exec_start}"; then + echo "[pingora-direct-enable] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start},请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2 + exit 1 + fi +} + +verify_systemd_dropin() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 systemd drop-in 生效核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl cat ${SERVICE_NAME}" + echo "[pingora-direct-enable] dry-run:--apply 后会校验 systemd 最终配置包含 CAP_NET_BIND_SERVICE。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl,无法核验 drop-in 是否生效。" >&2 + exit 1 + fi + + echo "+ systemctl cat ${SERVICE_NAME}" + unit_content="$(systemctl cat "${SERVICE_NAME}")" + if [[ "${unit_content}" != *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-enable] systemctl cat 未显示 AmbientCapabilities=CAP_NET_BIND_SERVICE,direct-entry drop-in 未生效。" >&2 + exit 1 + fi + if [[ "${unit_content}" != *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-enable] systemctl cat 未显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE,direct-entry drop-in 未生效。" >&2 + exit 1 + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" == "true" && -n "${PREFLIGHT_ENV_FILE}" ]]; then + if ! systemd_env_file_matches "${PREFLIGHT_ENV_FILE}"; then + echo "[pingora-direct-enable] systemctl cat 未显示 EnvironmentFile=${PREFLIGHT_ENV_FILE},service 实际读取的 env 与 preflight env 不一致。" >&2 + exit 1 + fi + fi +} + +verify_service_active_after_restart() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 Pingora 重启后状态核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl is-active ${SERVICE_NAME}" + echo "[pingora-direct-enable] dry-run:--apply 后会校验 Pingora service 为 active。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl,无法核验 Pingora 重启后状态。" >&2 + exit 1 + fi + + echo "+ systemctl is-active ${SERVICE_NAME}" + service_state="$(systemctl is-active "${SERVICE_NAME}")" + if [[ "${service_state}" != "active" ]]; then + echo "[pingora-direct-enable] Pingora 重启后状态不是 active: ${service_state}" >&2 + exit 1 + fi +} + +run_direct_live_smoke_after_enable() { + if [[ -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "+ node ${DIRECT_LIVE_SCRIPT} --https-base-url --http-base-url --host --redirect-host --spacetime-database --require-wss-upgrade" + echo "[pingora-direct-enable] dry-run:--apply 后会执行 direct live smoke;--apply 会强制要求 direct live 参数。" + return + fi + + direct_live_args=( + "${DIRECT_LIVE_SCRIPT}" + --https-base-url "${DIRECT_HTTPS_BASE_URL}" + ) + if [[ -n "${DIRECT_HTTP_BASE_URL}" ]]; then + direct_live_args+=(--http-base-url "${DIRECT_HTTP_BASE_URL}") + fi + if [[ -n "${DIRECT_HOST}" ]]; then + direct_live_args+=(--host "${DIRECT_HOST}") + fi + if [[ -n "${DIRECT_REDIRECT_HOST}" ]]; then + direct_live_args+=(--redirect-host "${DIRECT_REDIRECT_HOST}") + fi + if [[ -n "${DIRECT_REDIRECT_BASE_URL}" ]]; then + direct_live_args+=(--redirect-base-url "${DIRECT_REDIRECT_BASE_URL}") + fi + if [[ -n "${DIRECT_PROBE_TOKEN}" ]]; then + direct_live_args+=(--probe-token "${DIRECT_PROBE_TOKEN}") + fi + if [[ -n "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + direct_live_args+=(--pingora-access-log "${DIRECT_PINGORA_ACCESS_LOG}") + fi + if [[ -n "${DIRECT_ACCESS_LOG_SINCE_LINES}" ]]; then + direct_live_args+=(--access-log-since-lines "${DIRECT_ACCESS_LOG_SINCE_LINES}") + fi + if [[ -n "${DIRECT_SPACETIME_DATABASE}" ]]; then + direct_live_args+=(--spacetime-database "${DIRECT_SPACETIME_DATABASE}") + fi + direct_live_args+=(--require-wss-upgrade) + if [[ -n "${DIRECT_TIMEOUT_MS}" ]]; then + direct_live_args+=(--timeout-ms "${DIRECT_TIMEOUT_MS}") + fi + direct_live_args+=(--json) + + print_redacted_command node "${direct_live_args[@]}" + if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] dry-run:--apply 后会验证 Pingora HTTPS / HTTP redirect / WSS 直连入口,并解析 direct-access-log 结构化证据。" + return + fi + + set +e + direct_live_output="$(node "${direct_live_args[@]}")" + direct_live_status=$? + set -e + printf '%s\n' "${direct_live_output}" + if [[ "${direct_live_status}" -ne 0 ]]; then + exit "${direct_live_status}" + fi + DIRECT_LIVE_OUTPUT="${direct_live_output}" node -- <<'NODE' +const stdout = process.env.DIRECT_LIVE_OUTPUT || ''; + +const jsonText = extractFirstJsonObject(stdout); +if (!jsonText) { + console.error( + '[pingora-direct-enable] direct live stdout 中未找到 JSON 结果对象,无法确认 direct-access-log 结构化证据。', + ); + process.exit(1); +} + +let parsed; +try { + parsed = JSON.parse(jsonText); +} catch (error) { + console.error( + `[pingora-direct-enable] direct live JSON 解析失败: ${error.message}`, + ); + process.exit(1); +} + +const accessLogCheck = parsed?.results?.find( + (item) => item?.name === 'direct-access-log', +); +if (!accessLogCheck) { + console.error( + '[pingora-direct-enable] direct live JSON 缺少 direct-access-log 结构化结果。', + ); + process.exit(1); +} + +const checked = toNonNegativeInteger(accessLogCheck.checked); +const matchedCount = toNonNegativeInteger(accessLogCheck.matchedCount); +const missingCount = toNonNegativeInteger(accessLogCheck.missingCount); +const mismatchCount = toNonNegativeInteger(accessLogCheck.mismatchCount); +const missingDetailsCount = Array.isArray(accessLogCheck.missing) + ? accessLogCheck.missing.length + : null; +const mismatchDetailsCount = Array.isArray(accessLogCheck.mismatches) + ? accessLogCheck.mismatches.length + : null; + +if ( + checked === null || + matchedCount === null || + missingCount !== 0 || + mismatchCount !== 0 || + matchedCount !== checked || + missingDetailsCount !== 0 || + mismatchDetailsCount !== 0 +) { + console.error( + `[pingora-direct-enable] direct-access-log 结构化证据未通过: checked=${checked ?? '-'} matched=${matchedCount ?? '-'} missing=${missingCount ?? '-'} mismatches=${mismatchCount ?? '-'}`, + ); + process.exit(1); +} + +console.log( + `[pingora-direct-enable] direct-access-log evidence checked=${checked} matched=${matchedCount} missing=0 mismatches=0`, +); + +function extractFirstJsonObject(text) { + const startIndex = text.indexOf('{'); + if (startIndex < 0) { + return ''; + } + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + return ''; +} + +function toNonNegativeInteger(value) { + const number = Number(value); + if (!Number.isInteger(number) || number < 0) { + return null; + } + return number; +} +NODE +} + +echo "[pingora-direct-enable] apply=${APPLY} service=${SERVICE_NAME} template=${TEMPLATE_PATH} dropin=${DROPIN_PATH}" + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] 当前是 dry-run;确认 preflight 通过后追加 --apply。" + if [[ ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] dry-run 提示:当前模板不存在,--apply 会失败: ${TEMPLATE_PATH}" >&2 + fi +fi + +run_current_release_audit +run_preflight +run_cmd install -d -m 0755 "$(dirname "${DROPIN_PATH}")" +run_cmd install -m 0644 "${TEMPLATE_PATH}" "${DROPIN_PATH}" +run_systemctl daemon-reload +run_systemctl restart "${SERVICE_NAME}" +verify_systemd_dropin +verify_systemd_exec_start +verify_service_active_after_restart +run_direct_live_smoke_after_enable + +if [[ "${STATUS_AFTER}" == "true" ]]; then + run_systemctl status "${SERVICE_NAME}" --no-pager +fi + +cat <} --http-base-url ${DIRECT_HTTP_BASE_URL:-} --host ${DIRECT_HOST:-} --redirect-host ${DIRECT_REDIRECT_HOST:-} ${DIRECT_REDIRECT_BASE_URL:+--redirect-base-url ${DIRECT_REDIRECT_BASE_URL} }--pingora-access-log ${DIRECT_PINGORA_ACCESS_LOG:-} --spacetime-database ${DIRECT_SPACETIME_DATABASE:-} --require-wss-upgrade --json 已通过,且 direct-access-log 结构化证据 matched=checked、missing=0、mismatches=0 +EOF diff --git a/scripts/deploy/pingora-direct-rollback.sh b/scripts/deploy/pingora-direct-rollback.sh new file mode 100644 index 000000000..f5b56ba8a --- /dev/null +++ b/scripts/deploy/pingora-direct-rollback.sh @@ -0,0 +1,768 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)" +DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service" +SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_ROLLBACK_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}" +DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SMOKE_URL="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_URL:-}" +NGINX_SMOKE_HOST="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_HOST:-}" +NGINX_SMOKE_EXPECT_BODY="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_EXPECT_BODY:-}" +CURL_BINARY="${GENARRATIVE_CURL_BINARY:-curl}" +HEALTH_PATROL_ENV_FILE="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_FILE:-}" +HEALTH_PATROL_ENV_CHECK_SCRIPT="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_CHECK_SCRIPT:-}" +HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL:-}" +HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_HOST:-}" +HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST:-auto}" +PINGORA_SHADOW_PROBE_URL="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL:-}" +PINGORA_SHADOW_PROBE_TOKEN="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN:-}" +APPLY="false" +RELOAD_NGINX="false" +STATUS_AFTER="true" +VERIFY_SYSTEMD_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-direct-rollback.sh [--apply] [--reload-nginx] [--nginx-smoke-url ] [--nginx-smoke-host ] [--nginx-smoke-expect-body ] [--curl-binary ] [--health-patrol-env-file ] [--health-patrol-env-check-script ] [--health-patrol-expected-public-base-url ] [--health-patrol-expected-public-host |--health-patrol-require-empty-public-host] [--pingora-shadow-probe-url ] [--pingora-shadow-probe-token ] [--no-postcheck] [--service ] [--service-unit-path ] [--dropin-path ] [--nginx-service ] [--nginx-binary ] + +说明: + 回退 Pingora 直连低端口入口:移除 direct-entry systemd drop-in,执行 daemon-reload, + 重启 genarrative-pingora-gateway.service,让服务回到 shadow/high-port 口径。 + +默认是 dry-run,只打印将执行的命令;必须显式传 --apply 才会修改系统状态。 +--apply 必须同时传 --reload-nginx,让脚本执行 nginx reload, +脚本会先运行 nginx -t,通过后再 reload,并在 reload 后确认 Nginx 仍处于 active 状态。 +--apply 还必须提供 --nginx-smoke-url,reload 后用 curl 验证 Nginx 入口已经真实响应。 +若提供 --nginx-smoke-expect-body,脚本会要求 smoke 响应体包含该片段,避免 HTTP 200 命中错误入口。 +脚本会以 current release 随包主 service 模板的 ExecStart 为真相源, +在重启 Pingora 后核验 systemd 最终 ExecStart 仍指向同一个 pingora-gateway 二进制。 +如果提供 --health-patrol-env-file,脚本会在 Nginx smoke 之后复核 health patrol 已切回 nginx 模式, +并可校验回退后的 public base URL / Host 是否恢复为切换前记录值。 +如果同时提供 --pingora-shadow-probe-url 与 --pingora-shadow-probe-token,脚本会在回退后 +验证 Pingora shadow 高端口探针仍返回 gateway=pingora-shadow。 +EOF +} + +resolve_default_health_patrol_env_check_script() { + printf '%s\n' "${REPO_ROOT}/scripts/check-production-health-patrol-env.mjs" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --reload-nginx) + RELOAD_NGINX="true" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + --no-postcheck) + VERIFY_SYSTEMD_AFTER="false" + shift + ;; + --service) + SERVICE_NAME="${2:-}" + if [[ -z "${SERVICE_NAME}" ]]; then + echo "[pingora-direct-rollback] --service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --service-unit-path) + SERVICE_UNIT_PATH="${2:-}" + if [[ -z "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-rollback] --service-unit-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --dropin-path) + DROPIN_PATH="${2:-}" + if [[ -z "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] --dropin-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-direct-rollback] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-direct-rollback] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-url) + NGINX_SMOKE_URL="${2:-}" + if [[ -z "${NGINX_SMOKE_URL}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-host) + NGINX_SMOKE_HOST="${2:-}" + if [[ -z "${NGINX_SMOKE_HOST}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-expect-body) + NGINX_SMOKE_EXPECT_BODY="${2:-}" + if [[ -z "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-expect-body 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --curl-binary) + CURL_BINARY="${2:-}" + if [[ -z "${CURL_BINARY}" ]]; then + echo "[pingora-direct-rollback] --curl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-env-file) + HEALTH_PATROL_ENV_FILE="${2:-}" + if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-file 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-env-check-script) + HEALTH_PATROL_ENV_CHECK_SCRIPT="${2:-}" + if [[ -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-check-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-expected-public-base-url) + HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${2:-}" + if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-expected-public-host) + HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${2:-}" + if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-require-empty-public-host) + HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="true" + shift + ;; + --pingora-shadow-probe-url) + PINGORA_SHADOW_PROBE_URL="${2:-}" + if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --pingora-shadow-probe-token) + PINGORA_SHADOW_PROBE_TOKEN="${2:-}" + if [[ -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + *) + echo "[pingora-direct-rollback] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -n "${HEALTH_PATROL_ENV_FILE}" && -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + HEALTH_PATROL_ENV_CHECK_SCRIPT="$(resolve_default_health_patrol_env_check_script)" +fi + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-direct-rollback] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +reject_filesystem_root_path() { + local label="$1" + local value="$2" + if is_filesystem_root_path "${value}"; then + echo "[pingora-direct-rollback] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +reject_control_characters "--service" "${SERVICE_NAME}" +reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_control_characters "--dropin-path" "${DROPIN_PATH}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-smoke-url" "${NGINX_SMOKE_URL}" +reject_control_characters "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" +reject_control_characters "--nginx-smoke-expect-body" "${NGINX_SMOKE_EXPECT_BODY}" +reject_control_characters "--curl-binary" "${CURL_BINARY}" +reject_control_characters "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" +reject_control_characters "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" +reject_control_characters "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" +reject_control_characters "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" +reject_control_characters "--health-patrol-require-empty-public-host" "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" +reject_control_characters "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" +reject_control_characters "--pingora-shadow-probe-token" "${PINGORA_SHADOW_PROBE_TOKEN}" + +reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}" +reject_filesystem_root_path "--nginx-binary" "${NGINX_BINARY}" +reject_filesystem_root_path "--curl-binary" "${CURL_BINARY}" +reject_filesystem_root_path "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" +reject_filesystem_root_path "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" + +if [[ "${DROPIN_PATH}" != /* ]]; then + echo "[pingora-direct-rollback] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2 + exit 1 +fi + +if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then + echo "[pingora-direct-rollback] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_ENV_FILE}" && "${HEALTH_PATROL_ENV_FILE}" != /* ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-file 必须是绝对路径: ${HEALTH_PATROL_ENV_FILE}" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" && "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" != /* ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-check-script 必须是绝对路径: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 + exit 1 +fi + +require_absolute_path_if_path_like() { + local label="$1" + local value="$2" + if [[ "${value}" == */* && "${value}" != /* ]]; then + echo "[pingora-direct-rollback] ${label} 包含路径分隔符时必须是绝对路径: ${value}" >&2 + exit 1 + fi +} + +require_absolute_path_if_path_like "--nginx-binary" "${NGINX_BINARY}" +require_absolute_path_if_path_like "--curl-binary" "${CURL_BINARY}" + +if [[ -n "${PINGORA_SHADOW_PROBE_URL}" && -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-url 必须同时提供 --pingora-shadow-probe-token。" >&2 + exit 1 +fi + +if [[ -n "${PINGORA_SHADOW_PROBE_TOKEN}" && -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-token 必须同时提供 --pingora-shadow-probe-url。" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" && "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" == "true" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-host 和 --health-patrol-require-empty-public-host 不能同时使用。" >&2 + exit 1 +fi + +validate_nginx_smoke_host() { + validate_host_value "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" +} + +validate_host_value() { + local label="$1" + local value="$2" + if [[ "${value}" == *"://"* || "${value}" == */* || "${value}" == *\?* || "${value}" == *"#"* || "${value}" =~ [[:space:]] ]]; then + echo "[pingora-direct-rollback] ${label} 只能是 host 或 host:port,不能包含 URL、路径、查询、片段或空白字符: ${value}" >&2 + exit 1 + fi +} + +is_loopback_smoke_url() { + local url="$1" + local authority host + case "${url}" in + http://*|https://*) ;; + *) return 1 ;; + esac + authority="${url#*://}" + authority="${authority%%/*}" + authority="${authority%%\?*}" + authority="${authority%%#*}" + authority="${authority##*@}" + if [[ "${authority}" == \[*\]* ]]; then + host="${authority#\[}" + host="${host%%\]*}" + else + host="${authority%%:*}" + fi + host="${host,,}" + [[ "${host}" == "localhost" || "${host}" == "127."* || "${host}" == "::1" ]] +} + +validate_http_url() { + local label="$1" + local url="$2" + case "${url}" in + http://*|https://*) ;; + *) + echo "[pingora-direct-rollback] ${label} 必须是 http(s) URL: ${url}" >&2 + exit 1 + ;; + esac +} + +if [[ -n "${NGINX_SMOKE_HOST}" ]]; then + validate_nginx_smoke_host +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + validate_host_value "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" +fi + +if [[ -n "${PINGORA_SHADOW_PROBE_URL}" ]]; then + validate_http_url "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" +fi + +if [[ -n "${NGINX_SMOKE_URL}" ]]; then + validate_http_url "--nginx-smoke-url" "${NGINX_SMOKE_URL}" +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + validate_http_url "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" +fi + +if [[ "${APPLY}" == "true" && "${RELOAD_NGINX}" != "true" ]]; then + echo "[pingora-direct-rollback] --apply 必须同时提供 --reload-nginx,确保回退后 Nginx 公网入口重新加载并保持 active。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_URL}" ]]; then + echo "[pingora-direct-rollback] --apply 必须同时提供 --nginx-smoke-url,确保回退后 Nginx 入口真实可访问。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_HOST}" ]] && is_loopback_smoke_url "${NGINX_SMOKE_URL}"; then + echo "[pingora-direct-rollback] --nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host,避免回退 smoke 命中默认 vhost。" >&2 + exit 1 +fi + +run_cmd() { + echo "+ $*" + if [[ "${APPLY}" == "true" ]]; then + "$@" + fi +} + +run_systemctl() { + if [[ "${APPLY}" != "true" ]]; then + run_cmd systemctl "$@" + return + fi + if command -v systemctl >/dev/null 2>&1; then + run_cmd systemctl "$@" + else + echo "[pingora-direct-rollback] 未找到 systemctl,无法执行: systemctl $*" >&2 + exit 1 + fi +} + +read_expected_exec_start() { + local line value + if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then + return 1 + fi + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == ExecStart=* ]] || continue + value="${line#ExecStart=}" + [[ -n "${value}" ]] || continue + printf '%s\n' "${value}" + return 0 + done <"${SERVICE_UNIT_PATH}" + return 1 +} + +first_exec_word() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ "${value}" == \"* ]]; then + value="${value#\"}" + printf '%s\n' "${value%%\"*}" + return + fi + printf '%s\n' "${value%%[[:space:]]*}" +} + +systemd_show_exec_start_matches() { + local expected_exec="$1" + local expected_binary actual_show actual_binary + expected_binary="$(first_exec_word "${expected_exec}")" + [[ -n "${expected_binary}" ]] || return 1 + actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)" + actual_binary="${actual_show}" + if [[ "${actual_show}" == *"path="* ]]; then + actual_binary="${actual_show#*path=}" + actual_binary="${actual_binary%% ;*}" + elif [[ "${actual_show}" == ExecStart=* ]]; then + actual_binary="${actual_show#ExecStart=}" + actual_binary="$(first_exec_word "${actual_binary}")" + else + actual_binary="$(first_exec_word "${actual_binary}")" + fi + [[ "${actual_binary}" == "${expected_binary}" ]] +} + +verify_systemd_exec_start_after_rollback() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Pingora ExecStart 指向核验。" + return + fi + + expected_exec_start="$(read_expected_exec_start || true)" + if [[ -z "${expected_exec_start}" ]]; then + echo "[pingora-direct-rollback] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2 + exit 1 + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + echo "[pingora-direct-rollback] dry-run:--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 Pingora ExecStart。" >&2 + exit 1 + fi + + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + if ! systemd_show_exec_start_matches "${expected_exec_start}"; then + echo "[pingora-direct-rollback] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start},请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2 + exit 1 + fi +} + +verify_systemd_dropin_removed() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 systemd drop-in 移除核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl cat ${SERVICE_NAME}" + echo "[pingora-direct-rollback] dry-run:--apply 后会校验 systemd 最终配置不再包含 CAP_NET_BIND_SERVICE。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 drop-in 是否已移除。" >&2 + exit 1 + fi + + echo "+ systemctl cat ${SERVICE_NAME}" + unit_content="$(systemctl cat "${SERVICE_NAME}")" + if [[ "${unit_content}" == *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-rollback] systemctl cat 仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE,direct-entry drop-in 未移除。" >&2 + exit 1 + fi + if [[ "${unit_content}" == *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-rollback] systemctl cat 仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE,direct-entry drop-in 未移除。" >&2 + exit 1 + fi +} + +verify_nginx_active_after_reload() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Nginx reload 后状态核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl is-active ${NGINX_SERVICE}" + echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会 reload Nginx 并校验 service 仍为 active。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl,无法核验 Nginx reload 后状态。" >&2 + exit 1 + fi + + echo "+ systemctl is-active ${NGINX_SERVICE}" + nginx_state="$(systemctl is-active "${NGINX_SERVICE}")" + if [[ "${nginx_state}" != "active" ]]; then + echo "[pingora-direct-rollback] Nginx reload 后状态不是 active: ${nginx_state}" >&2 + exit 1 + fi +} + +verify_nginx_smoke_after_reload() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Nginx reload 后 smoke 核验。" + return + fi + + if [[ -z "${NGINX_SMOKE_URL}" ]]; then + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 " + echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会用 --nginx-smoke-url 验证 Nginx 入口。" + return + fi + + smoke_args=( + --fail + --silent + --show-error + --max-time + 5 + ) + if [[ -n "${NGINX_SMOKE_HOST}" ]]; then + smoke_args+=(-H "Host: ${NGINX_SMOKE_HOST}") + fi + smoke_args+=("${NGINX_SMOKE_URL}") + + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${CURL_BINARY} ${smoke_args[*]}" + if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会验证 Nginx smoke URL 可访问,且响应体包含预期片段。" + else + echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会验证 Nginx smoke URL 可访问。" + fi + return + fi + + if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 curl 可执行文件,无法验证 Nginx smoke URL: ${CURL_BINARY}" >&2 + exit 1 + fi + + echo "+ ${CURL_BINARY} ${smoke_args[*]}" + smoke_body="$("${CURL_BINARY}" "${smoke_args[@]}")" + if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + if [[ "${smoke_body}" != *"${NGINX_SMOKE_EXPECT_BODY}"* ]]; then + echo "[pingora-direct-rollback] Nginx smoke 响应缺少预期片段: ${NGINX_SMOKE_EXPECT_BODY}" >&2 + echo "[pingora-direct-rollback] Nginx smoke 响应前 500 字符: ${smoke_body:0:500}" >&2 + exit 1 + fi + echo "[pingora-direct-rollback] nginx smoke evidence matched expected body fragment." + else + echo "[pingora-direct-rollback] nginx smoke evidence curl --fail passed without body fragment check." + fi +} + +test_nginx_config_before_reload() { + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${NGINX_BINARY} -t" + echo "[pingora-direct-rollback] dry-run:--apply --reload-nginx 后会先执行 nginx -t,通过后再 reload。" + return + fi + + if ! command -v "${NGINX_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 Nginx 可执行文件,无法执行: ${NGINX_BINARY} -t" >&2 + exit 1 + fi + + echo "+ ${NGINX_BINARY} -t" + "${NGINX_BINARY}" -t +} + +validate_dropin_path_for_apply() { + if [[ "${APPLY}" != "true" ]]; then + return + fi + + DROPIN_DIR="$(dirname "${DROPIN_PATH}")" + if [[ -L "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-rollback] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-rollback] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -L "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2 + exit 1 + fi +} + +verify_health_patrol_env_after_rollback() { + if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then + return + fi + + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 health patrol env 回退后复核。" + return + fi + + health_patrol_args=( + "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" + --env-file + "${HEALTH_PATROL_ENV_FILE}" + --expected-gateway-mode + nginx + ) + if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + health_patrol_args+=( + --expected-public-base-url + "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" + ) + fi + if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + health_patrol_args+=( + --expected-public-host + "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" + ) + elif [[ "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" != "false" ]]; then + health_patrol_args+=(--require-empty-public-host) + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ node -- ${health_patrol_args[*]}" + echo "[pingora-direct-rollback] dry-run:--apply 后会复核 health patrol env 已切回 nginx,且 public base URL / Host 已恢复为预期值。" + return + fi + + if ! command -v node >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 node,无法复核 health patrol env。" >&2 + exit 1 + fi + if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + echo "[pingora-direct-rollback] health patrol env 复核脚本不存在: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 + exit 1 + fi + + echo "+ node -- ${health_patrol_args[*]}" + node -- "${health_patrol_args[@]}" +} + +verify_pingora_shadow_probe_after_rollback() { + if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + return + fi + + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Pingora shadow 探针回退后核验。" + return + fi + + shadow_args=( + --fail + --silent + --show-error + --max-time + 5 + -H + "X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_TOKEN}" + "${PINGORA_SHADOW_PROBE_URL}" + ) + + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" + echo "[pingora-direct-rollback] dry-run:--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。" + return + fi + + if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 curl 可执行文件,无法验证 Pingora shadow 探针: ${CURL_BINARY}" >&2 + exit 1 + fi + + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" + shadow_body="$("${CURL_BINARY}" "${shadow_args[@]}")" + if [[ "${shadow_body}" != *'"ok":true'* || "${shadow_body}" != *'"gateway":"pingora-shadow"'* ]]; then + echo "[pingora-direct-rollback] Pingora shadow 探针响应不是预期 JSON: ${shadow_body}" >&2 + exit 1 + fi +} + +echo "[pingora-direct-rollback] apply=${APPLY} service=${SERVICE_NAME} dropin=${DROPIN_PATH}" + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-rollback] 当前是 dry-run;确认无误后追加 --apply。" +fi + +validate_dropin_path_for_apply +test_nginx_config_before_reload + +if [[ -e "${DROPIN_PATH}" || -L "${DROPIN_PATH}" ]]; then + run_cmd rm -f "${DROPIN_PATH}" +else + echo "[pingora-direct-rollback] direct-entry drop-in 不存在,视为已移除: ${DROPIN_PATH}" +fi + +run_systemctl daemon-reload +run_systemctl restart "${SERVICE_NAME}" +verify_systemd_dropin_removed +verify_systemd_exec_start_after_rollback + +run_systemctl reload "${NGINX_SERVICE}" +verify_nginx_active_after_reload +verify_nginx_smoke_after_reload +verify_health_patrol_env_after_rollback +verify_pingora_shadow_probe_after_rollback + +if [[ "${STATUS_AFTER}" == "true" ]]; then + run_systemctl status "${SERVICE_NAME}" --no-pager +fi + +cat <} 已通过 +- 如传入 --nginx-smoke-expect-body,Nginx smoke 响应体已匹配预期片段 +- 如传入 --health-patrol-env-file,health patrol env 已复核为 nginx 模式且 public base URL / Host 符合预期 +- 如传入 --pingora-shadow-probe-url,Pingora shadow 高端口探针已复核为 gateway=pingora-shadow +- 公网入口已回到 Nginx 或其它正式前置入口 +EOF diff --git a/scripts/deploy/pingora-gateway-env-shadow-switch.mjs b/scripts/deploy/pingora-gateway-env-shadow-switch.mjs new file mode 100644 index 000000000..ab7ba188e --- /dev/null +++ b/scripts/deploy/pingora-gateway-env-shadow-switch.mjs @@ -0,0 +1,254 @@ +#!/usr/bin/env node + +import { + chownSync, + chmodSync, + lstatSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const TARGET_KEYS = [ + 'GENARRATIVE_PINGORA_GATEWAY_LISTEN', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE', +]; +const SHADOW_VALUES = { + GENARRATIVE_PINGORA_GATEWAY_LISTEN: '127.0.0.1:18081', + GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN: '', + GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN: '', + GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE: '', + GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE: '', +}; +const DRY_RUN_ENV_FILE_MODE = 0o600; + +const config = parseArgs(process.argv.slice(2)); +if (config.apply) { + assertEnvFileIsWritableRegularFile(config.envFile); +} + +const originalText = readFileSync(config.envFile, 'utf8'); +const nextText = rewriteEnvFile(originalText, SHADOW_VALUES); +const checkFile = writeDryRunEnv(nextText); + +try { + console.log( + `[pingora-gateway-env-shadow-switch] apply=${config.apply} env=${config.envFile}`, + ); + for (const key of TARGET_KEYS) { + console.log(`~ ${key}=${SHADOW_VALUES[key]}`); + } + assertShadowEnv(checkFile); + + if (!config.apply) { + console.log( + '[pingora-gateway-env-shadow-switch] 当前是 dry-run;追加 --apply 后才会写入 env 文件。', + ); + } else { + writeEnvFileAtomically(config.envFile, nextText); + } + + console.log('[pingora-gateway-env-shadow-switch] 完成'); +} finally { + rmSync(path.dirname(checkFile), { recursive: true, force: true }); +} + +function parseArgs(argv) { + const result = { + envFile: + process.env.GENARRATIVE_PINGORA_GATEWAY_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + apply: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--env-file': + result.envFile = requireValue(argv, ++index, arg); + break; + case '--apply': + result.apply = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.envFile)) { + throw new Error(`--env-file 必须是绝对路径: ${result.envFile}`); + } + validateNoControlCharacters(result.envFile, '--env-file'); + if (isFilesystemRootPath(result.envFile)) { + throw new Error('--env-file 不能是文件系统根目录。'); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/deploy/pingora-gateway-env-shadow-switch.mjs [options] + +Options: + --env-file Pingora gateway env 文件,默认 /etc/genarrative/pingora-gateway.env。 + --apply 写入 env 文件;默认只 dry-run 并用临时 env 复核。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function rewriteEnvFile(text, nextValuesByKey) { + const lines = text.replace(/\r\n/gu, '\n').split('\n'); + if (lines.at(-1) === '') { + lines.pop(); + } + + const seen = new Set(); + const output = []; + for (const line of lines) { + const assignment = parseAssignmentLine(line); + if (!assignment || !TARGET_KEYS.includes(assignment.key)) { + output.push(line); + continue; + } + if (seen.has(assignment.key)) { + throw new Error(`pingora gateway env 中存在重复配置: ${assignment.key}`); + } + seen.add(assignment.key); + output.push( + `${assignment.leading}${assignment.exportPrefix}${assignment.key}=${nextValuesByKey[assignment.key]}`, + ); + } + + for (const key of TARGET_KEYS) { + if (!seen.has(key)) { + output.push(`${key}=${nextValuesByKey[key]}`); + } + } + + return `${output.join('\n')}\n`; +} + +function parseAssignmentLine(line) { + const match = line.match(/^(\s*)(export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/u); + if (!match) { + return null; + } + return { + leading: match[1] || '', + exportPrefix: match[2] || '', + key: match[3], + }; +} + +function writeDryRunEnv(content) { + const dir = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-gateway-env-shadow-switch-'), + ); + const filePath = path.join(dir, 'pingora-gateway.env'); + writeFileSync(filePath, content, { + encoding: 'utf8', + mode: DRY_RUN_ENV_FILE_MODE, + }); + chmodSync(filePath, DRY_RUN_ENV_FILE_MODE); + return filePath; +} + +function assertShadowEnv(filePath) { + const values = parseEnvFile(readFileSync(filePath, 'utf8')); + if ( + values.GENARRATIVE_PINGORA_GATEWAY_LISTEN !== + SHADOW_VALUES.GENARRATIVE_PINGORA_GATEWAY_LISTEN + ) { + throw new Error( + 'shadow env 复核失败:GENARRATIVE_PINGORA_GATEWAY_LISTEN 必须是 127.0.0.1:18081。', + ); + } + for (const key of [ + 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE', + ]) { + if ((values[key] || '') !== '') { + throw new Error(`shadow env 复核失败:${key} 必须为空。`); + } + } +} + +function parseEnvFile(text) { + const values = {}; + for (const line of text.split(/\r?\n/u)) { + const assignment = parseAssignmentLine(line); + if (!assignment) { + continue; + } + values[assignment.key] = line + .slice(line.indexOf('=') + 1) + .trim() + .replace(/^['"]|['"]$/gu, ''); + } + return values; +} + +function writeEnvFileAtomically(filePath, content) { + const currentStat = assertEnvFileIsWritableRegularFile(filePath); + const currentMode = currentStat.mode & 0o777; + const tempFile = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp`, + ); + try { + writeFileSync(tempFile, content, { encoding: 'utf8', mode: currentMode }); + chownSync(tempFile, currentStat.uid, currentStat.gid); + chmodSync(tempFile, currentMode); + assertEnvFileIsWritableRegularFile(filePath); + renameSync(tempFile, filePath); + } catch (error) { + rmSync(tempFile, { force: true }); + throw error; + } +} + +function assertEnvFileIsWritableRegularFile(filePath) { + const currentStat = lstatSync(filePath); + if (currentStat.isSymbolicLink()) { + throw new Error( + `--env-file 不能是符号链接;请直接传真实 Pingora gateway env 文件路径: ${filePath}`, + ); + } + if (!currentStat.isFile()) { + throw new Error(`--env-file 必须指向普通文件: ${filePath}`); + } + return currentStat; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value ?? ''))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} diff --git a/scripts/deploy/pingora-health-patrol-env-switch.mjs b/scripts/deploy/pingora-health-patrol-env-switch.mjs new file mode 100644 index 000000000..60f5df19a --- /dev/null +++ b/scripts/deploy/pingora-health-patrol-env-switch.mjs @@ -0,0 +1,391 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + chownSync, + chmodSync, + lstatSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const TARGET_KEYS = [ + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', +]; +const DRY_RUN_ENV_FILE_MODE = 0o600; + +const config = parseArgs(process.argv.slice(2)); +if (config.apply) { + assertEnvFileIsWritableRegularFile(config.envFile); +} +const originalText = readFileSync(config.envFile, 'utf8'); +const nextValues = { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: config.gatewayMode, + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: normalizeBaseUrl( + config.publicBaseUrl, + '--public-base-url', + ), + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: config.clearPublicHost + ? '' + : config.publicHost, +}; +const nextText = rewriteEnvFile(originalText, nextValues); +const checkFile = writeDryRunEnv(nextText); + +try { + console.log( + `[pingora-health-patrol-env-switch] apply=${config.apply} env=${config.envFile} mode=${config.gatewayMode}`, + ); + for (const key of TARGET_KEYS) { + console.log(`~ ${key}=${nextValues[key]}`); + } + + runHealthPatrolEnvCheck(checkFile); + + if (!config.apply) { + console.log( + '[pingora-health-patrol-env-switch] 当前是 dry-run;追加 --apply 后才会写入 env 文件。', + ); + } else { + writeEnvFileAtomically(config.envFile, nextText); + } + + console.log('[pingora-health-patrol-env-switch] 完成'); +} finally { + rmSync(path.dirname(checkFile), { recursive: true, force: true }); +} + +function parseArgs(argv) { + const result = { + envFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + gatewayMode: process.env.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || '', + publicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '', + publicHost: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || null, + clearPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST', + ), + checkScript: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_CHECK_SCRIPT || + defaultEnvCheckScript(), + apply: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--env-file': + result.envFile = requireValue(argv, ++index, arg); + break; + case '--gateway-mode': + result.gatewayMode = requireValue(argv, ++index, arg); + break; + case '--public-base-url': + result.publicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--public-host': + result.publicHost = requireValue(argv, ++index, arg); + break; + case '--clear-public-host': + result.clearPublicHost = true; + break; + case '--check-script': + result.checkScript = requireValue(argv, ++index, arg); + break; + case '--apply': + result.apply = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.envFile)) { + throw new Error(`--env-file 必须是绝对路径: ${result.envFile}`); + } + validateNoControlCharacters(result.envFile, '--env-file'); + if (isFilesystemRootPath(result.envFile)) { + throw new Error('--env-file 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.checkScript)) { + throw new Error(`--check-script 必须是绝对路径: ${result.checkScript}`); + } + validateNoControlCharacters(result.checkScript, '--check-script'); + if (isFilesystemRootPath(result.checkScript)) { + throw new Error('--check-script 不能是文件系统根目录。'); + } + if (!GATEWAY_MODES.has(result.gatewayMode)) { + throw new Error( + `--gateway-mode 只支持 nginx 或 pingora-direct: ${result.gatewayMode}`, + ); + } + if (!result.publicBaseUrl) { + throw new Error('必须提供 --public-base-url。'); + } + normalizeBaseUrl(result.publicBaseUrl, '--public-base-url'); + if (result.publicHost !== null && result.clearPublicHost) { + throw new Error('--public-host 和 --clear-public-host 不能同时使用。'); + } + if (result.publicHost === null && !result.clearPublicHost) { + throw new Error( + '必须显式提供 --public-host 或 --clear-public-host,避免沿用旧 Host 覆盖。', + ); + } + if (result.publicHost !== null) { + validateHostOption(result.publicHost, '--public-host'); + } + if ( + result.gatewayMode === 'pingora-direct' && + isLoopbackUrl(result.publicBaseUrl) && + result.clearPublicHost + ) { + throw new Error( + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host,避免巡检命中默认 vhost。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/deploy/pingora-health-patrol-env-switch.mjs --gateway-mode --public-base-url (--public-host |--clear-public-host) [options] + +Options: + --env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 + --gateway-mode nginx 或 pingora-direct。 + --public-base-url 写入 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL。 + --public-host 写入 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST。 + --clear-public-host 清空 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST。 + --check-script env 复核脚本,默认使用同一 release 内的 check-production-health-patrol-env.mjs。 + --apply 写入 env 文件;默认只 dry-run 并用临时 env 复核。 +`); +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function defaultEnvCheckScript() { + const scriptPath = fileURLToPath(import.meta.url); + const releaseRoot = path.resolve(path.dirname(scriptPath), '../..'); + return path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function rewriteEnvFile(text, nextValuesByKey) { + const lines = text.replace(/\r\n/gu, '\n').split('\n'); + if (lines.at(-1) === '') { + lines.pop(); + } + + const seen = new Set(); + const output = []; + for (const line of lines) { + const assignment = parseAssignmentLine(line); + if (!assignment || !TARGET_KEYS.includes(assignment.key)) { + output.push(line); + continue; + } + if (seen.has(assignment.key)) { + throw new Error(`health patrol env 中存在重复配置: ${assignment.key}`); + } + seen.add(assignment.key); + output.push( + `${assignment.leading}${assignment.exportPrefix}${assignment.key}=${nextValuesByKey[assignment.key]}`, + ); + } + + for (const key of TARGET_KEYS) { + if (!seen.has(key)) { + output.push(`${key}=${nextValuesByKey[key]}`); + } + } + + return `${output.join('\n')}\n`; +} + +function parseAssignmentLine(line) { + const match = line.match(/^(\s*)(export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/u); + if (!match) { + return null; + } + return { + leading: match[1] || '', + exportPrefix: match[2] || '', + key: match[3], + }; +} + +function writeDryRunEnv(content) { + const dir = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-env-switch-'), + ); + const filePath = path.join(dir, 'health-patrol.env'); + writeFileSync(filePath, content, { + encoding: 'utf8', + mode: DRY_RUN_ENV_FILE_MODE, + }); + chmodSync(filePath, DRY_RUN_ENV_FILE_MODE); + return filePath; +} + +function writeEnvFileAtomically(filePath, content) { + const currentStat = assertEnvFileIsWritableRegularFile(filePath); + const currentMode = currentStat.mode & 0o777; + const tempFile = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp`, + ); + try { + writeFileSync(tempFile, content, { encoding: 'utf8', mode: currentMode }); + chownSync(tempFile, currentStat.uid, currentStat.gid); + chmodSync(tempFile, currentMode); + assertEnvFileIsWritableRegularFile(filePath); + renameSync(tempFile, filePath); + } catch (error) { + rmSync(tempFile, { force: true }); + throw error; + } +} + +function assertEnvFileIsWritableRegularFile(filePath) { + const currentStat = lstatSync(filePath); + if (currentStat.isSymbolicLink()) { + throw new Error( + `--env-file 不能是符号链接;请直接传真实 health patrol env 文件路径: ${filePath}`, + ); + } + if (!currentStat.isFile()) { + throw new Error(`--env-file 必须指向普通文件: ${filePath}`); + } + return currentStat; +} + +function runHealthPatrolEnvCheck(envFile) { + const checkArgs = [ + config.checkScript, + '--env-file', + envFile, + '--expected-gateway-mode', + config.gatewayMode, + '--expected-public-base-url', + normalizeBaseUrl(config.publicBaseUrl, '--public-base-url'), + ]; + if (config.clearPublicHost) { + checkArgs.push('--require-empty-public-host'); + } else { + checkArgs.push('--expected-public-host', config.publicHost); + } + + console.log(`+ node ${checkArgs.join(' ')}`); + const result = spawnSync(process.execPath, ['--', ...checkArgs], { + encoding: 'utf8', + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + if (result.status !== 0) { + process.exit(result.status || 1); + } +} + +function normalizeBaseUrl(value, label) { + validateNoControlCharacters(value, label); + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + return parsed.toString().replace(/\/+$/u, ''); + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + validateNoControlCharacters(raw, label); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/u.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符。`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value ?? ''))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function isLoopbackUrl(value) { + try { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === 'localhost' || + hostname === '::1' || + hostname === '[::1]' || + hostname.startsWith('127.') + ); + } catch { + return false; + } +} diff --git a/scripts/deploy/pingora-realpath-canary-disable.sh b/scripts/deploy/pingora-realpath-canary-disable.sh new file mode 100644 index 000000000..379048b8d --- /dev/null +++ b/scripts/deploy/pingora-realpath-canary-disable.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TARGET_PATH:-/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +SYSTEMCTL_BINARY="${GENARRATIVE_SYSTEMCTL_BINARY:-systemctl}" +APPLY="false" +STATUS_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-realpath-canary-disable.sh [--apply] [--target-path ] [--nginx-binary ] [--nginx-service ] [--systemctl-binary ] [--no-status] + +说明: + 关闭 Nginx -> Pingora 真实路径 canary:删除 + /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf,执行 nginx -t, + 再 reload nginx。默认 dry-run,必须显式传 --apply 才会修改系统。 + 如果 nginx -t 或 reload 失败,脚本会恢复删除前的目标文件。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --target-path) + TARGET_PATH="${2:-}" + if [[ -z "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] --target-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-realpath-canary-disable] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-realpath-canary-disable] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --systemctl-binary) + SYSTEMCTL_BINARY="${2:-}" + if [[ -z "${SYSTEMCTL_BINARY}" ]]; then + echo "[pingora-realpath-canary-disable] --systemctl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + *) + echo "[pingora-realpath-canary-disable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-realpath-canary-disable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +require_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" != /* ]]; then + echo "[pingora-realpath-canary-disable] ${label} 必须是绝对路径。" >&2 + exit 1 + fi + if is_filesystem_root_path "${value}"; then + echo "[pingora-realpath-canary-disable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +validate_command_or_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" == */* ]]; then + require_absolute_path "${label}" "${value}" + fi +} + +print_command() { + printf '+' + for arg in "$@"; do + printf ' %q' "${arg}" + done + printf '\n' +} + +run_or_print() { + if [[ "${APPLY}" == "true" ]]; then + "$@" + else + print_command "$@" + fi +} + +restore_previous() { + local backup_path="$1" + cp -p -- "${backup_path}" "${TARGET_PATH}" +} + +rollback_after_failure() { + local backup_path="$1" + local reason="$2" + echo "[pingora-realpath-canary-disable] ${reason},恢复删除前配置。" >&2 + restore_previous "${backup_path}" + "${NGINX_BINARY}" -t >/dev/null 2>&1 || true + "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" >/dev/null 2>&1 || true +} + +reject_control_characters "--target-path" "${TARGET_PATH}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--systemctl-binary" "${SYSTEMCTL_BINARY}" +require_absolute_path "--target-path" "${TARGET_PATH}" +validate_command_or_absolute_path "--nginx-binary" "${NGINX_BINARY}" +validate_command_or_absolute_path "--systemctl-binary" "${SYSTEMCTL_BINARY}" + +TARGET_DIR="$(dirname -- "${TARGET_PATH}")" +if [[ "${APPLY}" == "true" && ( -L "${TARGET_DIR}" || -L "${TARGET_PATH}" ) ]]; then + echo "[pingora-realpath-canary-disable] 目标目录或目标文件不能是符号链接。" >&2 + exit 1 +fi + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-realpath-canary-disable] dry-run:将删除 realpath canary ${TARGET_PATH}。" + print_command rm -f -- "${TARGET_PATH}" + run_or_print "${NGINX_BINARY}" -t + run_or_print "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" + exit 0 +fi + +if [[ ! -e "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] 目标文件不存在,视为已关闭: ${TARGET_PATH}" + if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" + fi + exit 0 +fi +if [[ ! -f "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] 目标存在但不是普通文件: ${TARGET_PATH}" >&2 + exit 1 +fi + +backup_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.disable.backup.XXXXXX")" +trap 'rm -f -- "${backup_path}"' EXIT +cp -p -- "${TARGET_PATH}" "${backup_path}" +rm -f -- "${TARGET_PATH}" + +if ! "${NGINX_BINARY}" -t; then + rollback_after_failure "${backup_path}" "nginx -t 失败" + exit 1 +fi +if ! "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}"; then + rollback_after_failure "${backup_path}" "reload ${NGINX_SERVICE} 失败" + exit 1 +fi + +if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" +fi + +echo "[pingora-realpath-canary-disable] 已关闭 realpath canary: ${TARGET_PATH}" diff --git a/scripts/deploy/pingora-realpath-canary-enable.sh b/scripts/deploy/pingora-realpath-canary-enable.sh new file mode 100644 index 000000000..dd22528f4 --- /dev/null +++ b/scripts/deploy/pingora-realpath-canary-enable.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" + +TEMPLATE_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TEMPLATE_PATH:-${REPO_ROOT}/deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf}" +TARGET_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TARGET_PATH:-/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf}" +PROBE_TOKEN="${GENARRATIVE_PINGORA_REALPATH_CANARY_PROBE_TOKEN:-${GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN:-}}" +BASE_URL="${GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL:-http://127.0.0.1:18083}" +HOST_HEADER="${GENARRATIVE_PINGORA_REALPATH_CANARY_HOST:-}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +SYSTEMCTL_BINARY="${GENARRATIVE_SYSTEMCTL_BINARY:-systemctl}" +LIVE_SCRIPT="${GENARRATIVE_PINGORA_REALPATH_CANARY_LIVE_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-canary-live.mjs}" +APPLY="false" +RUN_LIVE="true" +STATUS_AFTER="true" + +PLACEHOLDER="__GENARRATIVE_PINGORA_PROBE_TOKEN__" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-realpath-canary-enable.sh [--apply] [--probe-token ] [--host ] [--base-url ] [--template-path ] [--target-path ] [--nginx-binary ] [--nginx-service ] [--systemctl-binary ] [--live-script ] [--skip-live] [--no-status] + +说明: + 启用 Nginx -> Pingora 真实路径 canary。脚本会把随包 + deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf 渲染到 + /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf。 + +默认是 dry-run,只打印计划;必须显式传 --apply 才会写 Nginx 配置。 +--apply 会替换 probe token、写入目标文件、执行 nginx -t、reload nginx, +然后运行 realpath canary live smoke。任一阶段失败都会恢复写入前的目标文件状态。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --probe-token) + PROBE_TOKEN="${2:-}" + if [[ -z "${PROBE_TOKEN}" ]]; then + echo "[pingora-realpath-canary-enable] --probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --host) + HOST_HEADER="${2:-}" + if [[ -z "${HOST_HEADER}" ]]; then + echo "[pingora-realpath-canary-enable] --host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --base-url) + BASE_URL="${2:-}" + if [[ -z "${BASE_URL}" ]]; then + echo "[pingora-realpath-canary-enable] --base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --template-path) + TEMPLATE_PATH="${2:-}" + if [[ -z "${TEMPLATE_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] --template-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --target-path) + TARGET_PATH="${2:-}" + if [[ -z "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] --target-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-realpath-canary-enable] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-realpath-canary-enable] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --systemctl-binary) + SYSTEMCTL_BINARY="${2:-}" + if [[ -z "${SYSTEMCTL_BINARY}" ]]; then + echo "[pingora-realpath-canary-enable] --systemctl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --live-script) + LIVE_SCRIPT="${2:-}" + if [[ -z "${LIVE_SCRIPT}" ]]; then + echo "[pingora-realpath-canary-enable] --live-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --skip-live) + RUN_LIVE="false" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + *) + echo "[pingora-realpath-canary-enable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-realpath-canary-enable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +require_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" != /* ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须是绝对路径。" >&2 + exit 1 + fi + if is_filesystem_root_path "${value}"; then + echo "[pingora-realpath-canary-enable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +validate_command_or_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" == */* ]]; then + require_absolute_path "${label}" "${value}" + fi +} + +validate_host() { + local label="$1" + local value="$2" + if [[ -z "${value}" ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须提供。" >&2 + exit 1 + fi + if [[ "${value}" == http://* || "${value}" == https://* || "${value}" == */* || "${value}" == *\?* || "${value}" =~ [[:space:]] ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须是 Host 或 host:port,不能是 URL、路径、查询或包含空白。" >&2 + exit 1 + fi +} + +validate_base_url() { + if [[ "${BASE_URL}" != http://* && "${BASE_URL}" != https://* ]]; then + echo "[pingora-realpath-canary-enable] --base-url 必须以 http:// 或 https:// 开头。" >&2 + exit 1 + fi +} + +validate_probe_token() { + if [[ "${APPLY}" != "true" ]]; then + return + fi + if [[ -z "${PROBE_TOKEN}" ]]; then + echo "[pingora-realpath-canary-enable] --apply 必须提供 --probe-token 或 GENARRATIVE_PINGORA_REALPATH_CANARY_PROBE_TOKEN。" >&2 + exit 1 + fi + if [[ "${PROBE_TOKEN}" == "${PLACEHOLDER}" || "${#PROBE_TOKEN}" -lt 8 ]]; then + echo "[pingora-realpath-canary-enable] probe token 不能是占位值或过短。" >&2 + exit 1 + fi +} + +escape_nginx_double_quoted() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s' "${value}" +} + +render_template_to() { + local output_path="$1" + local content escaped_token + content="$(<"${TEMPLATE_PATH}")" + escaped_token="$(escape_nginx_double_quoted "${PROBE_TOKEN}")" + printf '%s' "${content//${PLACEHOLDER}/${escaped_token}}" >"${output_path}" +} + +print_command() { + printf '+' + for arg in "$@"; do + printf ' %q' "${arg}" + done + printf '\n' +} + +run_or_print() { + if [[ "${APPLY}" == "true" ]]; then + "$@" + else + print_command "$@" + fi +} + +restore_previous() { + local backup_path="$1" + local had_target="$2" + if [[ "${had_target}" == "true" ]]; then + cp -p -- "${backup_path}" "${TARGET_PATH}" + else + rm -f -- "${TARGET_PATH}" + fi +} + +rollback_after_failure() { + local backup_path="$1" + local had_target="$2" + local reason="$3" + echo "[pingora-realpath-canary-enable] ${reason},恢复写入前配置。" >&2 + restore_previous "${backup_path}" "${had_target}" + "${NGINX_BINARY}" -t >/dev/null 2>&1 || true + "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" >/dev/null 2>&1 || true +} + +reject_control_characters "--template-path" "${TEMPLATE_PATH}" +reject_control_characters "--target-path" "${TARGET_PATH}" +reject_control_characters "--probe-token" "${PROBE_TOKEN}" +reject_control_characters "--base-url" "${BASE_URL}" +reject_control_characters "--host" "${HOST_HEADER}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--systemctl-binary" "${SYSTEMCTL_BINARY}" +reject_control_characters "--live-script" "${LIVE_SCRIPT}" + +require_absolute_path "--template-path" "${TEMPLATE_PATH}" +require_absolute_path "--target-path" "${TARGET_PATH}" +if [[ "${RUN_LIVE}" == "true" ]]; then + require_absolute_path "--live-script" "${LIVE_SCRIPT}" +fi +validate_command_or_absolute_path "--nginx-binary" "${NGINX_BINARY}" +validate_command_or_absolute_path "--systemctl-binary" "${SYSTEMCTL_BINARY}" +validate_host "--host" "${HOST_HEADER}" +validate_base_url +validate_probe_token + +if [[ ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 未找到 realpath canary 模板: ${TEMPLATE_PATH}" >&2 + exit 1 +fi +if [[ "${RUN_LIVE}" == "true" && ! -f "${LIVE_SCRIPT}" ]]; then + echo "[pingora-realpath-canary-enable] 未找到 realpath canary live smoke 脚本: ${LIVE_SCRIPT}" >&2 + exit 1 +fi + +TARGET_DIR="$(dirname -- "${TARGET_PATH}")" +if [[ "${APPLY}" == "true" ]]; then + if [[ ! -d "${TARGET_DIR}" ]]; then + echo "[pingora-realpath-canary-enable] 目标目录不存在: ${TARGET_DIR}" >&2 + exit 1 + fi + if [[ -L "${TARGET_DIR}" || -L "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 目标目录或目标文件不能是符号链接。" >&2 + exit 1 + fi +fi + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-realpath-canary-enable] dry-run:将渲染 realpath canary 到 ${TARGET_PATH},probe token 输出已隐藏。" + print_command install -m 0644 "" "${TARGET_PATH}" + run_or_print "${NGINX_BINARY}" -t + run_or_print "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" + if [[ "${RUN_LIVE}" == "true" ]]; then + run_or_print node -- "${LIVE_SCRIPT}" --realpath --base-url "${BASE_URL}" --host "${HOST_HEADER}" + fi + exit 0 +fi + +tmp_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.XXXXXX")" +backup_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.backup.XXXXXX")" +had_target="false" +cleanup_paths=("${tmp_path}" "${backup_path}") +trap 'rm -f -- "${cleanup_paths[@]}"' EXIT + +render_template_to "${tmp_path}" +chmod 0644 "${tmp_path}" + +if [[ -e "${TARGET_PATH}" ]]; then + if [[ ! -f "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 目标已存在但不是普通文件: ${TARGET_PATH}" >&2 + exit 1 + fi + cp -p -- "${TARGET_PATH}" "${backup_path}" + had_target="true" +fi + +install -m 0644 "${tmp_path}" "${TARGET_PATH}" +if ! "${NGINX_BINARY}" -t; then + rollback_after_failure "${backup_path}" "${had_target}" "nginx -t 失败" + exit 1 +fi +if ! "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}"; then + rollback_after_failure "${backup_path}" "${had_target}" "reload ${NGINX_SERVICE} 失败" + exit 1 +fi + +if [[ "${RUN_LIVE}" == "true" ]]; then + if ! node -- "${LIVE_SCRIPT}" --realpath --base-url "${BASE_URL}" --host "${HOST_HEADER}"; then + rollback_after_failure "${backup_path}" "${had_target}" "realpath canary live smoke 失败" + exit 1 + fi +fi + +if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" +fi + +echo "[pingora-realpath-canary-enable] 已启用 realpath canary: ${TARGET_PATH}" diff --git a/scripts/deploy/pingora-tls-cert-sync.mjs b/scripts/deploy/pingora-tls-cert-sync.mjs new file mode 100644 index 000000000..63533eea7 --- /dev/null +++ b/scripts/deploy/pingora-tls-cert-sync.mjs @@ -0,0 +1,480 @@ +#!/usr/bin/env node + +import { randomBytes } from 'node:crypto'; +import { + accessSync, + chmodSync, + chownSync, + constants, + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + realpathSync, + renameSync, + rmSync, + statSync, +} from 'node:fs'; +import { execFileSync, spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { userInfo } from 'node:os'; + +const DEFAULT_SERVICE_USER = 'genarrative'; +const DEFAULT_SERVICE_GROUP = 'genarrative'; +const DEFAULT_TARGET_USER = 'root'; +const DEFAULT_DIRECTORY_MODE = '0750'; +const DEFAULT_FILE_MODE = '0640'; + +const config = parseArgs(process.argv.slice(2)); +const result = run(); + +if (config.json) { + console.log(`${JSON.stringify(result, null, 2)}\n`); +} else { + const mode = config.apply ? 'apply' : 'dry-run'; + console.log(`[pingora-tls-cert-sync] ${mode} OK`); + console.log(`[pingora-tls-cert-sync] target=${result.target.directory}`); + for (const item of result.files) { + console.log( + `[pingora-tls-cert-sync] ${item.kind}: ${item.source.path} -> ${item.target.path}`, + ); + } +} + +function usage() { + console.log(`Usage: + node scripts/deploy/pingora-tls-cert-sync.mjs --source-cert-file --source-key-file --target-dir [options] + +Options: + --source-cert-file 已存在的证书链文件,通常是 Let's Encrypt live/fullchain.pem。 + --source-key-file 已存在的私钥文件,通常是 Let's Encrypt live/privkey.pem。 + --target-dir Pingora 私有 TLS 目录,例如 /etc/genarrative/pingora-tls/。 + --service-user 目标服务用户,默认 genarrative。 + --service-group + 目标文件组,默认 genarrative。 + --target-user 目标文件 owner,默认 root。 + --directory-mode 目标目录权限,默认 0750。 + --file-mode 证书和私钥权限,默认 0640。 + --apply 执行复制;默认只做 dry-run 校验。 + --json 输出 JSON。 + +该脚本只把现有证书复制到 Pingora 私有目录,不修改 Let's Encrypt / Certbot / Nginx 原始文件。 +源路径可以是 Certbot live symlink,但解析后的目标必须是普通文件;目标目录和目标文件不允许是 symlink。 +`); +} + +function parseArgs(argv) { + const result = { + sourceCertFile: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SOURCE_CERT_FILE || '', + sourceKeyFile: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SOURCE_KEY_FILE || '', + targetDir: process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_TARGET_DIR || '', + serviceUser: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SERVICE_USER || + DEFAULT_SERVICE_USER, + serviceGroup: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SERVICE_GROUP || + DEFAULT_SERVICE_GROUP, + targetUser: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_TARGET_USER || + DEFAULT_TARGET_USER, + directoryMode: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_DIRECTORY_MODE || + DEFAULT_DIRECTORY_MODE, + fileMode: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_FILE_MODE || + DEFAULT_FILE_MODE, + apply: readBoolEnv( + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_APPLY, + false, + 'GENARRATIVE_PINGORA_TLS_CERT_SYNC_APPLY', + ), + json: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + usage(); + process.exit(0); + break; + case '--source-cert-file': + result.sourceCertFile = requireValue(argv, ++index, arg); + break; + case '--source-key-file': + result.sourceKeyFile = requireValue(argv, ++index, arg); + break; + case '--target-dir': + result.targetDir = requireValue(argv, ++index, arg); + break; + case '--service-user': + result.serviceUser = requireValue(argv, ++index, arg); + break; + case '--service-group': + result.serviceGroup = requireValue(argv, ++index, arg); + break; + case '--target-user': + result.targetUser = requireValue(argv, ++index, arg); + break; + case '--directory-mode': + result.directoryMode = requireValue(argv, ++index, arg); + break; + case '--file-mode': + result.fileMode = requireValue(argv, ++index, arg); + break; + case '--apply': + result.apply = true; + break; + case '--json': + result.json = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateSafeAbsoluteFilePath( + result.sourceCertFile, + '--source-cert-file', + ); + validateSafeAbsoluteFilePath(result.sourceKeyFile, '--source-key-file'); + validateSafeAbsoluteDirectoryPath(result.targetDir, '--target-dir'); + validateIdentity(result.serviceUser, '--service-user'); + validateIdentity(result.serviceGroup, '--service-group'); + validateIdentity(result.targetUser, '--target-user'); + result.directoryModeNumber = parseMode( + result.directoryMode, + '--directory-mode', + ); + result.fileModeNumber = parseMode(result.fileMode, '--file-mode'); + if ((result.fileModeNumber & 0o007) !== 0) { + throw new Error('--file-mode 不能允许 other 读取证书或私钥。'); + } + if ((result.directoryModeNumber & 0o007) !== 0) { + throw new Error('--directory-mode 不能允许 other 进入 TLS 私有目录。'); + } + return result; +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + validateNoControlCharacters(raw, label); + const normalized = String(raw).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${label} 必须是布尔值 true/false 或 1/0。`); +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateSafeAbsoluteDirectoryPath(value, label) { + validateSafeAbsoluteFilePath(value, label); + if (path.basename(path.resolve(value)) === '..') { + throw new Error(`${label} 不能以 .. 结尾。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateIdentity(value, label) { + validateNoControlCharacters(value, label); + if (!/^[A-Za-z0-9_.#-]+$/u.test(String(value))) { + throw new Error(`${label} 只能包含字母、数字、点、下划线、短横线或 #。`); + } +} + +function parseMode(value, label) { + validateNoControlCharacters(value, label); + const text = String(value).trim(); + if (!/^0?[0-7]{3}$/u.test(text)) { + throw new Error(`${label} 必须是三位八进制权限,例如 0640。`); + } + return Number.parseInt(text, 8); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function run() { + const certSource = inspectSourceFile( + config.sourceCertFile, + '--source-cert-file', + ); + const keySource = inspectSourceFile(config.sourceKeyFile, '--source-key-file'); + const certTarget = path.join(config.targetDir, 'fullchain.pem'); + const keyTarget = path.join(config.targetDir, 'privkey.pem'); + const owner = config.apply + ? resolveOwner(config.targetUser, config.serviceGroup) + : { uid: null, gid: null }; + + assertNoSymlinkAncestors(config.targetDir, '--target-dir'); + assertTargetPath(certTarget, '目标 fullchain.pem'); + assertTargetPath(keyTarget, '目标 privkey.pem'); + if (existsSync(config.targetDir)) { + assertTargetDirectory(config.targetDir); + } + + const planned = { + mode: config.apply ? 'apply' : 'dry-run', + target: { + directory: config.targetDir, + user: config.targetUser, + group: config.serviceGroup, + uid: owner.uid, + gid: owner.gid, + serviceUser: config.serviceUser, + directoryMode: formatMode(config.directoryModeNumber), + fileMode: formatMode(config.fileModeNumber), + }, + files: [ + { + kind: 'cert', + source: certSource, + target: { path: certTarget }, + }, + { + kind: 'key', + source: keySource, + target: { path: keyTarget }, + }, + ], + serviceUserReadable: false, + }; + + if (!config.apply) { + return planned; + } + + mkdirSync(config.targetDir, { recursive: true }); + assertTargetDirectory(config.targetDir); + chmodSync(config.targetDir, config.directoryModeNumber); + applyOwner(config.targetDir, owner); + + copyAtomic(certSource.resolvedPath, certTarget, owner); + copyAtomic(keySource.resolvedPath, keyTarget, owner); + + planned.serviceUserReadable = assertServiceUserReadable( + config.serviceUser, + [certTarget, keyTarget], + ); + return planned; +} + +function inspectSourceFile(filePath, label) { + const linkStat = lstatSync(filePath); + const sourceIsSymlink = linkStat.isSymbolicLink(); + const resolvedPath = realpathSync(filePath); + const fileStat = statSync(resolvedPath); + if (!fileStat.isFile()) { + throw new Error(`${label} 解析后必须是普通文件: ${resolvedPath}`); + } + return { + path: filePath, + resolvedPath, + sourceIsSymlink, + sizeBytes: fileStat.size, + }; +} + +function assertTargetDirectory(directory) { + const linkStat = lstatSync(directory); + if (linkStat.isSymbolicLink()) { + throw new Error(`--target-dir 不能是符号链接: ${directory}`); + } + if (!linkStat.isDirectory()) { + throw new Error(`--target-dir 已存在但不是目录: ${directory}`); + } +} + +function assertNoSymlinkAncestors(targetDir, label) { + const resolved = path.resolve(targetDir); + const parsed = path.parse(resolved); + let current = parsed.root; + for (const part of resolved.slice(parsed.root.length).split(path.sep)) { + if (!part) { + continue; + } + current = path.join(current, part); + if (!existsSync(current)) { + break; + } + const linkStat = lstatSync(current); + if (linkStat.isSymbolicLink()) { + throw new Error(`${label} 已存在路径不能包含符号链接: ${current}`); + } + if (!linkStat.isDirectory()) { + throw new Error(`${label} 已存在父路径不是目录: ${current}`); + } + } +} + +function assertTargetPath(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const linkStat = lstatSync(filePath); + if (linkStat.isSymbolicLink()) { + throw new Error(`${label} 不能是符号链接: ${filePath}`); + } + if (!linkStat.isFile()) { + throw new Error(`${label} 已存在但不是普通文件: ${filePath}`); + } +} + +function resolveOwner(user, group) { + const current = currentIdentity(); + const uid = resolveUserId(user, current); + const gid = resolveGroupId(group, current, user); + if (!isCurrentUserRoot()) { + if (uid !== current.uid || gid !== current.gid) { + throw new Error( + `当前用户不是 root,不能把证书授权给 ${user}:${group};请用 sudo -n 执行 --apply。`, + ); + } + } + return { uid, gid }; +} + +function resolveUserId(user, current) { + if (/^#?\d+$/u.test(user)) { + return Number.parseInt(user.replace(/^#/u, ''), 10); + } + if (user === current.username) { + return current.uid; + } + return Number.parseInt( + execFileSync('id', ['-u', user], { encoding: 'utf8' }).trim(), + 10, + ); +} + +function resolveGroupId(group, current, user) { + if (/^#?\d+$/u.test(group)) { + return Number.parseInt(group.replace(/^#/u, ''), 10); + } + if (group === current.groupname || group === String(current.gid)) { + return current.gid; + } + const getent = spawnSync('getent', ['group', group], { encoding: 'utf8' }); + if (getent.status === 0 && getent.stdout.trim()) { + const parts = getent.stdout.trim().split(':'); + return Number.parseInt(parts[2], 10); + } + if (group === user) { + return Number.parseInt( + execFileSync('id', ['-g', user], { encoding: 'utf8' }).trim(), + 10, + ); + } + throw new Error(`无法解析服务组: ${group}`); +} + +function currentIdentity() { + const uid = typeof process.getuid === 'function' ? process.getuid() : -1; + const gid = typeof process.getgid === 'function' ? process.getgid() : -1; + let username = ''; + try { + username = userInfo().username; + } catch { + username = process.env.USER || process.env.LOGNAME || ''; + } + return { + uid, + gid, + username, + groupname: '', + }; +} + +function isCurrentUserRoot() { + return typeof process.getuid === 'function' && process.getuid() === 0; +} + +function applyOwner(filePath, owner) { + const current = currentIdentity(); + if (!isCurrentUserRoot() && owner.uid === current.uid && owner.gid === current.gid) { + return; + } + chownSync(filePath, owner.uid, owner.gid); +} + +function copyAtomic(sourcePath, targetPath, owner) { + const tempPath = path.join( + path.dirname(targetPath), + `.${path.basename(targetPath)}.tmp-${process.pid}-${randomBytes(4).toString('hex')}`, + ); + try { + copyFileSync(sourcePath, tempPath, constants.COPYFILE_EXCL); + chmodSync(tempPath, config.fileModeNumber); + applyOwner(tempPath, owner); + renameSync(tempPath, targetPath); + chmodSync(targetPath, config.fileModeNumber); + applyOwner(targetPath, owner); + } catch (error) { + rmSync(tempPath, { force: true }); + throw error; + } +} + +function assertServiceUserReadable(serviceUser, files) { + if (/^#?\d+$/u.test(serviceUser)) { + for (const file of files) { + accessSync(file, constants.R_OK); + } + return true; + } + const current = currentIdentity(); + if (serviceUser === current.username && !isCurrentUserRoot()) { + for (const file of files) { + accessSync(file, constants.R_OK); + } + return true; + } + for (const file of files) { + const check = spawnSync('sudo', ['-n', '-u', serviceUser, 'test', '-r', file], { + encoding: 'utf8', + }); + if (check.status !== 0) { + const detail = (check.stderr || check.stdout || '').trim(); + throw new Error( + `服务用户 ${serviceUser} 不可读: ${file}${detail ? ` (${detail})` : ''}`, + ); + } + } + return true; +} + +function formatMode(mode) { + return `0${mode.toString(8).padStart(3, '0')}`; +} diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index e215b3e90..851d897a2 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,12 +5,13 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--worker-service-pattern 'genarrative-external-generation-worker@*.service'] [--no-worker-services] [--worker-controller-service genarrative-external-generation-controller.service] [--no-worker-controller] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] 说明: 进入维护模式,校验并发布 api-server 单文件,更新 current 链接,重启 systemd 服务并执行 readiness 检查。 默认同时重启外部生成 worker controller 和已加载的 worker 实例;未启用 worker 单元时会自动跳过。 若传入 --database,会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件,避免服务继续读取旧库。 + 若发布包包含 pingora-gateway,部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置,再启动或重启 Pingora 影子服务并复核 active。 失败时保留维护模式。 EOF } @@ -25,6 +26,16 @@ require_argument() { fi } +require_absolute_path() { + local value="$1" + local label="$2" + + if [[ ! "${value}" = /* ]]; then + echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2 + exit 1 + fi +} + validate_spacetime_database_name() { local database="$1" @@ -224,6 +235,144 @@ ensure_runtime_env_and_dirs() { fi } +extract_pingora_env_files_from_unit() { + local service_name="$1" + local unit_content + + if ! unit_content="$(systemctl cat "${service_name}")"; then + echo "[production-api-deploy] 无法读取 Pingora systemd 最终配置: ${service_name}" >&2 + exit 1 + fi + + if printf "%s\n" "${unit_content}" | grep -Eq '^[[:space:]]*(AmbientCapabilities|CapabilityBoundingSet)=.*CAP_NET_BIND_SERVICE'; then + echo "[production-api-deploy] Pingora systemd 已包含 CAP_NET_BIND_SERVICE,疑似直连入口配置;API deploy 不会自动启动或重启直连服务: ${service_name}" >&2 + exit 1 + fi + + printf "%s\n" "${unit_content}" | while IFS= read -r raw_line; do + local line value token env_file + line="${raw_line#"${raw_line%%[![:space:]]*}"}" + [[ "${line}" == EnvironmentFile=* ]] || continue + value="${line#EnvironmentFile=}" + for token in ${value}; do + env_file="${token#-}" + env_file="${env_file%\"}" + env_file="${env_file#\"}" + env_file="${env_file%\'}" + env_file="${env_file#\'}" + [[ -n "${env_file}" ]] && printf "%s\n" "${env_file}" + done + done +} + +find_pingora_gateway_env_file() { + local service_name="$1" + local env_file listen + + while IFS= read -r env_file; do + if [[ "${env_file}" != /* ]]; then + echo "[production-api-deploy] Pingora EnvironmentFile 必须使用绝对路径: ${env_file}" >&2 + exit 1 + fi + listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")" + if [[ -n "${listen}" ]]; then + printf "%s\n" "${env_file}" + return + fi + done < <(extract_pingora_env_files_from_unit "${service_name}") + + echo "[production-api-deploy] Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile: ${service_name}" >&2 + exit 1 +} + +require_pingora_shadow_env() { + local env_file="$1" + local listen tls_listen redirect_listen + + listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")" + tls_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN")" + redirect_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN")" + + if [[ "${listen}" != "127.0.0.1:18081" ]]; then + echo "[production-api-deploy] Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081,当前 GENARRATIVE_PINGORA_GATEWAY_LISTEN=${listen:-}" >&2 + exit 1 + fi + if [[ -n "${tls_listen}" ]]; then + echo "[production-api-deploy] Pingora 自动启动不允许启用 TLS_LISTEN,当前 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${tls_listen}" >&2 + exit 1 + fi + if [[ -n "${redirect_listen}" ]]; then + echo "[production-api-deploy] Pingora 自动启动不允许启用 HTTP_REDIRECT_LISTEN,当前 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${redirect_listen}" >&2 + exit 1 + fi +} + +check_pingora_shadow_service_config() { + local service_name="$1" + local env_file + + env_file="$(find_pingora_gateway_env_file "${service_name}")" + require_pingora_shadow_env "${env_file}" + printf "%s\n" "${env_file}" +} + +ensure_pingora_shadow_service() { + local service_name="$1" + local env_file="${2:-}" + + if [[ -z "${env_file}" ]]; then + env_file="$(check_pingora_shadow_service_config "${service_name}")" + else + require_pingora_shadow_env "${env_file}" + fi + + echo "[production-api-deploy] 发布包包含 Pingora,启动或重启 shadow 影子服务: ${service_name} (${env_file})" + systemctl restart "${service_name}" + if ! systemctl is-active --quiet "${service_name}"; then + echo "[production-api-deploy] Pingora shadow 影子服务启动或重启后不是 active: ${service_name}" >&2 + exit 1 + fi +} + +validate_release_manifest() { + local manifest_path="$1" + local require_pingora="$2" + + if [[ ! -f "${manifest_path}" ]]; then + echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2 + exit 1 + fi + + node - "${manifest_path}" "${require_pingora}" <<'NODE' +const fs = require('fs'); + +const manifestPath = process.argv[2]; +const requirePingora = process.argv[3] === '1'; +let manifest; +try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); +} catch (error) { + console.error(`[production-api-deploy] release-manifest.json 不是合法 JSON: ${error.message}`); + process.exit(1); +} + +const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; +const hasApiServer = artifacts.some((artifact) => artifact?.path === 'api-server'); +if (!hasApiServer) { + console.error('[production-api-deploy] release-manifest.json 缺少 api-server artifact。'); + process.exit(1); +} + +if (requirePingora) { + const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway'); + if (!hasPingora) { + console.error('[production-api-deploy] release-manifest.json 缺少 pingora-gateway artifact。'); + process.exit(1); + } +} +NODE +} + list_worker_services() { local pattern="$1" @@ -360,6 +509,7 @@ VERSION="" RELEASE_ROOT="/opt/genarrative/releases" CURRENT_LINK="/opt/genarrative/current" SERVICE_NAME="genarrative-api.service" +PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service" WORKER_SERVICE_PATTERN="genarrative-external-generation-worker@*.service" WORKER_CONTROLLER_SERVICE="genarrative-external-generation-controller.service" HEALTH_URL="http://127.0.0.1:8082/readyz" @@ -367,6 +517,9 @@ API_ENV_FILE="/etc/genarrative/api-server.env" DATABASE="" SPACETIME_SERVER_URL="" DEPLOY_COMPLETED=0 +PINGORA_INCLUDED=0 +RELEASE_DIR="" +STAGING_RELEASE_DIR="" while [[ $# -gt 0 ]]; do case "$1" in @@ -394,6 +547,10 @@ while [[ $# -gt 0 ]]; do SERVICE_NAME="${2:?缺少 --service 的值}" shift 2 ;; + --pingora-service) + PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}" + shift 2 + ;; --worker-service-pattern) WORKER_SERVICE_PATTERN="${2:?缺少 --worker-service-pattern 的值}" shift 2 @@ -435,6 +592,9 @@ while [[ $# -gt 0 ]]; do done require_argument "${SOURCE_DIR}" "--source-dir" +require_absolute_path "${RELEASE_ROOT}" "--release-root" +require_absolute_path "${CURRENT_LINK}" "--current-link" +require_absolute_path "${API_ENV_FILE}" "--api-env-file" if [[ -n "${DATABASE}" ]]; then validate_spacetime_database_name "${DATABASE}" @@ -448,8 +608,13 @@ fi SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)" VERSION="${VERSION:-$(basename "${SOURCE_DIR}")}" -if [[ ! "${VERSION}" =~ ^[0-9A-Za-z._-]+$ ]]; then - echo "[production-api-deploy] --version 只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2 +if [[ ! "${VERSION}" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "[production-api-deploy] --version 必须以数字或字母开头,且只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2 + exit 1 +fi + +if [[ "${VERSION}" == "." || "${VERSION}" == ".." ]]; then + echo "[production-api-deploy] --version 不能是点目录: ${VERSION}" >&2 exit 1 fi @@ -458,9 +623,34 @@ if [[ ! -f "${SOURCE_DIR}/api-server" || ! -f "${SOURCE_DIR}/api-server.sha256" exit 1 fi +RELEASE_DIR="${RELEASE_ROOT}/${VERSION}" +STAGING_RELEASE_DIR="${RELEASE_ROOT}/.${VERSION}.staging.$$" + +if [[ -e "${RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 目标 release 已存在,拒绝覆盖或合并旧文件: ${RELEASE_DIR}" >&2 + exit 1 +fi + +if [[ -e "${STAGING_RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 临时 staging release 已存在: ${STAGING_RELEASE_DIR}" >&2 + exit 1 +fi + +if [[ -e "${CURRENT_LINK}" && ! -L "${CURRENT_LINK}" ]]; then + echo "[production-api-deploy] current 链接路径已存在但不是符号链接,拒绝覆盖: ${CURRENT_LINK}" >&2 + exit 1 +fi + +cleanup_staging_release() { + if [[ -n "${STAGING_RELEASE_DIR:-}" && -d "${STAGING_RELEASE_DIR}" ]]; then + rm -rf "${STAGING_RELEASE_DIR}" + fi +} + on_exit() { local exit_code=$? if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then + cleanup_staging_release echo "[production-api-deploy] 部署失败,保持维护模式。" >&2 fi exit "${exit_code}" @@ -468,52 +658,225 @@ on_exit() { trap on_exit EXIT -"${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" +bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" echo "[production-api-deploy] 校验 api-server" ( cd "${SOURCE_DIR}" sha256sum -c api-server.sha256 + if [[ -f pingora-gateway || -f pingora-gateway.sha256 ]]; then + if [[ ! -f pingora-gateway || ! -f pingora-gateway.sha256 ]]; then + echo "[production-api-deploy] pingora-gateway 与 pingora-gateway.sha256 必须同时存在。" >&2 + exit 1 + fi + sha256sum -c pingora-gateway.sha256 + fi ) +if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then + PINGORA_INCLUDED=1 +fi +validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}" -RELEASE_DIR="${RELEASE_ROOT}/${VERSION}" -mkdir -p "${RELEASE_DIR}" -cp "${SOURCE_DIR}/api-server" "${RELEASE_DIR}/api-server" -chmod +x "${RELEASE_DIR}/api-server" +mkdir -p "${RELEASE_ROOT}" +mkdir "${STAGING_RELEASE_DIR}" +RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}" +cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server" +cp "${SOURCE_DIR}/api-server.sha256" "${RELEASE_CONTENT_DIR}/api-server.sha256" +chmod +x "${RELEASE_CONTENT_DIR}/api-server" +if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then + cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway" + cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256" + chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway" + echo "[production-api-deploy] 已复制 Pingora 影子网关;current 链接切换后将复核 shadow 配置并启动或重启 ${PINGORA_SERVICE_NAME}" +fi BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" -WORKSPACE_BACKUP_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/database-backup-to-oss.mjs" HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" -WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/ops/production-health-patrol.mjs" -mkdir -p "${RELEASE_DIR}/scripts" "${RELEASE_DIR}/scripts/ops" +PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" +PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" +PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" +PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" +PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" +PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" +PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" +PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" +PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" +PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" +PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" +PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" +PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +PINGORA_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/pingora" +SYSTEMD_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/systemd" +NGINX_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/nginx" +ENV_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/env" +mkdir -p "${RELEASE_CONTENT_DIR}/scripts" "${RELEASE_CONTENT_DIR}/scripts/deploy" "${RELEASE_CONTENT_DIR}/scripts/ops" "${RELEASE_CONTENT_DIR}/deploy" if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then - if [[ -f "${WORKSPACE_BACKUP_SCRIPT_SOURCE}" ]]; then - echo "[production-api-deploy] 发布产物缺少 scripts/database-backup-to-oss.mjs,回退使用部署工作区脚本;请重新触发包含该脚本的 API 构建。" >&2 - BACKUP_SCRIPT_SOURCE="${WORKSPACE_BACKUP_SCRIPT_SOURCE}" - else - echo "[production-api-deploy] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 - exit 1 - fi + echo "[production-api-deploy] 发布产物缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 + exit 1 fi -cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs" -chmod 0644 "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs" +cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - if [[ -f "${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - echo "[production-api-deploy] 发布产物缺少 scripts/ops/production-health-patrol.mjs,回退使用部署工作区脚本;请重新触发包含该脚本的 API 构建。" >&2 - HEALTH_PATROL_SCRIPT_SOURCE="${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}" - else - echo "[production-api-deploy] 未找到生产健康巡检脚本,跳过复制;genarrative-health-patrol.service 会因脚本缺失而跳过执行。" >&2 - HEALTH_PATROL_SCRIPT_SOURCE="" - fi + echo "[production-api-deploy] 发布产物缺少生产健康巡检脚本: ${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" >&2 + exit 1 fi -if [[ -n "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs" - chmod 0644 "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs" +cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" +if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora current release 自审脚本: ${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" >&2 + exit 1 fi +cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" +if [[ ! -f "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连彩排状态脚本: ${SOURCE_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" >&2 + exit 1 +fi +cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" +if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据包脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +if [[ ! -f "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换命令证据脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据验真脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据根目录审计脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少生产健康巡检 env 复核脚本: ${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" >&2 + exit 1 +fi +cp "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" +if [[ ! -f "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora release readiness 聚合门禁脚本: ${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" >&2 + exit 1 +fi +cp "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" +if [[ ! -f "${PINGORA_ENABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" >&2 + exit 1 +fi +cp "${PINGORA_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" +if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连回退脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" >&2 + exit 1 +fi +cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" +if [[ ! -f "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" >&2 + exit 1 +fi +cp "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +if [[ ! -f "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 关闭脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" >&2 + exit 1 +fi +cp "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" +if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2 + exit 1 +fi +cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +if [[ ! -f "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora gateway env shadow 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" >&2 + exit 1 +fi +cp "${PINGORA_GATEWAY_ENV_SHADOW_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" +if [[ ! -f "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora TLS 证书同步脚本: ${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" >&2 + exit 1 +fi +cp "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" +if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2 + exit 1 +fi +cp "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" +if [[ ! -f "${PINGORA_LIVE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连 live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" >&2 + exit 1 +fi +cp "${PINGORA_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" +if [[ ! -f "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora canary live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" +if [[ ! -f "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora canary access log 对账脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +if [[ ! -d "${PINGORA_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 配置目录: ${SOURCE_DIR}/deploy/pingora" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/pingora" +cp -R "${PINGORA_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/pingora" +if [[ ! -d "${SYSTEMD_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 systemd 配置目录: ${SOURCE_DIR}/deploy/systemd" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/systemd" +cp -R "${SYSTEMD_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/systemd" +if [[ ! -d "${NGINX_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Nginx 配置目录: ${SOURCE_DIR}/deploy/nginx" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/nginx" +cp -R "${NGINX_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/nginx" +if [[ ! -d "${ENV_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少环境变量示例目录: ${SOURCE_DIR}/deploy/env" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/env" +cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env" -if [[ -f "${SOURCE_DIR}/release-manifest.json" ]]; then - cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_DIR}/release-manifest.api-server.json" -fi +cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json" + +PINGORA_SHADOW_ENV_FILE="" if [[ -n "${DATABASE}" ]]; then echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}" @@ -527,8 +890,22 @@ fi ensure_runtime_env_and_dirs "${API_ENV_FILE}" +if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then + PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")" +fi + mkdir -p "$(dirname "${CURRENT_LINK}")" -ln -sfn "${RELEASE_DIR}" "${CURRENT_LINK}" +if [[ -e "${RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 目标 release 在发布过程中出现,拒绝合并 staging: ${RELEASE_DIR}" >&2 + exit 1 +fi +mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}" +STAGING_RELEASE_DIR="" +ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}" + +if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then + ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}" +fi echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}" systemctl restart "${SERVICE_NAME}" @@ -540,7 +917,7 @@ wait_for_worker_controller_service "${WORKER_CONTROLLER_SERVICE}" echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}" for _ in {1..30}; do if curl -fsS "${HEALTH_URL}" >/dev/null; then - "${SCRIPT_DIR}/maintenance-off.sh" + bash "${SCRIPT_DIR}/maintenance-off.sh" DEPLOY_COMPLETED=1 echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server" exit 0 diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index cdd60d41b..0f894adc7 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -174,7 +174,7 @@ prepare_async_backup() { ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)" echo "[production-stdb-publish] publish 前生成本地冷备份,随后会异步上传 OSS" - node "${ASYNC_BACKUP_SCRIPT}" \ + node -- "${ASYNC_BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ @@ -200,7 +200,7 @@ start_async_backup_upload() { mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")" ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log" echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}" - nohup node "${ASYNC_BACKUP_SCRIPT}" \ + nohup node -- "${ASYNC_BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --upload-archive "${ASYNC_BACKUP_ARCHIVE}" \ --manifest-file "${ASYNC_BACKUP_MANIFEST}" \ @@ -254,7 +254,7 @@ case "${BACKUP_MODE}" in fi echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份,失败会阻断发布" - node "${BACKUP_SCRIPT}" \ + node -- "${BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 279a4dac3..454611aa6 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -652,10 +652,15 @@ restore_nginx_default_sites_enabled() { install_nginx_config_with_rollback() { local config_target="/etc/nginx/conf.d/genarrative.conf" local snippet_target="/etc/nginx/snippets/genarrative-maintenance.conf" + local pingora_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-canary.conf" + local pingora_realpath_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-realpath-canary.conf" local config_source - local rendered_config rendered_snippet config_backup snippet_backup disabled_sites + local rendered_config rendered_snippet rendered_pingora_canary_snippet rendered_pingora_realpath_canary_snippet + local config_backup snippet_backup pingora_canary_snippet_backup pingora_realpath_canary_snippet_backup disabled_sites local had_config="false" local had_snippet="false" + local had_pingora_canary_snippet="false" + local had_pingora_realpath_canary_snippet="false" run_cmd mkdir -p /etc/nginx/snippets /etc/nginx/conf.d if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then @@ -669,6 +674,8 @@ install_nginx_config_with_rollback() { echo "+ render ${config_source} -> ${config_target}" echo "+ install -m 0644 deploy/nginx/snippets/genarrative-maintenance.conf ${snippet_target}" + echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-canary.conf ${pingora_canary_snippet_target}" + echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf ${pingora_realpath_canary_snippet_target}" if [[ "${DRY_RUN}" == "true" ]]; then echo "+ disable /etc/nginx/sites-enabled/default* if present" @@ -679,8 +686,12 @@ install_nginx_config_with_rollback() { rendered_config="$(mktemp)" rendered_snippet="$(mktemp)" + rendered_pingora_canary_snippet="$(mktemp)" + rendered_pingora_realpath_canary_snippet="$(mktemp)" config_backup="$(mktemp)" snippet_backup="$(mktemp)" + pingora_canary_snippet_backup="$(mktemp)" + pingora_realpath_canary_snippet_backup="$(mktemp)" disabled_sites="$(mktemp)" if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then validate_nginx_tls @@ -689,6 +700,8 @@ install_nginx_config_with_rollback() { render_nginx_development_http_config >"${rendered_config}" fi cp deploy/nginx/snippets/genarrative-maintenance.conf "${rendered_snippet}" + cp deploy/nginx/snippets/genarrative-pingora-canary.conf "${rendered_pingora_canary_snippet}" + cp deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf "${rendered_pingora_realpath_canary_snippet}" if [[ -f "${config_target}" ]]; then cp -p "${config_target}" "${config_backup}" @@ -698,9 +711,19 @@ install_nginx_config_with_rollback() { cp -p "${snippet_target}" "${snippet_backup}" had_snippet="true" fi + if [[ -f "${pingora_canary_snippet_target}" ]]; then + cp -p "${pingora_canary_snippet_target}" "${pingora_canary_snippet_backup}" + had_pingora_canary_snippet="true" + fi + if [[ -f "${pingora_realpath_canary_snippet_target}" ]]; then + cp -p "${pingora_realpath_canary_snippet_target}" "${pingora_realpath_canary_snippet_backup}" + had_pingora_realpath_canary_snippet="true" + fi install -m 0644 "${rendered_config}" "${config_target}" install -m 0644 "${rendered_snippet}" "${snippet_target}" + install -m 0644 "${rendered_pingora_canary_snippet}" "${pingora_canary_snippet_target}" + install -m 0644 "${rendered_pingora_realpath_canary_snippet}" "${pingora_realpath_canary_snippet_target}" disable_nginx_default_sites_enabled "${disabled_sites}" if ! nginx -t; then @@ -715,14 +738,24 @@ install_nginx_config_with_rollback() { else rm -f "${snippet_target}" fi + if [[ "${had_pingora_canary_snippet}" == "true" ]]; then + cp -p "${pingora_canary_snippet_backup}" "${pingora_canary_snippet_target}" + else + rm -f "${pingora_canary_snippet_target}" + fi + if [[ "${had_pingora_realpath_canary_snippet}" == "true" ]]; then + cp -p "${pingora_realpath_canary_snippet_backup}" "${pingora_realpath_canary_snippet_target}" + else + rm -f "${pingora_realpath_canary_snippet_target}" + fi restore_nginx_default_sites_enabled "${disabled_sites}" - rm -f "${rendered_config}" "${rendered_snippet}" "${config_backup}" "${snippet_backup}" "${disabled_sites}" + rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}" exit 1 fi echo "+ nginx -s reload" nginx -s reload - rm -f "${rendered_config}" "${rendered_snippet}" "${config_backup}" "${snippet_backup}" "${disabled_sites}" + rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}" } cleanup_placeholder_nginx_config() { @@ -824,11 +857,16 @@ require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer +require_path deploy/systemd/genarrative-pingora-gateway.service +require_path deploy/systemd/genarrative-pingora-gateway-direct-entry.conf require_path deploy/systemd/otelcol-contrib.service require_path deploy/otelcol/genarrative-debug.yaml require_path deploy/nginx/genarrative.conf require_path deploy/nginx/genarrative-dev-http.conf require_path deploy/nginx/snippets/genarrative-maintenance.conf +require_path deploy/nginx/snippets/genarrative-pingora-canary.conf +require_path deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf +require_path deploy/logrotate/genarrative-pingora-gateway require_path deploy/env/api-server.env.example require_path deploy/env/external-generation-worker.env.example require_path deploy/env/external-generation-controller.env.example @@ -844,7 +882,7 @@ echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_conf run_cmd id require_root_for_real_provision install_nginx_brotli_modules -run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol +run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative if ! id spacetimedb >/dev/null 2>&1; then run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb @@ -860,6 +898,7 @@ fi run_cmd chown -R spacetimedb:spacetimedb "${SPACETIME_ROOT}" run_cmd chown -R genarrative:genarrative /opt/genarrative /var/lib/genarrative /srv/genarrative +run_cmd chown -R genarrative:genarrative /var/log/genarrative install_genarrative_openssl_runtime if [[ ! -x "${SPACETIME_BIN_SOURCE}" ]]; then @@ -893,6 +932,9 @@ install_file "${database_backup_service}" /etc/systemd/system/genarrative-databa install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644 install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644 install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644 +install_file deploy/systemd/genarrative-pingora-gateway.service /etc/systemd/system/genarrative-pingora-gateway.service 0644 +install_file deploy/systemd/genarrative-pingora-gateway-direct-entry.conf /etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf 0644 +install_file deploy/logrotate/genarrative-pingora-gateway /etc/logrotate.d/genarrative-pingora-gateway 0644 rm -f "${spacetimedb_service}" "${api_service}" "${external_generation_worker_service}" "${external_generation_controller_service}" "${database_backup_service}" "${health_patrol_service}" if [[ ! -f "${API_ENV_FILE}" ]]; then diff --git a/scripts/ops/pingora-current-release-audit.mjs b/scripts/ops/pingora-current-release-audit.mjs new file mode 100644 index 000000000..cc49901f2 --- /dev/null +++ b/scripts/ops/pingora-current-release-audit.mjs @@ -0,0 +1,705 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, realpath, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const REQUIRED_ARTIFACTS = [ + { path: 'api-server', executable: true }, + { path: 'scripts/database-backup-to-oss.mjs' }, + { path: 'scripts/ops/production-health-patrol.mjs' }, + { path: 'scripts/ops/pingora-current-release-audit.mjs' }, + { path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' }, + { path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' }, + { path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' }, + { path: 'scripts/check-production-health-patrol-env.mjs' }, + { path: 'scripts/check-pingora-release-readiness.mjs' }, + { path: 'scripts/check-pingora-direct-preflight.mjs' }, + { path: 'scripts/check-pingora-direct-live.mjs' }, + { path: 'scripts/check-pingora-canary-live.mjs' }, + { path: 'scripts/check-pingora-canary-access-log-parity.mjs' }, + { path: 'scripts/deploy/pingora-direct-enable.sh', executable: true }, + { path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true }, + { path: 'scripts/deploy/pingora-realpath-canary-enable.sh', executable: true }, + { path: 'scripts/deploy/pingora-realpath-canary-disable.sh', executable: true }, + { path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', executable: true }, + { path: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', executable: true }, + { path: 'scripts/deploy/pingora-tls-cert-sync.mjs', executable: true }, + { path: 'deploy/systemd/genarrative-pingora-gateway.service' }, + { path: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf' }, + { path: 'deploy/nginx/snippets/genarrative-pingora-canary.conf' }, + { path: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf' }, + { path: 'deploy/env/health-patrol.env.example' }, + { path: 'deploy/env/pingora-direct-live.env.example' }, + { path: 'deploy/env/pingora-canary-live.env.example' }, + { path: 'deploy/pingora/pingora-gateway.env.example' }, + { path: 'deploy/pingora/nginx-route-parity.matrix.json' }, +]; +const REQUIRED_DIRS = [ + 'deploy/systemd', + 'deploy/nginx', + 'deploy/env', + 'deploy/pingora', +]; + +const config = parseArgs(process.argv.slice(2)); +const audit = await buildAudit(config); + +console.log(`${JSON.stringify(audit, null, 2)}\n`); + +if (audit.summary.status === 'CRITICAL' && !config.warnOnly) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY', + ), + systemdShow: readBoolEnv( + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW', + ), + systemdService: + process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SERVICE || + 'genarrative-pingora-gateway.service', + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS', + ), + warnOnly: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--systemd-show': + result.systemdShow = true; + break; + case '--systemd-service': + result.systemdService = requireValue(argv, ++index, arg); + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--warn-only': + result.warnOnly = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.releaseRoot)) { + throw new Error('--release-root 必须是绝对路径。'); + } + validateNoControlCharacters(result.releaseRoot, '--release-root'); + if (isFilesystemRootPath(result.releaseRoot)) { + throw new Error('--release-root 不能是文件系统根目录。'); + } + validateNoControlCharacters(result.systemdService, '--systemd-service'); + if (!result.systemdService || /[\s/]/u.test(result.systemdService)) { + throw new Error('--systemd-service 必须是 systemd unit 名称。'); + } + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-current-release-audit.mjs [options] + +Options: + --release-root current release 根目录,默认 /opt/genarrative/current。 + --require-pingora-gateway 要求 current release 已包含 pingora-gateway 且可执行。 + --systemd-show 读取 systemctl show,确认 service ExecStart 指向 current release 的 pingora-gateway。 + --systemd-service systemd service 名,默认 genarrative-pingora-gateway.service。 + --timeout-ms systemctl 超时,默认 5000。 + --warn-only 即使出现 CRITICAL 也以退出码 0 结束,只用于人工盘点。 + +该脚本只读检查 current release 自包含能力,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +CRITICAL 默认会让进程以退出码 1 结束,适合作为切换窗口前置门禁。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +async function buildAudit(input) { + const artifacts = await inspectArtifacts(input); + const directories = await inspectDirectories(input); + const pingoraGateway = await inspectPingoraGateway(input); + const checksums = await inspectChecksums(input, pingoraGateway); + const releaseManifest = await inspectReleaseManifest(input, pingoraGateway); + const systemd = input.systemdShow + ? await inspectSystemd(input) + : { + checked: false, + status: 'OK', + diagnostics: [], + }; + const summary = summarize([ + ...artifacts.map((artifact) => artifact.status), + ...directories.map((directory) => directory.status), + pingoraGateway.status, + ...checksums.map((checksum) => checksum.status), + releaseManifest.status, + systemd.status, + ]); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + releaseRoot: input.releaseRoot, + summary, + artifacts, + directories, + pingoraGateway, + checksums, + releaseManifest, + systemd, + }; +} + +async function inspectArtifacts(input) { + const artifacts = []; + for (const artifact of REQUIRED_ARTIFACTS) { + artifacts.push(await inspectArtifact(input.releaseRoot, artifact)); + } + return artifacts; +} + +async function inspectDirectories(input) { + const directories = []; + for (const relativePath of REQUIRED_DIRS) { + const fullPath = path.join(input.releaseRoot, relativePath); + const diagnostics = []; + let exists = false; + let status = 'OK'; + try { + const fileStat = await stat(fullPath); + exists = fileStat.isDirectory(); + if (!exists) { + diagnostics.push('路径存在但不是目录。'); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`目录不存在或不可读: ${error.message}`); + status = 'CRITICAL'; + } + directories.push({ + path: relativePath, + exists, + status, + diagnostics, + }); + } + return directories; +} + +async function inspectArtifact(releaseRoot, artifact) { + const fullPath = path.join(releaseRoot, artifact.path); + const diagnostics = []; + let exists = false; + let type = 'missing'; + let executable = false; + let status = 'OK'; + + try { + const fileStat = await stat(fullPath); + exists = true; + type = fileStat.isFile() + ? 'file' + : fileStat.isDirectory() + ? 'directory' + : 'other'; + if (type !== 'file') { + diagnostics.push('路径存在但不是文件。'); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`文件不存在或不可读: ${error.message}`); + status = 'CRITICAL'; + } + + if (exists && artifact.executable) { + try { + await access(fullPath, fsConstants.X_OK); + executable = true; + } catch (error) { + diagnostics.push(`文件不可执行: ${error.message}`); + status = 'CRITICAL'; + } + } + + return { + path: artifact.path, + expectedExecutable: Boolean(artifact.executable), + exists, + type, + executable, + status, + diagnostics, + }; +} + +async function inspectPingoraGateway(input) { + const binaryPath = path.join(input.releaseRoot, 'pingora-gateway'); + const diagnostics = []; + let included = false; + let executable = false; + let status = 'OK'; + + try { + const fileStat = await stat(binaryPath); + included = fileStat.isFile(); + if (!included) { + diagnostics.push('pingora-gateway 路径存在但不是文件。'); + status = 'CRITICAL'; + } + } catch (error) { + if (input.requirePingoraGateway) { + diagnostics.push(`缺少 pingora-gateway: ${error.message}`); + status = 'CRITICAL'; + } + } + + if (included) { + try { + await access(binaryPath, fsConstants.X_OK); + executable = true; + } catch (error) { + diagnostics.push(`pingora-gateway 不可执行: ${error.message}`); + status = 'CRITICAL'; + } + } + + return { + required: input.requirePingoraGateway, + included, + path: 'pingora-gateway', + absolutePath: binaryPath, + executable, + status, + diagnostics, + }; +} + +async function inspectChecksums(input, pingoraGateway) { + const checksumTargets = [ + { + path: 'api-server', + checksumPath: 'api-server.sha256', + required: true, + }, + { + path: 'pingora-gateway', + checksumPath: 'pingora-gateway.sha256', + required: input.requirePingoraGateway || pingoraGateway.included, + }, + ]; + + const checksums = []; + for (const target of checksumTargets) { + checksums.push(await inspectChecksum(input.releaseRoot, target)); + } + return checksums; +} + +async function inspectChecksum(releaseRoot, target) { + const diagnostics = []; + const artifactPath = path.join(releaseRoot, target.path); + const checksumPath = path.join(releaseRoot, target.checksumPath); + let status = 'OK'; + let checked = false; + let exists = false; + let expectedSha256 = ''; + let computedSha256 = ''; + let checksumFileTarget = ''; + + if (!target.required) { + return { + path: target.path, + checksumPath: target.checksumPath, + required: false, + checked, + exists, + checksumFileTarget, + expectedSha256, + computedSha256, + matches: null, + status, + diagnostics, + }; + } + + try { + const checksumText = await readFile(checksumPath, 'utf8'); + exists = true; + const parsed = parseSha256File(checksumText); + expectedSha256 = parsed.sha256; + checksumFileTarget = parsed.fileName; + if (path.basename(checksumFileTarget) !== path.basename(target.path)) { + diagnostics.push( + `${target.checksumPath} 指向了非预期文件: ${checksumFileTarget}`, + ); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`checksum 文件不存在、不可读或格式错误: ${error.message}`); + status = 'CRITICAL'; + } + + try { + computedSha256 = await sha256File(artifactPath); + checked = true; + } catch (error) { + diagnostics.push(`无法读取待校验文件: ${error.message}`); + status = 'CRITICAL'; + } + + const matches = + checked && Boolean(expectedSha256) && computedSha256 === expectedSha256; + if (checked && expectedSha256 && !matches) { + diagnostics.push( + `${target.path} sha256 不匹配: expected ${expectedSha256}, actual ${computedSha256}`, + ); + status = 'CRITICAL'; + } + + return { + path: target.path, + checksumPath: target.checksumPath, + required: target.required, + checked, + exists, + checksumFileTarget, + expectedSha256, + computedSha256, + matches, + status, + diagnostics, + }; +} + +function parseSha256File(content) { + const firstLine = content.split(/\r?\n/u).find((line) => line.trim()); + if (!firstLine) { + throw new Error('checksum 文件为空'); + } + const match = firstLine.match(/^([0-9a-fA-F]{64})\s+(.+)$/u); + if (!match) { + throw new Error('checksum 行必须是 sha256sum 输出格式'); + } + return { + sha256: match[1].toLowerCase(), + fileName: match[2].replace(/^\*/u, '').trim(), + }; +} + +async function sha256File(filePath) { + const content = await readFile(filePath); + return createHash('sha256').update(content).digest('hex'); +} + +async function inspectReleaseManifest(input, pingoraGateway) { + const candidates = [ + 'release-manifest.api-server.json', + 'release-manifest.json', + ]; + const diagnostics = []; + let status = 'OK'; + let pathUsed = ''; + let manifest = null; + + for (const candidate of candidates) { + try { + const content = await readFile(path.join(input.releaseRoot, candidate), 'utf8'); + manifest = JSON.parse(content); + pathUsed = candidate; + break; + } catch (error) { + diagnostics.push(`${candidate} 不可用: ${error.message}`); + } + } + + if (!manifest) { + return { + checked: true, + path: '', + candidates, + componentType: '', + artifacts: [], + status: 'CRITICAL', + diagnostics, + }; + } + + diagnostics.length = 0; + const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; + const apiArtifact = artifacts.find((artifact) => artifact?.path === 'api-server'); + const pingoraArtifact = artifacts.find( + (artifact) => artifact?.path === 'pingora-gateway', + ); + + if (manifest.component_type !== 'api-server') { + diagnostics.push( + `release manifest component_type 应为 api-server,实际 ${manifest.component_type || '(空)'}`, + ); + status = 'CRITICAL'; + } + if (!apiArtifact) { + diagnostics.push('release manifest 缺少 api-server artifact。'); + status = 'CRITICAL'; + } else if (apiArtifact.checksum_path !== 'api-server.sha256') { + diagnostics.push( + `api-server artifact checksum_path 应为 api-server.sha256,实际 ${apiArtifact.checksum_path || '(空)'}`, + ); + status = 'CRITICAL'; + } + + if (input.requirePingoraGateway || pingoraGateway.included) { + if (!pingoraArtifact) { + diagnostics.push('release manifest 缺少 pingora-gateway artifact。'); + status = 'CRITICAL'; + } else if (pingoraArtifact.checksum_path !== 'pingora-gateway.sha256') { + diagnostics.push( + `pingora-gateway artifact checksum_path 应为 pingora-gateway.sha256,实际 ${pingoraArtifact.checksum_path || '(空)'}`, + ); + status = 'CRITICAL'; + } + } else if (pingoraArtifact) { + diagnostics.push( + 'release manifest 登记了 pingora-gateway,但 current release 未包含该二进制。', + ); + status = 'CRITICAL'; + } + + return { + checked: true, + path: pathUsed, + candidates, + componentType: manifest.component_type || '', + artifacts: artifacts.map((artifact) => ({ + component: artifact?.component || '', + path: artifact?.path || '', + checksumPath: artifact?.checksum_path || '', + })), + status, + diagnostics, + }; +} + +async function inspectSystemd(input) { + const expectedBinary = path.join(input.releaseRoot, 'pingora-gateway'); + const expectedRealpath = await resolveOptionalRealpath(expectedBinary); + const result = await runCommand( + 'systemctl', + [ + 'show', + input.systemdService, + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + const show = parseSystemctlShow(result.stdout); + const diagnostics = []; + let status = 'OK'; + + if (result.code !== 0) { + diagnostics.push(`systemctl show 失败: ${result.stderr || result.error}`); + status = 'CRITICAL'; + } + + const execStart = show.ExecStart || ''; + const execStartBinary = parseExecStartBinary(execStart); + const execStartRealpath = execStartBinary + ? await resolveOptionalRealpath(execStartBinary) + : null; + const execStartMatches = + execStartBinary === expectedBinary || + (expectedRealpath && + execStartRealpath && + expectedRealpath === execStartRealpath); + + if (result.code === 0 && !execStartMatches) { + diagnostics.push( + `ExecStart 未指向 current release 网关二进制: expected ${expectedBinary}, actual ${execStart || '(空)'}`, + ); + status = 'CRITICAL'; + } + + return { + checked: true, + service: input.systemdService, + status, + expectedBinary, + expectedRealpath, + execStartBinary, + execStartRealpath, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart, + command: result.command, + diagnostics, + }; +} + +async function resolveOptionalRealpath(filePath) { + try { + return await realpath(filePath); + } catch { + return null; + } +} + +function parseExecStartBinary(execStart) { + const pathMatch = execStart.match(/\bpath=([^ ;]+)(?:\s|;|$)/u); + if (pathMatch) { + return pathMatch[1]; + } + const argvMatch = execStart.match(/\bargv\[\]=([^ ;]+)(?:\s|;|$)/u); + if (argvMatch) { + return argvMatch[1]; + } + return ''; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +function runCommand(command, args, input) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + + return new Promise((resolve) => { + const startedAt = new Date(); + const child = execFile( + command, + args, + { + encoding: 'utf8', + timeout: input.timeoutMs, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + const finishedAt = new Date(); + resolve({ + command: [command, ...args], + code: error?.code ?? 0, + signal: error?.signal ?? null, + error: error ? error.message : '', + stdout: stdout || '', + stderr: stderr || '', + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + }); + }, + ); + child.on('error', (error) => { + const finishedAt = new Date(); + resolve({ + command: [command, ...args], + code: 1, + signal: null, + error: error.message, + stdout: '', + stderr: '', + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + }); + }); + }); +} + +function summarize(statuses) { + const criticalCount = statuses.filter((status) => status === 'CRITICAL').length; + const warningCount = statuses.filter((status) => status === 'WARNING').length; + return { + status: criticalCount > 0 ? 'CRITICAL' : warningCount > 0 ? 'WARNING' : 'OK', + criticalCount, + warningCount, + }; +} diff --git a/scripts/ops/pingora-cutover-command-evidence.mjs b/scripts/ops/pingora-cutover-command-evidence.mjs new file mode 100644 index 000000000..9fdce149b --- /dev/null +++ b/scripts/ops/pingora-cutover-command-evidence.mjs @@ -0,0 +1,518 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { + access, + chmod, + lstat, + mkdir, + readFile, + stat, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; + +const BUNDLE_DIR_MODE = 0o750; +const EVIDENCE_FILE_MODE = 0o640; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); +const SECRET_ENV_KEYS = [ + 'GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN', + 'GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN', + 'GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN', + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN', +]; + +const config = parseArgs(process.argv.slice(2)); +const secretValues = collectSecretValues(config.commandArgs, process.env); +const startedAt = new Date(); +const bundleDir = await createBundleDir( + config.outputRoot, + config.phase, + config.commandName, + startedAt, +); +const run = await runCommand(config.command, config.commandArgs, secretValues); +const finishedAt = new Date(); + +const stdoutPath = path.join(bundleDir, 'command.stdout.txt'); +const stderrPath = path.join(bundleDir, 'command.stderr.txt'); +const recordPath = path.join(bundleDir, 'command-record.json'); +const manifestPath = path.join(bundleDir, 'manifest.json'); +const exitCode = + run.signal + ? run.exitCode ?? 1 + : run.exitCode === null || run.exitCode === undefined + ? run.error + ? 1 + : 0 + : run.exitCode; +const status = exitCode === 0 && !run.signal && !run.error ? 'OK' : 'FAILED'; + +await writeEvidenceFile(stdoutPath, redactSecrets(run.stdout, secretValues)); +await writeEvidenceFile(stderrPath, redactSecrets(run.stderr, secretValues)); + +const commandRecord = { + schemaVersion: 1, + name: config.commandName, + phase: config.phase, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + ...(config.expectedExecutable + ? { expectedExecutable: config.expectedExecutable } + : {}), + executable: redactSecrets(config.command, secretValues), + args: redactSecretArgs(config.commandArgs, secretValues), + command: formatCommand(config.command, config.commandArgs, secretValues), + cwd: process.cwd(), + exitCode, + signal: run.signal, + error: run.error || null, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + stdoutPath: path.basename(stdoutPath), + stderrPath: path.basename(stderrPath), +}; +await writeEvidenceFile( + recordPath, + `${JSON.stringify(commandRecord, null, 2)}\n`, +); +const artifactFiles = { + stdout: await buildEvidenceFileMetadata(stdoutPath), + stderr: await buildEvidenceFileMetadata(stderrPath), + commandRecord: await buildEvidenceFileMetadata(recordPath), +}; + +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: config.phase, + commandName: config.commandName, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + ...(config.expectedExecutable + ? { expectedExecutable: config.expectedExecutable } + : {}), + outputRoot: config.outputRoot, + bundleDir, + summary: { + status, + exitCode, + signal: run.signal, + }, + files: { + manifest: path.basename(manifestPath), + ...artifactFiles, + }, + command: commandRecord, +}; +await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + +console.log( + `${JSON.stringify( + { + ok: status === 'OK', + phase: config.phase, + commandName: config.commandName, + cutoverRunId: config.cutoverRunId || null, + expectedExecutable: config.expectedExecutable || null, + status, + exitCode, + bundleDir, + manifestPath, + stdoutPath, + stderrPath, + commandRecordPath: recordPath, + }, + null, + 2, + )}\n`, +); + +if (exitCode !== 0) { + process.exit(exitCode); +} + +function parseArgs(argv) { + const separatorIndex = argv.indexOf('--'); + const optionArgs = + separatorIndex < 0 ? argv : argv.slice(0, separatorIndex); + if (optionArgs.includes('-h') || optionArgs.includes('--help')) { + printUsage(); + process.exit(0); + } + if (separatorIndex < 0) { + throw new Error('必须使用 -- 分隔证据参数和真实命令。'); + } + + const command = argv[separatorIndex + 1] || ''; + const commandArgs = argv.slice(separatorIndex + 2); + const result = { + phase: + process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_PHASE || 'manual', + commandName: + process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_NAME || 'command', + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + expectedExecutable: + process.env.GENARRATIVE_PINGORA_CUTOVER_EXPECTED_EXECUTABLE || '', + requiredArgs: parseRequiredArgsEnv( + process.env.GENARRATIVE_PINGORA_CUTOVER_REQUIRED_ARGS || '', + ), + outputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + command, + commandArgs, + }; + + for (let index = 0; index < optionArgs.length; index += 1) { + const arg = optionArgs[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(optionArgs, ++index, arg); + break; + case '--command-name': + result.commandName = requireValue(optionArgs, ++index, arg); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(optionArgs, ++index, arg); + break; + case '--expected-executable': + result.expectedExecutable = requireValue(optionArgs, ++index, arg); + break; + case '--require-arg': + result.requiredArgs.push(requireAnyValue(optionArgs, ++index, arg)); + break; + case '--output-root': + result.outputRoot = requireValue(optionArgs, ++index, arg); + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-command-evidence.mjs [options] -- [args...] + +Options: + --phase 写入 manifest 的阶段标签,例如 enable-apply / rollback-apply。 + --command-name 写入 manifest 的命令名,例如 pingora-direct-rollback-apply。 + --cutover-run-id 可选,写入 manifest 的本次切换批次 ID;正式 runbook 会用同一 ID 串联所有阶段和命令证据。 + --expected-executable + 可选,要求 -- 后面的真实命令精确等于该绝对路径;正式 runbook 用于绑定 apply 证据和 current release 随包脚本。 + --require-arg 可选,要求 -- 后面的真实命令参数包含该值;正式 runbook 用于在执行前确认 apply 证据包含 --apply。 + --output-root 证据输出根目录,默认 /var/log/genarrative/pingora-cutover-evidence。 + +该脚本会执行 -- 后面的真实命令,并把 stdout、stderr、退出码、脱敏后的命令记录和 manifest 写入 --output-root 下的新证据目录;失败时仍保留证据并返回真实命令的退出码。 +`); +} + +function validateConfig(config) { + validateSafeName(config.phase, '--phase'); + validateSafeName(config.commandName, '--command-name'); + if (config.cutoverRunId) { + validateSafeName(config.cutoverRunId, '--cutover-run-id'); + } + if (config.expectedExecutable) { + validateSafeExecutablePath(config.expectedExecutable, '--expected-executable'); + } + if (!path.isAbsolute(config.outputRoot)) { + throw new Error('--output-root 必须是绝对路径。'); + } + if (path.resolve(config.outputRoot) === path.parse(path.resolve(config.outputRoot)).root) { + throw new Error('--output-root 不能是文件系统根目录。'); + } + validateNoControlCharacters(config.outputRoot, '--output-root'); + if (!config.command) { + throw new Error('-- 后必须提供真实命令。'); + } + validateCommandPath(config.command); + for (const commandArg of config.commandArgs) { + validateCommandArg(commandArg); + } + if ( + config.expectedExecutable && + config.command !== config.expectedExecutable + ) { + throw new Error( + `真实命令与 --expected-executable 不一致: expected ${config.expectedExecutable}, got ${config.command}`, + ); + } + for (const requiredArg of config.requiredArgs) { + validateRequiredArg(requiredArg, '--require-arg'); + if (!config.commandArgs.includes(requiredArg)) { + throw new Error(`真实命令参数缺少 --require-arg 要求的 ${requiredArg}`); + } + } +} + +function validateCommandPath(command) { + validateSafeExecutablePath(command, '真实命令'); +} + +function validateCommandArg(value) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error('真实命令参数不能包含换行或 NUL 字符。'); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateSafeExecutablePath(command, label) { + const prefix = label.startsWith('--') ? `${label} ` : label; + if (!path.isAbsolute(command)) { + throw new Error(`${prefix}必须是绝对路径。`); + } + if (path.resolve(command) === path.parse(path.resolve(command)).root) { + throw new Error(`${prefix}不能是文件系统根目录。`); + } + if (/[\0\r\n]/u.test(command)) { + throw new Error(`${prefix}不能包含换行或 NUL 字符。`); + } +} + +function parseRequiredArgsEnv(value) { + return String(value || '') + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +function validateRequiredArg(value, label) { + if ( + typeof value !== 'string' || + value.length === 0 || + /[\0\r\n]/u.test(value) + ) { + throw new Error(`${label} 必须是非空且不包含换行的字符串。`); + } +} + +function validateSafeName(value, label) { + const text = String(value || ''); + if (!/^[0-9A-Za-z._-]+$/u.test(text)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function requireAnyValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +async function createBundleDir(outputRoot, phase, commandName, date) { + await validateOutputRootForWriting(outputRoot); + await mkdir(outputRoot, { recursive: true }); + await validateOutputRootForWriting(outputRoot); + await access(outputRoot, fsConstants.W_OK); + const baseName = `${formatTimestampForPath(date)}-${phase}-${commandName}`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? '' : `-${attempt}`; + const bundleDir = path.join(outputRoot, `${baseName}${suffix}`); + try { + await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE }); + await chmod(bundleDir, BUNDLE_DIR_MODE); + return bundleDir; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + } + } + throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`); +} + +async function validateOutputRootForWriting(outputRoot) { + const target = path.resolve(outputRoot); + const root = path.parse(target).root; + let current = root; + const segments = path + .relative(root, target) + .split(path.sep) + .filter(Boolean); + + for (const segment of segments) { + current = path.join(current, segment); + let stats; + try { + stats = await lstat(current); + } catch (error) { + if (error?.code === 'ENOENT') { + return; + } + throw error; + } + + if (stats.isSymbolicLink()) { + if (current === target) { + throw new Error(`--output-root 不能是符号链接: ${current}`); + } + throw new Error(`--output-root 上级目录不能是符号链接: ${current}`); + } + + if (!stats.isDirectory()) { + if (current === target) { + throw new Error(`--output-root 已存在但不是目录: ${current}`); + } + throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`); + } + } +} + +async function writeEvidenceFile(filePath, content) { + await writeFile(filePath, content, { + encoding: 'utf8', + flag: 'wx', + mode: EVIDENCE_FILE_MODE, + }); + await chmod(filePath, EVIDENCE_FILE_MODE); +} + +async function buildEvidenceFileMetadata(filePath) { + const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + return { + path: path.basename(filePath), + sizeBytes: stats.size, + sha256: createHash('sha256').update(content).digest('hex'), + }; +} + +function runCommand(command, args, secrets) { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: process.cwd(), + env: process.env, + shell: false, + windowsHide: true, + }); + let stdout = ''; + let stderr = ''; + let spawnError = ''; + + child.stdout?.on('data', (chunk) => { + const text = String(chunk); + stdout += text; + process.stdout.write(redactSecrets(text, secrets)); + }); + child.stderr?.on('data', (chunk) => { + const text = String(chunk); + stderr += text; + process.stderr.write(redactSecrets(text, secrets)); + }); + child.on('error', (error) => { + spawnError = error.message; + }); + child.on('close', (exitCode, signal) => { + resolve({ + exitCode, + signal: signal || null, + error: spawnError, + stdout, + stderr, + }); + }); + }); +} + +function collectSecretValues(args, env) { + const values = []; + for (let index = 0; index < args.length; index += 1) { + const inlineValue = getInlineSecretFlagValue(args[index]); + if (inlineValue) { + values.push(inlineValue); + continue; + } + if (index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) && args[index]) { + values.push(args[index]); + } + } + for (const key of SECRET_ENV_KEYS) { + const value = String(env[key] || ''); + if (value) { + values.push(value); + } + } + return [...new Set(values)].filter((value) => value.length >= 4); +} + +function redactSecrets(text, secrets) { + let redacted = String(text || ''); + for (const secret of secrets) { + redacted = redacted.split(secret).join(''); + } + return redacted; +} + +function formatCommand(command, args, secrets) { + return [redactSecrets(command, secrets), ...redactSecretArgs(args, secrets)].join(' '); +} + +function redactSecretArgs(args, secrets = []) { + return args.map((arg, index) => { + const redacted = + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : redactInlineSecretArg(arg); + return redactSecrets(redacted, secrets); + }); +} + +function getInlineSecretFlagValue(arg) { + const text = String(arg || ''); + for (const flag of SECRET_VALUE_FLAGS) { + const prefix = `${flag}=`; + if (text.startsWith(prefix)) { + return text.slice(prefix.length); + } + } + return ''; +} + +function redactInlineSecretArg(arg) { + const text = String(arg || ''); + for (const flag of SECRET_VALUE_FLAGS) { + if (text.startsWith(`${flag}=`)) { + return `${flag}=`; + } + } + return arg; +} + +function formatTimestampForPath(date) { + return date + .toISOString() + .replace(/[-:]/gu, '') + .replace(/\.\d{3}Z$/u, 'Z'); +} diff --git a/scripts/ops/pingora-cutover-evidence-audit.mjs b/scripts/ops/pingora-cutover-evidence-audit.mjs new file mode 100644 index 000000000..43b73eba2 --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-audit.mjs @@ -0,0 +1,2035 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, lstat, readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const DEFAULT_EVIDENCE_ROOT = '/var/log/genarrative/pingora-cutover-evidence'; +const DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); +const CUTOVER_TIMELINE_ORDER = [ + { type: 'phase', phase: 'pre-cutover' }, + { + type: 'command', + phase: 'enable-apply', + commandName: 'pingora-direct-enable-apply', + }, + { + type: 'command', + phase: 'post-enable', + commandName: 'pingora-health-patrol-direct-env-switch', + }, + { type: 'phase', phase: 'post-enable' }, + { + type: 'command', + phase: 'rollback-prep', + commandName: 'pingora-gateway-shadow-env-switch', + }, + { + type: 'command', + phase: 'rollback-prep', + commandName: 'pingora-health-patrol-nginx-env-switch', + }, + { + type: 'command', + phase: 'rollback-apply', + commandName: 'pingora-direct-rollback-apply', + }, + { type: 'phase', phase: 'post-rollback' }, +]; + +const config = parseArgs(process.argv.slice(2)); +await validateReadOnlyDirectory(config.evidenceRoot, '--evidence-root'); +await validateVerifierScript(config.verifyScript); + +const discovery = await discoverEvidence(config.evidenceRoot, { + allowExtraRootEntries: config.allowExtraRootEntries, +}); +const audit = await buildAudit(config, discovery); + +console.log(`${JSON.stringify(audit, null, 2)}\n`); + +if (!audit.ok) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + evidenceRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + DEFAULT_EVIDENCE_ROOT, + verifyScript: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT || + path.join(SCRIPT_DIR, 'pingora-cutover-evidence-verify.mjs'), + timelineMaxSpanMs: parsePositiveInteger( + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS || + DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS, + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS', + ), + requiredCutoverRunId: + process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + allowExtraRootEntries: false, + requiredPhases: [], + requiredPhaseDirectLiveAccessLog: [], + requiredPhaseDirectLiveStaticHeaders: [], + requiredPhasePingoraEnvShadow: [], + requiredCommands: [], + requiredCommandExecutables: [], + requiredCommandArgs: [], + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--evidence-root': + result.evidenceRoot = requireValue(argv, ++index, arg); + break; + case '--verify-script': + result.verifyScript = requireValue(argv, ++index, arg); + break; + case '--phase': + case '--require-phase': + result.requiredPhases.push(requireValue(argv, ++index, arg)); + break; + case '--require-command': + result.requiredCommands.push(requireValue(argv, ++index, arg)); + break; + case '--require-phase-direct-live-access-log': + result.requiredPhaseDirectLiveAccessLog.push( + requireValue(argv, ++index, arg), + ); + break; + case '--require-phase-direct-live-static-headers': + result.requiredPhaseDirectLiveStaticHeaders.push( + requireValue(argv, ++index, arg), + ); + break; + case '--require-phase-pingora-env-shadow': + result.requiredPhasePingoraEnvShadow.push( + requireValue(argv, ++index, arg), + ); + break; + case '--require-command-executable': + result.requiredCommandExecutables.push(requireValue(argv, ++index, arg)); + break; + case '--require-command-arg': + result.requiredCommandArgs.push(requireValue(argv, ++index, arg)); + break; + case '--timeline-max-span-ms': + result.timelineMaxSpanMs = parsePositiveInteger( + requireValue(argv, ++index, arg), + '--timeline-max-span-ms', + ); + break; + case '--require-cutover-run-id': + result.requiredCutoverRunId = requireValue(argv, ++index, arg); + break; + case '--allow-extra-root-entries': + result.allowExtraRootEntries = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + for (const [label, value] of [ + ['--evidence-root', result.evidenceRoot], + ['--verify-script', result.verifyScript], + ]) { + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + validateNoControlCharacters(value, label); + } + result.requiredPhases = normalizeRequiredPhases(result.requiredPhases); + result.requiredPhaseDirectLiveAccessLog = normalizeRequiredPhases( + result.requiredPhaseDirectLiveAccessLog, + ); + result.requiredPhaseDirectLiveStaticHeaders = normalizeRequiredPhases( + result.requiredPhaseDirectLiveStaticHeaders, + ); + result.requiredPhasePingoraEnvShadow = normalizeRequiredPhases( + result.requiredPhasePingoraEnvShadow, + ); + const requiredCommands = normalizeRequiredCommands(result.requiredCommands); + result.requiredCommandExecutables = normalizeRequiredCommandExecutables( + result.requiredCommandExecutables, + ); + result.requiredCommandArgs = normalizeRequiredCommandArgs( + result.requiredCommandArgs, + ); + result.requiredCommands = mergeRequiredCommands( + requiredCommands, + result.requiredCommandExecutables, + result.requiredCommandArgs, + ); + if (result.requiredCutoverRunId) { + validateSafeCutoverRunId( + result.requiredCutoverRunId, + '--require-cutover-run-id', + ); + } + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-audit.mjs [options] + +Options: + --evidence-root 证据根目录,默认 ${DEFAULT_EVIDENCE_ROOT}。 + --verify-script 证据验真脚本,默认同目录 pingora-cutover-evidence-verify.mjs。 + --require-phase 要求并验真某个阶段的最新证据;可重复。--phase 是别名。 + --require-command : + 要求并验真某个切换命令证据;可重复。 + --require-phase-direct-live-access-log + 要求某个阶段 manifest.summary.directLiveAccessLog 已存在且 direct live request_id access log 对账成功;正式切换默认用于 post-enable。 + --require-phase-direct-live-static-headers + 要求某个阶段 manifest.summary.directLiveStaticHeaders 已存在且包含可判定的 direct live 静态响应头摘要;正式切换默认用于 post-enable。 + --require-phase-pingora-env-shadow + 要求某个阶段 manifest.summary.pingoraEnvShadow 已存在且证明 Pingora env 已恢复 shadow 高端口;正式切换默认用于 post-rollback。 + --require-command-executable :: + 要求某个命令证据的 manifest.expectedExecutable、manifest.command.executable 和 command-record.json executable 与该绝对路径一致;可重复。 + --require-command-arg :: + 要求某个命令证据的 manifest.command.args 和 command-record.json args 都包含该参数;可重复。 + --timeline-max-span-ms + 标准五段切换时间线允许的最大跨度,默认 ${DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS}ms,可用 GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS 覆盖。 + --require-cutover-run-id + 只接受 manifest.cutoverRunId 与该 ID 一致的阶段和命令证据,避免混入其它切换批次。 + --allow-extra-root-entries + 允许证据根目录中存在非证据目录条目;默认拒绝,正式切换归档不应使用。 + +该脚本只读扫描 Pingora 直连切换证据根目录,按 manifest.phase 找到每个阶段的最新证据目录,也可按 manifest.phase + manifest.commandName 找到真实切换命令证据,并调用随包证据验真脚本的 --require-summary-ok 严格模式校验 schemaVersion=1 manifest.files 的 sizeBytes / sha256 与 manifest.summary.status=OK;所有候选证据都必须带合法 manifest.generatedAt,最新证据选择和标准时间线证明只使用该字段,不用目录 mtime 兜底;若同一阶段或同一命令的最新 manifest.generatedAt 重复,则拒绝按目录名打平并要求重新归档或清理证据根目录;命令证据的 manifest.command 与 command-record.json 也必须是 schemaVersion=1;可用 --require-phase-direct-live-access-log 要求指定阶段必须带 direct live access log 对账摘要,防止旧 post-enable 证据包缺少 request_id 反查复盘入口;可用 --require-phase-direct-live-static-headers 要求指定阶段必须带 direct live 静态响应头摘要,防止旧 post-enable 证据包缺少 Cache-Control、ETag、Last-Modified、Range 和 304 复盘入口;可用 --require-phase-pingora-env-shadow 要求指定阶段必须带 Pingora env shadow 摘要,防止旧 post-rollback 证据包无法证明 active env 已从低端口 direct 恢复到 127.0.0.1:18081;证据根目录默认只能包含带 manifest.json 的证据目录,不允许夹带普通文件、无 manifest 目录或符号链接;不会修改证据目录、不会 reload systemd、不会访问 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function normalizeRequiredPhases(phases) { + const seen = new Set(); + const normalized = []; + for (const phase of phases) { + const text = String(phase || '').trim(); + validateSafePhase(text, '--require-phase'); + if (!seen.has(text)) { + seen.add(text); + normalized.push(text); + } + } + return normalized; +} + +function normalizeRequiredCommands(commands) { + const seen = new Set(); + const normalized = []; + for (const command of commands) { + const text = String(command || '').trim(); + const parts = text.split(':'); + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command 必须使用 : 格式。', + ); + } + const [phase, commandName] = parts; + validateSafePhase(phase, '--require-command phase'); + validateSafeCommandName(commandName, '--require-command commandName'); + const key = `${phase}:${commandName}`; + if (!seen.has(key)) { + seen.add(key); + normalized.push({ phase, commandName }); + } + } + return normalized; +} + +function normalizeRequiredCommandExecutables(commandExecutables) { + const seen = new Map(); + const normalized = []; + for (const item of commandExecutables) { + const text = String(item || '').trim(); + const parts = text.split(':'); + if (parts.length < 3 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command-executable 必须使用 :: 格式。', + ); + } + const [phase, commandName, ...pathParts] = parts; + const executable = pathParts.join(':'); + validateSafePhase(phase, '--require-command-executable phase'); + validateSafeCommandName( + commandName, + '--require-command-executable commandName', + ); + if (!path.isAbsolute(executable)) { + throw new Error('--require-command-executable executable 必须是绝对路径。'); + } + if (isFilesystemRootPath(executable)) { + throw new Error( + '--require-command-executable executable 不能是文件系统根目录。', + ); + } + if (/[\0\r\n]/u.test(executable)) { + throw new Error( + '--require-command-executable executable 不能包含换行或 NUL 字符。', + ); + } + const key = `${phase}:${commandName}`; + const previousExecutable = seen.get(key); + if (previousExecutable && previousExecutable !== executable) { + throw new Error( + '--require-command-executable 不能为同一个 : 指定多个不同路径。', + ); + } + if (!previousExecutable) { + seen.set(key, executable); + normalized.push({ phase, commandName, executable }); + } + } + return normalized; +} + +function normalizeRequiredCommandArgs(commandArgs) { + const seen = new Set(); + const normalized = []; + for (const item of commandArgs) { + const text = String(item || '').trim(); + const parts = text.split(':'); + if (parts.length < 3 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command-arg 必须使用 :: 格式。', + ); + } + const [phase, commandName, ...argParts] = parts; + const requiredArg = argParts.join(':'); + validateSafePhase(phase, '--require-command-arg phase'); + validateSafeCommandName(commandName, '--require-command-arg commandName'); + validateRequiredCommandArg(requiredArg, '--require-command-arg arg'); + const key = `${phase}:${commandName}:${requiredArg}`; + if (!seen.has(key)) { + seen.add(key); + normalized.push({ phase, commandName, arg: requiredArg }); + } + } + return normalized; +} + +function mergeRequiredCommands( + requiredCommands, + requiredCommandExecutables, + requiredCommandArgs, +) { + const seen = new Set(); + const merged = []; + for (const command of [ + ...requiredCommands, + ...requiredCommandExecutables, + ...requiredCommandArgs, + ]) { + const key = `${command.phase}:${command.commandName}`; + if (!seen.has(key)) { + seen.add(key); + merged.push({ + phase: command.phase, + commandName: command.commandName, + }); + } + } + return merged; +} + +function findRequiredCommandExecutable(requiredCommandExecutables, command) { + return ( + requiredCommandExecutables.find( + (item) => + item.phase === command.phase && item.commandName === command.commandName, + )?.executable || null + ); +} + +function findRequiredCommandArgs(requiredCommandArgs, command) { + return requiredCommandArgs + .filter( + (item) => + item.phase === command.phase && item.commandName === command.commandName, + ) + .map((item) => item.arg); +} + +function validateSafePhase(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateSafeCommandName(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateSafeCutoverRunId(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateRequiredCommandArg(value, label) { + if ( + typeof value !== 'string' || + value.length === 0 || + /[\0\r\n]/u.test(value) + ) { + throw new Error(`${label} 必须是非空且不包含换行的字符串。`); + } +} + +function isSafePhase(value) { + return /^[0-9A-Za-z._-]+$/u.test(String(value || '')); +} + +function parsePositiveInteger(value, label) { + const text = String(value || '').trim(); + if (!/^[1-9]\d*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label} 超出 JavaScript 安全整数范围。`); + } + return parsed; +} + +async function validateReadOnlyDirectory(dirPath, label) { + const stats = await lstat(dirPath); + if (stats.isSymbolicLink()) { + throw new Error(`${label} 不能是符号链接: ${dirPath}`); + } + if (!stats.isDirectory()) { + throw new Error(`${label} 必须是目录: ${dirPath}`); + } + await access(dirPath, fsConstants.R_OK | fsConstants.X_OK); +} + +async function validateVerifierScript(scriptPath) { + const stats = await lstat(scriptPath); + if (stats.isSymbolicLink()) { + throw new Error(`--verify-script 不能是符号链接: ${scriptPath}`); + } + if (!stats.isFile()) { + throw new Error(`--verify-script 必须是文件: ${scriptPath}`); + } + await access(scriptPath, fsConstants.R_OK); +} + +async function discoverEvidence(evidenceRoot, options = {}) { + const allowExtraRootEntries = Boolean(options.allowExtraRootEntries); + const entries = await readdir(evidenceRoot, { withFileTypes: true }); + const candidates = []; + const diagnostics = []; + + for (const entry of entries) { + const entryPath = path.join(evidenceRoot, entry.name); + const linkStats = await lstat(entryPath); + if (linkStats.isSymbolicLink()) { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: '证据根目录下不能包含符号链接条目。', + }); + } + continue; + } + if (!linkStats.isDirectory()) { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: linkStats.isFile() + ? '证据根目录只能包含证据目录,发现普通文件。' + : '证据根目录只能包含证据目录,发现非目录条目。', + }); + } + continue; + } + + const manifestPath = path.join(entryPath, 'manifest.json'); + let manifestStats; + try { + manifestStats = await lstat(manifestPath); + } catch (error) { + if (error?.code === 'ENOENT') { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: '证据根目录只能包含带 manifest.json 的证据目录。', + }); + } + } else { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: `读取 manifest 状态失败: ${error.message}`, + }); + } + continue; + } + if (manifestStats.isSymbolicLink()) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 不能是符号链接。', + }); + continue; + } + if (!manifestStats.isFile()) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 必须是普通文件。', + }); + continue; + } + + const manifest = await readJsonManifest(manifestPath, entry.name, diagnostics); + if (!manifest) { + continue; + } + if (manifest.schemaVersion !== 1) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest.schemaVersion 必须是 1。', + }); + continue; + } + if (!manifest.phase || typeof manifest.phase !== 'string') { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 缺少 phase 字符串。', + }); + continue; + } + if (!isSafePhase(manifest.phase)) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.phase 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + const topLevelCommandName = + manifest.commandName === undefined ? null : manifest.commandName; + const embeddedCommandName = + manifest.command?.name === undefined ? null : manifest.command.name; + if ( + topLevelCommandName !== null && + (typeof topLevelCommandName !== 'string' || + !isSafePhase(topLevelCommandName)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.commandName 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + if ( + embeddedCommandName !== null && + (typeof embeddedCommandName !== 'string' || + !isSafePhase(embeddedCommandName)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.command.name 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + if ( + topLevelCommandName !== null && + embeddedCommandName !== null && + topLevelCommandName !== embeddedCommandName + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: `manifest.commandName 与 manifest.command.name 必须一致,manifest.commandName=${formatNullable(topLevelCommandName)},manifest.command.name=${formatNullable(embeddedCommandName)}。`, + }); + continue; + } + const commandName = topLevelCommandName || embeddedCommandName; + const cutoverRunId = + manifest.cutoverRunId === undefined ? null : manifest.cutoverRunId; + if ( + cutoverRunId !== null && + (typeof cutoverRunId !== 'string' || !isSafePhase(cutoverRunId)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.cutoverRunId 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + const topLevelExpectedExecutable = + manifest.expectedExecutable === undefined + ? null + : manifest.expectedExecutable; + const embeddedExpectedExecutable = + manifest.command?.expectedExecutable === undefined + ? null + : manifest.command.expectedExecutable; + const expectedExecutableDiagnostics = [ + ...validateOptionalAbsolutePath( + topLevelExpectedExecutable, + 'manifest.expectedExecutable', + ), + ...validateOptionalAbsolutePath( + embeddedExpectedExecutable, + 'manifest.command.expectedExecutable', + ), + ]; + if (expectedExecutableDiagnostics.length > 0) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: expectedExecutableDiagnostics.join(' '), + }); + continue; + } + const generatedAt = normalizeIsoTime(manifest.generatedAt); + if (!generatedAt) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.generatedAt 必须是合法 ISO 时间;证据最新选择和时间线证明不能依赖目录 mtime。', + }); + continue; + } + candidates.push({ + phase: manifest.phase, + cutoverRunId, + bundleDir: entryPath, + manifestPath, + directoryName: entry.name, + generatedAt, + summaryStatus: manifest.summary?.status || null, + summaryExitCode: manifest.summary?.exitCode ?? null, + summarySignal: manifest.summary?.signal ?? null, + summaryDirectLiveAccessLog: manifest.summary?.directLiveAccessLog ?? null, + summaryDirectLiveStaticHeaders: + manifest.summary?.directLiveStaticHeaders ?? null, + summaryPingoraEnvShadow: manifest.summary?.pingoraEnvShadow ?? null, + commandName, + manifestCommand: manifest.command || null, + commandRecordFileName: manifest.files?.commandRecord?.path || null, + stdoutFileName: manifest.files?.stdout?.path || null, + stderrFileName: manifest.files?.stderr?.path || null, + commandStartedAt: normalizeIsoTime(manifest.command?.startedAt), + commandFinishedAt: normalizeIsoTime(manifest.command?.finishedAt), + commandDurationMs: manifest.command?.durationMs ?? null, + expectedExecutable: + topLevelExpectedExecutable ?? embeddedExpectedExecutable ?? null, + executable: manifest.command?.executable || null, + }); + } + + return { candidates, diagnostics }; +} + +function validateOptionalAbsolutePath(value, label) { + if (value === null) { + return []; + } + return validateRequiredAbsolutePath(value, label); +} + +function validateRequiredAbsolutePath(value, label) { + if (typeof value !== 'string' || value.length === 0) { + return [`${label} 必须是绝对路径字符串。`]; + } + if (!path.isAbsolute(value)) { + return [`${label} 必须是绝对路径。`]; + } + if (isFilesystemRootPath(value)) { + return [`${label} 不能是文件系统根目录。`]; + } + if (/[\0\r\n]/u.test(value)) { + return [`${label} 不能包含换行或 NUL 字符。`]; + } + return []; +} + +async function readJsonManifest(manifestPath, directoryName, diagnostics) { + let content; + try { + content = await readFile(manifestPath, 'utf8'); + } catch (error) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: `读取 manifest 失败: ${error.message}`, + }); + return null; + } + try { + const manifest = JSON.parse(content); + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 顶层必须是 JSON object。', + }); + return null; + } + return manifest; + } catch (error) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: `manifest 不是合法 JSON: ${error.message}`, + }); + return null; + } +} + +async function buildAudit(config, discovery) { + const phasesToCheck = + config.requiredPhases.length > 0 + ? config.requiredPhases + : config.requiredCommands.length > 0 + ? [] + : sortedUnique( + discovery.candidates + .filter((candidate) => candidate.commandName === null) + .map((candidate) => candidate.phase), + ); + const phases = []; + + for (const phase of phasesToCheck) { + const requiredAccessLog = config.requiredPhaseDirectLiveAccessLog.includes( + phase, + ); + const requiredStaticHeaders = config.requiredPhaseDirectLiveStaticHeaders.includes( + phase, + ); + const requiredPingoraEnvShadow = config.requiredPhasePingoraEnvShadow.includes( + phase, + ); + const allPhaseCandidates = discovery.candidates.filter( + (candidate) => candidate.phase === phase && candidate.commandName === null, + ); + const candidates = filterCandidatesByCutoverRunId( + allPhaseCandidates, + config.requiredCutoverRunId, + ) + .sort(compareEvidenceCandidates); + if (candidates.length === 0) { + phases.push({ + phase, + status: 'MISSING', + candidateCount: 0, + allCandidateCount: allPhaseCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + requiredPingoraEnvShadow, + directLiveAccessLog: null, + directLiveStaticHeaders: null, + pingoraEnvShadow: null, + diagnostics: [ + missingEvidenceDiagnostic( + '该阶段', + config.requiredCutoverRunId, + allPhaseCandidates.length, + ), + ], + }); + continue; + } + + const latestSelection = selectUniqueLatestEvidence(candidates); + if (!latestSelection.ok) { + phases.push({ + phase, + status: 'AMBIGUOUS_LATEST', + candidateCount: candidates.length, + allCandidateCount: allPhaseCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latestSelection.generatedAt, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + requiredPingoraEnvShadow, + directLiveAccessLog: null, + directLiveStaticHeaders: null, + pingoraEnvShadow: null, + ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, + diagnostics: latestSelection.diagnostics, + }); + continue; + } + + const latest = latestSelection.latest; + const verify = await runVerifier(config.verifyScript, latest.bundleDir); + const manifestCheck = checkPhaseManifest(latest, { + requiredAccessLog, + requiredStaticHeaders, + requiredPingoraEnvShadow, + }); + phases.push({ + phase, + status: phaseStatus(verify, manifestCheck), + candidateCount: candidates.length, + latestBundleDir: latest.bundleDir, + manifestPath: latest.manifestPath, + cutoverRunId: latest.cutoverRunId, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latest.generatedAt, + summaryStatus: latest.summaryStatus, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + requiredPingoraEnvShadow, + directLiveAccessLog: latest.summaryDirectLiveAccessLog, + directLiveStaticHeaders: latest.summaryDirectLiveStaticHeaders, + pingoraEnvShadow: latest.summaryPingoraEnvShadow, + commandName: latest.commandName, + verify, + manifestCheck, + diagnostics: [ + ...(verify.ok ? [] : ['最新证据目录 manifest 验真失败。']), + ...manifestCheck.diagnostics, + ], + }); + } + + const commands = []; + for (const requiredCommand of config.requiredCommands) { + const allCommandCandidates = discovery.candidates.filter( + (candidate) => + candidate.phase === requiredCommand.phase && + candidate.commandName === requiredCommand.commandName, + ); + const candidates = filterCandidatesByCutoverRunId( + allCommandCandidates, + config.requiredCutoverRunId, + ) + .sort(compareEvidenceCandidates); + if (candidates.length === 0) { + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: 'MISSING', + candidateCount: 0, + allCandidateCount: allCommandCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + diagnostics: [ + missingEvidenceDiagnostic( + '该切换命令', + config.requiredCutoverRunId, + allCommandCandidates.length, + ), + ], + }); + continue; + } + + const latestSelection = selectUniqueLatestEvidence(candidates); + if (!latestSelection.ok) { + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: 'AMBIGUOUS_LATEST', + candidateCount: candidates.length, + allCandidateCount: allCommandCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latestSelection.generatedAt, + ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, + diagnostics: latestSelection.diagnostics, + }); + continue; + } + + const latest = latestSelection.latest; + const verify = await runVerifier(config.verifyScript, latest.bundleDir); + const expectedExecutable = findRequiredCommandExecutable( + config.requiredCommandExecutables, + requiredCommand, + ); + const requiredArgs = findRequiredCommandArgs( + config.requiredCommandArgs, + requiredCommand, + ); + const manifestCheck = await checkCommandManifest( + latest, + expectedExecutable, + requiredArgs, + ); + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: commandStatus(verify, manifestCheck), + candidateCount: candidates.length, + latestBundleDir: latest.bundleDir, + manifestPath: latest.manifestPath, + cutoverRunId: latest.cutoverRunId, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latest.generatedAt, + summaryStatus: latest.summaryStatus, + summaryExitCode: latest.summaryExitCode, + summarySignal: latest.summarySignal, + expectedExecutable: latest.expectedExecutable, + executable: latest.executable, + commandRecordPath: manifestCheck.commandRecordPath, + stdoutPath: manifestCheck.stdoutPath, + stderrPath: manifestCheck.stderrPath, + commandStartedAt: manifestCheck.commandRecordStartedAt, + commandFinishedAt: manifestCheck.commandRecordFinishedAt, + commandDurationMs: manifestCheck.commandRecordDurationMs, + requiredExecutable: expectedExecutable, + requiredArgs, + verify, + manifestCheck, + diagnostics: [ + ...(verify.ok ? [] : ['最新命令证据目录 manifest 验真失败。']), + ...manifestCheck.diagnostics, + ], + }); + } + + const criticalDiagnostics = discovery.diagnostics.filter( + (item) => item.status === 'CRITICAL', + ); + const timeline = buildTimelineAudit( + phases, + commands, + config.timelineMaxSpanMs, + ); + const failedPhases = phases.filter((phase) => phase.status !== 'OK'); + const failedCommands = commands.filter((command) => command.status !== 'OK'); + const ok = + phases.length + commands.length > 0 && + failedPhases.length === 0 && + failedCommands.length === 0 && + criticalDiagnostics.length === 0 && + timeline.ok; + + return { + ok, + generatedAt: new Date().toISOString(), + evidenceRoot: config.evidenceRoot, + verifyScript: config.verifyScript, + requiredCutoverRunId: config.requiredCutoverRunId || null, + allowExtraRootEntries: config.allowExtraRootEntries, + timelineMaxSpanMs: config.timelineMaxSpanMs, + requiredPhases: config.requiredPhases, + requiredCommands: config.requiredCommands, + requiredCommandExecutables: config.requiredCommandExecutables, + requiredCommandArgs: config.requiredCommandArgs, + discoveredCount: discovery.candidates.length, + checkedCount: phases.length + commands.length, + failedCount: + failedPhases.length + + failedCommands.length + + criticalDiagnostics.length + + timeline.failedCount, + diagnostics: discovery.diagnostics, + summary: buildOperatorSummary({ + ok, + phases, + commands, + criticalDiagnostics, + timeline, + checkedCount: phases.length + commands.length, + failedCount: + failedPhases.length + + failedCommands.length + + criticalDiagnostics.length + + timeline.failedCount, + }), + timeline, + phases, + commands, + }; +} + +function buildOperatorSummary(input) { + const pingoraEnvShadowEvidence = input.phases + .filter((phase) => phase.requiredPingoraEnvShadow) + .map((phase) => { + const summary = phase.pingoraEnvShadow ?? null; + const firstDiagnostic = phase.diagnostics?.[0] ?? null; + return { + phase: phase.phase, + status: phase.status, + latestBundleDir: phase.latestBundleDir ?? null, + manifestPath: phase.manifestPath ?? null, + cutoverRunId: phase.cutoverRunId ?? null, + generatedAt: phase.generatedAt ?? null, + shadow: { + required: true, + ok: phase.manifestCheck?.pingoraEnvShadowCheck?.ok === true, + reason: + phase.manifestCheck?.pingoraEnvShadowCheck?.reason ?? + firstDiagnostic, + summary, + }, + }; + }); + const directLiveEvidence = input.phases + .filter( + (phase) => + phase.requiredDirectLiveAccessLog || + phase.requiredDirectLiveStaticHeaders, + ) + .map((phase) => { + const accessLogRequired = phase.requiredDirectLiveAccessLog === true; + const staticHeadersRequired = + phase.requiredDirectLiveStaticHeaders === true; + const accessLogSummary = phase.directLiveAccessLog ?? null; + const staticHeadersSummary = phase.directLiveStaticHeaders ?? null; + const accessLogChecked = accessLogRequired || accessLogSummary !== null; + const staticHeadersChecked = + staticHeadersRequired || staticHeadersSummary !== null; + const firstDiagnostic = phase.diagnostics?.[0] ?? null; + return { + phase: phase.phase, + status: phase.status, + latestBundleDir: phase.latestBundleDir ?? null, + manifestPath: phase.manifestPath ?? null, + cutoverRunId: phase.cutoverRunId ?? null, + generatedAt: phase.generatedAt ?? null, + accessLog: { + required: accessLogRequired, + ok: accessLogChecked + ? phase.manifestCheck?.directLiveAccessLogCheck?.ok === true + : null, + reason: accessLogChecked + ? phase.manifestCheck?.directLiveAccessLogCheck?.reason ?? + firstDiagnostic + : null, + summary: accessLogSummary, + }, + staticHeaders: { + required: staticHeadersRequired, + ok: staticHeadersChecked + ? phase.manifestCheck?.directLiveStaticHeadersCheck?.ok === true + : null, + reason: staticHeadersChecked + ? phase.manifestCheck?.directLiveStaticHeadersCheck?.reason ?? + firstDiagnostic + : null, + summary: staticHeadersSummary, + }, + }; + }); + + return { + status: input.ok ? 'OK' : 'CRITICAL', + checkedCount: input.checkedCount, + failedCount: input.failedCount, + failedItems: [ + ...input.criticalDiagnostics.map((diagnostic) => ({ + type: 'evidence-root', + status: diagnostic.status, + path: diagnostic.path ?? null, + manifestPath: diagnostic.manifestPath ?? null, + reason: diagnostic.reason ?? null, + })), + ...input.phases + .filter((phase) => phase.status !== 'OK') + .map((phase) => ({ + type: 'phase', + phase: phase.phase, + status: phase.status, + latestBundleDir: phase.latestBundleDir ?? null, + manifestPath: phase.manifestPath ?? null, + cutoverRunId: phase.cutoverRunId ?? null, + generatedAt: phase.generatedAt ?? null, + diagnostics: phase.diagnostics ?? [], + })), + ...input.commands + .filter((command) => command.status !== 'OK') + .map((command) => ({ + type: 'command', + phase: command.phase, + commandName: command.commandName, + status: command.status, + latestBundleDir: command.latestBundleDir ?? null, + manifestPath: command.manifestPath ?? null, + cutoverRunId: command.cutoverRunId ?? null, + generatedAt: command.generatedAt ?? null, + diagnostics: command.diagnostics ?? [], + })), + ...(input.timeline.ok + ? [] + : [ + { + type: 'timeline', + status: 'FAILED', + diagnostics: input.timeline.diagnostics ?? [], + failureBreakdown: input.timeline.failureBreakdown ?? null, + spanMs: input.timeline.spanMs ?? null, + maxSpanMs: input.timeline.maxSpanMs ?? null, + }, + ]), + ], + pingoraEnvShadowEvidence, + directLiveEvidence, + timeline: { + checked: input.timeline.checked, + ok: input.timeline.ok, + failedCount: input.timeline.failedCount, + spanMs: input.timeline.spanMs, + maxSpanMs: input.timeline.maxSpanMs, + firstGeneratedAt: input.timeline.firstGeneratedAt, + lastGeneratedAt: input.timeline.lastGeneratedAt, + }, + }; +} + +function filterCandidatesByCutoverRunId(candidates, requiredCutoverRunId) { + if (!requiredCutoverRunId) { + return candidates; + } + return candidates.filter( + (candidate) => candidate.cutoverRunId === requiredCutoverRunId, + ); +} + +function missingEvidenceDiagnostic(label, requiredCutoverRunId, allCandidateCount) { + if (!requiredCutoverRunId) { + return `没有找到${label}的证据 manifest。`; + } + if (allCandidateCount > 0) { + return `没有找到${label}且 cutoverRunId=${requiredCutoverRunId} 的证据 manifest;已有同名证据属于其它切换批次或缺少 cutoverRunId。`; + } + return `没有找到${label}的证据 manifest,无法满足 cutoverRunId=${requiredCutoverRunId}。`; +} + +function selectUniqueLatestEvidence(candidates) { + const latest = candidates.at(-1); + const latestGeneratedAt = latest?.generatedAt || null; + const ambiguousCandidates = candidates.filter( + (candidate) => candidate.generatedAt === latestGeneratedAt, + ); + if (ambiguousCandidates.length > 1) { + const ambiguousBundleDirs = ambiguousCandidates.map( + (candidate) => candidate.bundleDir, + ); + return { + ok: false, + latest: null, + generatedAt: latestGeneratedAt, + ambiguousBundleDirs, + diagnostics: [ + `最新证据 manifest.generatedAt=${latestGeneratedAt} 出现重复,不能按目录名打平选择;请重新归档或清理证据根目录。`, + `重复 generatedAt 的证据目录: ${ambiguousBundleDirs.join(', ')}`, + ], + }; + } + return { + ok: true, + latest, + generatedAt: latestGeneratedAt, + ambiguousBundleDirs: [], + diagnostics: [], + }; +} + +function checkPhaseManifest(candidate, options = {}) { + const { + requiredAccessLog = false, + requiredStaticHeaders = false, + requiredPingoraEnvShadow = false, + } = options; + const diagnostics = []; + if (candidate.commandName !== null) { + diagnostics.push( + `最新阶段证据不能带 manifest.commandName,实际为 ${formatNullable(candidate.commandName)}。`, + ); + } + if (candidate.summaryStatus !== 'OK') { + diagnostics.push( + `最新阶段证据 manifest.summary.status 必须是 OK,实际为 ${formatNullable(candidate.summaryStatus)}。`, + ); + } + const accessLogCheck = checkDirectLiveAccessLogSummary( + candidate.summaryDirectLiveAccessLog, + ); + if (requiredAccessLog && !accessLogCheck.ok) { + diagnostics.push( + `最新阶段证据 manifest.summary.directLiveAccessLog 必须包含 direct live access log 对账摘要:${accessLogCheck.reason}`, + ); + } + const staticHeadersCheck = checkDirectLiveStaticHeadersSummary( + candidate.summaryDirectLiveStaticHeaders, + ); + if (requiredStaticHeaders && !staticHeadersCheck.ok) { + diagnostics.push( + `最新阶段证据 manifest.summary.directLiveStaticHeaders 必须包含 direct live 静态响应头摘要:${staticHeadersCheck.reason}`, + ); + } + const pingoraEnvShadowCheck = checkPingoraEnvShadowSummary( + candidate.summaryPingoraEnvShadow, + ); + if (requiredPingoraEnvShadow && !pingoraEnvShadowCheck.ok) { + diagnostics.push( + `最新阶段证据 manifest.summary.pingoraEnvShadow 必须证明 Pingora env 已恢复 shadow:${pingoraEnvShadowCheck.reason}`, + ); + } + return { + ok: diagnostics.length === 0, + expectedSummaryStatus: 'OK', + actualSummaryStatus: candidate.summaryStatus, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + requiredPingoraEnvShadow, + directLiveAccessLogCheck: accessLogCheck, + directLiveStaticHeadersCheck: staticHeadersCheck, + pingoraEnvShadowCheck, + diagnostics, + }; +} + +function checkPingoraEnvShadowSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { + return { ok: false, reason: '字段缺失或不是对象。' }; + } + const failures = []; + if (summary.present !== true) { + failures.push('present 必须为 true'); + } + if (summary.ok !== true) { + failures.push('ok 必须为 true'); + } + if (summary.listen !== '127.0.0.1:18081') { + failures.push( + `listen 必须是 127.0.0.1:18081,实际为 ${formatNullable(summary.listen)}`, + ); + } + if (summary.tlsListen !== '') { + failures.push(`tlsListen 必须为空,实际为 ${formatNullable(summary.tlsListen)}`); + } + if (summary.httpRedirectListen !== '') { + failures.push( + `httpRedirectListen 必须为空,实际为 ${formatNullable(summary.httpRedirectListen)}`, + ); + } + if (summary.tlsCertFile !== '') { + failures.push( + `tlsCertFile 必须为空,实际为 ${formatNullable(summary.tlsCertFile)}`, + ); + } + if (summary.tlsKeyFile !== '') { + failures.push( + `tlsKeyFile 必须为空,实际为 ${formatNullable(summary.tlsKeyFile)}`, + ); + } + if (summary.mode && summary.mode !== 'shadow') { + failures.push(`mode 必须是 shadow,实际为 ${formatNullable(summary.mode)}`); + } + if (summary.shadowReady === false) { + failures.push('shadowReady 必须为 true'); + } + return failures.length === 0 + ? { ok: true, reason: null } + : { ok: false, reason: failures.join(';') }; +} + +function checkDirectLiveAccessLogSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { + return { ok: false, reason: '字段缺失或不是对象。' }; + } + if (summary.present !== true) { + return { ok: false, reason: 'present 必须为 true。' }; + } + const checked = Number(summary.checked); + const matchedCount = Number(summary.matchedCount); + const missingCount = Number(summary.missingCount); + const mismatchCount = Number(summary.mismatchCount); + const failures = []; + if (!Number.isInteger(checked) || checked <= 0) { + failures.push('checked 必须是正整数'); + } + if (!Number.isInteger(matchedCount) || matchedCount !== checked) { + failures.push('matchedCount 必须等于 checked'); + } + if (!Number.isInteger(missingCount) || missingCount !== 0) { + failures.push('missingCount 必须为 0'); + } + if (!Number.isInteger(mismatchCount) || mismatchCount !== 0) { + failures.push('mismatchCount 必须为 0'); + } + return failures.length === 0 + ? { ok: true, reason: null } + : { ok: false, reason: failures.join(';') }; +} + +function checkDirectLiveStaticHeadersSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { + return { ok: false, reason: '字段缺失或不是对象。' }; + } + if (summary.present !== true) { + return { ok: false, reason: 'present 必须为 true。' }; + } + if (summary.skipped) { + return { + ok: false, + reason: `静态检查被跳过:${formatNullable(summary.skipReason)}`, + }; + } + const targets = [summary.normal, summary.fingerprinted].filter( + (target) => target && target.present, + ); + if (targets.length === 0) { + return { ok: false, reason: '缺少 normal 或 fingerprinted 静态资产摘要。' }; + } + const failures = []; + for (const [label, target] of [ + ['normal', summary.normal], + ['fingerprinted', summary.fingerprinted], + ]) { + if (!target || !target.present) { + continue; + } + if (target.ok !== true) { + failures.push(`${label}.ok 不是 true`); + } + for (const key of [ + 'cacheControl', + 'etag', + 'lastModified', + 'acceptRanges', + 'rangeContentRange', + ]) { + if (typeof target[key] !== 'string' || target[key].length === 0) { + failures.push(`${label}.${key} 缺失`); + } + } + if (target.rangeStatusCode !== 206) { + failures.push(`${label}.rangeStatusCode 不是 206`); + } + if (target.etag304StatusCode !== 304) { + failures.push(`${label}.etag304StatusCode 不是 304`); + } + if (target.lastModified304StatusCode !== 304) { + failures.push(`${label}.lastModified304StatusCode 不是 304`); + } + } + return failures.length === 0 + ? { ok: true, reason: null } + : { ok: false, reason: failures.join(';') }; +} + +async function checkCommandManifest( + candidate, + expectedExecutable = null, + requiredArgs = [], +) { + const diagnostics = []; + if (candidate.summaryStatus !== 'OK') { + diagnostics.push( + `最新命令证据 manifest.summary.status 必须是 OK,实际为 ${formatNullable(candidate.summaryStatus)}。`, + ); + } + if (candidate.summaryExitCode !== 0) { + diagnostics.push( + `最新命令证据 manifest.summary.exitCode 必须是 0,实际为 ${formatNullable(candidate.summaryExitCode)}。`, + ); + } + if (candidate.summarySignal !== null) { + diagnostics.push( + `最新命令证据 manifest.summary.signal 必须为空,实际为 ${formatNullable(candidate.summarySignal)}。`, + ); + } + if (expectedExecutable) { + if (candidate.expectedExecutable !== expectedExecutable) { + diagnostics.push( + `最新命令证据 manifest.expectedExecutable 必须是 ${expectedExecutable},实际为 ${formatNullable(candidate.expectedExecutable)}。`, + ); + } + if (candidate.executable !== expectedExecutable) { + diagnostics.push( + `最新命令证据 command.executable 必须是 ${expectedExecutable},实际为 ${formatNullable(candidate.executable)}。`, + ); + } + } + diagnostics.push(...checkCommandOutputEvidenceFileNames(candidate)); + const embeddedCommandCheck = checkCommandRecordObject( + candidate.manifestCommand, + 'manifest.command', + candidate, + expectedExecutable, + requiredArgs, + ); + diagnostics.push(...embeddedCommandCheck.diagnostics); + + const commandRecordCheck = await readCommandRecord(candidate); + diagnostics.push(...commandRecordCheck.diagnostics); + if (commandRecordCheck.record) { + const fileCommandCheck = checkCommandRecordObject( + commandRecordCheck.record, + 'command-record.json', + candidate, + expectedExecutable, + requiredArgs, + ); + diagnostics.push(...fileCommandCheck.diagnostics); + if (embeddedCommandCheck.record) { + diagnostics.push( + ...compareCommandRecords( + embeddedCommandCheck.record, + commandRecordCheck.record, + ), + ); + } + } + return { + ok: diagnostics.length === 0, + expectedSummaryStatus: 'OK', + actualSummaryStatus: candidate.summaryStatus, + expectedExitCode: 0, + actualExitCode: candidate.summaryExitCode, + expectedSignal: null, + actualSignal: candidate.summarySignal, + expectedExecutable, + actualExpectedExecutable: candidate.expectedExecutable, + actualExecutable: candidate.executable, + stdoutPath: candidate.stdoutFileName, + stderrPath: candidate.stderrFileName, + commandRecordStdoutPath: commandRecordCheck.record?.stdoutPath ?? null, + commandRecordStderrPath: commandRecordCheck.record?.stderrPath ?? null, + commandRecordPath: commandRecordCheck.path, + commandRecordReadOk: commandRecordCheck.ok, + commandRecordExpectedExecutable: + commandRecordCheck.record?.expectedExecutable ?? null, + commandRecordExecutable: commandRecordCheck.record?.executable ?? null, + commandRecordStartedAt: normalizeIsoTime(commandRecordCheck.record?.startedAt), + commandRecordFinishedAt: normalizeIsoTime(commandRecordCheck.record?.finishedAt), + commandRecordDurationMs: commandRecordCheck.record?.durationMs ?? null, + requiredArgs, + diagnostics, + }; +} + +function checkCommandOutputEvidenceFileNames(candidate) { + const diagnostics = []; + for (const [field, fileName] of [ + ['stdout', candidate.stdoutFileName], + ['stderr', candidate.stderrFileName], + ]) { + if (!fileName) { + diagnostics.push( + `最新命令证据 manifest.files.${field}.path 必须存在。`, + ); + continue; + } + if (!isSafeEvidenceFileName(fileName)) { + diagnostics.push( + `最新命令证据 manifest.files.${field}.path 只能是证据目录内的普通文件名,实际为 ${formatNullable(fileName)}。`, + ); + } + } + return diagnostics; +} + +async function readCommandRecord(candidate) { + const fileName = candidate.commandRecordFileName; + if (!fileName) { + return { + ok: false, + path: null, + record: null, + diagnostics: ['最新命令证据 manifest.files.commandRecord.path 必须存在。'], + }; + } + if (!isSafeEvidenceFileName(fileName)) { + return { + ok: false, + path: null, + record: null, + diagnostics: [ + `最新命令证据 manifest.files.commandRecord.path 只能是证据目录内的普通文件名,实际为 ${formatNullable(fileName)}。`, + ], + }; + } + + const recordPath = path.join(candidate.bundleDir, fileName); + try { + const stats = await lstat(recordPath); + if (stats.isSymbolicLink()) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 不能是符号链接。'], + }; + } + if (!stats.isFile()) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 必须是普通文件。'], + }; + } + const content = await readFile(recordPath, 'utf8'); + const record = JSON.parse(content); + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 顶层必须是 JSON object。'], + }; + } + return { ok: true, path: recordPath, record, diagnostics: [] }; + } catch (error) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: [ + `读取最新命令证据 command-record.json 失败: ${error.message}`, + ], + }; + } +} + +function checkCommandRecordObject( + record, + label, + candidate, + expectedExecutable, + requiredArgs = [], +) { + const diagnostics = []; + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return { + ok: false, + record: null, + diagnostics: [`最新命令证据 ${label} 必须是 JSON object。`], + }; + } + const checks = [ + ['schemaVersion', 1], + ['phase', candidate.phase], + ['name', candidate.commandName], + ['exitCode', candidate.summaryExitCode], + ['signal', candidate.summarySignal], + ]; + if (candidate.cutoverRunId !== null || record.cutoverRunId !== undefined) { + checks.push(['cutoverRunId', candidate.cutoverRunId]); + } + if ( + candidate.expectedExecutable !== null || + record.expectedExecutable !== undefined + ) { + checks.push(['expectedExecutable', candidate.expectedExecutable]); + } + for (const [field, expected] of checks) { + const actual = record[field] ?? null; + if (actual !== expected) { + diagnostics.push( + `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)},实际为 ${formatNullable(actual)}。`, + ); + } + } + for (const [field, expected] of [ + ['stdoutPath', candidate.stdoutFileName], + ['stderrPath', candidate.stderrFileName], + ]) { + const actual = record[field] ?? null; + if (actual !== expected) { + diagnostics.push( + `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)},实际为 ${formatNullable(actual)}。`, + ); + } + } + diagnostics.push(...checkCommandRecordInvocation(record, label)); + diagnostics.push( + ...validateRequiredAbsolutePath(record.executable, `${label}.executable`), + ); + const effectiveExpectedExecutable = + expectedExecutable || candidate.expectedExecutable; + if (effectiveExpectedExecutable) { + if (record.expectedExecutable !== effectiveExpectedExecutable) { + diagnostics.push( + `最新命令证据 ${label}.expectedExecutable 必须是 ${effectiveExpectedExecutable},实际为 ${formatNullable(record.expectedExecutable)}。`, + ); + } + if (record.executable !== effectiveExpectedExecutable) { + diagnostics.push( + `最新命令证据 ${label}.executable 必须是 ${effectiveExpectedExecutable},实际为 ${formatNullable(record.executable)}。`, + ); + } + } + diagnostics.push(...checkRequiredCommandArgs(record, label, requiredArgs)); + diagnostics.push(...checkCommandRecordTimes(record, label, candidate)); + return { + ok: diagnostics.length === 0, + record, + diagnostics, + }; +} + +function checkRequiredCommandArgs(record, label, requiredArgs) { + const diagnostics = []; + if (requiredArgs.length === 0) { + return diagnostics; + } + if (!Array.isArray(record.args)) { + diagnostics.push( + `最新命令证据 ${label}.args 必须是字符串数组,才能校验必需参数 ${formatNullable(requiredArgs)}。`, + ); + return diagnostics; + } + for (const requiredArg of requiredArgs) { + if (!record.args.includes(requiredArg)) { + diagnostics.push( + `最新命令证据 ${label}.args 必须包含 ${formatNullable(requiredArg)}。`, + ); + } + } + return diagnostics; +} + +function checkCommandRecordInvocation(record, label) { + const diagnostics = []; + if (!Array.isArray(record.args)) { + diagnostics.push(`最新命令证据 ${label}.args 必须是字符串数组。`); + } else if (record.args.some((item) => typeof item !== 'string')) { + diagnostics.push(`最新命令证据 ${label}.args 必须只包含字符串。`); + } else if (record.args.some((item) => /[\0\r\n]/u.test(item))) { + diagnostics.push( + `最新命令证据 ${label}.args 不能包含换行或 NUL 字符。`, + ); + } + if (typeof record.command !== 'string' || record.command.length === 0) { + diagnostics.push(`最新命令证据 ${label}.command 必须是非空字符串。`); + } + if (typeof record.cwd !== 'string' || !path.isAbsolute(record.cwd)) { + diagnostics.push(`最新命令证据 ${label}.cwd 必须是绝对路径字符串。`); + } + if (record.error !== null && typeof record.error !== 'string') { + diagnostics.push(`最新命令证据 ${label}.error 必须是字符串或 null。`); + } + return diagnostics; +} + +function checkCommandRecordTimes(record, label, candidate) { + const diagnostics = []; + const startedAt = normalizeIsoTime(record.startedAt); + const finishedAt = normalizeIsoTime(record.finishedAt); + if (!startedAt) { + diagnostics.push(`最新命令证据 ${label}.startedAt 必须是合法 ISO 时间。`); + } + if (!finishedAt) { + diagnostics.push(`最新命令证据 ${label}.finishedAt 必须是合法 ISO 时间。`); + } + if (startedAt && finishedAt) { + const startedMs = Date.parse(startedAt); + const finishedMs = Date.parse(finishedAt); + if (finishedMs < startedMs) { + diagnostics.push( + `最新命令证据 ${label}.finishedAt 不能早于 startedAt。`, + ); + } + if (candidate.generatedAt) { + const generatedMs = Date.parse(candidate.generatedAt); + if (generatedMs < finishedMs) { + diagnostics.push( + `最新命令证据 manifest.generatedAt 不能早于 ${label}.finishedAt。`, + ); + } + } + const durationMs = record.durationMs; + if (!Number.isSafeInteger(durationMs) || durationMs < 0) { + diagnostics.push( + `最新命令证据 ${label}.durationMs 必须是非负安全整数。`, + ); + } else { + const actualDurationMs = finishedMs - startedMs; + if (durationMs !== actualDurationMs) { + diagnostics.push( + `最新命令证据 ${label}.durationMs 必须等于 finishedAt - startedAt,实际记录 ${durationMs}ms,计算值 ${actualDurationMs}ms。`, + ); + } + } + } + return diagnostics; +} + +function compareCommandRecords(embeddedRecord, fileRecord) { + const diagnostics = []; + const scalarFields = [ + 'schemaVersion', + 'name', + 'phase', + 'cutoverRunId', + 'expectedExecutable', + 'executable', + 'command', + 'cwd', + 'error', + 'stdoutPath', + 'stderrPath', + 'exitCode', + 'signal', + ]; + for (const field of scalarFields) { + const embedded = embeddedRecord[field] ?? null; + const file = fileRecord[field] ?? null; + if (embedded !== file) { + diagnostics.push( + `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致,manifest=${formatNullable(embedded)},command-record=${formatNullable(file)}。`, + ); + } + } + for (const field of ['args']) { + const embedded = embeddedRecord[field] ?? null; + const file = fileRecord[field] ?? null; + if (!jsonValuesEqual(embedded, file)) { + diagnostics.push( + `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致,manifest=${formatNullable(embedded)},command-record=${formatNullable(file)}。`, + ); + } + } + return diagnostics; +} + +function jsonValuesEqual(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function phaseStatus(verify, manifestCheck) { + if (!manifestCheck.ok) { + return 'MANIFEST_FAILED'; + } + if (!verify.ok) { + return 'VERIFY_FAILED'; + } + return 'OK'; +} + +function commandStatus(verify, manifestCheck) { + if (!manifestCheck.ok) { + return 'MANIFEST_FAILED'; + } + if (!verify.ok) { + return 'VERIFY_FAILED'; + } + return 'OK'; +} + +function formatNullable(value) { + return value === null || value === undefined ? '' : JSON.stringify(value); +} + +function isSafeEvidenceFileName(value) { + return ( + typeof value === 'string' && + value.length > 0 && + !/[\0\r\n]/u.test(value) && + !path.isAbsolute(value) && + !value.includes('/') && + !value.includes('\\') && + value !== '.' && + value !== '..' + ); +} + +function buildTimelineAudit(phases, commands, timelineMaxSpanMs) { + const selected = []; + const missing = []; + for (const expected of CUTOVER_TIMELINE_ORDER) { + const item = + expected.type === 'phase' + ? phases.find((phase) => phase.phase === expected.phase) + : commands.find( + (command) => + command.phase === expected.phase && + command.commandName === expected.commandName, + ); + if (!item) { + missing.push(expected); + continue; + } + selected.push({ + ...expected, + generatedAt: item.generatedAt, + latestBundleDir: item.latestBundleDir, + manifestPath: item.manifestPath, + cutoverRunId: item.cutoverRunId, + status: item.status, + }); + } + + if (selected.length === 0 || missing.length > 0) { + return { + ok: true, + checked: false, + failedCount: 0, + reason: + selected.length === 0 + ? '未要求标准直连切换时间线证据。' + : '仅在标准直连切换时间线的阶段和命令证据都被要求时检查顺序。', + maxSpanMs: timelineMaxSpanMs, + spanMs: null, + firstGeneratedAt: null, + lastGeneratedAt: null, + expectedOrder: CUTOVER_TIMELINE_ORDER, + missing, + items: selected, + diagnostics: [], + }; + } + + const diagnostics = []; + const failureBreakdown = { + nonOkItems: 0, + missingGeneratedAt: 0, + cutoverRunIdMismatch: 0, + outOfOrder: 0, + spanExceeded: 0, + }; + const nonOkItems = selected.filter((item) => item.status !== 'OK'); + failureBreakdown.nonOkItems = nonOkItems.length; + if (nonOkItems.length > 0) { + diagnostics.push( + `标准切换时间线包含非 OK 证据: ${nonOkItems + .map((item) => `${formatTimelineItem(item)}=${item.status}`) + .join(', ')}。`, + ); + } + const timelineCutoverRunIds = selected.map((item) => ({ + item: formatTimelineItem(item), + cutoverRunId: item.cutoverRunId, + })); + const hasDeclaredCutoverRunId = timelineCutoverRunIds.some( + (item) => item.cutoverRunId !== null, + ); + const distinctCutoverRunIds = new Set( + timelineCutoverRunIds.map((item) => item.cutoverRunId ?? ''), + ); + if (hasDeclaredCutoverRunId && distinctCutoverRunIds.size > 1) { + failureBreakdown.cutoverRunIdMismatch = 1; + diagnostics.push( + `标准切换时间线包含不同 cutoverRunId 或缺少 cutoverRunId: ${timelineCutoverRunIds + .map((item) => `${item.item}=${formatNullable(item.cutoverRunId)}`) + .join(', ')},疑似混入不同切换批次证据。`, + ); + } + const parsedTimes = []; + for (const item of selected) { + if (!item.generatedAt) { + failureBreakdown.missingGeneratedAt += 1; + diagnostics.push( + `${formatTimelineItem(item)} 缺少合法 manifest.generatedAt,无法证明切换时间线。`, + ); + parsedTimes.push(null); + continue; + } + parsedTimes.push(Date.parse(item.generatedAt)); + } + for (let index = 1; index < selected.length; index += 1) { + const previous = selected[index - 1]; + const current = selected[index]; + const previousTime = parsedTimes[index - 1]; + const currentTime = parsedTimes[index]; + if ( + previousTime !== null && + currentTime !== null && + currentTime < previousTime + ) { + failureBreakdown.outOfOrder += 1; + diagnostics.push( + `${formatTimelineItem(current)} 的 manifest.generatedAt 早于 ${formatTimelineItem(previous)},疑似混入不同切换窗口证据。`, + ); + } + } + + let firstGeneratedAt = null; + let lastGeneratedAt = null; + let spanMs = null; + if (parsedTimes.every((time) => time !== null && !Number.isNaN(time))) { + const firstTime = parsedTimes[0]; + const lastTime = parsedTimes.at(-1); + firstGeneratedAt = selected[0].generatedAt; + lastGeneratedAt = selected.at(-1).generatedAt; + spanMs = lastTime - firstTime; + if (spanMs > timelineMaxSpanMs) { + failureBreakdown.spanExceeded = 1; + diagnostics.push( + `标准切换时间线从 ${firstGeneratedAt} 到 ${lastGeneratedAt} 的时间跨度 ${spanMs}ms 超过 ${timelineMaxSpanMs}ms,疑似混入不同切换窗口证据。`, + ); + } + } + const timelineFailedCount = Object.values(failureBreakdown).reduce( + (sum, count) => sum + count, + 0, + ); + + return { + ok: diagnostics.length === 0, + checked: true, + failedCount: timelineFailedCount, + failureBreakdown, + maxSpanMs: timelineMaxSpanMs, + spanMs, + firstGeneratedAt, + lastGeneratedAt, + cutoverRunIds: timelineCutoverRunIds, + expectedOrder: CUTOVER_TIMELINE_ORDER, + items: selected, + diagnostics, + }; +} + +function formatTimelineItem(item) { + if (item.type === 'command') { + return `${item.phase}:${item.commandName}`; + } + return item.phase; +} + +function compareEvidenceCandidates(left, right) { + const leftTime = Date.parse(left.generatedAt); + const rightTime = Date.parse(right.generatedAt); + if (leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.directoryName.localeCompare(right.directoryName); +} + +function sortedUnique(values) { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function normalizeIsoTime(value) { + if ( + !value || + typeof value !== 'string' || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(value) + ) { + return null; + } + const time = Date.parse(value); + if (Number.isNaN(time)) { + return null; + } + const normalized = new Date(time).toISOString(); + return normalized === value ? normalized : null; +} + +async function runVerifier(verifyScript, bundleDir) { + const result = await execFileJson(process.execPath, [ + '--', + verifyScript, + '--bundle-dir', + bundleDir, + '--require-summary-ok', + ]); + const parsed = parseJsonOutput(result.stdout); + return { + ok: + result.exitCode === 0 && + parsed.ok && + parsed.value?.ok === true && + parsed.value?.requireSummaryOk === true, + exitCode: result.exitCode, + stdoutJson: parsed.ok, + checkedCount: parsed.value?.checkedCount ?? null, + failedCount: parsed.value?.failedCount ?? null, + requireSummaryOk: parsed.value?.requireSummaryOk ?? null, + summary: parsed.value?.summary ?? null, + stderr: result.stderr, + error: result.error, + }; +} + +function execFileJson(command, args) { + return new Promise((resolve) => { + execFile( + command, + args, + { + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + shell: false, + }, + (error, stdout, stderr) => { + resolve({ + exitCode: + typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: stdout || '', + stderr: stderr || '', + error: error?.message || null, + }); + }, + ); + }); +} + +function parseJsonOutput(stdout) { + try { + return { ok: true, value: JSON.parse(stdout) }; + } catch { + return { ok: false, value: null }; + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} diff --git a/scripts/ops/pingora-cutover-evidence-bundle.mjs b/scripts/ops/pingora-cutover-evidence-bundle.mjs new file mode 100644 index 000000000..bdc4c6f4e --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-bundle.mjs @@ -0,0 +1,1274 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { + access, + chmod, + lstat, + mkdir, + readFile, + stat, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const PINGORA_ENV_MODES = new Set(['shadow', 'direct']); +const BUNDLE_DIR_MODE = 0o750; +const EVIDENCE_FILE_MODE = 0o640; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); +const DIRECT_LIVE_JSON_PATTERN = /\{\s*"ok"\s*:/u; + +const config = parseArgs(process.argv.slice(2)); +const startedAt = new Date(); +const bundleDir = await createBundleDir( + config.outputRoot, + config.phase, + startedAt, +); +const snapshotArgs = buildSnapshotArgs(config); +const snapshotRun = await runEvidenceCommand({ + name: 'pingora-cutover-status-snapshot', + executable: 'node', + args: [config.snapshotScript, ...snapshotArgs], + stdoutFile: 'snapshot.stdout.txt', + stderrFile: 'snapshot.stderr.txt', + jsonFile: 'snapshot.json', + parseErrorFile: 'snapshot-parse-error.txt', + commandFile: 'snapshot-command.json', + parseJson: parseSnapshotJson, +}); +const manifestPath = path.join(bundleDir, 'manifest.json'); +const snapshot = snapshotRun.parsed; +const snapshotStatus = snapshot.ok + ? snapshot.value?.summary?.status || + (snapshotRun.result.code === 0 ? 'OK' : 'CRITICAL') + : 'CRITICAL'; + +let directLiveRun = null; +if ( + config.runDirectLive && + snapshotRun.result.code === 0 && + snapshot.ok && + (!config.failOnCritical || snapshotStatus !== 'CRITICAL') +) { + directLiveRun = await runEvidenceCommand({ + name: 'pingora-direct-live', + executable: 'node', + args: [config.directLiveScript, ...buildDirectLiveArgs(config)], + stdoutFile: 'direct-live.stdout.txt', + stderrFile: 'direct-live.stderr.txt', + jsonFile: 'direct-live.json', + parseErrorFile: 'direct-live-parse-error.txt', + commandFile: 'direct-live-command.json', + parseJson: parseDirectLiveJson, + }); +} + +const directLiveAccessLog = summarizeDirectLiveAccessLog(directLiveRun); +const directLiveStaticHeaders = summarizeDirectLiveStaticHeaders(directLiveRun); +const pingoraEnvShadow = summarizePingoraEnvShadow(snapshotRun); +const directLiveStatus = directLiveRun + ? isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, + ) + ? 'OK' + : 'CRITICAL' + : 'SKIPPED'; +const snapshotCriticalCount = snapshot.ok + ? Number(snapshot.value?.summary?.criticalCount || 0) + : 1; +const directLiveCriticalCount = directLiveStatus === 'CRITICAL' ? 1 : 0; +const bundleStatus = + snapshotStatus === 'CRITICAL' || directLiveStatus === 'CRITICAL' + ? 'CRITICAL' + : snapshotStatus; +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: config.phase, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + releaseRoot: config.releaseRoot, + outputRoot: config.outputRoot, + bundleDir, + summary: { + status: bundleStatus, + snapshotStatus, + snapshotExitCode: snapshotRun.result.code, + directLiveStatus, + directLiveExitCode: directLiveRun?.result.code ?? null, + directLiveAccessLog: directLiveAccessLog?.summary ?? null, + directLiveStaticHeaders: directLiveStaticHeaders?.summary ?? null, + pingoraEnvShadow: pingoraEnvShadow?.summary ?? null, + criticalCount: snapshotCriticalCount + directLiveCriticalCount, + warningCount: snapshot.ok + ? Number(snapshot.value?.summary?.warningCount || 0) + : 0, + }, + files: { + manifest: path.basename(manifestPath), + snapshot: snapshot.ok ? snapshotRun.files.json : null, + snapshotParseError: snapshot.ok ? null : snapshotRun.files.parseError, + snapshotStdout: snapshotRun.files.stdout, + snapshotStderr: snapshotRun.files.stderr, + snapshotCommand: snapshotRun.files.command, + directLive: directLiveRun?.parsed.ok ? directLiveRun.files.json : null, + directLiveParseError: + directLiveRun && !directLiveRun.parsed.ok + ? directLiveRun.files.parseError + : null, + directLiveStdout: directLiveRun ? directLiveRun.files.stdout : null, + directLiveStderr: directLiveRun ? directLiveRun.files.stderr : null, + directLiveCommand: directLiveRun ? directLiveRun.files.command : null, + }, + commands: [ + snapshotRun.commandRecord, + ...(directLiveRun ? [directLiveRun.commandRecord] : []), + ], +}; +await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + +const directLiveOk = + !config.runDirectLive || + isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, + ); +const result = { + ok: + snapshotRun.result.code === 0 && + snapshot.ok && + (!config.failOnCritical || snapshotStatus !== 'CRITICAL') && + directLiveOk, + phase: config.phase, + cutoverRunId: config.cutoverRunId || null, + status: bundleStatus, + bundleDir, + manifestPath, + snapshotPath: snapshot.ok ? snapshotRun.jsonPath : null, + snapshotParseErrorPath: snapshot.ok ? null : snapshotRun.parseErrorPath, + directLivePath: directLiveRun?.parsed.ok ? directLiveRun.jsonPath : null, + directLiveParseErrorPath: + directLiveRun && !directLiveRun.parsed.ok + ? directLiveRun.parseErrorPath + : null, +}; +console.log(`${JSON.stringify(result, null, 2)}\n`); + +if (!result.ok) { + process.exit(1); +} + +function summarizePingoraEnvShadow(snapshotRun) { + if (!snapshotRun.parsed.ok) { + return { + ok: false, + summary: { + present: false, + listen: '', + tlsListen: '', + httpRedirectListen: '', + ok: false, + diagnostics: ['snapshot JSON parse failed'], + }, + }; + } + + const pingoraEnv = snapshotRun.parsed.value?.pingoraEnv; + const values = pingoraEnv?.values; + if (!values || typeof values !== 'object' || Array.isArray(values)) { + return { + ok: false, + summary: { + present: false, + listen: '', + tlsListen: '', + httpRedirectListen: '', + ok: false, + diagnostics: ['snapshot JSON missing pingoraEnv.values'], + }, + }; + } + + const summary = { + present: true, + mode: String(pingoraEnv?.posture?.mode || ''), + shadowReady: pingoraEnv?.posture?.shadowReady === true, + directReady: pingoraEnv?.posture?.directReady === true, + listen: String(values.listen || ''), + tlsListen: String(values.tlsListen || ''), + httpRedirectListen: String(values.httpRedirectListen || ''), + tlsCertFile: String(values.tlsCertFile || ''), + tlsKeyFile: String(values.tlsKeyFile || ''), + ok: false, + diagnostics: [], + }; + if (summary.listen !== '127.0.0.1:18081') { + summary.diagnostics.push( + `listen 应为 127.0.0.1:18081,实际 ${summary.listen || '-'}`, + ); + } + if (summary.tlsListen !== '') { + summary.diagnostics.push( + `tlsListen 应为空,实际 ${summary.tlsListen}`, + ); + } + if (summary.httpRedirectListen !== '') { + summary.diagnostics.push( + `httpRedirectListen 应为空,实际 ${summary.httpRedirectListen}`, + ); + } + if (summary.tlsCertFile !== '') { + summary.diagnostics.push( + `tlsCertFile 应为空,实际 ${summary.tlsCertFile}`, + ); + } + if (summary.tlsKeyFile !== '') { + summary.diagnostics.push( + `tlsKeyFile 应为空,实际 ${summary.tlsKeyFile}`, + ); + } + if (summary.mode && summary.mode !== 'shadow') { + summary.diagnostics.push(`mode 应为 shadow,实际 ${summary.mode}`); + } + if (summary.shadowReady === false) { + summary.diagnostics.push('shadowReady 应为 true'); + } + summary.ok = summary.diagnostics.length === 0; + return { ok: summary.ok, summary }; +} + +function summarizeDirectLiveAccessLog(directLiveRun) { + if (!directLiveRun) { + return null; + } + if (!directLiveRun.parsed.ok) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON parse failed', + }, + }; + } + const accessLogCheck = directLiveRun.parsed.value?.results?.find( + (item) => item?.name === 'direct-access-log', + ); + if (!accessLogCheck) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON missing direct-access-log result', + }, + }; + } + + const summary = { + present: true, + logFile: accessLogCheck.logFile || '', + sinceLines: coerceNonNegativeInteger(accessLogCheck.sinceLines), + scannedLineCount: coerceNonNegativeInteger( + accessLogCheck.scannedLineCount, + ), + checked: coerceNonNegativeInteger(accessLogCheck.checked), + matchedCount: coerceNonNegativeInteger(accessLogCheck.matchedCount), + missingCount: coerceNonNegativeInteger(accessLogCheck.missingCount), + mismatchCount: coerceNonNegativeInteger(accessLogCheck.mismatchCount), + missingDetailsCount: Array.isArray(accessLogCheck.missing) + ? accessLogCheck.missing.length + : null, + mismatchDetailsCount: Array.isArray(accessLogCheck.mismatches) + ? accessLogCheck.mismatches.length + : null, + }; + const ok = + summary.checked !== null && + summary.matchedCount === summary.checked && + summary.missingCount === 0 && + summary.mismatchCount === 0 && + summary.missingDetailsCount === 0 && + summary.mismatchDetailsCount === 0; + + return { ok, summary }; +} + +function summarizeDirectLiveStaticHeaders(directLiveRun) { + if (!directLiveRun) { + return null; + } + if (!directLiveRun.parsed.ok) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON parse failed', + }, + }; + } + + const staticAssetCheck = directLiveRun.parsed.value?.results?.find( + (item) => item?.name === 'https-static-asset', + ); + if (!staticAssetCheck) { + return { + ok: false, + summary: { + present: false, + skipped: false, + reason: 'direct live JSON missing https-static-asset result', + }, + }; + } + if (staticAssetCheck.skipped) { + return { + ok: true, + summary: { + present: false, + skipped: true, + reason: staticAssetCheck.reason || '', + }, + }; + } + + const normal = summarizeStaticHeaderEntry(staticAssetCheck); + const fingerprinted = summarizeStaticHeaderEntry( + staticAssetCheck.fingerprinted, + { + expectedCacheControl: 'public, max-age=31536000, immutable', + }, + ); + const diagnostics = [ + ...normal.diagnostics.map((item) => `normal: ${item}`), + ...(fingerprinted.present + ? fingerprinted.diagnostics.map((item) => `fingerprinted: ${item}`) + : []), + ]; + const summary = { + present: true, + normal, + fingerprinted, + diagnostics, + }; + return { + ok: normal.ok && (!fingerprinted.present || fingerprinted.ok), + summary, + }; +} + +function summarizeStaticHeaderEntry(entry, options = {}) { + if (!entry || entry.skipped) { + return { + ok: true, + present: false, + skipped: Boolean(entry?.skipped), + reason: entry?.reason || '', + diagnostics: [], + }; + } + + const summary = { + present: true, + path: safePathFromUrl(entry.url), + statusCode: coerceNonNegativeInteger(entry.statusCode), + cacheControl: entry.headers?.['cache-control'] || '', + etag: entry.headers?.etag || '', + lastModified: entry.headers?.['last-modified'] || '', + acceptRanges: entry.headers?.['accept-ranges'] || '', + contentLength: entry.headers?.['content-length'] || '', + rangeStatusCode: coerceNonNegativeInteger(entry.range?.statusCode), + rangeContentRange: entry.range?.headers?.['content-range'] || '', + rangeContentLength: entry.range?.headers?.['content-length'] || '', + etag304StatusCode: coerceNonNegativeInteger( + entry.notModified?.etag?.statusCode, + ), + lastModified304StatusCode: coerceNonNegativeInteger( + entry.notModified?.lastModified?.statusCode, + ), + }; + const diagnostics = []; + if (summary.statusCode !== 200) { + diagnostics.push(`GET statusCode 应为 200,实际 ${formatValue(summary.statusCode)}`); + } + if (!summary.cacheControl) { + diagnostics.push('缺少 Cache-Control'); + } + if ( + options.expectedCacheControl && + summary.cacheControl !== options.expectedCacheControl + ) { + diagnostics.push( + `Cache-Control 应为 ${options.expectedCacheControl},实际 ${summary.cacheControl || '-'}`, + ); + } + if (!summary.etag) { + diagnostics.push('缺少 ETag'); + } + if (!summary.lastModified) { + diagnostics.push('缺少 Last-Modified'); + } + if (summary.acceptRanges !== 'bytes') { + diagnostics.push( + `Accept-Ranges 应为 bytes,实际 ${summary.acceptRanges || '-'}`, + ); + } + if (!summary.contentLength) { + diagnostics.push('缺少 Content-Length'); + } + if (summary.rangeStatusCode !== 206) { + diagnostics.push( + `Range statusCode 应为 206,实际 ${formatValue(summary.rangeStatusCode)}`, + ); + } + if (!/^bytes 0-0\/\d+$/u.test(summary.rangeContentRange)) { + diagnostics.push( + `Range Content-Range 应匹配 bytes 0-0/,实际 ${summary.rangeContentRange || '-'}`, + ); + } + if (summary.rangeContentLength !== '1') { + diagnostics.push( + `Range Content-Length 应为 1,实际 ${summary.rangeContentLength || '-'}`, + ); + } + if (summary.etag304StatusCode !== 304) { + diagnostics.push( + `ETag 304 statusCode 应为 304,实际 ${formatValue(summary.etag304StatusCode)}`, + ); + } + if (summary.lastModified304StatusCode !== 304) { + diagnostics.push( + `Last-Modified 304 statusCode 应为 304,实际 ${formatValue(summary.lastModified304StatusCode)}`, + ); + } + + return { + ok: diagnostics.length === 0, + ...summary, + diagnostics, + }; +} + +function safePathFromUrl(value) { + if (!value) { + return ''; + } + try { + return new URL(value).pathname; + } catch { + return ''; + } +} + +function coerceNonNegativeInteger(value) { + const number = Number(value); + if (!Number.isInteger(number) || number < 0) { + return null; + } + return number; +} + +function isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, +) { + return Boolean( + directLiveRun && + directLiveRun.result.code === 0 && + directLiveRun.parsed.ok && + directLiveRun.parsed.value?.ok !== false && + directLiveAccessLog?.ok && + directLiveStaticHeaders?.ok, + ); +} + +function formatValue(value) { + return value === null || value === undefined ? '-' : String(value); +} + +function parseArgs(argv) { + const result = { + phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual', + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + outputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + snapshotScript: '', + directLiveScript: '', + runDirectLive: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_RUN_DIRECT_LIVE', + ), + directHttpsBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL || '', + directHttpBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '', + directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', + directRedirectHost: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directRedirectBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '', + directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', + directSpacetimeDatabase: + process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '', + directPingoraAccessLog: + process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '', + directAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES || '2000', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPingoraEnvMode: + process.env.GENARRATIVE_PINGORA_EXPECTED_ENV_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST', + ), + runHealthPatrol: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL', + ), + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(argv, ++index, arg); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(argv, ++index, arg); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--output-root': + result.outputRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--snapshot-script': + result.snapshotScript = requireValue(argv, ++index, arg); + break; + case '--direct-live-script': + result.directLiveScript = requireValue(argv, ++index, arg); + break; + case '--run-direct-live': + result.runDirectLive = true; + break; + case '--direct-https-base-url': + result.directHttpsBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-http-base-url': + result.directHttpBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-host': + result.directHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-host': + result.directRedirectHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-base-url': + result.directRedirectBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-probe-token': + result.directProbeToken = requireValue(argv, ++index, arg); + break; + case '--direct-spacetime-database': + result.directSpacetimeDatabase = requireValue(argv, ++index, arg); + break; + case '--direct-pingora-access-log': + result.directPingoraAccessLog = requireValue(argv, ++index, arg); + break; + case '--direct-access-log-since-lines': + result.directAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-pingora-env-mode': + result.expectedPingoraEnvMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + case '--run-health-patrol': + result.runHealthPatrol = true; + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + result.snapshotScript = + result.snapshotScript || + path.join( + result.releaseRoot, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ); + result.directLiveScript = + result.directLiveScript || + path.join(result.releaseRoot, 'scripts/check-pingora-direct-live.mjs'); + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-bundle.mjs [options] + +Options: + --phase 写入 manifest 的阶段标签,例如 pre-cutover / post-enable / post-rollback。 + --cutover-run-id 可选,写入 manifest 的本次切换批次 ID;正式 runbook 会用同一 ID 串联所有阶段和命令证据。 + --release-root current release 根目录,默认 /opt/genarrative/current。 + --output-root 证据包输出根目录,默认 /var/log/genarrative/pingora-cutover-evidence。 + --health-patrol-env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件,默认 /etc/genarrative/pingora-gateway.env。 + --snapshot-script 状态快照脚本,默认 current release 随包脚本。 + --direct-live-script direct live smoke 脚本,默认 current release 随包脚本。 + --run-direct-live 额外执行 direct live smoke,并把 request_id / access log 结果归档。 + --direct-https-base-url 透传给 direct live smoke 的 HTTPS 入口。 + --direct-http-base-url 透传给 direct live smoke 的 HTTP redirect / ACME 入口。 + --direct-host 透传给 direct live smoke 的正式 Host/SNI。 + --direct-redirect-host 透传给 direct live smoke 的 redirect Location host。 + --direct-redirect-base-url 可选,透传给 direct live smoke 的 redirect Location HTTPS base URL。 + --direct-probe-token 可选,透传给 direct live smoke 的内部探针 token;证据中会脱敏。 + --direct-spacetime-database 透传给 direct live smoke 的 SpacetimeDB 数据库名。 + --direct-pingora-access-log + 透传给 direct live smoke 的 Pingora access log 路径。 + --direct-access-log-since-lines + 透传给 direct live smoke 的 access log tail 行数,默认 2000。 + --expected-gateway-mode 可选,nginx 或 pingora-direct;会透传给状态快照。 + --expected-pingora-env-mode + 可选,shadow 或 direct;会透传给状态快照。 + --expected-public-base-url 可选,要求 health-patrol public base URL 与该值一致。 + --expected-public-host 可选,要求 health-patrol public Host 与该值一致。 + --require-empty-public-host 可选,要求 health-patrol public Host 为空。 + --run-health-patrol 让状态快照执行 current release 随包生产巡检脚本并收录结果。 + --require-pingora-gateway 让状态快照自审 current release 的 Pingora 二进制、checksum 和 manifest。 + --timeout-ms 透传给状态快照脚本的 systemctl / 子检查超时,默认 5000。 + --fail-on-critical 如果快照中出现 CRITICAL,则证据包脚本也以退出码 1 结束。 + +该脚本只写 --output-root 下的新证据目录,且 --output-root 不能是符号链接或非目录路径;证据目录权限固定 0750,证据文件权限固定 0640;不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function validateConfig(config) { + validatePhase(config.phase); + if (config.cutoverRunId) { + validateSafeName(config.cutoverRunId, '--cutover-run-id'); + } + for (const [label, value] of [ + ['--release-root', config.releaseRoot], + ['--output-root', config.outputRoot], + ['--health-patrol-env-file', config.healthPatrolEnvFile], + ['--pingora-env-file', config.pingoraEnvFile], + ['--snapshot-script', config.snapshotScript], + ['--direct-live-script', config.directLiveScript], + ]) { + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (path.resolve(value) === path.parse(path.resolve(value)).root) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + } + if ( + config.expectedGatewayMode && + !GATEWAY_MODES.has(config.expectedGatewayMode) + ) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${config.expectedGatewayMode}`, + ); + } + if ( + config.expectedPingoraEnvMode && + !PINGORA_ENV_MODES.has(config.expectedPingoraEnvMode) + ) { + throw new Error( + `--expected-pingora-env-mode 只支持 shadow 或 direct: ${config.expectedPingoraEnvMode}`, + ); + } + if (config.expectedPublicHost !== null) { + validateHostOption(config.expectedPublicHost, '--expected-public-host'); + } + if (config.expectedPublicHost !== null && config.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + if (config.expectedPublicBaseUrl) { + validateHttpUrl(config.expectedPublicBaseUrl, '--expected-public-base-url'); + } + if (config.runDirectLive) { + validateDirectLiveConfig(config); + } + validateEvidenceCommandArgs('状态快照命令参数', buildSnapshotArgs(config)); + if (config.runDirectLive) { + validateEvidenceCommandArgs('direct live 命令参数', buildDirectLiveArgs(config)); + } +} + +function validateDirectLiveConfig(config) { + if (!config.directHttpsBaseUrl) { + throw new Error('--run-direct-live 必须提供 --direct-https-base-url。'); + } + validateHttpsUrl(config.directHttpsBaseUrl, '--direct-https-base-url'); + if (!config.directHttpBaseUrl) { + throw new Error('--run-direct-live 必须提供 --direct-http-base-url。'); + } + validateHttpOnlyUrl(config.directHttpBaseUrl, '--direct-http-base-url'); + if (!config.directHost) { + throw new Error('--run-direct-live 必须提供 --direct-host。'); + } + validateHostOption(config.directHost, '--direct-host'); + if (!config.directRedirectHost) { + throw new Error('--run-direct-live 必须提供 --direct-redirect-host。'); + } + validateHostOption(config.directRedirectHost, '--direct-redirect-host'); + if (config.directRedirectBaseUrl) { + validateHttpsBaseUrl(config.directRedirectBaseUrl, '--direct-redirect-base-url'); + } + if (!config.directSpacetimeDatabase) { + throw new Error('--run-direct-live 必须提供 --direct-spacetime-database。'); + } + validateNoControlCharacters( + config.directSpacetimeDatabase, + '--direct-spacetime-database', + ); + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(config.directSpacetimeDatabase)) { + throw new Error( + '--direct-spacetime-database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$。', + ); + } + if (config.directProbeToken) { + validateNoControlCharacters(config.directProbeToken, '--direct-probe-token'); + } + if (!config.directPingoraAccessLog) { + throw new Error('--run-direct-live 必须提供 --direct-pingora-access-log。'); + } + validateSafeAbsoluteFilePath( + config.directPingoraAccessLog, + '--direct-pingora-access-log', + ); + validateNoControlCharacters( + config.directAccessLogSinceLines, + '--direct-access-log-since-lines', + ); + if (!/^[1-9][0-9]*$/u.test(String(config.directAccessLogSinceLines))) { + throw new Error('--direct-access-log-since-lines 必须是正整数。'); + } +} + +function validatePhase(value) { + validateSafeName(value, '--phase'); +} + +function validateSafeName(value, label) { + const text = String(value || ''); + if (!/^[0-9A-Za-z._-]+$/u.test(text)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateEvidenceCommandArgs(label, args) { + for (const arg of args) { + if (/[\0\r\n]/u.test(String(arg))) { + throw new Error(`${label}不能包含换行或 NUL 字符。`); + } + } +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (path.resolve(value) === path.parse(path.resolve(value)).root) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + validateNoControlCharacters(raw, label); + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function validateHttpUrl(value, label) { + validateNoControlCharacters(value, label); + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHttpsUrl(value, label) { + validateHttpUrl(value, label); + if (new URL(value).protocol !== 'https:') { + throw new Error(`${label} 必须使用 https://。`); + } +} + +function validateHttpsBaseUrl(value, label) { + validateHttpsUrl(value, label); + const parsed = new URL(value); + if ( + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error( + `${label} 只能是 HTTPS base URL,不能包含路径、查询、片段或认证信息。`, + ); + } +} + +function validateHttpOnlyUrl(value, label) { + validateHttpUrl(value, label); + if (new URL(value).protocol !== 'http:') { + throw new Error(`${label} 必须使用 http://。`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + validateNoControlCharacters(raw, label); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port`); + } +} + +async function createBundleDir(outputRoot, phase, date) { + await validateOutputRootForWriting(outputRoot); + await mkdir(outputRoot, { recursive: true }); + await validateOutputRootForWriting(outputRoot); + await access(outputRoot, fsConstants.W_OK); + const baseName = `${formatTimestampForPath(date)}-${phase}`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? '' : `-${attempt}`; + const bundleDir = path.join(outputRoot, `${baseName}${suffix}`); + try { + await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE }); + await chmod(bundleDir, BUNDLE_DIR_MODE); + return bundleDir; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + } + } + throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`); +} + +async function validateOutputRootForWriting(outputRoot) { + const target = path.resolve(outputRoot); + const root = path.parse(target).root; + let current = root; + const segments = path + .relative(root, target) + .split(path.sep) + .filter(Boolean); + + for (const segment of segments) { + current = path.join(current, segment); + let stats; + try { + stats = await lstat(current); + } catch (error) { + if (error?.code === 'ENOENT') { + return; + } + throw error; + } + + if (stats.isSymbolicLink()) { + if (current === target) { + throw new Error(`--output-root 不能是符号链接: ${current}`); + } + throw new Error(`--output-root 上级目录不能是符号链接: ${current}`); + } + + if (!stats.isDirectory()) { + if (current === target) { + throw new Error(`--output-root 已存在但不是目录: ${current}`); + } + throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`); + } + } +} + +async function writeEvidenceFile(filePath, content) { + await writeFile(filePath, content, { + encoding: 'utf8', + flag: 'wx', + mode: EVIDENCE_FILE_MODE, + }); + await chmod(filePath, EVIDENCE_FILE_MODE); +} + +async function buildEvidenceFileMetadata(filePath) { + const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + return { + path: path.basename(filePath), + sizeBytes: stats.size, + sha256: createHash('sha256').update(content).digest('hex'), + }; +} + +function formatTimestampForPath(date) { + return date + .toISOString() + .replace(/[-:]/gu, '') + .replace(/\.\d{3}Z$/u, 'Z'); +} + +function buildSnapshotArgs(config) { + return [ + '--phase', + config.phase, + '--release-root', + config.releaseRoot, + '--health-patrol-env-file', + config.healthPatrolEnvFile, + '--pingora-env-file', + config.pingoraEnvFile, + ...(config.expectedGatewayMode + ? ['--expected-gateway-mode', config.expectedGatewayMode] + : []), + ...(config.expectedPingoraEnvMode + ? ['--expected-pingora-env-mode', config.expectedPingoraEnvMode] + : []), + ...(config.expectedPublicBaseUrl + ? ['--expected-public-base-url', config.expectedPublicBaseUrl] + : []), + ...(config.expectedPublicHost !== null + ? ['--expected-public-host', config.expectedPublicHost] + : []), + ...(config.requireEmptyPublicHost ? ['--require-empty-public-host'] : []), + ...(config.runHealthPatrol ? ['--run-health-patrol'] : []), + ...(config.requirePingoraGateway ? ['--require-pingora-gateway'] : []), + '--timeout-ms', + String(config.timeoutMs), + ...(config.failOnCritical ? ['--fail-on-critical'] : []), + ]; +} + +function buildDirectLiveArgs(config) { + return [ + '--https-base-url', + config.directHttpsBaseUrl, + '--http-base-url', + config.directHttpBaseUrl, + '--host', + config.directHost, + '--redirect-host', + config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + '--spacetime-database', + config.directSpacetimeDatabase, + '--require-wss-upgrade', + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + '--json', + ]; +} + +async function runEvidenceCommand({ + name, + executable, + args, + stdoutFile, + stderrFile, + jsonFile, + parseErrorFile, + commandFile, + parseJson, +}) { + const startedAt = new Date(); + const result = await runCommand(executable, args); + const finishedAt = new Date(); + const stdoutPath = path.join(bundleDir, stdoutFile); + const stderrPath = path.join(bundleDir, stderrFile); + const jsonPath = path.join(bundleDir, jsonFile); + const parseErrorPath = path.join(bundleDir, parseErrorFile); + const commandPath = path.join(bundleDir, commandFile); + + await writeEvidenceFile(stdoutPath, result.stdout); + await writeEvidenceFile(stderrPath, result.stderr || result.error); + + const parsed = parseJson(result.stdout); + let jsonMetadata = null; + let parseErrorMetadata = null; + if (parsed.ok) { + await writeEvidenceFile(jsonPath, `${JSON.stringify(parsed.value, null, 2)}\n`); + jsonMetadata = await buildEvidenceFileMetadata(jsonPath); + } else { + await writeEvidenceFile(parseErrorPath, `${parsed.error}\n`); + parseErrorMetadata = await buildEvidenceFileMetadata(parseErrorPath); + } + + const commandRecord = { + name, + executable, + args: redactSecretArgs(args), + command: formatCommand(executable, args), + exitCode: result.code, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + stdoutPath: path.basename(stdoutPath), + stderrPath: path.basename(stderrPath), + }; + await writeEvidenceFile( + commandPath, + `${JSON.stringify(commandRecord, null, 2)}\n`, + ); + const files = { + stdout: await buildEvidenceFileMetadata(stdoutPath), + stderr: await buildEvidenceFileMetadata(stderrPath), + json: jsonMetadata, + parseError: parseErrorMetadata, + command: await buildEvidenceFileMetadata(commandPath), + }; + + return { + result, + parsed, + stdoutPath, + stderrPath, + jsonPath, + parseErrorPath, + commandPath, + commandRecord, + files, + }; +} + +function runCommand(command, args) { + return new Promise((resolve) => { + execFile( + command, + args, + { + env: process.env, + windowsHide: true, + maxBuffer: 2 * 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function parseSnapshotJson(stdout) { + try { + return { ok: true, value: JSON.parse(stdout) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function parseDirectLiveJson(stdout) { + const match = DIRECT_LIVE_JSON_PATTERN.exec(stdout); + if (!match) { + return { + ok: false, + error: 'direct live stdout 中未找到 JSON 结果对象。', + }; + } + const jsonText = extractJsonObject(stdout, match.index); + if (!jsonText) { + return { + ok: false, + error: 'direct live stdout 中未找到完整 JSON 结果对象。', + }; + } + try { + return { ok: true, value: JSON.parse(jsonText) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function extractJsonObject(text, startIndex) { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + + return ''; +} + +function formatCommand(command, args) { + return [command, ...redactSecretArgs(args)].join(' '); +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) ? '' : arg, + ); +} diff --git a/scripts/ops/pingora-cutover-evidence-verify.mjs b/scripts/ops/pingora-cutover-evidence-verify.mjs new file mode 100644 index 000000000..8c920a414 --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-verify.mjs @@ -0,0 +1,341 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, lstat, readdir, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const config = parseArgs(process.argv.slice(2)); +const manifestPath = await resolveManifestPath(config); +const bundleDir = path.dirname(manifestPath); +await validateBundleDir(bundleDir); + +const manifest = await readManifest(manifestPath); +validateManifestSchemaVersion(manifest); +const { verifications, registeredFileNames } = + await verifyManifestFiles(bundleDir, manifest); +const extraFiles = config.allowExtraFiles + ? [] + : await findExtraEvidenceEntries(bundleDir, registeredFileNames); +const summaryVerification = config.requireSummaryOk + ? verifySummaryOk(manifest) + : null; +const failed = verifications.filter((item) => item.status !== 'OK'); +const failedCount = + failed.length + + extraFiles.length + + (summaryVerification && summaryVerification.status !== 'OK' ? 1 : 0); + +console.log( + `${JSON.stringify( + { + ok: failedCount === 0, + manifestPath, + bundleDir, + checkedCount: verifications.length, + failedCount, + allowExtraFiles: config.allowExtraFiles, + requireSummaryOk: config.requireSummaryOk, + summary: summaryVerification, + files: verifications, + extraFiles, + }, + null, + 2, + )}\n`, +); + +if (failedCount > 0) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + manifestPath: '', + bundleDir: '', + allowExtraFiles: false, + requireSummaryOk: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--manifest': + result.manifestPath = requireValue(argv, ++index, arg); + break; + case '--bundle-dir': + result.bundleDir = requireValue(argv, ++index, arg); + break; + case '--allow-extra-files': + result.allowExtraFiles = true; + break; + case '--require-summary-ok': + result.requireSummaryOk = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (result.manifestPath && result.bundleDir) { + throw new Error('--manifest 和 --bundle-dir 只能二选一。'); + } + if (!result.manifestPath && !result.bundleDir) { + throw new Error('必须提供 --manifest 或 --bundle-dir。'); + } + for (const [label, value] of [ + ['--manifest', result.manifestPath], + ['--bundle-dir', result.bundleDir], + ]) { + if (!value) { + continue; + } + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + validateNoControlCharacters(value, label); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-verify.mjs --manifest + node scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir + +Options: + --allow-extra-files 允许证据目录中存在 manifest.files 未登记的额外条目;默认拒绝,正式切换归档不应使用。 + --require-summary-ok 要求 manifest.summary.status 必须是 OK;正式切换 runbook 的即时验真步骤必须使用。 + +该脚本只读校验 Pingora 直连切换证据目录中的 schemaVersion=1 manifest.files 元数据,确认每个已登记文件的 path / sizeBytes / sha256 与实际文件一致,并默认拒绝证据目录中混入未登记文件、目录或符号链接;null 文件条目会跳过。正式切换 runbook 还会用 --require-summary-ok 把 manifest.summary.status 非 OK 的证据作为失败处理。脚本不会修改证据目录、不会 reload systemd、不会访问 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +async function resolveManifestPath(config) { + if (config.manifestPath) { + return path.resolve(config.manifestPath); + } + return path.join(path.resolve(config.bundleDir), 'manifest.json'); +} + +async function validateBundleDir(bundleDir) { + const resolved = path.resolve(bundleDir); + if (isFilesystemRootPath(resolved)) { + throw new Error('证据目录不能是文件系统根目录。'); + } + const stats = await lstat(resolved); + if (stats.isSymbolicLink()) { + throw new Error(`证据目录不能是符号链接: ${resolved}`); + } + if (!stats.isDirectory()) { + throw new Error(`证据目录必须是目录: ${resolved}`); + } + await access(resolved, fsConstants.R_OK | fsConstants.X_OK); +} + +async function readManifest(manifestPath) { + const stats = await lstat(manifestPath); + if (stats.isSymbolicLink()) { + throw new Error(`manifest 不能是符号链接: ${manifestPath}`); + } + if (!stats.isFile()) { + throw new Error(`manifest 必须是文件: ${manifestPath}`); + } + let content; + try { + content = await readFile(manifestPath, 'utf8'); + } catch (error) { + throw new Error(`读取 manifest 失败: ${error.message}`); + } + try { + return JSON.parse(content); + } catch (error) { + throw new Error(`manifest 不是合法 JSON: ${error.message}`); + } +} + +function validateManifestSchemaVersion(manifest) { + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + throw new Error('manifest 顶层必须是 JSON object。'); + } + if (manifest.schemaVersion !== 1) { + throw new Error('manifest.schemaVersion 必须是 1。'); + } +} + +async function verifyManifestFiles(bundleDir, manifest) { + if (!manifest.files || typeof manifest.files !== 'object') { + throw new Error('manifest 缺少 files object。'); + } + + const verifications = []; + const registeredFileNames = new Set(['manifest.json']); + for (const [key, value] of Object.entries(manifest.files)) { + if (value === null) { + continue; + } + if (typeof value === 'string') { + if (key === 'manifest' && value === 'manifest.json') { + continue; + } + throw new Error( + `manifest.files.${key} 必须使用 { path, sizeBytes, sha256 } 元数据对象。`, + ); + } + validateMetadata(key, value); + registeredFileNames.add(value.path); + verifications.push(await verifyMetadata(bundleDir, key, value)); + } + return { verifications, registeredFileNames }; +} + +function verifySummaryOk(manifest) { + const status = manifest?.summary?.status; + if (status === 'OK') { + return { + status: 'OK', + expectedStatus: 'OK', + actualStatus: status, + reason: null, + }; + } + return { + status: 'MISMATCH', + expectedStatus: 'OK', + actualStatus: status ?? null, + reason: 'manifest.summary.status is not OK', + }; +} + +async function findExtraEvidenceEntries(bundleDir, registeredFileNames) { + const entries = await readdir(bundleDir, { withFileTypes: true }); + const extras = []; + for (const entry of entries) { + if (registeredFileNames.has(entry.name)) { + continue; + } + extras.push({ + path: entry.name, + status: 'MISMATCH', + reason: entry.isSymbolicLink() + ? 'extra symlink not registered in manifest.files' + : entry.isDirectory() + ? 'extra directory not registered in manifest.files' + : 'extra file not registered in manifest.files', + }); + } + return extras.sort((left, right) => left.path.localeCompare(right.path)); +} + +function validateMetadata(key, value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`manifest.files.${key} 必须是元数据对象或 null。`); + } + if (typeof value.path !== 'string' || value.path.length === 0) { + throw new Error(`manifest.files.${key}.path 必须是非空字符串。`); + } + if ( + /[\0\r\n]/u.test(value.path) || + path.isAbsolute(value.path) || + value.path.includes('/') || + value.path.includes('\\') || + value.path === '.' || + value.path === '..' || + value.path.includes('..') + ) { + throw new Error( + `manifest.files.${key}.path 只能是证据目录内的安全文件名: ${value.path}`, + ); + } + if (!Number.isSafeInteger(value.sizeBytes) || value.sizeBytes < 0) { + throw new Error(`manifest.files.${key}.sizeBytes 必须是非负安全整数。`); + } + if (typeof value.sha256 !== 'string' || !/^[0-9a-f]{64}$/u.test(value.sha256)) { + throw new Error(`manifest.files.${key}.sha256 必须是 64 位小写十六进制。`); + } +} + +async function verifyMetadata(bundleDir, key, metadata) { + const filePath = path.join(bundleDir, metadata.path); + const expectedPath = path.resolve(bundleDir, metadata.path); + if (path.dirname(expectedPath) !== path.resolve(bundleDir)) { + throw new Error(`manifest.files.${key}.path 超出证据目录: ${metadata.path}`); + } + let stats; + let content; + try { + const linkStats = await lstat(filePath); + if (linkStats.isSymbolicLink()) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: 'file is symlink', + }; + } + if (!linkStats.isFile()) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: 'not a regular file', + }; + } + [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + } catch (error) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: error?.code === 'ENOENT' ? 'missing file' : error.message, + }; + } + + const actualSha256 = createHash('sha256').update(content).digest('hex'); + const sizeMatches = stats.size === metadata.sizeBytes; + const sha256Matches = actualSha256 === metadata.sha256; + return { + key, + path: metadata.path, + status: sizeMatches && sha256Matches ? 'OK' : 'MISMATCH', + expectedSizeBytes: metadata.sizeBytes, + actualSizeBytes: stats.size, + expectedSha256: metadata.sha256, + actualSha256, + reason: + sizeMatches && sha256Matches + ? null + : [ + ...(sizeMatches ? [] : ['size mismatch']), + ...(sha256Matches ? [] : ['sha256 mismatch']), + ].join(', '), + }; +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} diff --git a/scripts/ops/pingora-cutover-status-snapshot.mjs b/scripts/ops/pingora-cutover-status-snapshot.mjs new file mode 100644 index 000000000..5b35b5f5b --- /dev/null +++ b/scripts/ops/pingora-cutover-status-snapshot.mjs @@ -0,0 +1,1052 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const STATUS_RANK = { + OK: 0, + WARNING: 1, + CRITICAL: 2, +}; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const PINGORA_ENV_MODES = new Set(['shadow', 'direct']); +const SERVICES = [ + 'genarrative-api.service', + 'spacetimedb.service', + 'nginx.service', + 'genarrative-pingora-gateway.service', +]; +const RELEASE_ARTIFACTS = [ + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + 'scripts/ops/pingora-current-release-audit.mjs', + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + 'scripts/ops/pingora-cutover-command-evidence.mjs', + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + 'scripts/ops/production-health-patrol.mjs', + 'scripts/check-production-health-patrol-env.mjs', + 'scripts/check-pingora-release-readiness.mjs', + 'scripts/check-pingora-direct-preflight.mjs', + 'scripts/check-pingora-direct-live.mjs', + 'scripts/check-pingora-canary-live.mjs', + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'scripts/deploy/pingora-direct-enable.sh', + 'scripts/deploy/pingora-direct-rollback.sh', + 'scripts/deploy/pingora-realpath-canary-enable.sh', + 'scripts/deploy/pingora-realpath-canary-disable.sh', + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', + 'deploy/systemd/genarrative-pingora-gateway.service', + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + 'deploy/pingora/pingora-gateway.env.example', +]; +const SECRET_ENV_KEY_PATTERN = + /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu; + +const config = parseArgs(process.argv.slice(2)); +const snapshot = await buildSnapshot(config); + +console.log(`${JSON.stringify(snapshot, null, 2)}\n`); + +if (config.failOnCritical && snapshot.summary.status === 'CRITICAL') { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual', + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + expectedPingoraEnvMode: + process.env.GENARRATIVE_PINGORA_EXPECTED_ENV_MODE || '', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST', + ), + runHealthPatrol: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL', + ), + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(argv, ++index, arg); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--expected-pingora-env-mode': + result.expectedPingoraEnvMode = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + case '--run-health-patrol': + result.runHealthPatrol = true; + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.releaseRoot)) { + throw new Error('--release-root 必须是绝对路径。'); + } + validateNoControlCharacters(result.releaseRoot, '--release-root'); + if (isFilesystemRootPath(result.releaseRoot)) { + throw new Error('--release-root 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.healthPatrolEnvFile)) { + throw new Error('--health-patrol-env-file 必须是绝对路径。'); + } + validateNoControlCharacters( + result.healthPatrolEnvFile, + '--health-patrol-env-file', + ); + if (isFilesystemRootPath(result.healthPatrolEnvFile)) { + throw new Error('--health-patrol-env-file 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.pingoraEnvFile)) { + throw new Error('--pingora-env-file 必须是绝对路径。'); + } + validateNoControlCharacters(result.pingoraEnvFile, '--pingora-env-file'); + if (isFilesystemRootPath(result.pingoraEnvFile)) { + throw new Error('--pingora-env-file 不能是文件系统根目录。'); + } + if ( + result.expectedGatewayMode && + !GATEWAY_MODES.has(result.expectedGatewayMode) + ) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`, + ); + } + if ( + result.expectedPingoraEnvMode && + !PINGORA_ENV_MODES.has(result.expectedPingoraEnvMode) + ) { + throw new Error( + `--expected-pingora-env-mode 只支持 shadow 或 direct: ${result.expectedPingoraEnvMode}`, + ); + } + if (result.expectedPublicHost !== null) { + validateHostOption(result.expectedPublicHost, '--expected-public-host'); + } + if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + if (result.expectedPublicBaseUrl) { + validateHttpUrl(result.expectedPublicBaseUrl, '--expected-public-base-url'); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-status-snapshot.mjs [options] + +Options: + --phase 写入 JSON 的阶段标签,例如 pre-cutover / post-enable / post-rollback。 + --release-root current release 根目录,默认 /opt/genarrative/current。 + --health-patrol-env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件,默认 /etc/genarrative/pingora-gateway.env。 + --expected-gateway-mode 可选,nginx 或 pingora-direct;提供后会检查 env 和 systemd capability 方向。 + --expected-pingora-env-mode + 可选,shadow 或 direct;提供后会检查 active Pingora env 姿态。 + --expected-public-base-url 可选,要求 health-patrol public base URL 与该值一致。 + --expected-public-host 可选,要求 health-patrol public Host 与该值一致。 + --require-empty-public-host 可选,要求 health-patrol public Host 为空。 + --run-health-patrol 读取 env 后执行 current release 随包生产巡检脚本并收录结果。 + --require-pingora-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。 + --timeout-ms systemctl / 子检查超时,默认 5000。 + --fail-on-critical 如果快照中出现 CRITICAL,则以退出码 1 结束。 + +该脚本只读采集状态,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateHttpUrl(value, label) { + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port`); + } +} + +async function buildSnapshot(input) { + const context = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: input.phase, + releaseRoot: input.releaseRoot, + }; + const healthPatrolEnv = await inspectHealthPatrolEnv(input); + const pingoraEnv = await inspectPingoraEnv(input); + const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv); + delete healthPatrolEnv.secretValues; + delete pingoraEnv.secretValues; + const releaseArtifacts = await inspectReleaseArtifacts(input); + const systemd = await inspectSystemd(input); + const checks = await runChecks(input, healthPatrolEnv, secrets); + const summary = summarize([ + healthPatrolEnv.status, + pingoraEnv.status, + releaseArtifacts.status, + systemd.status, + ...checks.map((check) => check.status), + ]); + + return { + ...context, + summary, + healthPatrolEnv, + pingoraEnv, + releaseArtifacts, + systemd, + checks, + }; +} + +async function inspectHealthPatrolEnv(input) { + const parsed = await readEnvFile(input.healthPatrolEnvFile); + if (parsed.status === 'CRITICAL') { + return parsed; + } + + const values = parsed.values; + const publicHost = values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || ''; + const publicBaseUrl = + values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || ''; + const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || ''; + const diagnostics = []; + let status = 'OK'; + + if (input.expectedGatewayMode && gatewayMode !== input.expectedGatewayMode) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${input.expectedGatewayMode},实际 ${gatewayMode || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if ( + input.expectedPublicBaseUrl && + normalizeBaseUrl(publicBaseUrl) !== normalizeBaseUrl(input.expectedPublicBaseUrl) + ) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${normalizeBaseUrl(input.expectedPublicBaseUrl)},实际 ${publicBaseUrl || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if (input.expectedPublicHost !== null && publicHost !== input.expectedPublicHost) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${input.expectedPublicHost},实际 ${publicHost || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if (input.requireEmptyPublicHost && publicHost) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空,实际 ${publicHost}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + + return { + path: input.healthPatrolEnvFile, + status, + secretValues: collectSecretValuesFromEnv(values), + values: { + gatewayMode, + publicBaseUrl, + publicHost, + apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '', + spacetimeBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL || '', + pingoraBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + hasPingoraProbeToken: Boolean( + values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN, + ), + }, + diagnostics, + }; +} + +async function inspectPingoraEnv(input) { + const parsed = await readEnvFile(input.pingoraEnvFile); + if (parsed.status === 'CRITICAL') { + return { + ...parsed, + values: {}, + posture: { + expectedMode: input.expectedPingoraEnvMode || '', + mode: 'unknown', + shadowReady: false, + directReady: false, + diagnostics: parsed.diagnostics || [], + }, + }; + } + const values = parsed.values; + const envValues = { + listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '', + tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '', + httpRedirectListen: + values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '', + tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '', + tlsKeyFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE || '', + forwardedProto: + values.GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO || '', + trustXForwardedFor: + values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '', + trustedFrontProxyConfirmed: + values.GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED || '', + protectionEnabled: + values.GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED || '', + instanceCount: + values.GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT || '', + sharedProtectionConfirmed: + values.GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED || '', + hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN), + accessLogFile: + values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '', + }; + const posture = summarizePingoraEnvPosture( + envValues, + input.expectedPingoraEnvMode, + ); + const diagnostics = [...parsed.diagnostics, ...posture.diagnostics]; + const status = + input.expectedPingoraEnvMode && posture.diagnostics.length > 0 + ? 'CRITICAL' + : parsed.status; + return { + path: input.pingoraEnvFile, + status, + secretValues: collectSecretValuesFromEnv(values), + values: envValues, + posture, + diagnostics, + }; +} + +function summarizePingoraEnvPosture(values, expectedMode) { + const shadowReady = + values.listen === '127.0.0.1:18081' && + values.tlsListen === '' && + values.httpRedirectListen === '' && + values.tlsCertFile === '' && + values.tlsKeyFile === ''; + const directReady = + values.tlsListen !== '' && + values.httpRedirectListen !== '' && + values.tlsCertFile !== '' && + values.tlsKeyFile !== '' && + values.forwardedProto === 'https'; + const mode = directReady ? 'direct' : shadowReady ? 'shadow' : 'mixed'; + const diagnostics = []; + + if (expectedMode === 'shadow') { + if (values.listen !== '127.0.0.1:18081') { + diagnostics.push( + `Pingora shadow env 要求 listen=127.0.0.1:18081,实际 ${values.listen || '-'}`, + ); + } + for (const [label, value] of [ + ['tlsListen', values.tlsListen], + ['httpRedirectListen', values.httpRedirectListen], + ['tlsCertFile', values.tlsCertFile], + ['tlsKeyFile', values.tlsKeyFile], + ]) { + if (value !== '') { + diagnostics.push( + `Pingora shadow env 要求 ${label} 为空,实际 ${value}`, + ); + } + } + } + + if (expectedMode === 'direct') { + for (const [label, value] of [ + ['tlsListen', values.tlsListen], + ['httpRedirectListen', values.httpRedirectListen], + ['tlsCertFile', values.tlsCertFile], + ['tlsKeyFile', values.tlsKeyFile], + ]) { + if (value === '') { + diagnostics.push(`Pingora direct env 要求 ${label} 已配置。`); + } + } + if (values.forwardedProto !== 'https') { + diagnostics.push( + `Pingora direct env 要求 forwardedProto=https,实际 ${values.forwardedProto || '-'}`, + ); + } + } + + return { + expectedMode: expectedMode || '', + mode, + shadowReady, + directReady, + diagnostics, + }; +} + +async function readEnvFile(filePath) { + let text; + try { + text = await readFile(filePath, 'utf8'); + } catch (error) { + return { + path: filePath, + status: 'CRITICAL', + values: {}, + diagnostics: [`无法读取 env 文件: ${error.message}`], + }; + } + + const values = {}; + const diagnostics = []; + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + values[key] = stripQuotes(line.slice(equalsIndex + 1).trim()); + } + + return { + path: filePath, + status: diagnostics.length > 0 ? 'WARNING' : 'OK', + values, + diagnostics, + }; +} + +function collectSecretValuesFromEnv(values) { + const secrets = []; + for (const [key, value] of Object.entries(values || {})) { + if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') { + continue; + } + const trimmed = value.trim(); + if (trimmed.length > 0) { + secrets.push(trimmed); + } + } + return secrets; +} + +function collectSecretValues(...envSnapshots) { + const secrets = new Set(); + for (const snapshot of envSnapshots) { + for (const value of snapshot?.secretValues || []) { + if (typeof value === 'string' && value.trim()) { + secrets.add(value.trim()); + } + } + } + return [...secrets].sort((left, right) => right.length - left.length); +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function normalizeBaseUrl(value) { + if (!value) { + return ''; + } + try { + return new URL(value).toString().replace(/\/+$/u, ''); + } catch { + return value; + } +} + +async function inspectReleaseArtifacts(input) { + const artifacts = []; + for (const relativePath of RELEASE_ARTIFACTS) { + const fullPath = path.join(input.releaseRoot, relativePath); + const artifact = { + path: relativePath, + exists: false, + executable: false, + status: 'CRITICAL', + }; + try { + const fileStat = await stat(fullPath); + artifact.exists = fileStat.isFile() || fileStat.isDirectory(); + if (fileStat.isFile()) { + try { + await access(fullPath, fsConstants.X_OK); + artifact.executable = true; + } catch { + artifact.executable = false; + } + } + artifact.status = artifact.exists ? 'OK' : 'CRITICAL'; + } catch { + artifact.status = 'CRITICAL'; + } + artifacts.push(artifact); + } + + const status = artifacts.some((artifact) => artifact.status === 'CRITICAL') + ? 'CRITICAL' + : 'OK'; + return { + status, + artifacts, + }; +} + +async function inspectSystemd(input) { + const services = []; + for (const service of SERVICES) { + services.push(await inspectService(service, input)); + } + const pingoraUnit = await inspectPingoraUnit(input); + return { + status: summarize([ + ...services.map((service) => service.status), + pingoraUnit.status, + ]).status, + services, + pingoraUnit, + }; +} + +async function inspectService(service, input) { + const expected = isServiceExpected(service, input.expectedGatewayMode); + const result = await runCommand('systemctl', ['is-active', service], input); + const state = result.stdout.trim() || result.stderr.trim() || result.error; + let status = 'OK'; + let summary = state || 'unknown'; + if (result.code === 0 && state === 'active') { + summary = 'active'; + } else if (expected) { + status = 'CRITICAL'; + } else { + summary = state ? `${state} (not required)` : 'not required'; + } + return { + name: service, + expectedActive: expected, + activeState: state, + status, + summary, + command: result.command, + }; +} + +function isServiceExpected(service, gatewayMode) { + if ( + service === 'genarrative-api.service' || + service === 'spacetimedb.service' + ) { + return Boolean(gatewayMode); + } + if (service === 'nginx.service') { + return gatewayMode === 'nginx'; + } + if (service === 'genarrative-pingora-gateway.service') { + return gatewayMode === 'pingora-direct'; + } + return false; +} + +async function inspectPingoraUnit(input) { + const catResult = await runCommand( + 'systemctl', + ['cat', 'genarrative-pingora-gateway.service'], + input, + ); + const showResult = await runCommand( + 'systemctl', + [ + 'show', + 'genarrative-pingora-gateway.service', + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + + const unitText = catResult.stdout; + const hasAmbientCapability = unitText.includes( + 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + ); + const hasCapabilityBoundingSet = unitText.includes( + 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + ); + const environmentFiles = [ + ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu), + ].map((match) => match[1].trim()); + const environmentFileMatchesPingoraEnvFile = environmentFilesInclude( + environmentFiles, + input.pingoraEnvFile, + ); + const show = parseSystemctlShow(showResult.stdout); + const diagnostics = []; + let status = 'OK'; + + if (catResult.code !== 0) { + diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`); + status = 'CRITICAL'; + } + if (showResult.code !== 0) { + diagnostics.push( + `systemctl show 失败: ${showResult.stderr || showResult.error}`, + ); + status = 'CRITICAL'; + } + if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) { + diagnostics.push( + `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`, + ); + status = 'CRITICAL'; + } + if (input.expectedGatewayMode === 'pingora-direct') { + if (!hasAmbientCapability || !hasCapabilityBoundingSet) { + diagnostics.push( + 'pingora-direct 模式必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + } + if (input.expectedGatewayMode === 'nginx') { + if (hasAmbientCapability || hasCapabilityBoundingSet) { + diagnostics.push( + 'nginx 模式下 systemd 最终配置不应残留 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + } + + return { + status, + hasAmbientCapability, + hasCapabilityBoundingSet, + environmentFiles, + environmentFileMatchesPingoraEnvFile, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart: show.ExecStart || '', + diagnostics, + commands: [catResult.command, showResult.command], + }; +} + +function environmentFilesInclude(environmentFiles, expectedPath) { + return environmentFiles.some((entry) => + splitSystemdEnvironmentFileEntry(entry).includes(expectedPath), + ); +} + +function splitSystemdEnvironmentFileEntry(entry) { + const files = []; + for (const word of String(entry || '').split(/\s+/u)) { + if (!word) { + continue; + } + let normalized = word.trim(); + if (normalized.startsWith('-')) { + normalized = normalized.slice(1); + } + if ( + (normalized.startsWith('"') && normalized.endsWith('"')) || + (normalized.startsWith("'") && normalized.endsWith("'")) + ) { + normalized = normalized.slice(1, -1); + } + if (normalized) { + files.push(normalized); + } + } + return files; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +async function runChecks(input, healthPatrolEnv, secrets) { + const checks = []; + checks.push(await runCurrentReleaseAudit(input, secrets)); + if (input.expectedGatewayMode) { + checks.push(await runHealthPatrolEnvCheck(input, secrets)); + } + if (input.runHealthPatrol) { + checks.push(await runProductionHealthPatrol(input, healthPatrolEnv, secrets)); + } + return checks; +} + +async function runCurrentReleaseAudit(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + const result = await runCommand( + 'node', + [ + '--', + script, + '--release-root', + input.releaseRoot, + ...(input.requirePingoraGateway ? ['--require-pingora-gateway'] : []), + '--systemd-show', + '--timeout-ms', + String(input.timeoutMs), + ], + input, + ); + const parsed = parseJsonObject(result.stdout); + const details = parsed.ok + ? { + summary: parsed.value.summary, + pingoraGateway: parsed.value.pingoraGateway, + checksums: parsed.value.checksums, + releaseManifest: parsed.value.releaseManifest, + systemd: parsed.value.systemd, + } + : { + parseError: parsed.error, + }; + const check = commandCheck('current-release-audit', result, secrets, details); + if (!parsed.ok) { + check.status = 'CRITICAL'; + check.stderr = trimForJson( + [check.stderr, `无法解析 current release 自审 JSON: ${parsed.error}`] + .filter(Boolean) + .join('\n'), + ); + } + return check; +} + +async function runHealthPatrolEnvCheck(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/check-production-health-patrol-env.mjs', + ); + const args = [ + '--', + script, + '--env-file', + input.healthPatrolEnvFile, + '--expected-gateway-mode', + input.expectedGatewayMode, + ]; + if (input.expectedPublicBaseUrl) { + args.push('--expected-public-base-url', input.expectedPublicBaseUrl); + } + if (input.expectedPublicHost !== null) { + args.push('--expected-public-host', input.expectedPublicHost); + } + if (input.requireEmptyPublicHost) { + args.push('--require-empty-public-host'); + } + const result = await runCommand('node', args, input); + return commandCheck('health-patrol-env-check', result, secrets); +} + +async function runProductionHealthPatrol(input, healthPatrolEnv, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/production-health-patrol.mjs', + ); + const rawEnv = await readEnvFile(input.healthPatrolEnvFile); + const env = { + ...process.env, + ...readEnvValuesForChild(rawEnv.status === 'CRITICAL' ? healthPatrolEnv : rawEnv), + }; + const result = await runCommand( + 'node', + ['--', script, '--json'], + input, + env, + ); + return commandCheck('production-health-patrol', result, secrets); +} + +function readEnvValuesForChild(healthPatrolEnv) { + if (!healthPatrolEnv.values) { + return {}; + } + const result = {}; + for (const [key, value] of Object.entries(healthPatrolEnv.values)) { + if (typeof value === 'string') { + result[key] = value; + } + } + return result; +} + +function commandCheck(name, result, secrets = [], details = undefined) { + return { + name, + status: result.code === 0 ? 'OK' : 'CRITICAL', + code: result.code, + command: result.command, + stdout: trimForJson(redactSecrets(result.stdout, secrets)), + stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)), + ...(details === undefined ? {} : { details }), + }; +} + +function redactSecrets(value, secrets) { + let text = String(value || ''); + for (const secret of secrets) { + if (!secret) { + continue; + } + text = text.split(secret).join(''); + } + return text; +} + +function runCommand(command, args, input, env = process.env) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + return new Promise((resolve) => { + execFile( + command, + args, + { + env, + timeout: input.timeoutMs, + windowsHide: true, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + command: formatCommand(command, args), + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + timedOut: Boolean(error?.killed), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function formatCommand(command, args) { + const secretValueFlags = new Set([ + '--pingora-probe-token', + '--probe-token', + '--direct-probe-token', + ]); + const redactedArgs = args.map((arg, index) => + index > 0 && secretValueFlags.has(args[index - 1]) ? '' : arg, + ); + return [command, ...redactedArgs].join(' '); +} + +function trimForJson(value) { + const text = String(value || '').trim(); + if (text.length <= 1000) { + return text; + } + return `${text.slice(0, 1000)}...`; +} + +function parseJsonObject(value) { + try { + return { ok: true, value: JSON.parse(value) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function summarize(statuses) { + const normalized = statuses.filter(Boolean); + const status = normalized.reduce( + (current, item) => maxStatus(current, item), + 'OK', + ); + return { + status, + criticalCount: normalized.filter((item) => item === 'CRITICAL').length, + warningCount: normalized.filter((item) => item === 'WARNING').length, + }; +} + +function maxStatus(left, right) { + return STATUS_RANK[right] > STATUS_RANK[left] ? right : left; +} diff --git a/scripts/ops/pingora-direct-rehearsal-status.mjs b/scripts/ops/pingora-direct-rehearsal-status.mjs new file mode 100644 index 000000000..91dfe13bd --- /dev/null +++ b/scripts/ops/pingora-direct-rehearsal-status.mjs @@ -0,0 +1,966 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const STATUS_RANK = { + OK: 0, + WARNING: 1, + CRITICAL: 2, +}; + +const PUBLIC_GATEWAYS = new Set(['none', 'nginx', 'pingora-direct']); +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const DEFAULT_PORTS = [80, 443, 18081, 18083]; +const SECRET_ENV_KEY_PATTERN = + /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu; + +const config = parseArgs(process.argv.slice(2)); +const status = await buildStatus(config); + +console.log(`${JSON.stringify(status, null, 2)}\n`); + +if (config.failOnCritical && status.summary.status === 'CRITICAL') { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + releaseRoot: + process.env.GENARRATIVE_PINGORA_REHEARSAL_RELEASE_ROOT || + '/opt/genarrative/current', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + realpathConfigFile: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_CONFIG_FILE || + '/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf', + expectedPublicGateway: + process.env.GENARRATIVE_PINGORA_REHEARSAL_EXPECT_PUBLIC_GATEWAY || + 'none', + expectedHealthPatrolGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + requireRealpathCanary: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY', + ), + requirePingoraShadow: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_PINGORA_SHADOW', + ), + requireCurrentReleaseGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_CURRENT_RELEASE_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--realpath-config-file': + result.realpathConfigFile = requireValue(argv, ++index, arg); + break; + case '--expect-public-gateway': + result.expectedPublicGateway = requireValue(argv, ++index, arg); + break; + case '--expected-health-patrol-gateway-mode': + result.expectedHealthPatrolGatewayMode = requireValue( + argv, + ++index, + arg, + ); + break; + case '--require-realpath-canary': + result.requireRealpathCanary = true; + break; + case '--require-pingora-shadow': + result.requirePingoraShadow = true; + break; + case '--require-current-release-gateway': + result.requireCurrentReleaseGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-direct-rehearsal-status.mjs [options] + +Options: + --release-root current release 根目录,默认 /opt/genarrative/current。 + --health-patrol-env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件,默认 /etc/genarrative/pingora-gateway.env。 + --realpath-config-file realpath canary Nginx 配置,默认 /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf。 + --expect-public-gateway 可选,none / nginx / pingora-direct;nginx 模式会要求 80/443 仍由 Nginx 接流。 + --expected-health-patrol-gateway-mode + 可选,nginx 或 pingora-direct;不传时随 --expect-public-gateway 推导。 + --require-realpath-canary 要求 realpath canary 配置存在,且 127.0.0.1:18083 由 Nginx 监听。 + --require-pingora-shadow 要求 Pingora shadow 127.0.0.1:18081 正在监听。 + --require-current-release-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。 + --timeout-ms systemctl / ss / 子检查超时,默认 5000。 + --fail-on-critical 如果状态中出现 CRITICAL,则以退出码 1 结束。 + +该脚本只读采集 dev / release 上的 Pingora 直连切换彩排状态,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function validateConfig(input) { + for (const [label, value] of [ + ['--release-root', input.releaseRoot], + ['--health-patrol-env-file', input.healthPatrolEnvFile], + ['--pingora-env-file', input.pingoraEnvFile], + ['--realpath-config-file', input.realpathConfigFile], + ]) { + validateSafeAbsoluteFilePath(value, label); + } + + if (!PUBLIC_GATEWAYS.has(input.expectedPublicGateway)) { + throw new Error( + `--expect-public-gateway 只支持 none / nginx / pingora-direct: ${input.expectedPublicGateway}`, + ); + } + if ( + input.expectedHealthPatrolGatewayMode && + !GATEWAY_MODES.has(input.expectedHealthPatrolGatewayMode) + ) { + throw new Error( + `--expected-health-patrol-gateway-mode 只支持 nginx 或 pingora-direct: ${input.expectedHealthPatrolGatewayMode}`, + ); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +async function buildStatus(input) { + const healthPatrolEnv = await inspectHealthPatrolEnv(input); + const pingoraEnv = await inspectPingoraEnv(input); + const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv); + delete healthPatrolEnv.secretValues; + delete pingoraEnv.secretValues; + const services = await inspectServices(input); + const systemd = await inspectPingoraSystemd(input); + const ports = await inspectPorts(input); + const realpathCanary = await inspectRealpathCanary(input, ports); + const currentReleaseAudit = await inspectCurrentRelease(input, secrets); + const publicBoundary = inspectPublicBoundary(input, ports, systemd); + const checks = [ + healthPatrolEnv.status, + pingoraEnv.status, + services.status, + systemd.status, + ports.status, + realpathCanary.status, + currentReleaseAudit.status, + publicBoundary.status, + ]; + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + summary: summarize(checks), + releaseRoot: input.releaseRoot, + expectations: { + publicGateway: input.expectedPublicGateway, + healthPatrolGatewayMode: expectedHealthPatrolGatewayMode(input), + requireRealpathCanary: input.requireRealpathCanary, + requirePingoraShadow: input.requirePingoraShadow, + requireCurrentReleaseGateway: input.requireCurrentReleaseGateway, + }, + publicBoundary, + ports, + services, + systemd, + healthPatrolEnv, + pingoraEnv, + realpathCanary, + currentReleaseAudit, + }; +} + +async function inspectHealthPatrolEnv(input) { + const parsed = await readEnvFile(input.healthPatrolEnvFile); + if (parsed.status === 'CRITICAL') { + return parsed; + } + + const values = parsed.values; + const expectedMode = expectedHealthPatrolGatewayMode(input); + const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || ''; + const diagnostics = [...parsed.diagnostics]; + let status = parsed.status; + + if (expectedMode && gatewayMode !== expectedMode) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${expectedMode},实际 ${gatewayMode || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + + return { + path: input.healthPatrolEnvFile, + status, + secretValues: collectSecretValuesFromEnv(values), + values: { + gatewayMode, + publicBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '', + publicHost: values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '', + apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '', + pingoraBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + hasPingoraProbeToken: Boolean( + values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN, + ), + }, + diagnostics, + }; +} + +function expectedHealthPatrolGatewayMode(input) { + if (input.expectedHealthPatrolGatewayMode) { + return input.expectedHealthPatrolGatewayMode; + } + if (input.expectedPublicGateway === 'nginx') { + return 'nginx'; + } + if (input.expectedPublicGateway === 'pingora-direct') { + return 'pingora-direct'; + } + return ''; +} + +async function inspectPingoraEnv(input) { + const parsed = await readEnvFile(input.pingoraEnvFile); + if (parsed.status === 'CRITICAL') { + return { + ...parsed, + values: {}, + }; + } + const values = parsed.values; + return { + path: input.pingoraEnvFile, + status: parsed.status, + secretValues: collectSecretValuesFromEnv(values), + values: { + listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '', + tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '', + httpRedirectListen: + values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '', + tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '', + hasTlsKeyFile: Boolean( + values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE, + ), + hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN), + accessLogFile: + values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '', + compressionAlgorithms: + values.GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS || '', + trustXForwardedFor: + values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '', + }, + diagnostics: parsed.diagnostics, + }; +} + +async function readEnvFile(filePath) { + let text; + try { + text = await readFile(filePath, 'utf8'); + } catch (error) { + return { + path: filePath, + status: 'CRITICAL', + values: {}, + diagnostics: [`无法读取 env 文件: ${error.message}`], + }; + } + + const values = {}; + const diagnostics = []; + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + values[key] = stripQuotes(line.slice(equalsIndex + 1).trim()); + } + + return { + path: filePath, + status: diagnostics.length > 0 ? 'WARNING' : 'OK', + values, + diagnostics, + }; +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function collectSecretValuesFromEnv(values) { + const secrets = []; + for (const [key, value] of Object.entries(values || {})) { + if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') { + continue; + } + const trimmed = value.trim(); + if (trimmed.length > 0) { + secrets.push(trimmed); + } + } + return secrets; +} + +function collectSecretValues(...envSnapshots) { + const secrets = new Set(); + for (const snapshot of envSnapshots) { + for (const value of snapshot?.secretValues || []) { + if (typeof value === 'string' && value.trim()) { + secrets.add(value.trim()); + } + } + } + return [...secrets].sort((left, right) => right.length - left.length); +} + +async function inspectServices(input) { + const serviceNames = [ + 'nginx.service', + 'genarrative-pingora-gateway.service', + 'genarrative-api.service', + 'spacetimedb.service', + ]; + const services = []; + for (const service of serviceNames) { + services.push(await inspectService(service, input)); + } + return { + status: summarize(services.map((service) => service.status)).status, + services, + }; +} + +async function inspectService(service, input) { + const result = await runCommand('systemctl', ['is-active', service], input); + const state = result.stdout.trim() || result.stderr.trim() || result.error; + let status = 'OK'; + if (service === 'nginx.service' && input.expectedPublicGateway === 'nginx') { + status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL'; + } else if ( + service === 'genarrative-pingora-gateway.service' && + (input.requirePingoraShadow || + input.expectedPublicGateway === 'pingora-direct') + ) { + status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL'; + } else if ( + (service === 'genarrative-api.service' || + service === 'spacetimedb.service') && + result.code !== 0 + ) { + status = 'WARNING'; + } + return { + name: service, + activeState: state || 'unknown', + status, + command: result.command, + }; +} + +async function inspectPingoraSystemd(input) { + const catResult = await runCommand( + 'systemctl', + ['cat', 'genarrative-pingora-gateway.service'], + input, + ); + const showResult = await runCommand( + 'systemctl', + [ + 'show', + 'genarrative-pingora-gateway.service', + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + const unitText = catResult.stdout; + const hasAmbientCapability = unitText.includes( + 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + ); + const hasCapabilityBoundingSet = unitText.includes( + 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + ); + const environmentFiles = [ + ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu), + ].map((match) => match[1].trim()); + const environmentFileMatchesPingoraEnvFile = environmentFilesInclude( + environmentFiles, + input.pingoraEnvFile, + ); + const show = parseSystemctlShow(showResult.stdout); + const diagnostics = []; + let status = 'OK'; + + if (catResult.code !== 0) { + diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`); + status = 'CRITICAL'; + } + if (showResult.code !== 0) { + diagnostics.push( + `systemctl show 失败: ${showResult.stderr || showResult.error}`, + ); + status = 'CRITICAL'; + } + if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) { + diagnostics.push( + `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`, + ); + status = 'CRITICAL'; + } + if ( + input.expectedPublicGateway === 'nginx' && + (hasAmbientCapability || hasCapabilityBoundingSet) + ) { + diagnostics.push( + 'Nginx 接公网阶段不应残留 Pingora 低端口 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + if ( + input.expectedPublicGateway === 'pingora-direct' && + (!hasAmbientCapability || !hasCapabilityBoundingSet) + ) { + diagnostics.push( + 'pingora-direct 接公网阶段必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + + return { + status, + hasAmbientCapability, + hasCapabilityBoundingSet, + environmentFiles, + environmentFileMatchesPingoraEnvFile, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart: show.ExecStart || '', + diagnostics, + commands: [catResult.command, showResult.command], + }; +} + +function environmentFilesInclude(environmentFiles, expectedPath) { + return environmentFiles.some((entry) => + splitSystemdEnvironmentFileEntry(entry).includes(expectedPath), + ); +} + +function splitSystemdEnvironmentFileEntry(entry) { + const files = []; + for (const word of String(entry || '').split(/\s+/u)) { + if (!word) { + continue; + } + let normalized = word.trim(); + if (normalized.startsWith('-')) { + normalized = normalized.slice(1); + } + if ( + (normalized.startsWith('"') && normalized.endsWith('"')) || + (normalized.startsWith("'") && normalized.endsWith("'")) + ) { + normalized = normalized.slice(1, -1); + } + if (normalized) { + files.push(normalized); + } + } + return files; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +async function inspectPorts(input) { + const result = await runCommand('ss', ['-H', '-ltnp'], input); + const listeners = result.code === 0 ? parseSsOutput(result.stdout) : []; + const byPort = {}; + const statuses = []; + for (const port of DEFAULT_PORTS) { + const portListeners = listeners.filter((item) => item.port === port); + const entry = inspectPort(port, portListeners, input); + byPort[String(port)] = entry; + statuses.push(entry.status); + } + if (result.code !== 0) { + statuses.push('CRITICAL'); + } + return { + status: summarize(statuses).status, + command: result.command, + error: result.code === 0 ? '' : trimForJson(result.stderr || result.error), + byPort, + }; +} + +function parseSsOutput(stdout) { + const listeners = []; + for (const line of String(stdout || '').split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + const parts = trimmed.split(/\s+/u); + const local = parts[3] || ''; + const port = parsePortFromLocalAddress(local); + if (!port) { + continue; + } + const processText = parts.slice(5).join(' '); + const processNames = [ + ...new Set([...processText.matchAll(/"([^"]+)"/gu)].map((match) => match[1])), + ]; + listeners.push({ + localAddress: local, + port, + loopback: isLoopbackLocalAddress(local), + processNames, + }); + } + return listeners; +} + +function parsePortFromLocalAddress(value) { + const bracketMatch = String(value).match(/^\[[^\]]+\]:(\d+)$/u); + const plainMatch = String(value).match(/:(\d+)$/u); + const portText = bracketMatch?.[1] || plainMatch?.[1] || ''; + const port = Number.parseInt(portText, 10); + return Number.isInteger(port) && port > 0 ? port : null; +} + +function isLoopbackLocalAddress(value) { + const text = String(value || '').toLowerCase(); + return ( + text.startsWith('127.') || + text.startsWith('[::1]') || + text.startsWith('::1') || + text.startsWith('localhost:') + ); +} + +function inspectPort(port, listeners, input) { + const hasNginx = listeners.some((listener) => + listener.processNames.some((name) => name.toLowerCase().includes('nginx')), + ); + const hasPingora = listeners.some((listener) => + listener.processNames.some((name) => name.toLowerCase().includes('pingora')), + ); + const hasUnknownProcess = + listeners.length > 0 && + listeners.some((listener) => listener.processNames.length === 0); + const loopbackOnly = + listeners.length > 0 && listeners.every((listener) => listener.loopback); + const diagnostics = []; + let status = 'OK'; + + if ([80, 443].includes(port)) { + if (input.expectedPublicGateway === 'nginx') { + if (listeners.length === 0) { + diagnostics.push(`${port} 未监听,Nginx 公网入口未就绪。`); + status = 'CRITICAL'; + } + if (!hasNginx) { + diagnostics.push(`${port} 未看到 Nginx 监听。`); + status = 'CRITICAL'; + } + if (hasPingora) { + diagnostics.push(`${port} 已被 Pingora 监听,不能作为未切公网彩排状态。`); + status = 'CRITICAL'; + } + } + if (input.expectedPublicGateway === 'pingora-direct') { + if (!hasPingora) { + diagnostics.push(`${port} 未看到 Pingora 监听。`); + status = 'CRITICAL'; + } + if (hasNginx) { + diagnostics.push(`${port} 仍由 Nginx 监听。`); + status = 'CRITICAL'; + } + } + } + + if (port === 18081 && input.requirePingoraShadow) { + if (!hasPingora) { + diagnostics.push('18081 未看到 Pingora shadow 监听。'); + status = 'CRITICAL'; + } + if (!loopbackOnly) { + diagnostics.push('Pingora shadow 18081 必须只监听 loopback。'); + status = 'CRITICAL'; + } + } + + if (port === 18083 && input.requireRealpathCanary) { + if (!hasNginx) { + diagnostics.push('18083 未看到 Nginx realpath canary 监听。'); + status = 'CRITICAL'; + } + if (!loopbackOnly) { + diagnostics.push('Nginx realpath canary 18083 必须只监听 loopback。'); + status = 'CRITICAL'; + } + } + + if ( + hasUnknownProcess && + ([80, 443].includes(port) || + (port === 18081 && input.requirePingoraShadow) || + (port === 18083 && input.requireRealpathCanary)) + ) { + diagnostics.push('ss 未返回进程名,无法确认监听归属。'); + status = maxStatus(status, 'WARNING'); + } + + return { + port, + status, + listening: listeners.length > 0, + loopbackOnly, + hasNginx, + hasPingora, + hasUnknownProcess, + listeners, + diagnostics, + }; +} + +async function inspectRealpathCanary(input, ports) { + const diagnostics = []; + let status = 'OK'; + let exists = false; + let templateLooksValid = false; + try { + const fileStat = await stat(input.realpathConfigFile); + exists = fileStat.isFile(); + } catch { + exists = false; + } + + if (!exists) { + diagnostics.push(`realpath canary 配置不存在: ${input.realpathConfigFile}`); + status = input.requireRealpathCanary ? 'CRITICAL' : 'WARNING'; + } else { + const content = await readFile(input.realpathConfigFile, 'utf8'); + const requiredSnippets = [ + 'listen 127.0.0.1:18083', + 'genarrative-pingora-realpath-canary.access.log', + 'X-Genarrative-Nginx-Handoff pingora-realpath-canary', + ]; + const missing = requiredSnippets.filter((snippet) => !content.includes(snippet)); + templateLooksValid = missing.length === 0; + if (missing.length > 0) { + diagnostics.push(`realpath canary 配置缺少关键片段: ${missing.join(', ')}`); + status = 'CRITICAL'; + } + } + + const portStatus = ports.byPort['18083']; + if (input.requireRealpathCanary && portStatus?.status === 'CRITICAL') { + status = 'CRITICAL'; + } + + return { + path: input.realpathConfigFile, + status, + exists, + templateLooksValid, + port: portStatus, + diagnostics, + }; +} + +async function inspectCurrentRelease(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + try { + await access(script, fsConstants.R_OK); + } catch (error) { + return { + status: 'CRITICAL', + script, + code: null, + diagnostics: [`无法读取 current release 自审脚本: ${error.message}`], + }; + } + + const args = [ + '--', + script, + '--release-root', + input.releaseRoot, + '--systemd-show', + '--timeout-ms', + String(input.timeoutMs), + ]; + if (input.requireCurrentReleaseGateway) { + args.push('--require-pingora-gateway'); + } + const result = await runCommand('node', args, input); + const parsed = parseJsonObject(result.stdout); + let status = result.code === 0 ? 'OK' : 'CRITICAL'; + const diagnostics = []; + if (!parsed.ok) { + diagnostics.push(`无法解析 current release 自审 JSON: ${parsed.error}`); + status = 'CRITICAL'; + } + return { + status, + script, + code: result.code, + command: result.command, + summary: parsed.ok ? parsed.value?.summary || null : null, + pingoraGateway: parsed.ok ? parsed.value?.pingoraGateway || null : null, + releaseManifest: parsed.ok ? parsed.value?.releaseManifest || null : null, + stdout: trimForJson(redactSecrets(result.stdout, secrets)), + stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)), + diagnostics, + }; +} + +function inspectPublicBoundary(input, ports, systemd) { + const diagnostics = []; + let status = 'OK'; + if (input.expectedPublicGateway === 'none') { + return { + status, + expectedPublicGateway: input.expectedPublicGateway, + diagnostics, + }; + } + + for (const port of [80, 443]) { + const portStatus = ports.byPort[String(port)]; + if (portStatus?.status === 'CRITICAL') { + status = 'CRITICAL'; + diagnostics.push(...portStatus.diagnostics); + } + } + if (systemd.status === 'CRITICAL') { + status = 'CRITICAL'; + diagnostics.push(...systemd.diagnostics); + } + + return { + status, + expectedPublicGateway: input.expectedPublicGateway, + diagnostics, + }; +} + +function runCommand(command, args, input, env = process.env) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + return new Promise((resolve) => { + execFile( + command, + args, + { + env, + timeout: input.timeoutMs, + windowsHide: true, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + command: formatCommand(command, args), + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + timedOut: Boolean(error?.killed), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function formatCommand(command, args) { + return [command, ...args].join(' '); +} + +function redactSecrets(value, secrets) { + let text = String(value || ''); + for (const secret of secrets) { + if (!secret) { + continue; + } + text = text.split(secret).join(''); + } + return text; +} + +function trimForJson(value) { + const text = String(value || '').trim(); + if (text.length <= 1000) { + return text; + } + return `${text.slice(0, 1000)}...`; +} + +function parseJsonObject(value) { + try { + return { ok: true, value: JSON.parse(value) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function summarize(statuses) { + const normalized = statuses.filter(Boolean); + const status = normalized.reduce( + (current, item) => maxStatus(current, item), + 'OK', + ); + return { + status, + criticalCount: normalized.filter((item) => item === 'CRITICAL').length, + warningCount: normalized.filter((item) => item === 'WARNING').length, + }; +} + +function maxStatus(left, right) { + return STATUS_RANK[right] > STATUS_RANK[left] ? right : left; +} diff --git a/scripts/ops/production-health-patrol.mjs b/scripts/ops/production-health-patrol.mjs index 01a1265c7..60eab89f9 100644 --- a/scripts/ops/production-health-patrol.mjs +++ b/scripts/ops/production-health-patrol.mjs @@ -1,10 +1,10 @@ #!/usr/bin/env node -import {execFile} from 'node:child_process'; +import { execFile } from 'node:child_process'; import http from 'node:http'; import https from 'node:https'; -import {mkdir, writeFile} from 'node:fs/promises'; -import {dirname} from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; const STATUS_RANK = { OK: 0, @@ -24,7 +24,14 @@ const DEFAULT_SERVICES = [ 'spacetimedb.service', 'nginx.service', ]; +const PINGORA_DIRECT_SERVICES = [ + 'genarrative-api.service', + 'genarrative-external-generation-controller.service', + 'spacetimedb.service', + 'genarrative-pingora-gateway.service', +]; const WORKER_SERVICE_PATTERN = 'genarrative-external-generation-worker@*.service'; +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); function usage() { console.log(`Usage: @@ -34,7 +41,11 @@ Options: --api-base-url API direct base URL, default http://127.0.0.1:8082 --spacetime-base-url SpacetimeDB base URL, default http://127.0.0.1:3101 --public-base-url Nginx/public base URL, default http://127.0.0.1 + --public-host Optional public Host header, useful when probing 127.0.0.1 --public-path Public API path to probe; repeatable + --gateway-mode nginx or pingora-direct, default nginx + --pingora-base-url Optional Pingora shadow gateway base URL + --pingora-probe-token Optional Pingora shadow probe token --status-file Write the last patrol result as JSON --timeout-ms HTTP/command timeout, default 5000 --slow-ms Mark successful probes slower than this as WARNING, default 3000 @@ -46,15 +57,32 @@ Options: function readBoolEnv(name, fallback = false) { const value = process.env[name]; - if (!value) { + if (value === undefined || value === null || String(value).trim() === '') { return fallback; } - return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()); + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); } -function parsePositiveInt(raw, fallback) { - const value = Number.parseInt(String(raw ?? ''), 10); - return Number.isFinite(value) && value > 0 ? value : fallback; +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const valueText = String(raw ?? '').trim(); + if (!/^[1-9]\d*$/.test(valueText)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(valueText, 10); } function parseArgs(argv) { @@ -67,17 +95,26 @@ function parseArgs(argv) { 'http://127.0.0.1:3101', publicBaseUrl: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || - process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || - 'http://127.0.0.1:8082', + 'http://127.0.0.1', + publicHost: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '', publicPaths: [], + gatewayMode: process.env.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || 'nginx', + pingoraBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + pingoraProbeToken: + process.env.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + process.env.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN || + '', statusFile: process.env.GENARRATIVE_HEALTH_PATROL_STATUS_FILE || '', - timeoutMs: parsePositiveInt( + timeoutMs: parseOptionalPositiveInt( process.env.GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS, 5000, + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS', ), - slowMs: parsePositiveInt( + slowMs: parseOptionalPositiveInt( process.env.GENARRATIVE_HEALTH_PATROL_SLOW_MS, 3000, + 'GENARRATIVE_HEALTH_PATROL_SLOW_MS', ), failOnWarning: readBoolEnv('GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING'), skipJournal: readBoolEnv('GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL'), @@ -102,17 +139,35 @@ function parseArgs(argv) { case '--public-base-url': config.publicBaseUrl = requireValue(argv, ++index, arg); break; + case '--public-host': + config.publicHost = requireValue(argv, ++index, arg); + break; case '--public-path': config.publicPaths.push(requireValue(argv, ++index, arg)); break; + case '--gateway-mode': + config.gatewayMode = requireValue(argv, ++index, arg); + break; + case '--pingora-base-url': + config.pingoraBaseUrl = requireValue(argv, ++index, arg); + break; + case '--pingora-probe-token': + config.pingoraProbeToken = requireValue(argv, ++index, arg); + break; case '--status-file': config.statusFile = requireValue(argv, ++index, arg); break; case '--timeout-ms': - config.timeoutMs = parsePositiveInt(requireValue(argv, ++index, arg), 5000); + config.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + '--timeout-ms', + ); break; case '--slow-ms': - config.slowMs = parsePositiveInt(requireValue(argv, ++index, arg), 3000); + config.slowMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + '--slow-ms', + ); break; case '--fail-on-warning': config.failOnWarning = true; @@ -131,6 +186,15 @@ function parseArgs(argv) { if (config.publicPaths.length === 0) { config.publicPaths = DEFAULT_PUBLIC_PATHS; } + config.gatewayMode = config.gatewayMode.trim(); + if (!GATEWAY_MODES.has(config.gatewayMode)) { + throw new Error( + `--gateway-mode 只支持 nginx 或 pingora-direct: ${config.gatewayMode}`, + ); + } + if (config.publicHost) { + validateHostOption(config.publicHost, '--public-host'); + } return config; } @@ -149,9 +213,22 @@ function joinUrl(baseUrl, path) { return `${base}${suffix}`; } +function validateHostOption(value, flag) { + if ( + value.includes('://') || + value.includes('/') || + value.includes('?') || + value.includes('#') + ) { + throw new Error(`${flag} 只接受域名或 host:port,不要填 URL、路径或查询。`); + } +} + function maxStatus(checks) { return checks.reduce((current, check) => { - return STATUS_RANK[check.status] > STATUS_RANK[current] ? check.status : current; + return STATUS_RANK[check.status] > STATUS_RANK[current] + ? check.status + : current; }, 'OK'); } @@ -177,12 +254,7 @@ function runCommand(command, args, timeoutMs) { (error, stdout, stderr) => { resolve({ command: [command, ...args].join(' '), - code: - typeof error?.code === 'number' - ? error.code - : error - ? 1 - : 0, + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, signal: error?.signal || '', stdout: String(stdout || ''), stderr: String(stderr || ''), @@ -273,7 +345,7 @@ async function checkActiveWorkerInstances(config) { ); } -function requestUrl(url, timeoutMs) { +function requestUrl(url, timeoutMs, headers = {}) { return new Promise((resolve) => { const startedAt = Date.now(); const parsed = new URL(url); @@ -286,6 +358,7 @@ function requestUrl(url, timeoutMs) { headers: { 'User-Agent': 'genarrative-health-patrol/1.0', Accept: 'application/json,text/plain,*/*', + ...headers, }, }, (response) => { @@ -319,9 +392,17 @@ function requestUrl(url, timeoutMs) { }); } -async function checkHttp(name, url, config) { - const result = await requestUrl(url, config.timeoutMs); - const curlCommand = `curl -fsS --max-time ${Math.ceil(config.timeoutMs / 1000)} ${url}`; +async function checkHttp(name, url, config, headers = {}) { + const result = await requestUrl(url, config.timeoutMs, headers); + const headerArgs = Object.entries(headers) + .map( + ([key, value]) => + ` -H '${key}: ${key.toLowerCase() === 'host' ? value : '***'}'`, + ) + .join(''); + const curlCommand = `curl -fsS --max-time ${Math.ceil( + config.timeoutMs / 1000, + )}${headerArgs} ${url}`; if (result.error) { return checkResult(name, 'CRITICAL', `请求失败: ${result.error}`, { @@ -356,24 +437,125 @@ async function checkHttp(name, url, config) { ); } - return checkResult(name, 'OK', `HTTP ${result.statusCode} ${result.elapsedMs}ms`, { - command: curlCommand, - elapsedMs: result.elapsedMs, + return checkResult( + name, + 'OK', + `HTTP ${result.statusCode} ${result.elapsedMs}ms`, + { + command: curlCommand, + elapsedMs: result.elapsedMs, + }, + ); +} + +async function checkPingoraShadow(config) { + if (!config.pingoraBaseUrl || !config.pingoraProbeToken) { + return checkResult( + 'pingora:shadow-probe', + 'WARNING', + 'Pingora shadow 巡检配置不完整,需同时配置 base URL 与 probe token', + ); + } + + const path = '/__genarrative_pingora/healthz'; + const url = joinUrl(config.pingoraBaseUrl, path); + const result = await requestUrl(url, config.timeoutMs, { + 'X-Genarrative-Pingora-Probe': config.pingoraProbeToken, }); + const command = `curl -fsS --max-time ${Math.ceil( + config.timeoutMs / 1000, + )} -H 'X-Genarrative-Pingora-Probe: ***' ${url}`; + + if (result.error) { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `请求失败: ${result.error}`, + { + command, + elapsedMs: result.elapsedMs, + }, + ); + } + + if (result.statusCode < 200 || result.statusCode >= 300) { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `HTTP ${result.statusCode},耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + let payload; + try { + payload = JSON.parse(result.body); + } catch { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `响应不是合法 JSON,耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + if (payload?.ok !== true || payload?.gateway !== 'pingora-shadow') { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `响应不是 Pingora shadow 探针,耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + if (result.elapsedMs > config.slowMs) { + return checkResult( + 'pingora:shadow-probe', + 'WARNING', + `HTTP ${result.statusCode} 但耗时偏高: ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + }, + ); + } + + return checkResult( + 'pingora:shadow-probe', + 'OK', + `HTTP ${result.statusCode} ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + maintenance: Boolean(payload.maintenance), + }, + ); +} + +function gatewayServiceNames(config) { + return config.gatewayMode === 'pingora-direct' + ? PINGORA_DIRECT_SERVICES + : DEFAULT_SERVICES; } async function checkRecentJournal(config) { + const serviceNames = [ + ...new Set([...gatewayServiceNames(config), WORKER_SERVICE_PATTERN]), + ]; const args = [ - '-u', - 'genarrative-api.service', - '-u', - 'genarrative-external-generation-controller.service', - '-u', - WORKER_SERVICE_PATTERN, - '-u', - 'spacetimedb.service', - '-u', - 'nginx.service', + ...serviceNames.flatMap((serviceName) => ['-u', serviceName]), '--since', '15 minutes ago', '-p', @@ -387,10 +569,15 @@ async function checkRecentJournal(config) { const result = await runCommand('journalctl', args, config.timeoutMs); if (result.code !== 0) { - return checkResult('journal:recent-errors', 'WARNING', '无法读取最近错误日志', { - command: result.command, - stderr: result.stderr.trim() || result.error, - }); + return checkResult( + 'journal:recent-errors', + 'WARNING', + '无法读取最近错误日志', + { + command: result.command, + stderr: result.stderr.trim() || result.error, + }, + ); } const lines = result.stdout @@ -399,9 +586,14 @@ async function checkRecentJournal(config) { .filter((line) => line && line !== '-- No entries --'); if (lines.length === 0) { - return checkResult('journal:recent-errors', 'OK', '最近 15 分钟无 err..alert 日志', { - command: result.command, - }); + return checkResult( + 'journal:recent-errors', + 'OK', + '最近 15 分钟无 err..alert 日志', + { + command: result.command, + }, + ); } return checkResult( @@ -419,7 +611,7 @@ async function writeStatusFile(statusFile, payload) { if (!statusFile) { return; } - await mkdir(dirname(statusFile), {recursive: true}); + await mkdir(dirname(statusFile), { recursive: true }); await writeFile(statusFile, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } @@ -484,13 +676,25 @@ async function main() { const config = parseArgs(process.argv.slice(2)); const checks = []; - for (const serviceName of DEFAULT_SERVICES) { + for (const serviceName of gatewayServiceNames(config)) { checks.push(await checkService(serviceName, config.timeoutMs)); } checks.push(await checkActiveWorkerInstances(config)); - checks.push(await checkHttp('api:/healthz', joinUrl(config.apiBaseUrl, '/healthz'), config)); - checks.push(await checkHttp('api:/readyz', joinUrl(config.apiBaseUrl, '/readyz'), config)); + checks.push( + await checkHttp( + 'api:/healthz', + joinUrl(config.apiBaseUrl, '/healthz'), + config, + ), + ); + checks.push( + await checkHttp( + 'api:/readyz', + joinUrl(config.apiBaseUrl, '/readyz'), + config, + ), + ); checks.push( await checkHttp( 'spacetimedb:/v1/ping', @@ -500,11 +704,21 @@ async function main() { ); for (const path of config.publicPaths) { + const headers = config.publicHost ? { Host: config.publicHost } : {}; checks.push( - await checkHttp(`public:${path}`, joinUrl(config.publicBaseUrl, path), config), + await checkHttp( + `public:${path}`, + joinUrl(config.publicBaseUrl, path), + config, + headers, + ), ); } + if (config.pingoraBaseUrl || config.pingoraProbeToken) { + checks.push(await checkPingoraShadow(config)); + } + if (!config.skipJournal) { checks.push(await checkRecentJournal(config)); } @@ -513,6 +727,7 @@ async function main() { status: maxStatus(checks), checkedAt: new Date().toISOString(), host: process.env.HOSTNAME || '', + gatewayMode: config.gatewayMode, checks, }; @@ -534,6 +749,8 @@ async function main() { } main().catch((error) => { - console.error(`[health-patrol] CRITICAL ${error instanceof Error ? error.message : String(error)}`); + console.error( + `[health-patrol] CRITICAL ${error instanceof Error ? error.message : String(error)}`, + ); process.exit(2); }); diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index a9d508fa5..1e719b24d 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -18,6 +18,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -66,6 +75,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -74,13 +89,19 @@ checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" [[package]] name = "alloc-stdlib" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-activity" version = "0.6.1" @@ -88,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f2a1bb052857d5dd49572219344a7332b31b76405648eabac5bc68978251bcd" dependencies = [ "android-properties", - "bitflags 2.11.1", + "bitflags 2.13.0", "cc", "jni", "libc", @@ -115,6 +136,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -123,9 +194,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "anymap3" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170433209e817da6aae2c51aa0dd443009a613425dd041ebfb2492d1c4c11a25" +checksum = "fb5dfbc6d8d2675589ccbe4d0fd61df2419075625f8c1a62325e718e2b0049f9" [[package]] name = "api-server" @@ -181,7 +252,7 @@ dependencies = [ "shared-contracts", "shared-kernel", "shared-logging", - "socket2 0.6.3", + "socket2 0.6.4", "spacetime-client", "time", "tokio", @@ -231,10 +302,19 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "x11rb", ] +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -265,6 +345,45 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -284,7 +403,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -295,7 +414,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -306,9 +425,31 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] [[package]] name = "axum" @@ -321,10 +462,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-util", "itoa", "matchit", @@ -354,7 +495,7 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "mime", @@ -365,6 +506,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.21.7" @@ -415,9 +571,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "blake2" @@ -430,9 +586,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", @@ -491,10 +647,30 @@ dependencies = [ ] [[package]] -name = "bumpalo" -version = "3.20.2" +name = "bs58" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" @@ -513,9 +689,15 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -530,9 +712,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "bytestring" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "113b4343b5f6617e7ad401ced8de3cc8b012e73a594347c307b90db3e9271289" +checksum = "86566c496f2f47d9b8147a4c8b02ffdb69c919fe0c2b2e7195d22cbba0e635c9" dependencies = [ "bytes", "serde_core", @@ -544,7 +726,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b99da2f8558ca23c71f4fd15dc57c906239752dd27ff3c00a1d56b685b7cbfec" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "log", "polling", "rustix 0.38.44", @@ -558,7 +740,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "polling", "rustix 1.1.4", "slab", @@ -609,9 +791,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -619,6 +801,34 @@ dependencies = [ "shlex", ] +[[package]] +name = "cf-rustracing" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f85c3824e4191621dec0551e3cef3d511f329da9a8990bf3e450a85651d97e" +dependencies = [ + "backtrace", + "rand 0.8.6", + "tokio", + "trackable", +] + +[[package]] +name = "cf-rustracing-jaeger" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a5f80d44c257c3300a7f45ada676c211e64bbbac591bbec19344a8f61fbcab" +dependencies = [ + "cf-rustracing", + "hostname", + "local-ip-address", + "percent-encoding", + "rand 0.9.4", + "thrift_codec", + "tokio", + "trackable", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -642,9 +852,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -664,6 +874,46 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -673,6 +923,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "codespan-reporting" version = "0.12.0" @@ -682,6 +941,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.7" @@ -883,24 +1148,24 @@ dependencies = [ [[package]] name = "curl" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79fc3b6dd0b87ba36e565715bf9a2ced221311db47bd18011676f24a6066edbc" +checksum = "3a45ee8994e5307cb4c60cfc1c20bf7263ffb771ddc135c9f768a14bcbc15b09" dependencies = [ "curl-sys", "libc", "openssl-probe", "openssl-sys", "schannel", - "socket2 0.6.3", - "windows-sys 0.59.0", + "socket2 0.6.4", + "windows-sys 0.61.2", ] [[package]] name = "curl-sys" -version = "0.4.88+curl-8.20.0" +version = "0.4.89+curl-8.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644816de6547255eff4e491a1dda1c19b7237f00b62a61e6e64859ce4f2906d0" +checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" dependencies = [ "cc", "libc", @@ -917,14 +1182,56 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f27ae1dd37df86211c42e150270f82743308803d90a6f6e6651cd730d5e1732f" +[[package]] +name = "daemonize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" +dependencies = [ + "libc", +] + +[[package]] +name = "daggy" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70def8d72740e44d9f676d8dab2c933a236663d86dd24319b57a2bed4d694774" +dependencies = [ + "petgraph 0.7.1", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", ] [[package]] @@ -937,7 +1244,18 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.118", ] [[package]] @@ -946,16 +1264,16 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "decorum" @@ -967,16 +1285,40 @@ dependencies = [ "num-traits", ] +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -985,7 +1327,38 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.118", ] [[package]] @@ -998,7 +1371,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.118", ] [[package]] @@ -1024,19 +1397,19 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "objc2 0.6.4", ] [[package]] name = "displaydoc" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1075,6 +1448,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1133,7 +1512,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a9b567d356674e9a5121ed3fedfb0a7c31e059fe71f6972b691bcd0bfc284e3" dependencies = [ "ahash", - "bitflags 2.11.1", + "bitflags 2.13.0", "emath", "epaint", "log", @@ -1194,7 +1573,7 @@ dependencies = [ "egui", "glow", "log", - "memoffset", + "memoffset 0.9.1", "profiling", "wasm-bindgen", "web-sys", @@ -1203,9 +1582,9 @@ dependencies = [ [[package]] name = "either" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "emath" @@ -1240,7 +1619,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1260,7 +1639,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1360,6 +1739,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -1367,6 +1752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", + "libz-ng-sys", "miniz_oxide", ] @@ -1415,7 +1801,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1439,6 +1825,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -1495,7 +1887,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -1587,6 +1979,23 @@ dependencies = [ "wasip3", ] +[[package]] +name = "getset" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf442baaabe4213ce7d1239afc26c039180b6456da2cededa316ae2c8a77a77" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "gl_generator" version = "0.14.0" @@ -1622,7 +2031,7 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12124de845cacfebedff80e877bb37b5b75c34c5a4c89e47e1cdd67fb6041325" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg_aliases", "cgl", "dispatch2", @@ -1701,6 +2110,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1734,6 +2162,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", "equivalent", "foldhash 0.2.0", "rayon", @@ -1743,9 +2172,14 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -1795,6 +2229,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "0.2.12" @@ -1808,9 +2253,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1834,7 +2279,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.4.0", + "http 1.4.2", ] [[package]] @@ -1845,7 +2290,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "pin-project-lite", ] @@ -1878,7 +2323,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1894,15 +2339,15 @@ dependencies = [ [[package]] name = "hyper" -version = "1.9.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "httparse", "httpdate", @@ -1919,14 +2364,14 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.0", - "hyper 1.9.0", + "http 1.4.2", + "hyper 1.10.1", "hyper-util", "rustls", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -1952,14 +2397,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", - "hyper 1.9.0", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -2096,9 +2541,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -2149,7 +2594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -2166,15 +2611,16 @@ dependencies = [ [[package]] name = "insta" -version = "1.47.2" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +checksum = "86f0f8fee8c926415c58d6ae43a08523a26faccb2323f5e6b644fe7dd4ef6b82" dependencies = [ "console", "once_cell", "regex", "serde", "similar", + "strip-ansi-escapes", "tempfile", "toml_edit", "toml_writer", @@ -2187,14 +2633,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] -name = "iri-string" -version = "0.7.12" +name = "is_terminal_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -2205,6 +2647,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2238,7 +2689,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.118", ] [[package]] @@ -2266,7 +2717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2281,13 +2732,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", - "once_cell", "wasm-bindgen", ] @@ -2323,9 +2773,9 @@ checksum = "e2db585e1d738fc771bf08a151420d3ed193d9d895a36df7f6f8a9456b911ddc" [[package]] name = "langchainrust" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e783bd3734eb269382df16f3018d3b5e31827c6f8a2bdf7d4bf3ddb7b9bb6039" +checksum = "ac5ddb421c9c0311ee154af906747ba1fb5c3db83d7d71dcb38f40b557eb28e4" dependencies = [ "async-trait", "bincode", @@ -2368,9 +2818,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -2394,7 +2844,7 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "libc", "plain", "redox_syscall 0.8.1", @@ -2410,6 +2860,16 @@ dependencies = [ "glob", ] +[[package]] +name = "libz-ng-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879917b256f6317769b9f374b435805a8697013098aacce5a38ac106cd6a9469" +dependencies = [ + "cmake", + "libc", +] + [[package]] name = "libz-sys" version = "1.1.29" @@ -2446,6 +2906,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" +[[package]] +name = "local-ip-address" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa08fb2b1ec3ea84575e94b489d06d4ce0cbf052d12acd515838f50e3c3d63e3" +dependencies = [ + "libc", + "neli", + "windows-sys 0.61.2", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -2457,9 +2928,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lopdf" @@ -2479,6 +2950,15 @@ dependencies = [ "weezl", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2512,9 +2992,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memmap2" @@ -2525,6 +3005,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2568,9 +3057,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2822,7 +3311,7 @@ checksum = "066cf25f0e8b11ee0df221219010f213ad429855f57c494f995590c861a9a7d8" dependencies = [ "arrayvec", "bit-set", - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "codespan-reporting", @@ -2862,7 +3351,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "jni-sys 0.3.1", "log", "ndk-sys", @@ -2886,6 +3375,53 @@ dependencies = [ "jni-sys 0.3.1", ] +[[package]] +name = "neli" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87" +dependencies = [ + "bitflags 2.13.0", + "byteorder", + "derive_builder", + "getset", + "libc", + "log", + "neli-proc-macros", + "parking_lot", +] + +[[package]] +name = "neli-proc-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d8d08c6e98f20a62417478ebf7be8e1425ec9acecc6f63e22da633f6b71609" +dependencies = [ + "either", + "proc-macro2", + "quote", + "serde", + "syn 2.0.118", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "no_debug" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f23a60c850e1144fc1dd9435152e0cfdc7dd18725350b4243584118013a52a4" + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2923,9 +3459,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2965,7 +3501,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -2999,7 +3535,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "libc", "objc2 0.5.2", @@ -3015,7 +3551,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "objc2 0.6.4", "objc2-core-foundation", "objc2-core-graphics", @@ -3028,7 +3564,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-core-location", @@ -3052,7 +3588,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-foundation 0.2.2", @@ -3064,7 +3600,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "dispatch2", "objc2 0.6.4", ] @@ -3075,7 +3611,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "dispatch2", "objc2 0.6.4", "objc2-core-foundation", @@ -3118,7 +3654,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "dispatch", "libc", @@ -3131,7 +3667,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "objc2 0.6.4", "objc2-core-foundation", ] @@ -3142,7 +3678,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "objc2 0.6.4", "objc2-core-foundation", ] @@ -3165,7 +3701,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-foundation 0.2.2", @@ -3177,7 +3713,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-foundation 0.2.2", @@ -3200,7 +3736,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-cloud-kit", @@ -3232,13 +3768,31 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "objc2 0.5.2", "objc2-core-location", "objc2-foundation 0.2.2", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3246,16 +3800,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] -name = "openssl" -version = "0.10.78" +name = "once_cell_polyfill" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "foreign-types 0.3.2", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -3268,7 +3827,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3279,9 +3838,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.114" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -3324,7 +3883,7 @@ checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", - "http 1.4.0", + "http 1.4.2", "opentelemetry", "reqwest 0.12.28", ] @@ -3335,7 +3894,7 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" dependencies = [ - "http 1.4.0", + "http 1.4.2", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", @@ -3383,6 +3942,30 @@ dependencies = [ "libredox", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.118", +] + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -3469,7 +4052,17 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap 2.14.0", ] @@ -3490,7 +4083,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -3499,6 +4092,286 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pingora" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a75f2ff8e122aa80ab202dc865294fe59cd856c2a5dab2d3df6e122c93b941" +dependencies = [ + "pingora-cache", + "pingora-core", + "pingora-http", + "pingora-load-balancing", + "pingora-proxy", + "pingora-timeout", +] + +[[package]] +name = "pingora-cache" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527735ac204efb9fa3884bfd9224d016c5735fabe1d394ebed145b40e7545b99" +dependencies = [ + "ahash", + "async-trait", + "blake2", + "bstr", + "bytes", + "cf-rustracing", + "cf-rustracing-jaeger", + "hex", + "http 1.4.2", + "httparse", + "httpdate", + "indexmap 1.9.3", + "log", + "lru", + "once_cell", + "parking_lot", + "pingora-core", + "pingora-error", + "pingora-header-serde", + "pingora-http", + "pingora-lru", + "pingora-timeout", + "rand 0.8.6", + "regex", + "rmp", + "rmp-serde", + "serde", + "strum 0.26.3", + "tokio", +] + +[[package]] +name = "pingora-core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a7ffe2f5acf9f94fd255cfd1438866bc9124f8f0c7d42562bd3f853df2094b7" +dependencies = [ + "ahash", + "async-trait", + "brotli", + "bstr", + "bytes", + "chrono", + "clap", + "daemonize", + "daggy", + "derivative", + "flate2", + "futures", + "h2 0.4.15", + "http 1.4.2", + "httparse", + "httpdate", + "libc", + "log", + "nix", + "once_cell", + "openssl-probe", + "ouroboros", + "parking_lot", + "percent-encoding", + "pingora-error", + "pingora-http", + "pingora-pool", + "pingora-runtime", + "pingora-rustls", + "pingora-timeout", + "prometheus 0.13.4", + "rand 0.8.6", + "regex", + "serde", + "serde_yaml", + "sfv", + "socket2 0.6.4", + "strum 0.26.3", + "strum_macros 0.26.4", + "tokio", + "tokio-stream", + "tokio-test", + "unicase", + "windows-sys 0.59.0", + "x509-parser", + "zstd", +] + +[[package]] +name = "pingora-error" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23f7bc013de67e44ed902a82843f6157460b89d11da882bcc6f09f8ae380af1" + +[[package]] +name = "pingora-gateway" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "httpdate", + "mime_guess", + "pingora", + "pingora-http", + "pingora-proxy", + "serde", + "serde_json", + "shared-logging", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "pingora-header-serde" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "828c0e53e74160cbfe8e67dd3a811eb6a253c36acbaf7a39a01d9aacfb9ac139" +dependencies = [ + "bytes", + "http 1.4.2", + "httparse", + "pingora-error", + "pingora-http", + "thread_local", + "zstd", + "zstd-safe", +] + +[[package]] +name = "pingora-http" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d553d310a15ec88107b9388a02885f798efc57764d8e9bdaaf32a76722927a10" +dependencies = [ + "bytes", + "http 1.4.2", + "pingora-error", +] + +[[package]] +name = "pingora-ketama" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2a2e43a14f1d291fba7905542c7c1d1f89528f470b3cd48b6806e702ea772f" +dependencies = [ + "crc32fast", +] + +[[package]] +name = "pingora-load-balancing" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361b69af0234d2e4d10234e2efd106bb3b8147c575d52f45604a46aaf26def7a" +dependencies = [ + "arc-swap", + "async-trait", + "derivative", + "fnv", + "futures", + "http 1.4.2", + "log", + "pingora-core", + "pingora-error", + "pingora-http", + "pingora-ketama", + "pingora-runtime", + "rand 0.8.6", + "tokio", +] + +[[package]] +name = "pingora-lru" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6705a26ad89d241a989a5395641931ba37076f5ab5fbd19ee92402414a43af32" +dependencies = [ + "arrayvec", + "hashbrown 0.17.1", + "parking_lot", + "rand 0.8.6", +] + +[[package]] +name = "pingora-pool" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb1237893b15a9cf6b371bee8d7e2e1c10742e4be6eb00ed38cfe87fd1363f8" +dependencies = [ + "crossbeam-queue", + "log", + "lru", + "parking_lot", + "pingora-timeout", + "thread_local", + "tokio", +] + +[[package]] +name = "pingora-proxy" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a92ee756ecf6ecb6419864da651cad6cecd933b6d420a26877031efa16bef57" +dependencies = [ + "async-trait", + "bytes", + "clap", + "futures", + "h2 0.4.15", + "http 1.4.2", + "log", + "once_cell", + "pingora-cache", + "pingora-core", + "pingora-error", + "pingora-http", + "rand 0.8.6", + "regex", + "tokio", +] + +[[package]] +name = "pingora-runtime" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41815a13691a3e7d9ad0e34767d4140284132e31b95a4481f5e73ab6f407f834" +dependencies = [ + "once_cell", + "rand 0.8.6", + "thread_local", + "tokio", +] + +[[package]] +name = "pingora-rustls" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2c12f20ac2eb8d708763aba7ce3c6a21bfed113b07cee79906225da5a500d28" +dependencies = [ + "log", + "no_debug", + "pingora-error", + "ring", + "rustls", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "tokio-rustls", +] + +[[package]] +name = "pingora-timeout" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e321452eaa461e0b6c5aaa35b7e42527ee89df33710279f37fae7f066b68e" +dependencies = [ + "once_cell", + "parking_lot", + "pin-project-lite", + "thread_local", + "tokio", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -3650,7 +4523,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "crc32fast", "fdeflate", "flate2", @@ -3729,7 +4602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.118", ] [[package]] @@ -3750,12 +4623,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "version_check", + "yansi", +] + [[package]] name = "profiling" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf 2.28.0", + "thiserror 1.0.69", +] + [[package]] name = "prometheus" version = "0.14.0" @@ -3767,15 +4668,15 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "protobuf", + "protobuf 3.7.2", "thiserror 2.0.18", ] [[package]] name = "prost" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" dependencies = [ "bytes", "prost-derive", @@ -3783,17 +4684,23 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.3" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "protobuf" version = "3.7.2" @@ -3848,7 +4755,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls", - "socket2 0.6.3", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -3885,9 +4792,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.6.4", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -4017,7 +4924,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4026,7 +4933,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", ] [[package]] @@ -4046,14 +4953,14 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -4074,9 +4981,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "renderdoc-sys" @@ -4095,7 +5002,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", @@ -4109,7 +5016,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -4138,10 +5045,10 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "http-body-util", - "hyper 1.9.0", + "hyper 1.10.1", "hyper-rustls", "hyper-util", "js-sys", @@ -4167,7 +5074,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -4184,6 +5091,41 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +dependencies = [ + "arrayvec", + "num-traits", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -4205,13 +5147,22 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4224,7 +5175,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "errno", "libc", "linux-raw-sys 0.12.1", @@ -4233,10 +5184,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ + "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -4245,6 +5198,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -4255,10 +5221,19 @@ dependencies = [ ] [[package]] -name = "rustls-pki-types" -version = "1.14.0" +name = "rustls-pemfile" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -4266,10 +5241,11 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.12" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4350,7 +5326,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.118", ] [[package]] @@ -4377,7 +5353,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4427,7 +5403,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -4438,14 +5414,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -4488,11 +5464,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.18.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -4507,14 +5484,27 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.18.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", ] [[package]] @@ -4524,6 +5514,17 @@ dependencies = [ "eframe", ] +[[package]] +name = "sfv" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa1f336066b758b7c9df34ed049c0e693a426afe2b27ff7d5b14f410ab1a132" +dependencies = [ + "base64 0.22.1", + "indexmap 2.14.0", + "rust_decimal", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4596,9 +5597,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -4667,9 +5668,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.1" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "smithay-client-toolkit" @@ -4677,7 +5678,7 @@ version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3457dea1f0eb631b4034d61d4d8c32074caa6cd1ab2d59f2327bd8461e2c0016" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "calloop 0.13.0", "calloop-wayland-source 0.3.0", "cursor-icon", @@ -4702,7 +5703,7 @@ version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0512da38f5e2b31201a93524adb8d3136276fa4fe4aafab4e1f727a82b534cc0" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "calloop 0.14.4", "calloop-wayland-source 0.4.1", "cursor-icon", @@ -4755,9 +5756,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", "windows-sys 0.61.2", @@ -4840,7 +5841,7 @@ dependencies = [ "bytes", "derive_more", "getrandom 0.2.17", - "http 1.4.0", + "http 1.4.2", "log", "rand 0.8.6", "scoped-tls", @@ -4863,7 +5864,7 @@ dependencies = [ "proc-macro2", "quote", "spacetimedb-primitives", - "syn", + "syn 2.0.118", ] [[package]] @@ -4893,7 +5894,7 @@ dependencies = [ "spacetimedb-lib", "spacetimedb-primitives", "spacetimedb-sats", - "strum", + "strum 0.25.0", "thiserror 1.0.69", ] @@ -4919,7 +5920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523a8d4a746bb4403fe3e5241e3a72204fc1358e3b118b4f827de7673b6aabcb" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "blake3", "bytes", "chrono", @@ -4927,7 +5928,7 @@ dependencies = [ "enum-as-inner", "enum-map", "hex", - "itertools", + "itertools 0.12.1", "log", "serde", "spacetimedb-bindings-macro", @@ -4954,9 +5955,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "226d91f133dcb792dd04ec3870828c4c1d7815a33646e8226894087f1680f8a9" dependencies = [ "arrayvec", - "itertools", + "itertools 0.12.1", "paste", - "prometheus", + "prometheus 0.14.0", ] [[package]] @@ -4965,10 +5966,10 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f625481d6715f9b0aba612599be6c4ab1028ab99425d23d75a268a49628a43f8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "either", "enum-as-inner", - "itertools", + "itertools 0.12.1", "nohash-hasher", "spacetimedb-memory-usage", ] @@ -4990,7 +5991,7 @@ checksum = "449ff63e22853eeaf903563f3cfaf8557ba0c84d0a62abcc388ac429670d321c" dependencies = [ "anyhow", "arrayvec", - "bitflags 2.11.1", + "bitflags 2.13.0", "bytemuck", "bytes", "bytestring", @@ -5000,7 +6001,7 @@ dependencies = [ "enum-as-inner", "ethnum", "hex", - "itertools", + "itertools 0.12.1", "lean_string", "rand 0.9.4", "second-stack", @@ -5028,10 +6029,10 @@ dependencies = [ "enum-map", "indexmap 2.14.0", "insta", - "itertools", + "itertools 0.12.1", "lazy_static", "lean_string", - "petgraph", + "petgraph 0.6.5", "serde_json", "smallvec", "spacetimedb-data-structures", @@ -5060,11 +6061,11 @@ dependencies = [ "futures", "futures-channel", "home", - "http 1.4.0", + "http 1.4.2", "log", "native-tls", "once_cell", - "prometheus", + "prometheus 0.14.0", "rand 0.9.4", "spacetimedb-client-api-messages", "spacetimedb-data-structures", @@ -5111,6 +6112,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strip-ansi-escapes" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025" +dependencies = [ + "vte", +] + [[package]] name = "strsim" version = "0.11.1" @@ -5123,7 +6133,16 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", ] [[package]] @@ -5136,7 +6155,20 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.118", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.118", ] [[package]] @@ -5147,9 +6179,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.117" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -5179,7 +6222,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5255,7 +6298,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5266,7 +6309,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5278,6 +6321,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift_codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d957f535b242b91aa9f47bde08080f9a6fef276477e55b0079979d002759d5" +dependencies = [ + "byteorder", + "trackable", +] + [[package]] name = "tiff" version = "0.11.3" @@ -5294,12 +6347,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde_core", @@ -5309,15 +6361,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -5350,9 +6402,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -5360,7 +6412,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.3", + "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", ] @@ -5373,7 +6425,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5407,6 +6459,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-tungstenite" version = "0.27.0" @@ -5461,9 +6524,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.11+spec-1.1.0" +version = "0.25.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" dependencies = [ "indexmap 2.14.0", "serde_core", @@ -5498,7 +6561,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", "http-body-util", "percent-encoding", @@ -5539,21 +6602,21 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "bytes", "futures-util", - "http 1.4.0", + "http 1.4.2", "http-body 1.0.1", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -5588,7 +6651,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -5646,6 +6709,25 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trackable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15bd114abb99ef8cee977e517c8f37aee63f184f2d08e3e6ceca092373369ae" +dependencies = [ + "trackable_derive", +] + +[[package]] +name = "trackable_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -5666,7 +6748,7 @@ checksum = "eadc29d668c91fcc564941132e17b28a7ceb2f3ebf0b9dae3e03fd7a6748eb0d" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.2", "httparse", "log", "native-tls", @@ -5687,7 +6769,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http 1.4.0", + "http 1.4.2", "httparse", "log", "rand 0.9.4", @@ -5715,9 +6797,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.20.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicase" @@ -5742,9 +6824,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" [[package]] name = "unicode-width" @@ -5758,6 +6840,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -5795,10 +6883,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] -name = "uuid" -version = "1.23.1" +name = "utf8parse" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -5824,6 +6918,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vte" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077" +dependencies = [ + "memchr", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5851,9 +6954,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.3+wasi-0.2.9" +version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ "wit-bindgen 0.57.1", ] @@ -5869,9 +6972,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -5882,9 +6985,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -5892,9 +6995,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5902,22 +7005,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] @@ -5963,7 +7066,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -5989,7 +7092,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "rustix 1.1.4", "wayland-backend", "wayland-scanner", @@ -6001,7 +7104,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "625c5029dbd43d25e6aa9615e88b829a5cad13b2819c4ae129fdbb7c31ab4c7e" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cursor-icon", "wayland-backend", ] @@ -6023,7 +7126,7 @@ version = "0.32.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "wayland-backend", "wayland-client", "wayland-scanner", @@ -6035,7 +7138,7 @@ version = "20250721.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40a1f863128dcaaec790d7b4b396cc9b9a7a079e878e18c47e6c2d2c5a8dcbb1" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "wayland-backend", "wayland-client", "wayland-protocols", @@ -6048,7 +7151,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9567599ef23e09b8dad6e429e5738d4509dfc46b3b21f32841a304d16b29c8" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "wayland-backend", "wayland-client", "wayland-protocols", @@ -6061,7 +7164,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b6d8cf1eb2c1c31ed1f5643c88a6e53538129d4af80030c8cabd1f9fa884d91" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "wayland-backend", "wayland-client", "wayland-protocols", @@ -6074,7 +7177,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "wayland-backend", "wayland-client", "wayland-protocols", @@ -6106,9 +7209,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6156,14 +7259,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -6181,7 +7284,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfe68bac7cde125de7a731c3400723cadaaf1703795ad3f4805f187459cd7a77" dependencies = [ "arrayvec", - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "document-features", @@ -6206,7 +7309,7 @@ dependencies = [ "arrayvec", "bit-set", "bit-vec", - "bitflags 2.11.1", + "bitflags 2.13.0", "bytemuck", "cfg_aliases", "document-features", @@ -6242,7 +7345,7 @@ version = "27.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b21cb61c57ee198bc4aff71aeadff4cbb80b927beb912506af9c780d64313ce" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "cfg-if", "cfg_aliases", "libloading", @@ -6262,7 +7365,7 @@ version = "27.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afdcf84c395990db737f2dd91628706cb31e86d72e53482320d368e52b5da5eb" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "bytemuck", "js-sys", "log", @@ -6300,7 +7403,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6311,7 +7414,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6365,6 +7468,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -6398,13 +7510,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -6417,6 +7546,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -6429,6 +7564,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -6441,12 +7582,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -6459,6 +7612,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -6471,6 +7630,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -6483,6 +7648,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -6495,6 +7666,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winit" version = "0.30.13" @@ -6504,7 +7681,7 @@ dependencies = [ "ahash", "android-activity", "atomic-waker", - "bitflags 2.11.1", + "bitflags 2.13.0", "block2", "bytemuck", "calloop 0.13.0", @@ -6548,9 +7725,9 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" dependencies = [ "memchr", ] @@ -6601,7 +7778,7 @@ dependencies = [ "heck 0.5.0", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.118", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -6617,7 +7794,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.118", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -6629,7 +7806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.1", + "bitflags 2.13.0", "indexmap 2.14.0", "log", "serde", @@ -6697,6 +7874,23 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xcursor" version = "0.3.10" @@ -6709,7 +7903,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d039de8032a9a8856a6be89cea3e5d12fdd82306ab7c94d74e6deab2460651c5" dependencies = [ - "bitflags 2.11.1", + "bitflags 2.13.0", "dlib", "log", "once_cell", @@ -6729,10 +7923,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f" [[package]] -name = "yoke" -version = "0.8.2" +name = "yansi" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6747,35 +7947,35 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.48" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] name = "zerofrom" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" dependencies = [ "zerofrom-derive", ] @@ -6788,15 +7988,15 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" @@ -6828,7 +8028,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.118", ] [[package]] @@ -6866,6 +8066,34 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index 2a6229c39..a7c331ee5 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -40,6 +40,7 @@ members = [ "crates/platform-wechat", "crates/platform-speech", "crates/platform-agent", + "crates/pingora-gateway", "crates/server-manager-panel", "crates/shared-contracts", "crates/shared-kernel", @@ -88,6 +89,7 @@ platform-llm = { path = "crates/platform-llm", default-features = false } platform-oss = { path = "crates/platform-oss", default-features = false } platform-speech = { path = "crates/platform-speech", default-features = false } platform-wechat = { path = "crates/platform-wechat", default-features = false } +pingora-gateway = { path = "crates/pingora-gateway", default-features = false } shared-contracts = { path = "crates/shared-contracts", default-features = false } shared-kernel = { path = "crates/shared-kernel", default-features = false } shared-logging = { path = "crates/shared-logging", default-features = false } @@ -107,11 +109,16 @@ flate2 = "1" futures-util = "0.3" hmac = "0.12" http-body-util = "0.1" +httpdate = "1" hex = "0.4" image = { version = "0.25", default-features = false } jsonwebtoken = "9" langchainrust = "0.2.18" log = "0.4" +mime_guess = "2.0.5" +pingora = { version = "0.8.1", default-features = false, features = ["proxy", "rustls"] } +pingora-http = { version = "0.8.1", default-features = false } +pingora-proxy = { version = "0.8.1", default-features = false } rand_core = "0.6" reqwest = { version = "0.12", default-features = false } ring = "0.17" diff --git a/server-rs/crates/api-server/src/creative_agent.rs b/server-rs/crates/api-server/src/creative_agent.rs index 65dc28126..f83344861 100644 --- a/server-rs/crates/api-server/src/creative_agent.rs +++ b/server-rs/crates/api-server/src/creative_agent.rs @@ -1431,4 +1431,62 @@ mod tests { assert_eq!(next["workTitle"], json!("轻松家庭拼图")); assert_eq!(next["levels"][0]["pictureDescription"], json!("旧图面")); } + + #[test] + fn draft_edit_writeback_uses_stable_puzzle_profile_from_target_session() { + let target_session_id = "puzzle-session-b1cf819ea2d944c4a7e34ca668f8a97c"; + let profile_id = build_puzzle_result_profile_id(target_session_id); + let draft = json!({ + "workTitle": "雨夜猫街", + "workDescription": "在灯牌下寻找猫的拼图", + "summary": "雨夜猫街拼图", + "themeTags": ["猫", "雨夜"], + "coverImageSrc": "/generated-puzzle-assets/cat.png", + "coverAssetId": "asset-cat", + "levels": [{ + "levelId": "puzzle-level-1", + "levelName": "灯牌下的猫", + "pictureDescription": "一只猫在雨夜灯牌下回头", + "pictureReference": null, + "selectedCandidateId": "candidate-1", + "coverImageSrc": "/generated-puzzle-assets/cat.png", + "coverAssetId": "asset-cat", + "generationStatus": "ready", + "candidates": [] + }] + }); + + let input = build_puzzle_work_update_from_draft(profile_id, "user-1".to_string(), &draft) + .expect("writeback payload should build"); + + assert_eq!( + input.profile_id, + "puzzle-profile-b1cf819ea2d944c4a7e34ca668f8a97c" + ); + assert_eq!(input.owner_user_id, "user-1"); + assert_eq!(input.work_title, "雨夜猫街"); + assert_eq!(input.work_description, "在灯牌下寻找猫的拼图"); + assert_eq!(input.level_name, "灯牌下的猫"); + assert_eq!(input.summary, "雨夜猫街拼图"); + assert_eq!(input.theme_tags, vec!["猫".to_string(), "雨夜".to_string()]); + assert_eq!( + input.cover_image_src.as_deref(), + Some("/generated-puzzle-assets/cat.png") + ); + assert_eq!(input.cover_asset_id.as_deref(), Some("asset-cat")); + + let levels: Vec = serde_json::from_str( + input + .levels_json + .as_deref() + .expect("levels should be serialized"), + ) + .expect("levels json should parse"); + assert_eq!(levels[0]["level_id"], json!("puzzle-level-1")); + assert_eq!(levels[0]["level_name"], json!("灯牌下的猫")); + assert_eq!( + levels[0]["picture_description"], + json!("一只猫在雨夜灯牌下回头") + ); + } } diff --git a/server-rs/crates/api-server/src/external_generation_worker.rs b/server-rs/crates/api-server/src/external_generation_worker.rs index ec90016af..fe110c2ed 100644 --- a/server-rs/crates/api-server/src/external_generation_worker.rs +++ b/server-rs/crates/api-server/src/external_generation_worker.rs @@ -744,6 +744,7 @@ mod tests { started_at: Some("2026-06-03T00:00:00Z".to_string()), completed_at: None, updated_at: "2026-06-03T00:00:00Z".to_string(), + updated_at_micros: 1_748_908_800_000_000, lease_token: lease_token.map(ToOwned::to_owned), } } diff --git a/server-rs/crates/api-server/src/main.rs b/server-rs/crates/api-server/src/main.rs index 38e0f23b9..90af7ba2f 100644 --- a/server-rs/crates/api-server/src/main.rs +++ b/server-rs/crates/api-server/src/main.rs @@ -116,7 +116,7 @@ use tracing::{error, info, warn}; use crate::{ app::{build_router, build_spacetime_unavailable_router}, - config::AppConfig, + config::{AppConfig, ProcessRole}, external_generation_worker::run_external_generation_worker, external_generation_worker_controller::run_external_generation_worker_controller, state::{AppState, AppStateInitError}, @@ -181,13 +181,11 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> { async fn run_worker_only(config: AppConfig) -> Result<(), io::Error> { let process_role = config.process_role; - let state = restore_app_state_for_startup(config) - .await - .map_err(|error| { - io::Error::other(format!( - "初始化 external generation worker 状态失败:{error}" - )) - })?; + let state = build_non_http_app_state_for_startup(config).map_err(|error| { + io::Error::other(format!( + "初始化 external generation worker 状态失败:{error}" + )) + })?; spawn_app_state_background_workers(&state); info!( process_role = process_role.as_str(), @@ -205,6 +203,22 @@ async fn run_worker_only(config: AppConfig) -> Result<(), io::Error> { } } +fn build_non_http_app_state_for_startup( + config: AppConfig, +) -> Result { + let process_role = config.process_role; + debug_assert!(!should_restore_auth_store_for_startup(process_role)); + info!( + process_role = process_role.as_str(), + "非 HTTP 进程跳过 SpacetimeDB 认证快照恢复" + ); + AppState::new_with_empty_auth_store(config) +} + +fn should_restore_auth_store_for_startup(process_role: ProcessRole) -> bool { + process_role.runs_http() +} + async fn run_http_role(config: AppConfig) -> Result<(), io::Error> { let bind_address = config.bind_socket_addr(); let listen_backlog = config.listen_backlog; @@ -520,8 +534,9 @@ fn is_valid_env_key(key: &str) -> bool { mod tests { use super::{ AUTH_STORE_STARTUP_RETRY_INTERVAL, is_valid_env_key, protected_env_keys_from, - strip_env_value, + should_restore_auth_store_for_startup, strip_env_value, }; + use crate::config::ProcessRole; #[test] fn strip_env_value_removes_wrapping_quotes() { @@ -567,4 +582,16 @@ mod tests { fn startup_dependency_retry_interval_is_short_enough_for_service_recovery() { assert_eq!(AUTH_STORE_STARTUP_RETRY_INTERVAL.as_secs(), 5); } + + #[test] + fn auth_store_startup_restore_is_limited_to_http_roles() { + assert!(should_restore_auth_store_for_startup(ProcessRole::Api)); + assert!(should_restore_auth_store_for_startup(ProcessRole::All)); + assert!(!should_restore_auth_store_for_startup( + ProcessRole::ExternalGenerationWorker + )); + assert!(!should_restore_auth_store_for_startup( + ProcessRole::ExternalGenerationController + )); + } } diff --git a/server-rs/crates/api-server/src/puzzle.rs b/server-rs/crates/api-server/src/puzzle.rs index cc9d0237e..e7f5ca4b1 100644 --- a/server-rs/crates/api-server/src/puzzle.rs +++ b/server-rs/crates/api-server/src/puzzle.rs @@ -25,6 +25,7 @@ use platform_oss::{OssHeadObjectRequest, OssObjectAccess, OssPutObjectRequest}; use serde_json::{Value, json}; use shared_contracts::{ creation_audio::CreationAudioAsset, + external_generation::{ExternalGenerationJobStatus, ExternalGenerationJobStatusRecord}, puzzle_agent::{ CreatePuzzleAgentSessionRequest, ExecutePuzzleAgentActionRequest, PuzzleAgentActionResponse, PuzzleAgentMessageResponse, PuzzleAgentOperationResponse, @@ -52,7 +53,7 @@ use shared_contracts::{ }; use shared_kernel::{build_prefixed_uuid_id, format_timestamp_micros}; use spacetime_client::{ - ExternalGenerationJobEnqueueRecordInput, PuzzleAgentMessageRecord, + ExternalGenerationJobEnqueueRecordInput, ExternalGenerationJobRecord, PuzzleAgentMessageRecord, PuzzleAgentMessageSubmitRecordInput, PuzzleAgentSessionCreateRecordInput, PuzzleAgentSessionRecord, PuzzleAgentSuggestedActionRecord, PuzzleAnchorItemRecord, PuzzleAnchorPackRecord, PuzzleAudioAssetRecord, PuzzleBackgroundCompileTaskClaimRecordInput, @@ -238,6 +239,32 @@ fn mark_puzzle_initial_generation_started_snapshot( session } +fn map_puzzle_queue_job_status( + job: &ExternalGenerationJobRecord, + fallback_progress: u32, +) -> ExternalGenerationJobStatusRecord { + let (status, phase_detail, default_progress) = match job.status.as_str() { + "completed" => (ExternalGenerationJobStatus::Completed, "生成已完成。", 100), + "running" => (ExternalGenerationJobStatus::Running, "正在生成。", 35), + "failed" => (ExternalGenerationJobStatus::Failed, "生成失败。", 0), + _ => (ExternalGenerationJobStatus::Queued, "排队中。", 8), + }; + + ExternalGenerationJobStatusRecord { + operation_id: job.job_id.clone(), + status, + phase_label: job.request_label.clone(), + phase_detail: phase_detail.to_string(), + progress: fallback_progress + .max(default_progress) + .min(100) + .try_into() + .unwrap_or(100), + error: job.last_error_message.clone(), + updated_at_micros: job.updated_at_micros, + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct ExternalGenerationWriteLeaseGuard { pub(crate) job_id: Option, diff --git a/server-rs/crates/api-server/src/puzzle/handlers.rs b/server-rs/crates/api-server/src/puzzle/handlers.rs index e5e24ddad..721dc729a 100644 --- a/server-rs/crates/api-server/src/puzzle/handlers.rs +++ b/server-rs/crates/api-server/src/puzzle/handlers.rs @@ -718,6 +718,7 @@ pub async fn execute_puzzle_agent_action( phase_detail: "首关草稿生成已在后台处理中。".to_string(), progress: session.progress_percent.max(10), error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, @@ -774,6 +775,7 @@ pub async fn execute_puzzle_agent_action( }, progress: 100, error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, @@ -861,7 +863,7 @@ pub async fn execute_puzzle_agent_action( Some(&request_context), PuzzleAgentActionResponse { operation: PuzzleAgentOperationResponse { - operation_id: job.job_id, + operation_id: job.job_id.clone(), operation_type: "compile_puzzle_draft".to_string(), status: status.to_string(), phase_label: "首关拼图草稿".to_string(), @@ -871,7 +873,8 @@ pub async fn execute_puzzle_agent_action( "首关草稿编译已进入后台队列。".to_string() }, progress, - error: job.last_error_message, + error: job.last_error_message.clone(), + queue_state: Some(map_puzzle_queue_job_status(&job, progress)), }, session: map_puzzle_agent_session_response(session), }, @@ -1005,6 +1008,7 @@ pub async fn execute_puzzle_agent_action( phase_detail: "关卡图片生成已完成。".to_string(), progress: 100, error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, @@ -1071,13 +1075,14 @@ pub async fn execute_puzzle_agent_action( Some(&request_context), PuzzleAgentActionResponse { operation: PuzzleAgentOperationResponse { - operation_id: job.job_id, + operation_id: job.job_id.clone(), operation_type: "generate_puzzle_images".to_string(), status: status.to_string(), phase_label: "拼图图片生成".to_string(), phase_detail: "关卡图片生成已进入后台队列。".to_string(), progress, - error: job.last_error_message, + error: job.last_error_message.clone(), + queue_state: Some(map_puzzle_queue_job_status(&job, progress)), }, session: map_puzzle_agent_session_response(session), }, @@ -1145,6 +1150,7 @@ pub async fn execute_puzzle_agent_action( phase_detail: "拼图 UI 背景图生成已完成。".to_string(), progress: 100, error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, @@ -1211,13 +1217,14 @@ pub async fn execute_puzzle_agent_action( Some(&request_context), PuzzleAgentActionResponse { operation: PuzzleAgentOperationResponse { - operation_id: job.job_id, + operation_id: job.job_id.clone(), operation_type: "generate_puzzle_ui_background".to_string(), status: status.to_string(), phase_label: "UI 背景图生成".to_string(), phase_detail: "拼图 UI 背景图生成已进入后台队列。".to_string(), progress, - error: job.last_error_message, + error: job.last_error_message.clone(), + queue_state: Some(map_puzzle_queue_job_status(&job, progress)), }, session: map_puzzle_agent_session_response(session), }, @@ -1390,6 +1397,7 @@ pub async fn execute_puzzle_agent_action( phase_detail: "拼图作品已发布到广场。".to_string(), progress: 100, error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, @@ -1416,6 +1424,7 @@ pub async fn execute_puzzle_agent_action( phase_detail: phase_detail.to_string(), progress: 100, error: None, + queue_state: None, }, session: map_puzzle_agent_session_response(session), }, diff --git a/server-rs/crates/pingora-gateway/Cargo.toml b/server-rs/crates/pingora-gateway/Cargo.toml new file mode 100644 index 000000000..1c159c717 --- /dev/null +++ b/server-rs/crates/pingora-gateway/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pingora-gateway" +edition.workspace = true +version.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +httpdate = { workspace = true } +mime_guess = { workspace = true } +pingora = { workspace = true } +pingora-http = { workspace = true } +pingora-proxy = { workspace = true } +shared-logging = { workspace = true } +tokio = { workspace = true, features = ["fs", "io-util"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["v4"] } + +[dev-dependencies] +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/server-rs/crates/pingora-gateway/src/main.rs b/server-rs/crates/pingora-gateway/src/main.rs new file mode 100644 index 000000000..4978a84c4 --- /dev/null +++ b/server-rs/crates/pingora-gateway/src/main.rs @@ -0,0 +1,3950 @@ +use std::{ + collections::HashMap, + env, + fs::OpenOptions, + io, + io::SeekFrom, + io::Write, + net::SocketAddr, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use httpdate::{fmt_http_date, parse_http_date}; +use pingora::{ + ConnectTimedout, ConnectionClosed, Error, ErrorSource, HTTPStatus, ReadError, ReadTimedout, + Result as PingoraResult, WriteError, WriteTimedout, + listeners::tls::TlsSettings, + modules::http::{HttpModule, HttpModuleBuilder, HttpModules, Module}, + prelude::HttpPeer, + protocols::http::compression::ResponseCompressionCtx, + server::{Server, configuration::Opt}, +}; +use pingora_http::{Method, RequestHeader, ResponseHeader}; +use pingora_proxy::{FailToProxy, ProxyHttp, Session, http_proxy_service}; +use shared_logging::{OtelConfig, init_tracing}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tracing::{error, info, warn}; +use uuid::Uuid; + +const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:18081"; +const DEFAULT_API_UPSTREAM: &str = "127.0.0.1:8082"; +const DEFAULT_SPACETIME_UPSTREAM: &str = "127.0.0.1:3101"; +const DEFAULT_WEB_ROOT: &str = "/srv/genarrative/web"; +const DEFAULT_ACME_ROOT: &str = "/var/www/html"; +const DEFAULT_MAINTENANCE_FILE: &str = "/var/lib/genarrative/maintenance/enabled"; +const DEFAULT_MAX_API_BODY_BYTES: u64 = 64 * 1024 * 1024; +const DEFAULT_GZIP_LEVEL: u32 = 5; +const DEFAULT_GZIP_MIN_LENGTH_BYTES: u64 = 1024; +const DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS: u64 = 3_000; +const DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS: u64 = 60; +const DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_HTML_CACHE_CONTROL: &str = "no-cache"; +const DEFAULT_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable"; +const DEFAULT_STATIC_CACHE_CONTROL: &str = "no-cache"; +const DEFAULT_INSTANCE_COUNT: u32 = 1; +const DEFAULT_LOG_FILTER: &str = "info,pingora=info,pingora_gateway=info"; +const SHADOW_PROBE_PATH: &str = "/__genarrative_pingora/healthz"; +const SHADOW_PROBE_HEADER: &str = "x-genarrative-pingora-probe"; +const PAYLOAD_TOO_LARGE_CONTEXT: &str = "genarrative_payload_too_large"; +const PROTECTION_STATE_TTL: Duration = Duration::from_secs(600); +const PROTECTION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Clone, Debug)] +struct GatewayConfig { + listen_addr: String, + tls_listen_addr: Option, + tls_cert_file: Option, + tls_key_file: Option, + http_redirect_listen_addr: Option, + http_redirect_target_scheme: String, + api_upstream: SocketAddr, + spacetime_upstream: SocketAddr, + gitea_hosts: Vec, + gitea_upstream: Option, + web_root: PathBuf, + acme_root: PathBuf, + maintenance_file: PathBuf, + forwarded_proto: String, + max_api_body_bytes: u64, + gzip_enabled: bool, + gzip_level: u32, + gzip_min_length_bytes: u64, + compression: CompressionConfig, + static_cache: StaticCacheConfig, + upstream_timeouts: UpstreamTimeoutConfig, + shadow_probe_token: Option, + trust_x_forwarded_for: bool, + instance_count: u32, + shared_protection_confirmed: bool, + protection: ProtectionConfig, + log_filter: String, + access_log_file: Option, + otel_enabled: bool, +} + +impl GatewayConfig { + fn from_env() -> io::Result { + let config = Self { + listen_addr: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_LISTEN", + DEFAULT_LISTEN_ADDR, + ), + tls_listen_addr: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN"), + tls_cert_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE") + .map(PathBuf::from), + tls_key_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE") + .map(PathBuf::from), + http_redirect_listen_addr: read_optional_env( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + ), + http_redirect_target_scheme: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME", + "https", + ), + api_upstream: read_socket_addr_env( + "GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM", + DEFAULT_API_UPSTREAM, + )?, + spacetime_upstream: read_socket_addr_env( + "GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM", + DEFAULT_SPACETIME_UPSTREAM, + )?, + gitea_hosts: read_host_list_env("GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS")?, + gitea_upstream: read_optional_socket_addr_env( + "GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM", + )?, + web_root: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT", + DEFAULT_WEB_ROOT, + )), + acme_root: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT", + DEFAULT_ACME_ROOT, + )), + maintenance_file: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE", + DEFAULT_MAINTENANCE_FILE, + )), + forwarded_proto: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO", + "http", + ), + max_api_body_bytes: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES", + DEFAULT_MAX_API_BODY_BYTES, + )?, + gzip_enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED", true), + gzip_level: read_u32_env("GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL", DEFAULT_GZIP_LEVEL)?, + gzip_min_length_bytes: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES", + DEFAULT_GZIP_MIN_LENGTH_BYTES, + )?, + compression: CompressionConfig::from_env()?, + static_cache: StaticCacheConfig::from_env(), + upstream_timeouts: UpstreamTimeoutConfig::from_env()?, + shadow_probe_token: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN"), + trust_x_forwarded_for: read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR", + false, + ), + instance_count: read_u32_env( + "GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT", + DEFAULT_INSTANCE_COUNT, + )?, + shared_protection_confirmed: read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED", + false, + ), + protection: ProtectionConfig::from_env()?, + log_filter: read_env_or_default("GENARRATIVE_PINGORA_GATEWAY_LOG", DEFAULT_LOG_FILTER), + access_log_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE") + .map(PathBuf::from), + otel_enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED", false), + }; + config.validate()?; + Ok(config) + } + + fn validate(&self) -> io::Result<()> { + let listen_addr = + parse_socket_addr_config("GENARRATIVE_PINGORA_GATEWAY_LISTEN", &self.listen_addr)?; + + if listen_addr == self.api_upstream || listen_addr == self.spacetime_upstream { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_LISTEN 不能与上游地址相同", + )); + } + self.validate_gitea_config(listen_addr, None, None)?; + + let tls_listen_addr = if let Some(tls_listen_addr) = &self.tls_listen_addr { + let tls_listen_addr = parse_socket_addr_config( + "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + tls_listen_addr, + )?; + validate_distinct_listen_addr( + "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + tls_listen_addr, + listen_addr, + self.api_upstream, + self.spacetime_upstream, + self.gitea_upstream, + )?; + self.validate_gitea_config(listen_addr, Some(tls_listen_addr), None)?; + validate_tls_file_config(&self.tls_cert_file, &self.tls_key_file)?; + Some(tls_listen_addr) + } else if self.tls_cert_file.is_some() || self.tls_key_file.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 TLS_CERT_FILE / TLS_KEY_FILE 时必须同时设置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + )); + } else { + None + }; + + if let Some(http_redirect_listen_addr) = &self.http_redirect_listen_addr { + let Some(tls_listen_addr) = tls_listen_addr else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN 时必须同时设置 TLS_LISTEN / TLS_CERT_FILE / TLS_KEY_FILE", + )); + }; + let http_redirect_listen_addr = parse_socket_addr_config( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + http_redirect_listen_addr, + )?; + validate_distinct_listen_addr( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + http_redirect_listen_addr, + listen_addr, + self.api_upstream, + self.spacetime_upstream, + self.gitea_upstream, + )?; + if http_redirect_listen_addr == tls_listen_addr { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN 不能与 TLS_LISTEN 相同", + )); + } + self.validate_gitea_config( + listen_addr, + Some(tls_listen_addr), + Some(http_redirect_listen_addr), + )?; + validate_redirect_target_scheme(&self.http_redirect_target_scheme)?; + } + + if self.max_api_body_bytes == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES 必须大于 0", + )); + } + + if self.gzip_level > 9 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL 必须在 0..=9 之间", + )); + } + if self.gzip_min_length_bytes == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES 必须大于 0", + )); + } + self.compression.validate()?; + self.static_cache.validate()?; + self.upstream_timeouts.validate()?; + + if let Some(token) = self.shadow_probe_token.as_deref() { + validate_shadow_probe_token(token)?; + } + + if self.trust_x_forwarded_for + && !read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED", + false, + ) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "启用 GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=true 前必须同时设置 GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=true,确认前置代理会清洗 X-Forwarded-For", + )); + } + + if let Some(access_log_file) = &self.access_log_file { + validate_access_log_file(access_log_file)?; + } + + self.protection.validate()?; + validate_instance_protection_boundary( + self.instance_count, + self.protection.enabled, + self.shared_protection_confirmed, + )?; + Ok(()) + } + + fn validate_gitea_config( + &self, + listen_addr: SocketAddr, + tls_listen_addr: Option, + http_redirect_listen_addr: Option, + ) -> io::Result<()> { + for host in &self.gitea_hosts { + validate_configured_gateway_host("GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS", host)?; + } + + match (self.gitea_hosts.is_empty(), self.gitea_upstream) { + (true, Some(_)) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM 时必须同时设置 GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS", + )); + } + (false, None) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS 时必须同时设置 GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM", + )); + } + _ => {} + } + + let Some(gitea_upstream) = self.gitea_upstream else { + return Ok(()); + }; + if gitea_upstream == self.api_upstream || gitea_upstream == self.spacetime_upstream { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM 不能与 API 或 SpacetimeDB 上游地址相同", + )); + } + if gitea_upstream == listen_addr + || Some(gitea_upstream) == tls_listen_addr + || Some(gitea_upstream) == http_redirect_listen_addr + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM 不能与 Pingora 监听地址相同", + )); + } + Ok(()) + } + + fn matches_gitea_host(&self, host: Option<&str>) -> bool { + let Some(host) = host.and_then(normalize_gateway_host) else { + return false; + }; + self.gitea_hosts.iter().any(|candidate| candidate == &host) + } +} + +#[derive(Clone, Debug)] +struct CompressionConfig { + gzip: bool, + raw_algorithms: String, +} + +impl CompressionConfig { + fn from_env() -> io::Result { + let raw_algorithms = + read_env_or_default("GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS", "gzip"); + Self::parse(raw_algorithms) + } + + fn parse(raw_algorithms: String) -> io::Result { + let mut gzip = false; + + for algorithm in raw_algorithms + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + match algorithm.to_ascii_lowercase().as_str() { + "gzip" => gzip = true, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS 当前只支持 gzip,收到 {algorithm:?}" + ), + )); + } + } + } + + Ok(Self { + gzip, + raw_algorithms, + }) + } + + fn validate(&self) -> io::Result<()> { + if !self.gzip && !self.raw_algorithms.trim().is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS 未启用任何受支持算法", + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct UpstreamTimeoutConfig { + connect_timeout_ms: u64, + default_read_timeout_seconds: u64, + api_read_timeout_seconds: u64, + long_read_timeout_seconds: u64, + write_timeout_seconds: u64, +} + +impl UpstreamTimeoutConfig { + fn from_env() -> io::Result { + Ok(Self { + connect_timeout_ms: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS", + DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + )?, + default_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + )?, + api_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + )?, + long_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + )?, + write_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + )?, + }) + } + + fn validate(&self) -> io::Result<()> { + for (name, value) in [ + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS", + self.connect_timeout_ms, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS", + self.default_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS", + self.api_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS", + self.long_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS", + self.write_timeout_seconds, + ), + ] { + if value == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} 必须大于 0"), + )); + } + } + + Ok(()) + } + + fn read_timeout_for_route(&self, _route: &RouteDecision, path: &str) -> Duration { + let seconds = if is_long_upstream_route(path) { + self.long_read_timeout_seconds + } else if is_generic_api_proxy_path(path) { + self.api_read_timeout_seconds + } else { + self.default_read_timeout_seconds + }; + Duration::from_secs(seconds) + } + + fn connect_timeout(&self) -> Duration { + Duration::from_millis(self.connect_timeout_ms) + } + + fn write_timeout(&self) -> Duration { + Duration::from_secs(self.write_timeout_seconds) + } +} + +#[derive(Clone, Debug)] +struct StaticCacheConfig { + html_cache_control: String, + asset_cache_control: String, + static_cache_control: String, +} + +impl StaticCacheConfig { + fn from_env() -> Self { + Self { + html_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL", + DEFAULT_HTML_CACHE_CONTROL, + ), + asset_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL", + DEFAULT_ASSET_CACHE_CONTROL, + ), + static_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL", + DEFAULT_STATIC_CACHE_CONTROL, + ), + } + } + + fn validate(&self) -> io::Result<()> { + for (name, value) in [ + ( + "GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL", + self.html_cache_control.as_str(), + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL", + self.asset_cache_control.as_str(), + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL", + self.static_cache_control.as_str(), + ), + ] { + if value.contains('\n') || value.contains('\r') || value.contains('\0') { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} 不能包含换行或 NUL 字符"), + )); + } + } + + Ok(()) + } +} + +#[derive(Clone)] +struct GenarrativeGateway { + config: Arc, + protection: Arc, + mode: GatewayMode, +} + +#[derive(Debug)] +struct RequestContext { + request_id: String, + route: RouteDecision, + request_body_bytes_seen: u64, + protection_class: Option, + protection_client: String, + protection_key: Option, + started_at: Instant, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GatewayMode { + Proxy, + HttpRedirect, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CompressionAlgorithm { + Gzip, +} + +impl CompressionAlgorithm { + fn header_value(self) -> &'static str { + match self { + Self::Gzip => "gzip", + } + } +} + +#[derive(Clone, Copy)] +struct CompressionRequestConfig { + enabled: bool, + gzip: bool, + min_length_bytes: u64, + gzip_level: u32, +} + +struct GatewayResponseCompressionBuilder { + config: CompressionRequestConfig, +} + +struct GatewayResponseCompression { + config: CompressionRequestConfig, + inner: ResponseCompressionCtx, +} + +impl HttpModuleBuilder for GatewayResponseCompressionBuilder { + fn init(&self) -> Module { + Box::new(GatewayResponseCompression { + config: self.config, + inner: ResponseCompressionCtx::new(self.config.gzip_level, false, false), + }) + } + + fn order(&self) -> i16 { + i16::MIN / 2 + } +} + +#[async_trait] +impl HttpModule for GatewayResponseCompression { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + async fn request_header_filter(&mut self, req: &mut RequestHeader) -> PingoraResult<()> { + normalize_accept_encoding_for_gateway_compression(req, self.config)?; + self.inner.request_filter(req); + Ok(()) + } + + async fn response_header_filter( + &mut self, + resp: &mut ResponseHeader, + _end_of_stream: bool, + ) -> PingoraResult<()> { + if !should_allow_gateway_compression_for_response(resp, self.config.min_length_bytes) { + self.inner.adjust_level(0); + } + self.inner.response_header_filter(resp, _end_of_stream); + Ok(()) + } + + fn response_body_filter( + &mut self, + body: &mut Option, + end_of_stream: bool, + ) -> PingoraResult<()> { + if !self.inner.is_enabled() { + return Ok(()); + } + if let Some(compressed) = self + .inner + .response_body_filter(body.as_ref(), end_of_stream) + { + *body = Some(compressed); + } + Ok(()) + } + + fn response_done_filter(&mut self) -> PingoraResult> { + if !self.inner.is_enabled() { + return Ok(None); + } + Ok(self.inner.response_body_filter(None, true)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProxyTarget { + Api, + Spacetime, + Gitea, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ProtectionClass { + AdminApi, + GalleryList, + GalleryDetail, + Api, + Spacetime, +} + +impl ProtectionClass { + fn as_str(self) -> &'static str { + match self { + ProtectionClass::AdminApi => "admin_api", + ProtectionClass::GalleryList => "gallery_list", + ProtectionClass::GalleryDetail => "gallery_detail", + ProtectionClass::Api => "api", + ProtectionClass::Spacetime => "spacetime", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RejectReason { + Concurrent, + Rate, +} + +impl RejectReason { + fn code(self) -> &'static str { + match self { + RejectReason::Concurrent => "GATEWAY_CONCURRENCY_LIMITED", + RejectReason::Rate => "GATEWAY_RATE_LIMITED", + } + } + + fn message(self) -> &'static str { + match self { + RejectReason::Concurrent => "服务繁忙,请稍后重试", + RejectReason::Rate => "请求过于频繁,请稍后重试", + } + } +} + +#[derive(Clone, Copy, Debug)] +struct ProtectionClassConfig { + max_concurrent: u32, + rate_per_second: u32, + burst: u32, +} + +#[derive(Clone, Debug)] +struct ProtectionConfig { + enabled: bool, + admin_api: ProtectionClassConfig, + gallery_list: ProtectionClassConfig, + gallery_detail: ProtectionClassConfig, + api: ProtectionClassConfig, + spacetime: ProtectionClassConfig, +} + +impl ProtectionConfig { + fn from_env() -> io::Result { + Ok(Self { + enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED", true), + admin_api: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_ADMIN_API", + 64, + 30, + 16, + )?, + gallery_list: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST", + 320, + 5000, + 4096, + )?, + gallery_detail: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL", + 32, + 300, + 32, + )?, + api: ProtectionClassConfig::from_env("GENARRATIVE_PINGORA_GATEWAY_API", 64, 300, 64)?, + spacetime: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_SPACETIME", + 256, + 1000, + 256, + )?, + }) + } + + fn class_config(&self, class: ProtectionClass) -> ProtectionClassConfig { + match class { + ProtectionClass::AdminApi => self.admin_api, + ProtectionClass::GalleryList => self.gallery_list, + ProtectionClass::GalleryDetail => self.gallery_detail, + ProtectionClass::Api => self.api, + ProtectionClass::Spacetime => self.spacetime, + } + } + + fn validate(&self) -> io::Result<()> { + for (name, config) in [ + ("admin_api", self.admin_api), + ("gallery_list", self.gallery_list), + ("gallery_detail", self.gallery_detail), + ("api", self.api), + ("spacetime", self.spacetime), + ] { + config.validate(name)?; + } + Ok(()) + } +} + +impl ProtectionClassConfig { + fn from_env( + prefix: &str, + max_concurrent: u32, + rate_per_second: u32, + burst: u32, + ) -> io::Result { + Ok(Self { + max_concurrent: read_u32_env(&format!("{prefix}_MAX_CONCURRENT"), max_concurrent)?, + rate_per_second: read_u32_env(&format!("{prefix}_RATE_PER_SECOND"), rate_per_second)?, + burst: read_u32_env(&format!("{prefix}_BURST"), burst)?, + }) + } + + fn is_disabled(&self) -> bool { + self.max_concurrent == 0 && self.rate_per_second == 0 + } + + fn bucket_capacity(&self) -> u32 { + self.rate_per_second.saturating_add(self.burst).max(1) + } + + fn validate(&self, name: &str) -> io::Result<()> { + if self.burst > 0 && self.rate_per_second == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{name} 配置了 burst 但 RATE_PER_SECOND=0;若要关闭 RPS,请同时把 BURST 设为 0" + ), + )); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticRoot { + Web, + Acme, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticMode { + Exact, + SpaFallback, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum LocalResponse { + RedirectPermanent { location: &'static str }, + HttpToHttpsRedirect, + ShadowProbe, + Static { root: StaticRoot, mode: StaticMode }, + NotFound, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RouteDecision { + Proxy { + target: ProxyTarget, + body_limit: Option, + }, + Local(LocalResponse), +} + +impl RouteDecision { + fn applies_maintenance_gate(&self) -> bool { + matches!( + self, + RouteDecision::Proxy { + target: ProxyTarget::Api | ProxyTarget::Spacetime, + .. + } | RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + .. + }) + ) + } + + fn is_api_like(&self) -> bool { + matches!( + self, + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } + ) + } + + fn proxy_target(&self) -> Option { + match self { + RouteDecision::Proxy { target, .. } => Some(*target), + RouteDecision::Local(_) => None, + } + } + + fn body_limit(&self) -> Option { + match self { + RouteDecision::Proxy { body_limit, .. } => *body_limit, + RouteDecision::Local(_) => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct ProtectionKey { + class: ProtectionClass, + client: String, +} + +#[derive(Debug)] +struct ClientProtectionState { + in_flight: u32, + tokens: f64, + last_refill: Instant, + last_seen: Instant, +} + +#[derive(Debug)] +struct ProtectionState { + clients: HashMap, + last_cleanup: Instant, +} + +struct GatewayProtection { + config: ProtectionConfig, + state: Mutex, +} + +impl GatewayProtection { + fn new(config: ProtectionConfig) -> Self { + Self { + config, + state: Mutex::new(ProtectionState { + clients: HashMap::new(), + last_cleanup: Instant::now(), + }), + } + } + + fn try_acquire( + self: &Arc, + class: ProtectionClass, + client: String, + ) -> Result, RejectReason> { + if !self.config.enabled { + return Ok(None); + } + + let class_config = self.config.class_config(class); + if class_config.is_disabled() { + return Ok(None); + } + + let now = Instant::now(); + let key = ProtectionKey { class, client }; + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + cleanup_protection_state(&mut state, now); + + let entry = state + .clients + .entry(key.clone()) + .or_insert_with(|| ClientProtectionState { + in_flight: 0, + tokens: class_config.bucket_capacity() as f64, + last_refill: now, + last_seen: now, + }); + refill_tokens(entry, class_config, now); + entry.last_seen = now; + + if class_config.max_concurrent > 0 && entry.in_flight >= class_config.max_concurrent { + return Err(RejectReason::Concurrent); + } + + if class_config.rate_per_second > 0 && entry.tokens < 1.0 { + return Err(RejectReason::Rate); + } + + if class_config.rate_per_second > 0 { + entry.tokens -= 1.0; + } + entry.in_flight = entry.in_flight.saturating_add(1); + + Ok(Some(key)) + } + + fn release(&self, key: &ProtectionKey) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(entry) = state.clients.get_mut(key) { + entry.in_flight = entry.in_flight.saturating_sub(1); + entry.last_seen = Instant::now(); + } + } +} + +fn refill_tokens( + entry: &mut ClientProtectionState, + class_config: ProtectionClassConfig, + now: Instant, +) { + if class_config.rate_per_second == 0 { + entry.last_refill = now; + return; + } + + let elapsed = now.saturating_duration_since(entry.last_refill); + let refill = elapsed.as_secs_f64() * class_config.rate_per_second as f64; + if refill > 0.0 { + entry.tokens = (entry.tokens + refill).min(class_config.bucket_capacity() as f64); + entry.last_refill = now; + } +} + +fn cleanup_protection_state(state: &mut ProtectionState, now: Instant) { + if now.saturating_duration_since(state.last_cleanup) < PROTECTION_CLEANUP_INTERVAL { + return; + } + + state.clients.retain(|_, entry| { + entry.in_flight > 0 + || now.saturating_duration_since(entry.last_seen) <= PROTECTION_STATE_TTL + }); + state.last_cleanup = now; +} + +fn main() -> std::result::Result<(), Box> { + let config = Arc::new(GatewayConfig::from_env()?); + init_tracing( + &config.log_filter, + OtelConfig { + enabled: config.otel_enabled, + }, + )?; + + info!( + listen = %config.listen_addr, + tls_listen = ?config.tls_listen_addr, + http_redirect_listen = ?config.http_redirect_listen_addr, + api_upstream = %config.api_upstream, + spacetime_upstream = %config.spacetime_upstream, + gitea_hosts = ?config.gitea_hosts, + gitea_upstream = ?config.gitea_upstream, + web_root = %config.web_root.display(), + "Pingora shadow gateway 启动" + ); + + let opt = Opt::parse_args(); + let mut server = Server::new(Some(opt))?; + server.bootstrap(); + + let protection = Arc::new(GatewayProtection::new(config.protection.clone())); + let gateway = GenarrativeGateway { + config: config.clone(), + protection: protection.clone(), + mode: GatewayMode::Proxy, + }; + let listen_addr = gateway.config.listen_addr.clone(); + let mut proxy = http_proxy_service(&server.configuration, gateway); + proxy.add_tcp(&listen_addr); + if let Some(tls_listen_addr) = config.tls_listen_addr.as_deref() { + let cert_path = config + .tls_cert_file + .as_deref() + .expect("TLS cert file validated") + .to_string_lossy() + .into_owned(); + let key_path = config + .tls_key_file + .as_deref() + .expect("TLS key file validated") + .to_string_lossy() + .into_owned(); + let mut tls_settings = TlsSettings::intermediate(&cert_path, &key_path)?; + tls_settings.enable_h2(); + proxy.add_tls_with_settings(tls_listen_addr, None, tls_settings); + } + server.add_service(proxy); + + if let Some(http_redirect_listen_addr) = config.http_redirect_listen_addr.clone() { + let redirect_gateway = GenarrativeGateway { + config, + protection, + mode: GatewayMode::HttpRedirect, + }; + let mut redirect_service = http_proxy_service(&server.configuration, redirect_gateway); + redirect_service.add_tcp(&http_redirect_listen_addr); + server.add_service(redirect_service); + } + + server.run_forever(); +} + +#[async_trait] +impl ProxyHttp for GenarrativeGateway { + type CTX = RequestContext; + + fn init_downstream_modules(&self, modules: &mut HttpModules) { + let gzip_level = if self.config.gzip_enabled { + self.config.gzip_level + } else { + 0 + }; + modules.add_module(Box::new(GatewayResponseCompressionBuilder { + config: CompressionRequestConfig { + enabled: gzip_level > 0, + gzip: self.config.compression.gzip, + min_length_bytes: self.config.gzip_min_length_bytes, + gzip_level, + }, + })); + } + + fn new_ctx(&self) -> Self::CTX { + RequestContext { + request_id: Uuid::new_v4().to_string(), + route: RouteDecision::Local(LocalResponse::NotFound), + request_body_bytes_seen: 0, + protection_class: None, + protection_client: String::new(), + protection_key: None, + started_at: Instant::now(), + } + } + + async fn request_filter( + &self, + session: &mut Session, + ctx: &mut Self::CTX, + ) -> PingoraResult + where + Self::CTX: Send + Sync, + { + let path = session.req_header().uri.path(); + let host = host_or_authority(session); + ctx.route = self.classify_request(host.as_deref(), path); + apply_configured_body_limit(&mut ctx.route, self.config.max_api_body_bytes); + ctx.request_id = resolve_request_id(session); + + if self.is_maintenance_enabled() && ctx.route.applies_maintenance_gate() { + respond_maintenance(session, ctx.route.is_api_like(), &self.config.web_root).await?; + return Ok(true); + } + + if let RouteDecision::Proxy { body_limit, .. } = ctx.route + && let Some(limit) = body_limit + && let Some(content_length) = content_length(session) + && content_length > limit + { + respond_json(session, 413, payload_too_large_body()).await?; + return Ok(true); + } + + if let Some(protection_class) = protection_class_for_route(&ctx.route, path) { + let protection_client = + protection_client_id(session, self.config.trust_x_forwarded_for); + match self + .protection + .try_acquire(protection_class, protection_client.clone()) + { + Ok(key) => { + ctx.protection_class = Some(protection_class); + ctx.protection_client = protection_client; + ctx.protection_key = key; + } + Err(reason) => { + ctx.protection_class = Some(protection_class); + ctx.protection_client = protection_client; + respond_too_many_requests(session, reason).await?; + return Ok(true); + } + } + } + + match &ctx.route { + RouteDecision::Proxy { .. } => Ok(false), + RouteDecision::Local(LocalResponse::RedirectPermanent { location }) => { + respond_redirect(session, location).await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) => { + respond_http_to_https_redirect( + session, + self.config.http_redirect_target_scheme.as_str(), + ) + .await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::ShadowProbe) => { + if !self.is_shadow_probe_authorized(session) { + respond_not_found(session).await?; + return Ok(true); + } + + respond_shadow_probe(session, self.is_maintenance_enabled()).await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::Static { root, mode }) => { + let root_path = match root { + StaticRoot::Web => &self.config.web_root, + StaticRoot::Acme => &self.config.acme_root, + }; + serve_static(session, root_path, *root, *mode, &self.config.static_cache).await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::NotFound) => { + respond_not_found(session).await?; + Ok(true) + } + } + } + + async fn request_body_filter( + &self, + session: &mut Session, + body: &mut Option, + _end_of_stream: bool, + ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + let Some(limit) = ctx.route.body_limit() else { + return Ok(()); + }; + + if let Some(body) = body.as_ref() { + ctx.request_body_bytes_seen = ctx + .request_body_bytes_seen + .saturating_add(body.len() as u64); + } + + if ctx.request_body_bytes_seen > limit { + respond_json(session, 413, payload_too_large_body()).await?; + return Error::e_explain(HTTPStatus(413), PAYLOAD_TOO_LARGE_CONTEXT); + } + + Ok(()) + } + + async fn upstream_peer( + &self, + _session: &mut Session, + ctx: &mut Self::CTX, + ) -> PingoraResult> { + let upstream = match ctx.route.proxy_target().unwrap_or(ProxyTarget::Api) { + ProxyTarget::Api => self.config.api_upstream, + ProxyTarget::Spacetime => self.config.spacetime_upstream, + ProxyTarget::Gitea => self + .config + .gitea_upstream + .expect("Gitea route requires configured upstream"), + }; + + let mut peer = HttpPeer::new(upstream, false, String::new()); + peer.options.connection_timeout = Some(self.config.upstream_timeouts.connect_timeout()); + peer.options.read_timeout = Some( + self.config + .upstream_timeouts + .read_timeout_for_route(&ctx.route, _session.req_header().uri.path()), + ); + peer.options.write_timeout = Some(self.config.upstream_timeouts.write_timeout()); + + Ok(Box::new(peer)) + } + + async fn upstream_request_filter( + &self, + session: &mut Session, + upstream_request: &mut pingora_http::RequestHeader, + ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + upstream_request.insert_header("X-Request-Id", ctx.request_id.as_str())?; + upstream_request + .insert_header("X-Forwarded-Proto", self.config.forwarded_proto.as_str())?; + + if let Some(host) = host_or_authority(session) { + upstream_request.insert_header("Host", host.clone())?; + upstream_request.insert_header("X-Forwarded-Host", host)?; + } + + if let Some(client_ip) = client_ip(session) { + upstream_request.insert_header("X-Real-IP", client_ip.as_str())?; + let forwarded_for = append_forwarded_for(session, &client_ip); + upstream_request.insert_header("X-Forwarded-For", forwarded_for.as_str())?; + } + + // 中文注释:SpacetimeDB 订阅走 WebSocket Upgrade;普通 API 连接头保持干净,贴近当前 Nginx 模板。 + if ctx.route.proxy_target() == Some(ProxyTarget::Spacetime) && is_upgrade_request(session) { + upstream_request.insert_header("Connection", "Upgrade")?; + } else { + upstream_request.remove_header("connection"); + } + + Ok(()) + } + + async fn response_filter( + &self, + _session: &mut Session, + upstream_response: &mut ResponseHeader, + ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + upstream_response.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + if should_disable_accel_buffering(&ctx.route) { + upstream_response.insert_header("X-Accel-Buffering", "no")?; + } + Ok(()) + } + + async fn logging( + &self, + session: &mut Session, + error: Option<&pingora::Error>, + ctx: &mut Self::CTX, + ) where + Self::CTX: Send + Sync, + { + let status = session + .response_written() + .map_or(0, |response| response.status.as_u16()); + let elapsed_ms = ctx.started_at.elapsed().as_millis(); + let method = session.req_header().method.as_str(); + let path = session.req_header().uri.path(); + let uri = session.req_header().uri.to_string(); + let host = header_value(session, "host").unwrap_or_default(); + let client_ip = client_ip(session).unwrap_or_default(); + let content_length = content_length(session); + let protection_class = ctx.protection_class.map(|class| class.as_str()); + let protection_client = ctx.protection_client.as_str(); + let proxy_target = ctx + .route + .proxy_target() + .map(|target| format!("{target:?}")) + .unwrap_or_else(|| "Local".to_string()); + let upstream = match ctx.route.proxy_target() { + Some(ProxyTarget::Api) => self.config.api_upstream.to_string(), + Some(ProxyTarget::Spacetime) => self.config.spacetime_upstream.to_string(), + Some(ProxyTarget::Gitea) => self + .config + .gitea_upstream + .map(|upstream| upstream.to_string()) + .unwrap_or_default(), + None => String::new(), + }; + let route = format!("{:?}", ctx.route); + if let Some(key) = ctx.protection_key.take() { + self.protection.release(&key); + } + + write_access_log( + self.config.access_log_file.as_deref(), + AccessLogRecord { + request_id: &ctx.request_id, + method, + path, + uri: &uri, + host: &host, + client_ip: &client_ip, + status, + route: &route, + proxy_target: &proxy_target, + upstream: &upstream, + content_length, + body_bytes_seen: ctx.request_body_bytes_seen, + protection_class, + protection_client, + elapsed_ms, + error: error.map(ToString::to_string), + }, + ); + + if let Some(error) = error { + if is_payload_too_large_error(error) { + warn!( + request_id = %ctx.request_id, + method, + path, + uri = %uri, + host = %host, + client_ip = %client_ip, + status, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, + elapsed_ms, + %error, + "Pingora gateway request rejected" + ); + } else { + error!( + request_id = %ctx.request_id, + method, + path, + uri = %uri, + host = %host, + client_ip = %client_ip, + status, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, + elapsed_ms, + %error, + "Pingora gateway request failed" + ); + } + } else { + info!( + request_id = %ctx.request_id, + method, + path, + uri = %uri, + host = %host, + client_ip = %client_ip, + status, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, + elapsed_ms, + "Pingora gateway request completed" + ); + } + } + + fn suppress_error_log( + &self, + _session: &Session, + _ctx: &Self::CTX, + error: &pingora::Error, + ) -> bool { + is_payload_too_large_error(error) + } + + async fn fail_to_proxy( + &self, + session: &mut Session, + error: &pingora::Error, + ctx: &mut Self::CTX, + ) -> FailToProxy + where + Self::CTX: Send + Sync, + { + if is_payload_too_large_error(error) { + if session.response_written().is_none() { + respond_json(session, 413, payload_too_large_body()) + .await + .unwrap_or_else(|error| { + warn!(%error, "请求体超限响应写入失败"); + }); + } + + return FailToProxy { + error_code: 413, + can_reuse_downstream: false, + }; + } + + let status = match error.etype() { + ConnectTimedout | ReadTimedout | WriteTimedout => 504, + HTTPStatus(code) => *code, + _ => match error.esource() { + ErrorSource::Upstream => 502, + ErrorSource::Downstream => match error.etype() { + WriteError | ReadError | ConnectionClosed => 0, + _ => 400, + }, + ErrorSource::Internal | ErrorSource::Unset => 500, + }, + }; + if status > 0 { + let response_result = if ctx.route.proxy_target().is_some() { + respond_gateway_proxy_error(session, status).await + } else { + session.respond_error(status).await + }; + response_result.unwrap_or_else(|error| { + warn!(%error, "Pingora 错误响应写入失败"); + }); + } + + FailToProxy { + error_code: status, + can_reuse_downstream: false, + } + } +} + +struct AccessLogRecord<'a> { + request_id: &'a str, + method: &'a str, + path: &'a str, + uri: &'a str, + host: &'a str, + client_ip: &'a str, + status: u16, + route: &'a str, + proxy_target: &'a str, + upstream: &'a str, + content_length: Option, + body_bytes_seen: u64, + protection_class: Option<&'static str>, + protection_client: &'a str, + elapsed_ms: u128, + error: Option, +} + +fn write_access_log(path: Option<&Path>, record: AccessLogRecord<'_>) { + let Some(path) = path else { + return; + }; + + if let Some(parent) = path.parent() + && let Err(error) = std::fs::create_dir_all(parent) + { + warn!( + file = %path.display(), + %error, + "Pingora access log 目录创建失败" + ); + return; + } + + let line = format_access_log_line(&record); + match OpenOptions::new().create(true).append(true).open(path) { + Ok(mut file) => { + if let Err(error) = file.write_all(line.as_bytes()) { + warn!( + file = %path.display(), + %error, + "Pingora access log 写入失败" + ); + } + } + Err(error) => { + warn!( + file = %path.display(), + %error, + "Pingora access log 打开失败" + ); + } + } +} + +fn format_access_log_line(record: &AccessLogRecord<'_>) -> String { + format!( + concat!( + "request_id={}\tmethod={}\tpath={}\turi={}\thost={}\tclient_ip={}\tstatus={}\t", + "route={}\tproxy_target={}\tupstream={}\tcontent_length={}\tbody_bytes_seen={}\t", + "protection_class={}\tprotection_client={}\telapsed_ms={}\terror={}\n" + ), + escape_access_log_value(record.request_id), + escape_access_log_value(record.method), + escape_access_log_value(record.path), + escape_access_log_value(record.uri), + escape_access_log_value(record.host), + escape_access_log_value(record.client_ip), + record.status, + escape_access_log_value(record.route), + escape_access_log_value(record.proxy_target), + escape_access_log_value(record.upstream), + record + .content_length + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()), + record.body_bytes_seen, + record.protection_class.unwrap_or("-"), + escape_access_log_value(record.protection_client), + record.elapsed_ms, + record + .error + .as_deref() + .map(escape_access_log_value) + .unwrap_or_else(|| "-".to_string()) + ) +} + +fn escape_access_log_value(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('\t', "\\t") + .replace('\n', "\\n") + .replace('\r', "\\r") +} + +fn should_disable_accel_buffering(route: &RouteDecision) -> bool { + matches!( + route.proxy_target(), + Some(ProxyTarget::Api | ProxyTarget::Gitea) + ) +} + +fn is_long_upstream_route(path: &str) -> bool { + is_spacetime_subscribe_path(path) || is_gallery_list_path(path) || is_gallery_detail_path(path) +} + +fn is_generic_api_proxy_path(path: &str) -> bool { + path == "/api" || path.starts_with("/api/") +} + +fn protection_class_for_route(route: &RouteDecision, path: &str) -> Option { + match route { + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if path.starts_with("/admin/api/") => Some(ProtectionClass::AdminApi), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if is_gallery_list_path(path) => Some(ProtectionClass::GalleryList), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if is_gallery_detail_path(path) => Some(ProtectionClass::GalleryDetail), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } => Some(ProtectionClass::Api), + RouteDecision::Proxy { + target: ProxyTarget::Spacetime, + .. + } => Some(ProtectionClass::Spacetime), + RouteDecision::Proxy { + target: ProxyTarget::Gitea, + .. + } => None, + RouteDecision::Local(_) => None, + } +} + +fn validate_shadow_probe_token(token: &str) -> io::Result<()> { + if token == "__GENARRATIVE_PINGORA_PROBE_TOKEN__" + || token.eq_ignore_ascii_case("changeme") + || token.len() < 16 + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN 必须使用 16 字符以上的非占位 token", + )); + } + Ok(()) +} + +fn validate_access_log_file(path: &Path) -> io::Result<()> { + if path.as_os_str().is_empty() { + return Ok(()); + } + + if path.exists() && path.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE 不能指向目录:{}", + path.display() + ), + )); + } + + Ok(()) +} + +fn validate_instance_protection_boundary( + instance_count: u32, + protection_enabled: bool, + shared_protection_confirmed: bool, +) -> io::Result<()> { + if instance_count == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT 必须大于 0", + )); + } + + if protection_enabled && instance_count > 1 && !shared_protection_confirmed { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Pingora 接流保护当前默认是进程内状态;GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1 时必须设置 GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true,或关闭网关保护并由前置层承担", + )); + } + + Ok(()) +} + +fn parse_socket_addr_config(key: &str, value: &str) -> io::Result { + value.parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效 socket 地址 {value:?}:{error}"), + ) + }) +} + +fn validate_distinct_listen_addr( + key: &str, + listen_addr: SocketAddr, + primary_listen_addr: SocketAddr, + api_upstream: SocketAddr, + spacetime_upstream: SocketAddr, + gitea_upstream: Option, +) -> io::Result<()> { + if listen_addr == primary_listen_addr { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不能与 GENARRATIVE_PINGORA_GATEWAY_LISTEN 相同"), + )); + } + + if listen_addr == api_upstream + || listen_addr == spacetime_upstream + || Some(listen_addr) == gitea_upstream + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不能与上游地址相同"), + )); + } + + Ok(()) +} + +fn validate_tls_file_config( + cert_file: &Option, + key_file: &Option, +) -> io::Result<()> { + let cert_file = cert_file.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN 时必须设置 GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE", + ) + })?; + let key_file = key_file.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN 时必须设置 GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE", + ) + })?; + + validate_existing_file( + "GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE", + cert_file.as_path(), + )?; + validate_existing_file( + "GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE", + key_file.as_path(), + )?; + Ok(()) +} + +fn validate_existing_file(key: &str, path: &Path) -> io::Result<()> { + let metadata = std::fs::metadata(path).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 无法读取 {}:{error}", path.display()), + ) + })?; + if !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 必须指向文件:{}", path.display()), + )); + } + Ok(()) +} + +fn validate_redirect_target_scheme(scheme: &str) -> io::Result<()> { + if scheme == "https" { + return Ok(()); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME 当前只允许 https", + )) +} + +impl GenarrativeGateway { + fn is_maintenance_enabled(&self) -> bool { + self.config.maintenance_file.exists() + } + + fn is_shadow_probe_authorized(&self, session: &Session) -> bool { + let Some(expected) = self.config.shadow_probe_token.as_deref() else { + return false; + }; + + session + .req_header() + .headers + .get(SHADOW_PROBE_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|actual| actual == expected) + } + + fn classify_request(&self, host: Option<&str>, path: &str) -> RouteDecision { + match self.mode { + GatewayMode::Proxy if self.config.matches_gitea_host(host) => RouteDecision::Proxy { + target: ProxyTarget::Gitea, + body_limit: None, + }, + GatewayMode::Proxy => classify_path(path), + GatewayMode::HttpRedirect => classify_http_redirect_path(path), + } + } +} + +fn classify_path(path: &str) -> RouteDecision { + if path == SHADOW_PROBE_PATH { + return RouteDecision::Local(LocalResponse::ShadowProbe); + } + + if path == "/admin" { + return RouteDecision::Local(LocalResponse::RedirectPermanent { + location: "/admin/", + }); + } + + if path.starts_with("/.well-known/acme-challenge/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Acme, + mode: StaticMode::Exact, + }); + } + + if path.starts_with("/admin/api/") { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if is_gallery_list_path(path) { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if is_gallery_detail_path(path) { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if path == "/api" || path.starts_with("/api/") { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: Some(DEFAULT_MAX_API_BODY_BYTES), + }; + } + + // 中文注释:公网只转发前端 SDK 必需的 SpacetimeDB subscribe / identity 路由,其它 /v1 继续关闭。 + if is_spacetime_subscribe_path(path) || path.starts_with("/v1/identity") { + return RouteDecision::Proxy { + target: ProxyTarget::Spacetime, + body_limit: None, + }; + } + + if path.starts_with("/v1/") + || path.starts_with("/generated-") + || path.starts_with("/healthz") + || path.starts_with("/readyz") + { + return RouteDecision::Local(LocalResponse::NotFound); + } + + if path.starts_with("/admin/assets/") || path.starts_with("/assets/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::Exact, + }); + } + + if path.starts_with("/admin/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }); + } + + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }) +} + +fn classify_http_redirect_path(path: &str) -> RouteDecision { + if path.starts_with("/.well-known/acme-challenge/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Acme, + mode: StaticMode::Exact, + }); + } + + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) +} + +fn apply_configured_body_limit(route: &mut RouteDecision, max_api_body_bytes: u64) { + if let RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: Some(body_limit), + } = route + { + *body_limit = max_api_body_bytes; + } +} + +fn payload_too_large_body() -> &'static str { + r#"{"ok":false,"error":{"code":"PAYLOAD_TOO_LARGE","message":"请求体过大"}}"# +} + +fn gateway_proxy_error_body(status: u16) -> &'static str { + match status { + 502 => { + r#"{"ok":false,"error":{"code":"GATEWAY_UPSTREAM_ERROR","message":"上游服务不可用"}}"# + } + 504 => { + r#"{"ok":false,"error":{"code":"GATEWAY_UPSTREAM_TIMEOUT","message":"上游服务请求超时"}}"# + } + _ => r#"{"ok":false,"error":{"code":"GATEWAY_PROXY_ERROR","message":"网关代理失败"}}"#, + } +} + +fn is_payload_too_large_error(error: &pingora::Error) -> bool { + matches!(error.etype(), HTTPStatus(413)) + && error + .context + .as_ref() + .is_some_and(|context| context.as_str() == PAYLOAD_TOO_LARGE_CONTEXT) +} + +fn is_gallery_list_path(path: &str) -> bool { + matches!( + path, + "/api/runtime/puzzle/gallery" | "/api/runtime/custom-world-gallery" + ) +} + +fn is_gallery_detail_path(path: &str) -> bool { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + matches!( + parts.as_slice(), + ["api", "runtime", "puzzle", "gallery", _] + | ["api", "runtime", "custom-world-gallery", _, _] + ) +} + +fn is_spacetime_subscribe_path(path: &str) -> bool { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + matches!(parts.as_slice(), ["v1", "database", _, "subscribe"]) +} + +struct StaticCandidate { + path: PathBuf, + cache_kind: StaticCacheKind, +} + +struct StaticResponseMetadata { + len: u64, + etag: String, + last_modified: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticRangeDecision { + Full, + Partial { start: u64, end: u64 }, + Unsatisfiable, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticCacheKind { + Html, + FingerprintedAsset, + Other, +} + +async fn serve_static( + session: &mut Session, + root: &Path, + root_kind: StaticRoot, + mode: StaticMode, + cache_config: &StaticCacheConfig, +) -> PingoraResult<()> { + let path = session.req_header().uri.path(); + let Some(candidate) = resolve_static_candidate(root, root_kind, path, mode).await else { + respond_not_found(session).await?; + return Ok(()); + }; + if !is_static_read_method(&session.req_header().method) { + respond_static_method_not_allowed(session).await?; + return Ok(()); + } + + let metadata = match static_response_metadata(&candidate.path).await { + Ok(metadata) => metadata, + Err(error) => { + warn!( + path = %path, + file = %candidate.path.display(), + %error, + "静态文件 metadata 读取失败" + ); + respond_not_found(session).await?; + return Ok(()); + } + }; + let content_type = mime_guess::from_path(&candidate.path) + .first_or_octet_stream() + .essence_str() + .to_string(); + let cache_control = cache_control_for_static_candidate(&candidate, cache_config); + let last_modified = metadata.last_modified.map(fmt_http_date); + let mut headers = vec![ + ("Cache-Control".to_string(), cache_control.to_string()), + ("ETag".to_string(), metadata.etag.clone()), + ("Accept-Ranges".to_string(), "bytes".to_string()), + ]; + if let Some(last_modified) = last_modified.as_deref() { + headers.push(("Last-Modified".to_string(), last_modified.to_string())); + } + + if static_not_modified(session, &metadata) { + return respond_empty_with_headers(session, 304, Some(&content_type), &headers).await; + } + + match static_range_decision(session, &metadata) { + StaticRangeDecision::Full => {} + StaticRangeDecision::Partial { start, end } => { + let range_len = end - start + 1; + let mut range_headers = headers.clone(); + range_headers.push(( + "Content-Range".to_string(), + format!("bytes {start}-{end}/{}", metadata.len), + )); + if session.req_header().method == Method::HEAD { + return respond_head_with_headers( + session, + 206, + Some(&content_type), + &range_headers, + range_len, + ) + .await; + } + + let body = match read_static_range(&candidate.path, start, range_len).await { + Ok(bytes) => bytes, + Err(error) => { + warn!( + path = %path, + file = %candidate.path.display(), + %error, + "静态文件 Range 读取失败" + ); + respond_not_found(session).await?; + return Ok(()); + } + }; + + return respond_bytes_with_headers( + session, + 206, + Some(&content_type), + &range_headers, + body, + ) + .await; + } + StaticRangeDecision::Unsatisfiable => { + let mut range_headers = headers.clone(); + range_headers.push(( + "Content-Range".to_string(), + format!("bytes */{}", metadata.len), + )); + return respond_head_with_headers(session, 416, Some(&content_type), &range_headers, 0) + .await; + } + } + + if session.req_header().method == Method::HEAD { + return respond_head_with_headers( + session, + 200, + Some(&content_type), + &headers, + metadata.len, + ) + .await; + } + + let body = match tokio::fs::read(&candidate.path).await { + Ok(bytes) => Bytes::from(bytes), + Err(error) => { + warn!( + path = %path, + file = %candidate.path.display(), + %error, + "静态文件读取失败" + ); + respond_not_found(session).await?; + return Ok(()); + } + }; + + respond_bytes_with_headers(session, 200, Some(&content_type), &headers, body).await +} + +fn is_static_read_method(method: &Method) -> bool { + *method == Method::GET || *method == Method::HEAD +} + +async fn read_static_range(path: &Path, start: u64, len: u64) -> io::Result { + let mut file = tokio::fs::File::open(path).await?; + file.seek(SeekFrom::Start(start)).await?; + let mut reader = file.take(len); + let capacity = usize::try_from(len) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Range 长度超过平台上限"))?; + let mut body = Vec::with_capacity(capacity); + reader.read_to_end(&mut body).await?; + Ok(Bytes::from(body)) +} + +async fn static_response_metadata(path: &Path) -> io::Result { + let metadata = tokio::fs::metadata(path).await?; + let len = metadata.len(); + let last_modified = metadata.modified().ok(); + Ok(StaticResponseMetadata { + len, + etag: build_static_etag(len, last_modified), + last_modified, + }) +} + +fn build_static_etag(len: u64, last_modified: Option) -> String { + let modified_secs = last_modified + .and_then(|modified| modified.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_secs()); + format!("W/\"{len:x}-{modified_secs:x}\"") +} + +fn static_not_modified(session: &Session, metadata: &StaticResponseMetadata) -> bool { + if session.req_header().method != Method::GET && session.req_header().method != Method::HEAD { + return false; + } + + if let Some(if_none_match) = header_value(session, "if-none-match") { + return if_none_match_matches(&if_none_match, &metadata.etag); + } + + let Some(last_modified) = metadata.last_modified else { + return false; + }; + header_value(session, "if-modified-since") + .and_then(|value| parse_http_date(&value).ok()) + .is_some_and(|since| truncate_system_time_to_seconds(last_modified) <= since) +} + +fn static_range_decision( + session: &Session, + metadata: &StaticResponseMetadata, +) -> StaticRangeDecision { + if session.req_header().method != Method::GET && session.req_header().method != Method::HEAD { + return StaticRangeDecision::Full; + } + + let Some(range) = header_value(session, "range") else { + return StaticRangeDecision::Full; + }; + if !static_if_range_matches(session, metadata) { + return StaticRangeDecision::Full; + } + parse_static_range(&range, metadata.len) +} + +fn static_if_range_matches(session: &Session, metadata: &StaticResponseMetadata) -> bool { + let Some(if_range) = header_value(session, "if-range") else { + return true; + }; + static_if_range_value_matches(&if_range, metadata) +} + +fn static_if_range_value_matches(if_range: &str, metadata: &StaticResponseMetadata) -> bool { + if let Ok(if_range_date) = parse_http_date(&if_range) { + return metadata.last_modified.is_some_and(|last_modified| { + truncate_system_time_to_seconds(last_modified) <= if_range_date + }); + } + + strong_etag_opaque_value(&if_range) + .is_some_and(|candidate| strong_etag_opaque_value(&metadata.etag) == Some(candidate)) +} + +fn parse_static_range(range: &str, file_len: u64) -> StaticRangeDecision { + let range = range.trim(); + let Some(unit) = range.get(..6) else { + return StaticRangeDecision::Full; + }; + if !unit.eq_ignore_ascii_case("bytes=") { + return StaticRangeDecision::Full; + } + let spec = &range[6..]; + if spec.contains(',') { + return StaticRangeDecision::Full; + } + if file_len == 0 { + return StaticRangeDecision::Unsatisfiable; + } + + let Some((start, end)) = spec.split_once('-') else { + return StaticRangeDecision::Full; + }; + let start = start.trim(); + let end = end.trim(); + + if start.is_empty() { + let Ok(suffix_len) = end.parse::() else { + return StaticRangeDecision::Full; + }; + if suffix_len == 0 { + return StaticRangeDecision::Unsatisfiable; + } + let start = file_len.saturating_sub(suffix_len); + return StaticRangeDecision::Partial { + start, + end: file_len - 1, + }; + } + + let Ok(start) = start.parse::() else { + return StaticRangeDecision::Full; + }; + if start >= file_len { + return StaticRangeDecision::Unsatisfiable; + } + if end.is_empty() { + return StaticRangeDecision::Partial { + start, + end: file_len - 1, + }; + } + + let Ok(mut end) = end.parse::() else { + return StaticRangeDecision::Full; + }; + if start > end { + return StaticRangeDecision::Unsatisfiable; + } + end = end.min(file_len - 1); + StaticRangeDecision::Partial { start, end } +} + +fn if_none_match_matches(value: &str, etag: &str) -> bool { + let expected = weak_etag_opaque_value(etag); + value + .split(',') + .map(str::trim) + .any(|candidate| candidate == "*" || weak_etag_opaque_value(candidate) == expected) +} + +fn weak_etag_opaque_value(value: &str) -> &str { + value.strip_prefix("W/").unwrap_or(value) +} + +fn strong_etag_opaque_value(value: &str) -> Option<&str> { + if value.trim().starts_with("W/") { + return None; + } + Some(value.trim()) +} + +fn truncate_system_time_to_seconds(value: SystemTime) -> SystemTime { + value + .duration_since(UNIX_EPOCH) + .map(|duration| UNIX_EPOCH + Duration::from_secs(duration.as_secs())) + .unwrap_or(UNIX_EPOCH) +} + +async fn resolve_static_candidate( + root: &Path, + root_kind: StaticRoot, + path: &str, + mode: StaticMode, +) -> Option { + let candidate = sanitize_static_path(root, path)?; + if is_regular_file(&candidate).await { + return Some(StaticCandidate { + cache_kind: classify_static_cache_kind(root_kind, path, &candidate), + path: candidate, + }); + } + + if is_directory(&candidate).await { + let index = candidate.join("index.html"); + if is_regular_file(&index).await { + return Some(StaticCandidate { + path: index, + cache_kind: StaticCacheKind::Html, + }); + } + } + + match mode { + StaticMode::Exact => None, + StaticMode::SpaFallback if path.starts_with("/admin/") => { + let fallback = root.join("admin").join("index.html"); + is_regular_file(&fallback).await.then_some(StaticCandidate { + path: fallback, + cache_kind: StaticCacheKind::Html, + }) + } + StaticMode::SpaFallback => { + let fallback = root.join("index.html"); + is_regular_file(&fallback).await.then_some(StaticCandidate { + path: fallback, + cache_kind: StaticCacheKind::Html, + }) + } + } +} + +fn classify_static_cache_kind( + root_kind: StaticRoot, + request_path: &str, + file_path: &Path, +) -> StaticCacheKind { + if is_html_path(file_path) { + return StaticCacheKind::Html; + } + + if root_kind == StaticRoot::Web + && (request_path.starts_with("/assets/") || request_path.starts_with("/admin/assets/")) + && has_fingerprinted_file_name(file_path) + { + return StaticCacheKind::FingerprintedAsset; + } + + StaticCacheKind::Other +} + +fn cache_control_for_static_candidate<'a>( + candidate: &StaticCandidate, + config: &'a StaticCacheConfig, +) -> &'a str { + match candidate.cache_kind { + StaticCacheKind::Html => config.html_cache_control.as_str(), + StaticCacheKind::FingerprintedAsset => config.asset_cache_control.as_str(), + StaticCacheKind::Other => config.static_cache_control.as_str(), + } +} + +fn is_html_path(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("html")) +} + +fn has_fingerprinted_file_name(path: &Path) -> bool { + let Some(file_stem) = path.file_stem().and_then(|value| value.to_str()) else { + return false; + }; + file_stem + .rsplit_once('-') + .is_some_and(|(_, suffix)| is_asset_fingerprint(suffix)) +} + +fn is_asset_fingerprint(value: &str) -> bool { + value.len() >= 8 && value.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + +fn sanitize_static_path(root: &Path, path: &str) -> Option { + let mut result = root.to_path_buf(); + for raw_segment in path.trim_start_matches('/').split('/') { + if raw_segment.is_empty() { + continue; + } + + let segment = decode_static_path_segment(raw_segment)?; + match segment.as_str() { + "." => continue, + ".." => return None, + _ => result.push(segment), + } + } + Some(result) +} + +fn decode_static_path_segment(segment: &str) -> Option { + let bytes = segment.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + + while index < bytes.len() { + if bytes[index] == b'%' { + let high = *bytes.get(index + 1)?; + let low = *bytes.get(index + 2)?; + let value = decode_percent_hex(high)? << 4 | decode_percent_hex(low)?; + if value == b'/' || value == b'\\' || value == b'\0' { + return None; + } + decoded.push(value); + index += 3; + } else { + if bytes[index] == b'\\' || bytes[index] == b'\0' { + return None; + } + decoded.push(bytes[index]); + index += 1; + } + } + + String::from_utf8(decoded).ok() +} + +fn decode_percent_hex(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} + +async fn is_regular_file(path: &Path) -> bool { + tokio::fs::metadata(path) + .await + .map(|metadata| metadata.is_file()) + .unwrap_or(false) +} + +async fn is_directory(path: &Path) -> bool { + tokio::fs::metadata(path) + .await + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) +} + +async fn respond_maintenance( + session: &mut Session, + api_like: bool, + web_root: &Path, +) -> PingoraResult<()> { + if api_like { + return respond_json( + session, + 503, + r#"{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}"#, + ) + .await; + } + + let maintenance_page = web_root.join("maintenance.html"); + if is_regular_file(&maintenance_page).await { + let body = Bytes::from(tokio::fs::read(&maintenance_page).await.unwrap_or_default()); + respond_bytes(session, 503, Some("text/html; charset=utf-8"), body).await + } else { + respond_bytes( + session, + 503, + Some("text/plain; charset=utf-8"), + Bytes::from_static("服务维护中".as_bytes()), + ) + .await + } +} + +async fn respond_redirect(session: &mut Session, location: &str) -> PingoraResult<()> { + let mut header = ResponseHeader::build(301, Some(0))?; + header.insert_header("Location", location)?; + header.insert_header("Content-Length", "0")?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_http_to_https_redirect( + session: &mut Session, + target_scheme: &str, +) -> PingoraResult<()> { + let Some(host) = host_or_authority(session).filter(|value| is_valid_redirect_host(value)) + else { + respond_json( + session, + 400, + r#"{"ok":false,"error":{"code":"BAD_REQUEST","message":"无效请求"}}"#, + ) + .await?; + return Ok(()); + }; + let location = format!( + "{target_scheme}://{host}{}", + session + .req_header() + .uri + .path_and_query() + .map_or("/", |value| value.as_str()) + ); + let mut header = ResponseHeader::build(301, Some(0))?; + header.insert_header("Location", location.as_str())?; + header.insert_header("Content-Length", "0")?; + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_json(session: &mut Session, status: u16, body: &str) -> PingoraResult<()> { + respond_bytes( + session, + status, + Some("application/json; charset=utf-8"), + Bytes::copy_from_slice(body.as_bytes()), + ) + .await +} + +async fn respond_not_found(session: &mut Session) -> PingoraResult<()> { + respond_bytes(session, 404, None, Bytes::new()).await +} + +async fn respond_static_method_not_allowed(session: &mut Session) -> PingoraResult<()> { + respond_head_with_headers( + session, + 405, + None, + &[("Allow".to_string(), "GET, HEAD".to_string())], + 0, + ) + .await +} + +async fn respond_gateway_proxy_error(session: &mut Session, status: u16) -> PingoraResult<()> { + respond_json(session, status, gateway_proxy_error_body(status)).await +} + +async fn respond_too_many_requests( + session: &mut Session, + reason: RejectReason, +) -> PingoraResult<()> { + let body = format!( + r#"{{"ok":false,"error":{{"code":"{}","message":"{}"}}}}"#, + reason.code(), + reason.message() + ); + let is_head = session.req_header().method == Method::HEAD; + let mut header = ResponseHeader::build(429, Some(body.len()))?; + header.insert_header("Content-Length", body.len().to_string())?; + header.insert_header("Content-Type", "application/json; charset=utf-8")?; + header.insert_header("Retry-After", "1")?; + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session + .write_response_header(Box::new(header), is_head) + .await?; + if !is_head { + session + .write_response_body(Some(Bytes::from(body)), true) + .await?; + } + Ok(()) +} + +async fn respond_shadow_probe( + session: &mut Session, + maintenance_enabled: bool, +) -> PingoraResult<()> { + let maintenance = if maintenance_enabled { "true" } else { "false" }; + let body = format!(r#"{{"ok":true,"gateway":"pingora-shadow","maintenance":{maintenance}}}"#); + respond_bytes( + session, + 200, + Some("application/json; charset=utf-8"), + Bytes::from(body), + ) + .await +} + +async fn respond_bytes( + session: &mut Session, + status: u16, + content_type: Option<&str>, + body: Bytes, +) -> PingoraResult<()> { + respond_bytes_with_headers(session, status, content_type, &[], body).await +} + +async fn respond_bytes_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], + body: Bytes, +) -> PingoraResult<()> { + let is_head = session.req_header().method == Method::HEAD; + let mut header = ResponseHeader::build(status, Some(body.len()))?; + header.insert_header("Content-Length", body.len().to_string())?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session + .write_response_header(Box::new(header), is_head || body.is_empty()) + .await?; + if !is_head && !body.is_empty() { + session.write_response_body(Some(body), true).await?; + } + Ok(()) +} + +async fn respond_head_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], + content_length: u64, +) -> PingoraResult<()> { + let mut header = ResponseHeader::build(status, Some(headers.len() + 3))?; + header.insert_header("Content-Length", content_length.to_string())?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_empty_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], +) -> PingoraResult<()> { + let mut header = ResponseHeader::build(status, Some(headers.len() + 2))?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session.write_response_header(Box::new(header), true).await +} + +fn resolve_request_id(session: &Session) -> String { + session + .req_header() + .headers + .get("x-request-id") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| Uuid::new_v4().to_string()) +} + +fn content_length(session: &Session) -> Option { + header_value(session, "content-length")?.parse().ok() +} + +fn header_value(session: &Session, name: &str) -> Option { + session + .req_header() + .headers + .get(name) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) +} + +fn host_or_authority(session: &Session) -> Option { + header_value(session, "host").or_else(|| { + session + .req_header() + .uri + .authority() + .map(|authority| authority.as_str().to_string()) + }) +} + +fn is_valid_redirect_host(host: &str) -> bool { + !host.is_empty() + && host.len() <= 255 + && host.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b':' | b'[' | b']') + }) +} + +fn client_ip(session: &Session) -> Option { + session + .as_downstream() + .client_addr() + .and_then(|addr| addr.as_inet()) + .map(|addr| addr.ip().to_string()) +} + +fn append_forwarded_for(session: &Session, client_ip: &str) -> String { + session + .req_header() + .headers + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(|existing| format!("{existing}, {client_ip}")) + .unwrap_or_else(|| client_ip.to_string()) +} + +fn protection_client_id(session: &Session, trust_x_forwarded_for: bool) -> String { + if trust_x_forwarded_for && let Some(forwarded_for) = first_forwarded_for(session) { + return forwarded_for; + } + + client_ip(session).unwrap_or_else(|| "unknown".to_string()) +} + +fn first_forwarded_for(session: &Session) -> Option { + session + .req_header() + .headers + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(',').next()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + +fn is_upgrade_request(session: &Session) -> bool { + session + .req_header() + .headers + .get("upgrade") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("websocket")) +} + +fn normalize_accept_encoding_for_gateway_compression( + req: &mut RequestHeader, + config: CompressionRequestConfig, +) -> PingoraResult<()> { + let algorithm = if config.enabled { + req.headers + .get("accept-encoding") + .and_then(|value| value.to_str().ok()) + .and_then(|value| select_compression_algorithm(value, config)) + } else { + None + }; + + if let Some(algorithm) = algorithm { + req.insert_header("Accept-Encoding", algorithm.header_value())?; + } else { + req.remove_header("accept-encoding"); + } + + Ok(()) +} + +fn should_allow_gateway_compression_for_response( + resp: &ResponseHeader, + min_length_bytes: u64, +) -> bool { + if resp.status.as_u16() != 200 + || resp.headers.get("content-range").is_some() + || resp.headers.get("content-encoding").is_some() + { + return false; + } + + resp.headers + .get("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_none_or(|content_length| content_length >= min_length_bytes) +} + +fn select_compression_algorithm( + value: &str, + config: CompressionRequestConfig, +) -> Option { + parse_accepted_encodings(value) + .into_iter() + .find_map(|coding| { + if coding.eq_ignore_ascii_case("gzip") && config.gzip { + Some(CompressionAlgorithm::Gzip) + } else { + None + } + }) +} + +fn parse_accepted_encodings(value: &str) -> Vec { + value + .split(',') + .filter_map(|item| { + let mut parts = item.split(';'); + let coding = parts.next()?.trim(); + if coding.is_empty() { + return None; + } + + let enabled = parts + .map(str::trim) + .filter_map(|param| param.split_once('=')) + .filter(|(name, _)| name.trim().eq_ignore_ascii_case("q")) + .all(|(_, value)| value.trim() != "0" && value.trim() != "0.0"); + if enabled { + Some(coding.to_string()) + } else { + None + } + }) + .collect() +} + +fn read_env_or_default(key: &str, default_value: &str) -> String { + env::var(key) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| default_value.to_string()) +} + +fn read_optional_env(key: &str) -> Option { + env::var(key) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +fn read_bool_env(key: &str, default_value: bool) -> bool { + env::var(key) + .ok() + .and_then(|value| match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => None, + }) + .unwrap_or(default_value) +} + +fn read_u32_env(key: &str, default_value: u32) -> io::Result { + match env::var(key) { + Ok(value) if !value.trim().is_empty() => value.trim().parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效整数:{error}"), + ) + }), + _ => Ok(default_value), + } +} + +fn read_u64_env(key: &str, default_value: u64) -> io::Result { + match env::var(key) { + Ok(value) if !value.trim().is_empty() => value.trim().parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效整数:{error}"), + ) + }), + _ => Ok(default_value), + } +} + +fn read_socket_addr_env(key: &str, default_value: &str) -> io::Result { + let value = read_env_or_default(key, default_value); + value.parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效 socket 地址 {value:?}:{error}"), + ) + }) +} + +fn read_optional_socket_addr_env(key: &str) -> io::Result> { + read_optional_env(key) + .map(|value| { + value.parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效 socket 地址 {value:?}:{error}"), + ) + }) + }) + .transpose() +} + +fn read_host_list_env(key: &str) -> io::Result> { + let Some(raw) = read_optional_env(key) else { + return Ok(Vec::new()); + }; + let mut result = Vec::new(); + for value in raw + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let host = normalize_gateway_host(value).ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 包含无效 Host:{value:?}"), + ) + })?; + if !result.contains(&host) { + result.push(host); + } + } + Ok(result) +} + +fn normalize_gateway_host(value: &str) -> Option { + let value = value.trim(); + if value.is_empty() || value.contains(['/', '\\', '\n', '\r', '\0']) { + return None; + } + if value.starts_with('[') { + return None; + } + let host = value + .rsplit_once(':') + .and_then(|(host, port)| { + (!port.is_empty() && port.bytes().all(|byte| byte.is_ascii_digit())).then_some(host) + }) + .unwrap_or(value) + .trim_end_matches('.'); + if host.is_empty() || host.len() > 253 { + return None; + } + if !host + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-')) + { + return None; + } + Some(host.to_ascii_lowercase()) +} + +fn validate_configured_gateway_host(key: &str, host: &str) -> io::Result<()> { + if normalize_gateway_host(host).as_deref() == Some(host) { + return Ok(()); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 包含无效 Host:{host:?}"), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde::Deserialize; + + const ROUTE_PARITY_MATRIX_JSON: &str = + include_str!("../../../../deploy/pingora/nginx-route-parity.matrix.json"); + + #[derive(Deserialize)] + struct RouteParityMatrix { + version: u32, + routes: Vec, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RouteParityCase { + id: String, + sample_path: String, + expect: RouteExpectation, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RouteExpectation { + kind: String, + target: Option, + body_limit: Option, + root: Option, + mode: Option, + location: Option, + protection_class: Option, + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum BodyLimitExpectation { + Named(String), + Bytes(u64), + } + + fn assert_route_matches_expectation(case: &RouteParityCase, route: &RouteDecision) { + match (case.expect.kind.as_str(), route) { + ("proxy", RouteDecision::Proxy { target, body_limit }) => { + assert_eq!( + Some(proxy_target_name(*target)), + case.expect.target.as_deref(), + "route parity proxy target mismatch: {} {}", + case.id, + case.sample_path + ); + assert_eq!( + *body_limit, + expected_body_limit(case.expect.body_limit.as_ref()), + "route parity body limit mismatch: {} {}", + case.id, + case.sample_path + ); + } + ("static", RouteDecision::Local(LocalResponse::Static { root, mode })) => { + assert_eq!( + Some(static_root_name(*root)), + case.expect.root.as_deref(), + "route parity static root mismatch: {} {}", + case.id, + case.sample_path + ); + assert_eq!( + Some(static_mode_name(*mode)), + case.expect.mode.as_deref(), + "route parity static mode mismatch: {} {}", + case.id, + case.sample_path + ); + } + ( + "redirect_permanent", + RouteDecision::Local(LocalResponse::RedirectPermanent { location }), + ) => { + assert_eq!( + Some(*location), + case.expect.location.as_deref(), + "route parity redirect location mismatch: {} {}", + case.id, + case.sample_path + ); + } + ("shadow_probe", RouteDecision::Local(LocalResponse::ShadowProbe)) => {} + ("not_found", RouteDecision::Local(LocalResponse::NotFound)) => {} + _ => panic!( + "route parity kind mismatch: {} {} expected {} got {:?}", + case.id, case.sample_path, case.expect.kind, route + ), + } + } + + fn proxy_target_name(target: ProxyTarget) -> &'static str { + match target { + ProxyTarget::Api => "api", + ProxyTarget::Spacetime => "spacetime", + ProxyTarget::Gitea => "gitea", + } + } + + fn static_root_name(root: StaticRoot) -> &'static str { + match root { + StaticRoot::Web => "web", + StaticRoot::Acme => "acme", + } + } + + fn static_mode_name(mode: StaticMode) -> &'static str { + match mode { + StaticMode::Exact => "exact", + StaticMode::SpaFallback => "spa_fallback", + } + } + + fn expected_body_limit(expectation: Option<&BodyLimitExpectation>) -> Option { + match expectation { + None => None, + Some(BodyLimitExpectation::Bytes(bytes)) => Some(*bytes), + Some(BodyLimitExpectation::Named(name)) if name == "default" => { + Some(DEFAULT_MAX_API_BODY_BYTES) + } + Some(BodyLimitExpectation::Named(name)) => { + panic!("unknown route parity body limit expectation: {name}") + } + } + } + + fn test_gateway_config() -> GatewayConfig { + GatewayConfig { + listen_addr: "127.0.0.1:18081".to_string(), + tls_listen_addr: None, + tls_cert_file: None, + tls_key_file: None, + http_redirect_listen_addr: None, + http_redirect_target_scheme: "https".to_string(), + api_upstream: "127.0.0.1:8082".parse().unwrap(), + spacetime_upstream: "127.0.0.1:3101".parse().unwrap(), + gitea_hosts: Vec::new(), + gitea_upstream: None, + web_root: PathBuf::from("/srv/genarrative/web"), + acme_root: PathBuf::from("/var/www/html"), + maintenance_file: PathBuf::from("/var/lib/genarrative/maintenance/enabled"), + forwarded_proto: "http".to_string(), + max_api_body_bytes: DEFAULT_MAX_API_BODY_BYTES, + gzip_enabled: true, + gzip_level: DEFAULT_GZIP_LEVEL, + gzip_min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + compression: CompressionConfig { + gzip: true, + raw_algorithms: "gzip".to_string(), + }, + static_cache: StaticCacheConfig { + html_cache_control: DEFAULT_HTML_CACHE_CONTROL.to_string(), + asset_cache_control: DEFAULT_ASSET_CACHE_CONTROL.to_string(), + static_cache_control: DEFAULT_STATIC_CACHE_CONTROL.to_string(), + }, + upstream_timeouts: UpstreamTimeoutConfig { + connect_timeout_ms: DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + }, + shadow_probe_token: None, + trust_x_forwarded_for: false, + instance_count: DEFAULT_INSTANCE_COUNT, + shared_protection_confirmed: false, + protection: ProtectionConfig { + enabled: true, + admin_api: ProtectionClassConfig { + max_concurrent: 64, + rate_per_second: 30, + burst: 16, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 320, + rate_per_second: 5000, + burst: 4096, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 32, + rate_per_second: 300, + burst: 32, + }, + api: ProtectionClassConfig { + max_concurrent: 64, + rate_per_second: 300, + burst: 64, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 256, + rate_per_second: 1000, + burst: 256, + }, + }, + log_filter: DEFAULT_LOG_FILTER.to_string(), + access_log_file: None, + otel_enabled: false, + } + } + + #[test] + fn matches_nginx_route_parity_matrix() { + let matrix: RouteParityMatrix = serde_json::from_str(ROUTE_PARITY_MATRIX_JSON) + .expect("route parity matrix should be valid JSON"); + assert_eq!(matrix.version, 1); + + for case in matrix.routes { + let route = classify_path(&case.sample_path); + assert_route_matches_expectation(&case, &route); + + let expected_protection_class = case.expect.protection_class.as_deref(); + let actual_protection_class = + protection_class_for_route(&route, &case.sample_path).map(ProtectionClass::as_str); + assert_eq!( + actual_protection_class, expected_protection_class, + "route parity protection class mismatch: {} {}", + case.id, case.sample_path + ); + } + } + + #[test] + fn applies_configured_body_limit_to_generic_api_routes_only() { + let mut generic_api = classify_path("/api/creation-entry/config"); + apply_configured_body_limit(&mut generic_api, 1024); + assert_eq!(generic_api.body_limit(), Some(1024)); + + let mut gallery = classify_path("/api/runtime/puzzle/gallery"); + apply_configured_body_limit(&mut gallery, 1024); + assert_eq!(gallery.body_limit(), None); + + let mut gitea = RouteDecision::Proxy { + target: ProxyTarget::Gitea, + body_limit: None, + }; + apply_configured_body_limit(&mut gitea, 1024); + assert_eq!(gitea.body_limit(), None); + } + + #[test] + fn routes_configured_gitea_hosts_to_gitea_upstream() { + let config = GatewayConfig { + gitea_hosts: vec!["git.genarrative.world".to_string()], + gitea_upstream: Some("127.0.0.1:3000".parse().unwrap()), + ..test_gateway_config() + }; + let gateway = GenarrativeGateway { + config: Arc::new(config), + protection: Arc::new(GatewayProtection::new(test_gateway_config().protection)), + mode: GatewayMode::Proxy, + }; + + let route = RouteDecision::Proxy { + target: ProxyTarget::Gitea, + body_limit: None, + }; + assert_eq!( + gateway.classify_request(Some("git.genarrative.world"), "/"), + route + ); + assert_eq!( + gateway.classify_request(Some("git.genarrative.world:443"), "/api/v1/repos"), + route + ); + assert_eq!( + gateway.classify_request(Some("GIT.GENARRATIVE.WORLD"), "/assets/app.js"), + route + ); + assert_eq!( + gateway.classify_request(Some("dev.genarrative.world"), "/api/creation-entry/config"), + classify_path("/api/creation-entry/config") + ); + } + + #[test] + fn gitea_routes_bypass_maintenance_body_limit_and_protection() { + let route = RouteDecision::Proxy { + target: ProxyTarget::Gitea, + body_limit: None, + }; + + assert!(!route.applies_maintenance_gate()); + assert!(!route.is_api_like()); + assert_eq!(route.body_limit(), None); + assert_eq!(protection_class_for_route(&route, "/api/v1/repos"), None); + assert!(should_disable_accel_buffering(&route)); + } + + #[test] + fn normalizes_gateway_hosts_for_matching() { + assert_eq!( + normalize_gateway_host("Git.Genarrative.World:443"), + Some("git.genarrative.world".to_string()) + ); + assert_eq!( + normalize_gateway_host("git.genarrative.world."), + Some("git.genarrative.world".to_string()) + ); + assert!(normalize_gateway_host("git.genarrative.world/path").is_none()); + assert!(normalize_gateway_host("[::1]:3000").is_none()); + assert!(normalize_gateway_host("bad\r\nhost").is_none()); + } + + #[test] + fn validates_gitea_host_routing_config() { + let config = GatewayConfig { + gitea_hosts: vec!["git.genarrative.world".to_string()], + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + gitea_upstream: Some("127.0.0.1:3000".parse().unwrap()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + gitea_hosts: vec!["git.genarrative.world".to_string()], + gitea_upstream: Some("127.0.0.1:8082".parse().unwrap()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + gitea_hosts: vec!["git.genarrative.world".to_string()], + gitea_upstream: Some("127.0.0.1:3000".parse().unwrap()), + ..test_gateway_config() + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn http_redirect_mode_redirects_non_acme_routes_only() { + assert_eq!( + classify_http_redirect_path("/api/creation-entry/config"), + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) + ); + assert_eq!( + classify_http_redirect_path("/.well-known/acme-challenge/token"), + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Acme, + mode: StaticMode::Exact, + }) + ); + } + + #[test] + fn identifies_streaming_payload_limit_errors() { + let error = Error::explain(HTTPStatus(413), PAYLOAD_TOO_LARGE_CONTEXT); + assert!(is_payload_too_large_error(&error)); + + let unrelated = Error::explain(HTTPStatus(413), "other"); + assert!(!is_payload_too_large_error(&unrelated)); + } + + #[test] + fn maps_gateway_proxy_errors_to_stable_json_codes() { + assert!(gateway_proxy_error_body(502).contains("GATEWAY_UPSTREAM_ERROR")); + assert!(gateway_proxy_error_body(504).contains("GATEWAY_UPSTREAM_TIMEOUT")); + assert!(gateway_proxy_error_body(500).contains("GATEWAY_PROXY_ERROR")); + } + + #[test] + fn formats_access_log_as_tab_separated_key_values() { + let line = format_access_log_line(&AccessLogRecord { + request_id: "req-1", + method: "GET", + path: "/api/test", + uri: "/api/test?a=1\t2", + host: "example.test", + client_ip: "127.0.0.1", + status: 200, + route: "Proxy", + proxy_target: "Api", + upstream: "127.0.0.1:8082", + content_length: None, + body_bytes_seen: 0, + protection_class: Some("api"), + protection_client: "127.0.0.1", + elapsed_ms: 12, + error: None, + }); + + assert!(line.contains("request_id=req-1\tmethod=GET\tpath=/api/test")); + assert!(line.contains("uri=/api/test?a=1\\t2")); + assert!(line.contains("content_length=-")); + assert!(line.ends_with("error=-\n")); + } + + #[test] + fn rejects_placeholder_or_short_shadow_probe_tokens() { + assert!(validate_shadow_probe_token("__GENARRATIVE_PINGORA_PROBE_TOKEN__").is_err()); + assert!(validate_shadow_probe_token("changeme").is_err()); + assert!(validate_shadow_probe_token("too-short").is_err()); + assert!(validate_shadow_probe_token("0123456789abcdef").is_ok()); + } + + #[test] + fn rejects_burst_without_rate_limit() { + let config = ProtectionClassConfig { + max_concurrent: 1, + rate_per_second: 0, + burst: 1, + }; + assert!(config.validate("api").is_err()); + } + + #[test] + fn rejects_access_log_directory_paths() { + let temp_dir = std::env::temp_dir(); + assert!(validate_access_log_file(&temp_dir).is_err()); + assert!(validate_access_log_file(&temp_dir.join("pingora-access.log")).is_ok()); + } + + #[test] + fn rejects_incomplete_tls_and_redirect_config() { + let config = GatewayConfig { + tls_listen_addr: Some("127.0.0.1:18443".to_string()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + http_redirect_listen_addr: Some("127.0.0.1:18080".to_string()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + tls_listen_addr: Some("127.0.0.1:18081".to_string()), + tls_cert_file: Some(PathBuf::from("/missing/cert.pem")), + tls_key_file: Some(PathBuf::from("/missing/key.pem")), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + } + + #[test] + fn validates_redirect_target_host_and_scheme() { + assert!(validate_redirect_target_scheme("https").is_ok()); + assert!(validate_redirect_target_scheme("http").is_err()); + assert!(is_valid_redirect_host("genarrative.world")); + assert!(is_valid_redirect_host("localhost:8443")); + assert!(is_valid_redirect_host("[::1]:8443")); + assert!(!is_valid_redirect_host("evil.test/path")); + assert!(!is_valid_redirect_host("evil.test\r\nx: y")); + } + + #[test] + fn builds_stable_static_validators() { + let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_123); + + assert_eq!( + build_static_etag(4096, Some(modified)), + "W/\"1000-6553f17b\"" + ); + assert_eq!(build_static_etag(0, None), "W/\"0-0\""); + assert_eq!( + truncate_system_time_to_seconds(modified + Duration::from_millis(900)), + modified + ); + } + + #[test] + fn matches_static_if_none_match_values() { + assert!(if_none_match_matches("*", "W/\"1000-6553f17b\"")); + assert!(if_none_match_matches( + "\"other\", W/\"1000-6553f17b\"", + "W/\"1000-6553f17b\"" + )); + assert!(if_none_match_matches( + "\"1000-6553f17b\"", + "W/\"1000-6553f17b\"" + )); + assert!(!if_none_match_matches( + "\"other\", W/\"bad\"", + "W/\"1000-6553f17b\"" + )); + } + + #[test] + fn parses_static_byte_ranges() { + assert_eq!( + parse_static_range("bytes=0-3", 10), + StaticRangeDecision::Partial { start: 0, end: 3 } + ); + assert_eq!( + parse_static_range("Bytes=5-", 10), + StaticRangeDecision::Partial { start: 5, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=-4", 10), + StaticRangeDecision::Partial { start: 6, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=-20", 10), + StaticRangeDecision::Partial { start: 0, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=8-20", 10), + StaticRangeDecision::Partial { start: 8, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=10-11", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=4-3", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=-0", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=0-1,3-4", 10), + StaticRangeDecision::Full + ); + assert_eq!( + parse_static_range("items=0-1", 10), + StaticRangeDecision::Full + ); + assert_eq!( + parse_static_range("bytes=0-1", 0), + StaticRangeDecision::Unsatisfiable + ); + } + + #[test] + fn matches_static_if_range_values() { + let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_123); + let metadata = StaticResponseMetadata { + len: 4096, + etag: build_static_etag(4096, Some(modified)), + last_modified: Some(modified), + }; + + assert!(static_if_range_value_matches( + &fmt_http_date(modified + Duration::from_secs(30)), + &metadata + )); + assert!(!static_if_range_value_matches( + &fmt_http_date(modified - Duration::from_secs(30)), + &metadata + )); + assert!(!static_if_range_value_matches(&metadata.etag, &metadata)); + assert!(!static_if_range_value_matches( + "\"1000-6553f17b\"", + &metadata + )); + } + + #[test] + fn only_allows_static_read_methods() { + assert!(is_static_read_method(&Method::GET)); + assert!(is_static_read_method(&Method::HEAD)); + assert!(!is_static_read_method(&Method::POST)); + assert!(!is_static_read_method(&Method::PUT)); + } + + #[test] + fn rejects_invalid_gzip_level() { + let config = GatewayConfig { + gzip_level: 10, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_invalid_gzip_min_length() { + let config = GatewayConfig { + gzip_min_length_bytes: 0, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_static_cache_control_header_injection() { + let config = GatewayConfig { + static_cache: StaticCacheConfig { + html_cache_control: "no-cache\nX-Bad: yes".to_string(), + asset_cache_control: DEFAULT_ASSET_CACHE_CONTROL.to_string(), + static_cache_control: DEFAULT_STATIC_CACHE_CONTROL.to_string(), + }, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_unconfirmed_multi_instance_protection() { + let config = GatewayConfig { + instance_count: 0, + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + instance_count: 2, + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + instance_count: 2, + shared_protection_confirmed: true, + ..test_gateway_config() + }; + assert!(config.validate().is_ok()); + + let config = GatewayConfig { + instance_count: 2, + protection: ProtectionConfig { + enabled: false, + ..test_gateway_config().protection + }, + ..test_gateway_config() + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn rejects_zero_upstream_timeouts() { + assert!( + UpstreamTimeoutConfig { + connect_timeout_ms: 0, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + } + .validate() + .is_err() + ); + assert!( + UpstreamTimeoutConfig { + connect_timeout_ms: DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: 0, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + } + .validate() + .is_err() + ); + } + + #[test] + fn chooses_long_upstream_timeout_for_gallery_and_spacetime_routes() { + let timeouts = UpstreamTimeoutConfig { + connect_timeout_ms: 100, + default_read_timeout_seconds: 60, + api_read_timeout_seconds: 10, + long_read_timeout_seconds: 3600, + write_timeout_seconds: 20, + }; + let generic_api = classify_path("/api/creation-entry/config"); + let gallery = classify_path("/api/runtime/puzzle/gallery"); + let subscribe = classify_path("/v1/database/genarrative/subscribe"); + + assert_eq!( + timeouts.read_timeout_for_route(&generic_api, "/api/creation-entry/config"), + Duration::from_secs(10) + ); + assert_eq!( + timeouts.read_timeout_for_route(&gallery, "/api/runtime/puzzle/gallery"), + Duration::from_secs(3600) + ); + assert_eq!( + timeouts.read_timeout_for_route(&subscribe, "/v1/database/genarrative/subscribe"), + Duration::from_secs(3600) + ); + } + + #[test] + fn rejects_unsupported_compression_algorithms() { + assert!(CompressionConfig::parse("zstd".to_string()).is_err()); + assert!(CompressionConfig::parse("gzip,br".to_string()).is_err()); + } + + #[test] + fn selects_configured_gateway_compression_algorithm() { + let gzip_only = CompressionRequestConfig { + enabled: true, + gzip: true, + min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + gzip_level: DEFAULT_GZIP_LEVEL, + }; + + let mut req = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + req.insert_header("Accept-Encoding", "br, gzip").unwrap(); + normalize_accept_encoding_for_gateway_compression(&mut req, gzip_only).unwrap(); + assert_eq!( + req.headers + .get("accept-encoding") + .and_then(|value| value.to_str().ok()), + Some("gzip") + ); + + let mut br_only = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + br_only.insert_header("Accept-Encoding", "br").unwrap(); + normalize_accept_encoding_for_gateway_compression(&mut br_only, gzip_only).unwrap(); + assert!(br_only.headers.get("accept-encoding").is_none()); + + let mut gzip_disabled = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + gzip_disabled + .insert_header("Accept-Encoding", "gzip") + .unwrap(); + normalize_accept_encoding_for_gateway_compression( + &mut gzip_disabled, + CompressionRequestConfig { + enabled: false, + gzip: true, + min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + gzip_level: DEFAULT_GZIP_LEVEL, + }, + ) + .unwrap(); + assert!(gzip_disabled.headers.get("accept-encoding").is_none()); + + assert_eq!(parse_accepted_encodings("br, gzip;q=1"), ["br", "gzip"]); + assert_eq!(parse_accepted_encodings("br, gzip;q=0"), ["br"]); + } + + #[test] + fn honors_configured_gzip_min_length() { + let mut small = ResponseHeader::build(200, Some(1023)).unwrap(); + small.insert_header("Content-Length", "1023").unwrap(); + small + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(!should_allow_gateway_compression_for_response( + &small, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut exact = ResponseHeader::build(200, Some(1024)).unwrap(); + exact.insert_header("Content-Length", "1024").unwrap(); + exact + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(should_allow_gateway_compression_for_response( + &exact, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut unknown_length = ResponseHeader::build(200, None).unwrap(); + unknown_length + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(should_allow_gateway_compression_for_response( + &unknown_length, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut partial = ResponseHeader::build(206, Some(1024)).unwrap(); + partial.insert_header("Content-Length", "1024").unwrap(); + partial + .insert_header("Content-Range", "bytes 0-1023/4096") + .unwrap(); + assert!(!should_allow_gateway_compression_for_response( + &partial, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut not_modified = ResponseHeader::build(304, Some(0)).unwrap(); + not_modified.insert_header("Content-Length", "0").unwrap(); + assert!(!should_allow_gateway_compression_for_response( + ¬_modified, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + } + + #[test] + fn disables_accel_buffering_for_api_proxy_only() { + assert!(should_disable_accel_buffering(&classify_path( + "/api/creation-entry/config" + ))); + assert!(should_disable_accel_buffering(&classify_path( + "/api/runtime/puzzle/gallery" + ))); + assert!(!should_disable_accel_buffering(&classify_path( + "/v1/database/genarrative/subscribe" + ))); + assert!(!should_disable_accel_buffering(&classify_path( + "/assets/app.js" + ))); + } + + #[test] + fn maps_proxy_routes_to_protection_classes() { + let cases = [ + ("/admin/api/users", Some(ProtectionClass::AdminApi)), + ( + "/api/runtime/puzzle/gallery", + Some(ProtectionClass::GalleryList), + ), + ( + "/api/runtime/custom-world-gallery/profile-a/owner-b", + Some(ProtectionClass::GalleryDetail), + ), + ("/api/creation-entry/config", Some(ProtectionClass::Api)), + ( + "/v1/database/genarrative/subscribe", + Some(ProtectionClass::Spacetime), + ), + ("/assets/app.js", None), + ]; + + for (path, expected) in cases { + assert_eq!( + protection_class_for_route(&classify_path(path), path), + expected, + "path: {path}" + ); + } + } + + #[test] + fn protection_rejects_when_concurrency_is_exhausted() { + let config = ProtectionConfig { + enabled: true, + api: ProtectionClassConfig { + max_concurrent: 1, + rate_per_second: 0, + burst: 0, + }, + admin_api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + }; + let protection = Arc::new(GatewayProtection::new(config)); + let first = protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("first request should be accepted"); + + assert!(matches!( + protection.try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()), + Err(RejectReason::Concurrent) + )); + + if let Some(key) = first { + protection.release(&key); + } + assert!( + protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("request should be accepted after release") + .is_some() + ); + } + + #[test] + fn protection_rejects_when_rate_bucket_is_empty() { + let config = ProtectionConfig { + enabled: true, + api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 1, + burst: 0, + }, + admin_api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + }; + let protection = Arc::new(GatewayProtection::new(config)); + let first = protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("first request should pass"); + if let Some(key) = first { + protection.release(&key); + } + + assert!(matches!( + protection.try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()), + Err(RejectReason::Rate) + )); + assert!( + protection + .try_acquire(ProtectionClass::Api, "127.0.0.2".to_string()) + .expect("different client should have an independent bucket") + .is_some() + ); + } + + #[test] + fn rejects_path_traversal_for_static_files() { + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/app.js").is_some()); + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/../secret").is_none()); + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/%2e%2e/secret").is_none(),); + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets%2fsecret").is_none()); + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/%GG").is_none()); + } + + #[test] + fn decodes_percent_encoded_static_path_segments() { + assert_eq!( + sanitize_static_path( + Path::new("/srv/web"), + "/Icons/Admurin%27s%20Pixel%20Items/General/Singles/499_Iron_Gear.png", + ), + Some(PathBuf::from( + "/srv/web/Icons/Admurin's Pixel Items/General/Singles/499_Iron_Gear.png", + )), + ); + } + + #[test] + fn classifies_static_cache_policy() { + let root = StaticRoot::Web; + assert_eq!( + classify_static_cache_kind(root, "/", Path::new("/srv/web/index.html")), + StaticCacheKind::Html + ); + assert_eq!( + classify_static_cache_kind( + root, + "/assets/index-B4dmVw0r.js", + Path::new("/srv/web/assets/index-B4dmVw0r.js"), + ), + StaticCacheKind::FingerprintedAsset + ); + assert_eq!( + classify_static_cache_kind( + root, + "/admin/assets/admin-Ca8f3012.css", + Path::new("/srv/web/admin/assets/admin-Ca8f3012.css"), + ), + StaticCacheKind::FingerprintedAsset + ); + assert_eq!( + classify_static_cache_kind(root, "/assets/app.js", Path::new("/srv/web/assets/app.js")), + StaticCacheKind::Other + ); + assert_eq!( + classify_static_cache_kind( + StaticRoot::Acme, + "/.well-known/acme-challenge/token-12345678", + Path::new("/var/www/html/.well-known/acme-challenge/token-12345678"), + ), + StaticCacheKind::Other + ); + } +} diff --git a/server-rs/crates/shared-contracts/src/puzzle_agent.rs b/server-rs/crates/shared-contracts/src/puzzle_agent.rs index b417ca801..6d585f5b6 100644 --- a/server-rs/crates/shared-contracts/src/puzzle_agent.rs +++ b/server-rs/crates/shared-contracts/src/puzzle_agent.rs @@ -1,6 +1,7 @@ use serde::{Deserialize, Serialize}; use crate::creation_audio::CreationAudioAsset; +use crate::external_generation::ExternalGenerationJobStatusRecord; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] @@ -280,6 +281,8 @@ pub struct PuzzleAgentOperationResponse { pub progress: u32, #[serde(default)] pub error: Option, + #[serde(default)] + pub queue_state: Option, } #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] diff --git a/server-rs/crates/spacetime-module/src/puzzle.rs b/server-rs/crates/spacetime-module/src/puzzle.rs index 8b253e7e7..2a568294e 100644 --- a/server-rs/crates/spacetime-module/src/puzzle.rs +++ b/server-rs/crates/spacetime-module/src/puzzle.rs @@ -1874,12 +1874,7 @@ fn update_puzzle_work_tx( ctx: &TxContext, input: PuzzleWorkUpsertInput, ) -> Result { - let row = ctx - .db - .puzzle_work_profile() - .profile_id() - .find(&input.profile_id) - .ok_or_else(|| "拼图作品不存在".to_string())?; + let row = find_or_recover_puzzle_work_profile_row_for_update(ctx, &input)?; if row.owner_user_id != input.owner_user_id { return Err("无权修改该拼图作品".to_string()); } @@ -1957,6 +1952,97 @@ fn update_puzzle_work_tx( ) } +fn find_or_recover_puzzle_work_profile_row_for_update( + ctx: &TxContext, + input: &PuzzleWorkUpsertInput, +) -> Result { + if let Some(row) = ctx + .db + .puzzle_work_profile() + .profile_id() + .find(&input.profile_id) + { + return Ok(row); + } + + let recovery = resolve_stable_puzzle_work_profile_recovery_ids(&input.profile_id) + .ok_or_else(|| "拼图作品不存在".to_string())?; + let row = get_owned_session_row(ctx, &recovery.session_id, &input.owner_user_id).map_err( + |error| { + if error.contains("无权") { + "无权修改该拼图作品".to_string() + } else { + "拼图作品不存在".to_string() + } + }, + )?; + if row.stage == PuzzleAgentStage::Published + || row.published_profile_id.as_deref() == Some(input.profile_id.as_str()) + { + return Err("拼图作品不存在".to_string()); + } + let recovered_profile_id = recovery.profile_id.clone(); + let draft = deserialize_draft_required(&row.draft_json)?; + let profile = build_recovered_puzzle_work_profile_from_session( + recovery, + &input.owner_user_id, + &draft, + input.updated_at_micros, + )?; + upsert_puzzle_work_profile(ctx, profile)?; + ctx.db + .puzzle_work_profile() + .profile_id() + .find(&recovered_profile_id) + .ok_or_else(|| "拼图作品不存在".to_string()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PuzzleWorkProfileRecoveryIds { + work_id: String, + profile_id: String, + session_id: String, +} + +fn resolve_stable_puzzle_work_profile_recovery_ids( + profile_id: &str, +) -> Option { + let normalized_profile_id = profile_id.trim(); + let stable_suffix = normalized_profile_id.strip_prefix("puzzle-profile-")?; + if stable_suffix.trim().is_empty() { + return None; + } + let session_id = format!("puzzle-session-{stable_suffix}"); + let (work_id, derived_profile_id) = build_puzzle_work_ids_from_session_id(&session_id); + if derived_profile_id != normalized_profile_id { + return None; + } + Some(PuzzleWorkProfileRecoveryIds { + work_id, + profile_id: derived_profile_id, + session_id, + }) +} + +fn build_recovered_puzzle_work_profile_from_session( + recovery: PuzzleWorkProfileRecoveryIds, + owner_user_id: &str, + draft: &PuzzleResultDraft, + updated_at_micros: i64, +) -> Result { + // 中文注释:update 可能晚于草稿投影落库执行;只恢复稳定 session 派生的草稿 profile。 + create_work_profile( + recovery.work_id, + recovery.profile_id, + owner_user_id.to_string(), + Some(recovery.session_id), + "陶泥儿主".to_string(), + draft, + updated_at_micros, + ) + .map_err(|error| error.to_string()) +} + fn sync_puzzle_source_session_draft_from_work( ctx: &TxContext, work_row: &PuzzleWorkProfileRow, @@ -4243,6 +4329,62 @@ mod tests { assert_eq!(draft.work_title, "我的猫街合集"); } + #[test] + fn stable_puzzle_profile_id_recovers_source_session_ids() { + let ids = resolve_stable_puzzle_work_profile_recovery_ids( + "puzzle-profile-b1cf819ea2d944c4a7e34ca668f8a97c", + ) + .expect("stable puzzle profile should be recoverable"); + + assert_eq!( + ids.session_id, + "puzzle-session-b1cf819ea2d944c4a7e34ca668f8a97c" + ); + assert_eq!(ids.work_id, "puzzle-work-b1cf819ea2d944c4a7e34ca668f8a97c"); + assert_eq!( + ids.profile_id, + "puzzle-profile-b1cf819ea2d944c4a7e34ca668f8a97c" + ); + assert!(resolve_stable_puzzle_work_profile_recovery_ids("").is_none()); + assert!(resolve_stable_puzzle_work_profile_recovery_ids("onboarding-profile-1").is_none()); + assert!(resolve_stable_puzzle_work_profile_recovery_ids("puzzle-profile-").is_none()); + } + + #[test] + fn recovered_puzzle_work_profile_keeps_draft_identity() { + let anchor_pack = infer_anchor_pack("画面描述:一只猫在雨夜灯牌下回头。", None); + let draft = compile_result_draft_from_seed( + &anchor_pack, + &[], + Some("画面描述:一只猫在雨夜灯牌下回头。"), + ); + let recovery = resolve_stable_puzzle_work_profile_recovery_ids( + "puzzle-profile-b1cf819ea2d944c4a7e34ca668f8a97c", + ) + .expect("stable ids should resolve"); + + let profile = + build_recovered_puzzle_work_profile_from_session(recovery, "user-1", &draft, 1_000) + .expect("profile should recover from draft"); + + assert_eq!( + profile.work_id, + "puzzle-work-b1cf819ea2d944c4a7e34ca668f8a97c" + ); + assert_eq!( + profile.profile_id, + "puzzle-profile-b1cf819ea2d944c4a7e34ca668f8a97c" + ); + assert_eq!( + profile.source_session_id.as_deref(), + Some("puzzle-session-b1cf819ea2d944c4a7e34ca668f8a97c") + ); + assert_eq!(profile.owner_user_id, "user-1"); + assert_eq!(profile.publication_status, PuzzlePublicationStatus::Draft); + assert_eq!(profile.updated_at_micros, 1_000); + assert_eq!(profile.levels.len(), draft.levels.len()); + } + #[test] fn failed_generation_returns_result_draft_stage_instead_of_refining() { let anchor_pack = infer_anchor_pack("画面描述:一只猫在雨夜灯牌下回头。", None); diff --git a/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx b/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx index f35b2358c..ba5eccb4f 100644 --- a/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx +++ b/src/components/platform-entry/PlatformEntryFlowShellImpl.tsx @@ -165,6 +165,7 @@ import { fetchCreationEntryConfig, } from '../../services/creationEntryConfigService'; import { + type CreationUrlState, clearCreationUrlState, readCreationUrlState, writeCreationUrlState, @@ -499,6 +500,7 @@ import { type PendingDraftShelfKind, type PendingDraftShelfMap, type PendingDraftShelfMetadata, + buildPuzzleDraftOpenNoticeKeys, resolveBigFishDraftOpenIntent, resolveJumpHopDraftOpenIntent, resolveMatch3DDraftOpenIntent, @@ -909,6 +911,29 @@ function pushPuzzleResultHistoryEntry( writeCreationUrlState(buildPuzzleCreationUrlState(session)); } +function pushPuzzleGeneratingHistoryEntry( + session: PuzzleAgentSessionSnapshot | null, + fallbackState?: CreationUrlState, +) { + pushAppHistoryPath('/creation/puzzle/generating'); + writeCreationUrlState( + session ? buildPuzzleCreationUrlState(session) : (fallbackState ?? {}), + ); +} + +function buildPuzzleCreationUrlStateFromWork( + item: PuzzleWorkSummary, +): CreationUrlState { + const sourceSessionId = item.sourceSessionId?.trim() || null; + + return { + sessionId: sourceSessionId, + profileId: + item.profileId?.trim() || buildPuzzleResultProfileId(sourceSessionId), + workId: item.workId?.trim() || buildPuzzleResultWorkId(sourceSessionId), + }; +} + const PUZZLE_DRAFT_GENERATION_POINT_COST = 2; const PUZZLE_BACKGROUND_ACTION_POLL_INTERVAL_MS = 3000; const PUZZLE_BACKGROUND_ACTION_MAX_POLL_ATTEMPTS = 160; @@ -4247,6 +4272,7 @@ export function PlatformEntryFlowShellImpl({ ); activePuzzleGenerationSessionIdRef.current = response.session.sessionId; + pushPuzzleGeneratingHistoryEntry(response.session); setSelectionStage('puzzle-generating'); markDraftGenerating('puzzle', [ response.session.sessionId, @@ -4391,6 +4417,7 @@ export function PlatformEntryFlowShellImpl({ ); selectionStageRef.current = 'puzzle-generating'; activePuzzleGenerationSessionIdRef.current = session.sessionId; + pushPuzzleGeneratingHistoryEntry(session); setSelectionStage('puzzle-generating'); const nextGenerationState = createPuzzleDraftGenerationStateFromPayload( formPayload ?? buildPuzzleFormPayloadFromSession(session), @@ -5657,6 +5684,7 @@ export function PlatformEntryFlowShellImpl({ } selectionStageRef.current = 'puzzle-generating'; activePuzzleGenerationSessionIdRef.current = nextSession.sessionId; + pushPuzzleGeneratingHistoryEntry(nextSession); setSelectionStage('puzzle-generating'); try { @@ -12060,6 +12088,66 @@ export function PlatformEntryFlowShellImpl({ setPuzzleRun(null); setPuzzleRuntimeAuthMode('default'); setSelectedPuzzleDetail(null); + const restorePuzzleGeneratingSession = ( + latestSession: PuzzleAgentSessionSnapshot, + ) => { + const payload = buildPuzzleFormPayloadFromSession(latestSession); + const startedAtMs = resolveMiniGameDraftGenerationStartedAtMs( + latestSession.updatedAt, + ); + const baseGenerationState = + createMiniGameDraftGenerationStateForRestoredDraft( + 'puzzle', + { + puzzleAiRedraw: payload.aiRedraw ?? true, + puzzleProgressPercent: + latestSession.draft && !latestSession.draft.formDraft + ? latestSession.progressPercent + : undefined, + }, + startedAtMs, + ); + const generationState = mergePuzzleSessionProgressIntoGenerationState( + baseGenerationState, + latestSession, + ); + puzzleFlow.setSession(latestSession); + setPuzzleFormDraftPayload(payload); + setPuzzleGenerationState(generationState); + setPuzzleBackgroundCompileTasks((current) => ({ + ...current, + [latestSession.sessionId]: { + session: latestSession, + payload, + generationState, + error: null, + }, + })); + enterCreateTab(); + selectionStageRef.current = 'puzzle-generating'; + activePuzzleGenerationSessionIdRef.current = latestSession.sessionId; + pushPuzzleGeneratingHistoryEntry(latestSession); + setSelectionStage('puzzle-generating'); + }; + const openPuzzleReadyResultSession = ( + latestSession: PuzzleAgentSessionSnapshot, + ) => { + const payload = buildPuzzleFormPayloadFromSession(latestSession); + puzzleFlow.setSession(latestSession); + setPuzzleFormDraftPayload(payload); + setPuzzleGenerationState(null); + setPuzzleBackgroundCompileTasks((current) => { + const next = { ...current }; + delete next[latestSession.sessionId]; + return next; + }); + enterCreateTab(); + selectionStageRef.current = 'puzzle-result'; + activePuzzleGenerationSessionIdRef.current = null; + markDraftNoticeSeen(buildPuzzleDraftOpenNoticeKeys(item)); + pushPuzzleResultHistoryEntry(latestSession); + setSelectionStage('puzzle-result'); + }; if (openIntent.type === 'open-published-detail') { await openPuzzleDetail(item.profileId, { tab: 'create' }); @@ -12125,6 +12213,10 @@ export function PlatformEntryFlowShellImpl({ enterCreateTab(); selectionStageRef.current = 'puzzle-generating'; activePuzzleGenerationSessionIdRef.current = sourceSessionId; + pushPuzzleGeneratingHistoryEntry( + failedSession, + buildPuzzleCreationUrlStateFromWork(item), + ); setPuzzleGenerationState(failedGenerationState); setSelectionStage('puzzle-generating'); return; @@ -12139,6 +12231,10 @@ export function PlatformEntryFlowShellImpl({ enterCreateTab(); selectionStageRef.current = 'puzzle-generating'; activePuzzleGenerationSessionIdRef.current = sourceSessionId; + pushPuzzleGeneratingHistoryEntry( + puzzleSession ?? backgroundTask?.session ?? null, + buildPuzzleCreationUrlStateFromWork(item), + ); setPuzzleGenerationState(rebasedGenerationState); if (backgroundTask) { setPuzzleBackgroundCompileTasks((current) => ({ @@ -12169,6 +12265,7 @@ export function PlatformEntryFlowShellImpl({ enterCreateTab(); selectionStageRef.current = 'puzzle-generating'; activePuzzleGenerationSessionIdRef.current = sourceSessionId; + pushPuzzleGeneratingHistoryEntry(rebasedTask.session); setSelectionStage('puzzle-generating'); return; } @@ -12177,42 +12274,14 @@ export function PlatformEntryFlowShellImpl({ try { const { session: latestSession } = await getPuzzleAgentSession(sourceSessionId); - const payload = buildPuzzleFormPayloadFromSession(latestSession); - const startedAtMs = resolveMiniGameDraftGenerationStartedAtMs( - latestSession.updatedAt, - ); - const baseGenerationState = - createMiniGameDraftGenerationStateForRestoredDraft( - 'puzzle', - { - puzzleAiRedraw: payload.aiRedraw ?? true, - puzzleProgressPercent: - latestSession.draft && !latestSession.draft.formDraft - ? latestSession.progressPercent - : undefined, - }, - startedAtMs, - ); - const generationState = mergePuzzleSessionProgressIntoGenerationState( - baseGenerationState, - latestSession, - ); - puzzleFlow.setSession(latestSession); - setPuzzleFormDraftPayload(payload); - setPuzzleGenerationState(generationState); - setPuzzleBackgroundCompileTasks((current) => ({ - ...current, - [latestSession.sessionId]: { - session: latestSession, - payload, - generationState, - error: null, - }, - })); - enterCreateTab(); - selectionStageRef.current = 'puzzle-generating'; - activePuzzleGenerationSessionIdRef.current = sourceSessionId; - setSelectionStage('puzzle-generating'); + if ( + !isPuzzleFormOnlyDraft(latestSession) && + isPuzzleCompileActionReady(latestSession) + ) { + openPuzzleReadyResultSession(latestSession); + return; + } + restorePuzzleGeneratingSession(latestSession); return; } catch (error) { setPuzzleError( @@ -12238,6 +12307,10 @@ export function PlatformEntryFlowShellImpl({ setSelectionStage('puzzle-agent-workspace'); } else { setPuzzleFormDraftPayload(null); + pushPuzzleResultHistoryEntry(restoredSession); + selectionStageRef.current = 'puzzle-result'; + activePuzzleGenerationSessionIdRef.current = null; + setSelectionStage('puzzle-result'); } }, [ diff --git a/src/components/platform-entry/platformDraftGenerationShelfModel.test.ts b/src/components/platform-entry/platformDraftGenerationShelfModel.test.ts index ac5118783..7d3d54f73 100644 --- a/src/components/platform-entry/platformDraftGenerationShelfModel.test.ts +++ b/src/components/platform-entry/platformDraftGenerationShelfModel.test.ts @@ -99,6 +99,58 @@ describe('platformDraftGenerationShelfModel', () => { }); }); + test('resolvePuzzleDraftOpenIntent ignores stale generating notice after puzzle draft is ready', () => { + expect( + resolvePuzzleDraftOpenIntent({ + item: buildPuzzleWork({ + generationStatus: 'ready', + coverImageSrc: '/generated-puzzle-assets/session/cover.png', + levels: [ + { + levelId: 'level-1', + levelName: '金色数字舞台', + pictureDescription: '123456', + pictureReference: null, + candidates: [], + selectedCandidateId: null, + coverImageSrc: '/generated-puzzle-assets/session/cover.png', + coverAssetId: 'asset-1', + generationStatus: 'ready', + }, + ], + }), + notices: { + 'puzzle:puzzle-session-base': { + status: 'generating', + seen: false, + }, + }, + generation: emptyGenerationFacts(), + }), + ).toMatchObject({ + type: 'restore-draft', + }); + }); + + test('resolvePuzzleDraftOpenIntent restores generating page from notice even before persisted status catches up', () => { + expect( + resolvePuzzleDraftOpenIntent({ + item: buildPuzzleWork({ + generationStatus: 'idle', + }), + notices: { + 'puzzle:puzzle-session-base': { + status: 'generating', + seen: false, + }, + }, + generation: emptyGenerationFacts(), + }), + ).toMatchObject({ + type: 'restore-generating', + }); + }); + test('resolveMatch3DDraftOpenIntent opens published work detail unless forced into draft', () => { const item = buildMatch3DWork({ publicationStatus: 'published', diff --git a/src/components/puzzle-result/PuzzleResultView.test.tsx b/src/components/puzzle-result/PuzzleResultView.test.tsx index ad6d3514c..6fc05b575 100644 --- a/src/components/puzzle-result/PuzzleResultView.test.tsx +++ b/src/components/puzzle-result/PuzzleResultView.test.tsx @@ -595,7 +595,7 @@ describe('PuzzleResultView', () => { ); }); - test('adds and deletes levels from the list', async () => { + test('keeps a blank new level local until the creator edits it', async () => { vi.useFakeTimers(); vi.mocked(puzzleWorksService.updatePuzzleWork).mockResolvedValue({ item: {} as never, @@ -621,25 +621,36 @@ describe('PuzzleResultView', () => { expect( within(dialog).queryByRole('button', { name: /关卡测试/u }), ).toBeNull(); - fireEvent.click(screen.getByLabelText('关闭关卡详情')); - expect(screen.getAllByText('第2关').length).toBeGreaterThan(0); await act(async () => { await vi.runAllTimersAsync(); }); - expect(puzzleWorksService.updatePuzzleWork).toHaveBeenLastCalledWith( + expect(screen.getByRole('dialog', { name: '关卡详情' })).toBeTruthy(); + expect(screen.queryByText('保存中')).toBeNull(); + expect(puzzleWorksService.updatePuzzleWork).not.toHaveBeenCalled(); + + fireEvent.change(within(dialog).getByLabelText('关卡名称'), { + target: { value: '钟楼猫街' }, + }); + + await act(async () => { + await vi.runAllTimersAsync(); + }); + + expect(puzzleWorksService.updatePuzzleWork).toHaveBeenCalledWith( 'puzzle-profile-session-1', expect.objectContaining({ levels: expect.arrayContaining([ expect.objectContaining({ levelId: 'puzzle-level-1' }), - expect.objectContaining({ levelName: '' }), + expect.objectContaining({ levelName: '钟楼猫街' }), ]), }), ); - fireEvent.click(screen.getByLabelText('删除关卡 第2关')); - expect(screen.queryByText('第2关')).toBeNull(); + fireEvent.click(screen.getByLabelText('关闭关卡详情')); + fireEvent.click(screen.getByLabelText('删除关卡 钟楼猫街')); + expect(screen.queryByText('钟楼猫街')).toBeNull(); await act(async () => { await vi.runAllTimersAsync(); @@ -657,6 +668,48 @@ describe('PuzzleResultView', () => { ); }); + test('keeps the blank level dialog open across incoming draft refreshes', async () => { + vi.useFakeTimers(); + const base = createSession(); + const { rerender } = render( + {}} + onExecuteAction={() => {}} + />, + ); + + openPuzzleLevelsTab(); + fireEvent.click(screen.getByRole('button', { name: /新增关卡/u })); + expect(screen.getByRole('dialog', { name: '关卡详情' })).toBeTruthy(); + + rerender( + {}} + onExecuteAction={() => {}} + />, + ); + + const dialog = screen.getByRole('dialog', { name: '关卡详情' }); + expect( + (within(dialog).getByLabelText('关卡名称') as HTMLInputElement).value, + ).toBe(''); + expect(screen.getByLabelText('删除关卡 第2关')).toBeTruthy(); + expect(screen.queryByText('保存中')).toBeNull(); + + await act(async () => { + await vi.runAllTimersAsync(); + }); + + expect(puzzleWorksService.updatePuzzleWork).not.toHaveBeenCalled(); + }); + test('generates image for a newly added level with the current levels snapshot', () => { vi.spyOn(Date, 'now').mockReturnValue(1_775_000_000_000); const onExecuteAction = vi.fn(); diff --git a/src/components/puzzle-result/PuzzleResultView.tsx b/src/components/puzzle-result/PuzzleResultView.tsx index ea2ee1e9c..684e872da 100644 --- a/src/components/puzzle-result/PuzzleResultView.tsx +++ b/src/components/puzzle-result/PuzzleResultView.tsx @@ -166,6 +166,96 @@ function resolveLevelFormalImageSrc(level: PuzzleDraftLevel) { ); } +function normalizeOptionalPuzzleLevelText(value?: string | null) { + return value?.trim() || null; +} + +function normalizePuzzleLevelForSave(level: PuzzleDraftLevel) { + return { + ...level, + levelName: level.levelName.trim(), + pictureDescription: level.pictureDescription.trim(), + pictureReference: normalizeOptionalPuzzleLevelText(level.pictureReference), + uiBackgroundPrompt: normalizeOptionalPuzzleLevelText( + level.uiBackgroundPrompt, + ), + uiBackgroundImageSrc: normalizeOptionalPuzzleLevelText( + level.uiBackgroundImageSrc, + ), + uiBackgroundImageObjectKey: normalizeOptionalPuzzleLevelText( + level.uiBackgroundImageObjectKey, + ), + levelSceneImageSrc: normalizeOptionalPuzzleLevelText( + level.levelSceneImageSrc, + ), + levelSceneImageObjectKey: normalizeOptionalPuzzleLevelText( + level.levelSceneImageObjectKey, + ), + uiSpritesheetImageSrc: normalizeOptionalPuzzleLevelText( + level.uiSpritesheetImageSrc, + ), + uiSpritesheetImageObjectKey: normalizeOptionalPuzzleLevelText( + level.uiSpritesheetImageObjectKey, + ), + levelBackgroundImageSrc: normalizeOptionalPuzzleLevelText( + level.levelBackgroundImageSrc, + ), + levelBackgroundImageObjectKey: normalizeOptionalPuzzleLevelText( + level.levelBackgroundImageObjectKey, + ), + generationStatus: level.generationStatus || 'idle', + }; +} + +function normalizeDraftEditStateForSave(state: DraftEditState): DraftEditState { + return { + ...state, + workTitle: state.workTitle.trim(), + workDescription: state.workDescription.trim(), + themeTags: normalizeThemeTagInput(state.themeTags.join(',')), + levels: state.levels.map(normalizePuzzleLevelForSave), + }; +} + +function isBlankPuzzleLevel(level: PuzzleDraftLevel) { + return ( + !level.levelName.trim() && + !level.pictureDescription.trim() && + !level.pictureReference?.trim() && + !level.uiBackgroundPrompt?.trim() && + !level.uiBackgroundImageSrc?.trim() && + !level.uiBackgroundImageObjectKey?.trim() && + !level.levelSceneImageSrc?.trim() && + !level.levelSceneImageObjectKey?.trim() && + !level.uiSpritesheetImageSrc?.trim() && + !level.uiSpritesheetImageObjectKey?.trim() && + !level.levelBackgroundImageSrc?.trim() && + !level.levelBackgroundImageObjectKey?.trim() && + !level.selectedCandidateId?.trim() && + !level.coverImageSrc?.trim() && + !level.coverAssetId?.trim() && + !level.backgroundMusic && + level.candidates.length === 0 && + (level.generationStatus || 'idle') === 'idle' + ); +} + +function removeUnsavedBlankPuzzleLevels( + state: DraftEditState, + originalState: DraftEditState, +) { + const originalLevelIds = new Set( + originalState.levels.map((level) => level.levelId), + ); + return { + ...state, + levels: state.levels.filter( + (level) => + originalLevelIds.has(level.levelId) || !isBlankPuzzleLevel(level), + ), + }; +} + function buildFallbackLevelFromDraft( draft: PuzzleResultDraft, ): PuzzleDraftLevel { @@ -261,13 +351,20 @@ function mergeDraftEditStateWithIncomingState( incomingLevel?.generationStatus !== 'generating' ); }); + const hasLocalBlankLevel = currentState.levels.some( + (level) => + !incomingLevelsById.has(level.levelId) && isBlankPuzzleLevel(level), + ); - if (!shouldPreserveLocalEdits) { + if (!shouldPreserveLocalEdits && !hasLocalBlankLevel) { return incomingState; } const mergedLevels = currentState.levels.map((level) => { const incomingLevel = incomingLevelsById.get(level.levelId); + if (!incomingLevel && isBlankPuzzleLevel(level)) { + return level; + } if ( !incomingLevel || level.generationStatus !== 'generating' || @@ -1411,36 +1508,17 @@ export function PuzzleResultView({ return; } - const normalizedState: DraftEditState = { - ...editState, - workTitle: editState.workTitle.trim(), - workDescription: editState.workDescription.trim(), - themeTags: normalizeThemeTagInput(editState.themeTags.join(',')), - levels: editState.levels.map((level) => ({ - ...level, - levelName: level.levelName.trim(), - pictureDescription: level.pictureDescription.trim(), - pictureReference: level.pictureReference?.trim() || null, - uiBackgroundPrompt: level.uiBackgroundPrompt?.trim() || null, - uiBackgroundImageSrc: level.uiBackgroundImageSrc?.trim() || null, - uiBackgroundImageObjectKey: - level.uiBackgroundImageObjectKey?.trim() || null, - levelSceneImageSrc: level.levelSceneImageSrc?.trim() || null, - levelSceneImageObjectKey: - level.levelSceneImageObjectKey?.trim() || null, - uiSpritesheetImageSrc: level.uiSpritesheetImageSrc?.trim() || null, - uiSpritesheetImageObjectKey: - level.uiSpritesheetImageObjectKey?.trim() || null, - levelBackgroundImageSrc: level.levelBackgroundImageSrc?.trim() || null, - levelBackgroundImageObjectKey: - level.levelBackgroundImageObjectKey?.trim() || null, - generationStatus: level.generationStatus || 'idle', - })), - }; const originalState = savedEditStateRef.current ?? createDraftEditState(draft); + const normalizedOriginalState = + normalizeDraftEditStateForSave(originalState); + const normalizedState = removeUnsavedBlankPuzzleLevels( + normalizeDraftEditStateForSave(editState), + normalizedOriginalState, + ); const changed = - JSON.stringify(normalizedState) !== JSON.stringify(originalState); + JSON.stringify(normalizedState) !== + JSON.stringify(normalizedOriginalState); if (!changed || normalizedState.levels.length <= 0) { return;