From f7ca9d0672c1d3be9cb388757bd4be5bf501809f Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 11 Jun 2026 16:51:28 +0800 Subject: [PATCH 01/22] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20Pingora=20=E7=8B=AC?= =?UTF-8?q?=E7=AB=8B=E7=BD=91=E5=85=B3=E8=AF=95=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 pingora-gateway 独立二进制 crate，复刻当前 Nginx 核心路由口径 接入 Pingora 依赖、workspace 成员和锁文件 补充 Pingora 影子网关运维文档、环境变量示例和 Hermes 决策记录 --- .hermes/shared-memory/decision-log.md | 8 + deploy/pingora/pingora-gateway.env.example | 16 + docs/README.md | 2 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 66 ++ server-rs/Cargo.lock | 1010 ++++++++++++++++- server-rs/Cargo.toml | 6 + server-rs/crates/pingora-gateway/Cargo.toml | 17 + server-rs/crates/pingora-gateway/src/main.rs | 818 +++++++++++++ 8 files changed, 1884 insertions(+), 59 deletions(-) create mode 100644 deploy/pingora/pingora-gateway.env.example create mode 100644 docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md create mode 100644 server-rs/crates/pingora-gateway/Cargo.toml create mode 100644 server-rs/crates/pingora-gateway/src/main.rs diff --git a/.hermes/shared-memory/decision-log.md b/.hermes/shared-memory/decision-log.md index 2044455fd..53b8a753d 100644 --- a/.hermes/shared-memory/decision-log.md +++ b/.hermes/shared-memory/decision-log.md @@ -1728,3 +1728,11 @@ - 影响范围：`shared-contracts` 默认 spec、`module-runtime` 入口配置响应、`spacetime-module` 后台保存校验、后台入口开关页摘要和前端 fallback spec。 - 验证方式：`GET /api/creation-entry/config` 中各玩法 `unifiedCreationSpec.title` 等于已保存契约内容；后台只修改入口名称时不应隐式改写已保存的统一创作页表头。 - 关联文档：`docs/【玩法创作】平台入口与玩法链路-2026-05-15.md`。 + +## 2026-06-11 Pingora 网关先独立二进制试点 + +- 背景：评估 Pingora 是否逐步替代当前生产 Nginx 时，需要先验证 Genarrative 的现有反向代理、静态资源、维护模式和最小 SpacetimeDB 公网路由口径。 +- 决策：新增 `server-rs/crates/pingora-gateway` 作为独立 binary crate，默认监听 `127.0.0.1:18081` 做影子网关；当前不绑定 `80/443`，不替代 `deploy/nginx/genarrative.conf`，生产仍以 Nginx 为公网入口。 +- 影响范围：Rust workspace、Pingora 依赖、网关环境变量、`deploy/pingora/pingora-gateway.env.example` 和运维技术方案。 +- 验证方式：先执行 `cargo fmt --manifest-path server-rs/Cargo.toml -p pingora-gateway`、`cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml`；替换前必须补齐路由 parity、压缩、TLS、限流、systemd、Jenkins 和健康巡检。 +- 关联文档：`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 diff --git a/deploy/pingora/pingora-gateway.env.example b/deploy/pingora/pingora-gateway.env.example new file mode 100644 index 000000000..d53174ae8 --- /dev/null +++ b/deploy/pingora/pingora-gateway.env.example @@ -0,0 +1,16 @@ +# Pingora 独立网关试点配置示例。 +# 当前只用于本机影子验证，不绑定 80/443，也不替代生产 Nginx。 + +GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081 +GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM=127.0.0.1:8082 +GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM=127.0.0.1:3101 + +GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT=/srv/genarrative/web +GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT=/var/www/html +GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE=/var/lib/genarrative/maintenance/enabled + +GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO=http +GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES=67108864 + +GENARRATIVE_PINGORA_GATEWAY_LOG=info,pingora=info,pingora_gateway=info +GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED=false diff --git a/docs/README.md b/docs/README.md index 00c2f8431..37cb061cd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,7 +21,7 @@ 微信小程序虚拟支付接入、`wechat_mp_virtual` 渠道、`wx.requestVirtualPayment` 承接页和后端签名配置见 [【技术方案】微信虚拟支付接入-2026-05-26.md](./%E3%80%90%E6%8A%80%E6%9C%AF%E6%96%B9%E6%A1%88%E3%80%91%E5%BE%AE%E4%BF%A1%E8%99%9A%E6%8B%9F%E6%94%AF%E4%BB%98%E6%8E%A5%E5%85%A5-2026-05-26.md)。 -生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md)，该文档也是当前生产 Jenkinsfile 的唯一入口。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)；private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md)；后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。 +生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md)，该文档也是当前生产 Jenkinsfile 的唯一入口。Pingora 只作为独立二进制影子网关试点时，边界、路由口径与替换前验收见 [【开发运维】Pingora独立网关试点-2026-06-11.md](./technical/【开发运维】Pingora独立网关试点-2026-06-11.md)。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)；private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md)；后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。 SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md new file mode 100644 index 000000000..c58a39de3 --- /dev/null +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -0,0 +1,66 @@ +# Pingora 独立网关试点 + +## 目标 + +本试点新增 `server-rs/crates/pingora-gateway` 独立二进制 crate，用 Pingora 复刻当前生产 Nginx 的核心反向代理与静态路由口径。当前阶段只作为影子网关验证，不绑定公网 `80/443`，不替代 `deploy/nginx/genarrative.conf`。 + +## 运行边界 + +- 默认监听 `127.0.0.1:18081`，只用于本机或容器内 smoke。 +- 默认转发 `api-server` 到 `127.0.0.1:8082`。 +- 默认转发最小 SpacetimeDB 公网路由到 `127.0.0.1:3101`。 +- 默认静态目录为 `/srv/genarrative/web`，维护开关文件为 `/var/lib/genarrative/maintenance/enabled`。 +- 当前不承接 TLS、HTTP 到 HTTPS 跳转、gzip/Brotli、完整连接数限制、完整 RPS 限流、Certbot ACME 自动化、Jenkins 产物发布和 systemd 健康巡检。 + +## 启动命令 + +本机影子验证时先启动 SpacetimeDB 与 `api-server`，再运行： + +```bash +GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT=/srv/genarrative/web \ +cargo run -p pingora-gateway --manifest-path server-rs/Cargo.toml +``` + +也可以复制 `deploy/pingora/pingora-gateway.env.example` 到部署环境的非 Git 配置文件，由 systemd 或容器注入。 + +## 环境变量 + +| 变量 | 默认值 | 说明 | +| --- | --- | --- | +| `GENARRATIVE_PINGORA_GATEWAY_LISTEN` | `127.0.0.1:18081` | Pingora 监听地址。 | +| `GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM` | `127.0.0.1:8082` | `api-server` 上游地址。 | +| `GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM` | `127.0.0.1:3101` | SpacetimeDB 上游地址。 | +| `GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT` | `/srv/genarrative/web` | 前端静态文件根目录。 | +| `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` | `/var/www/html` | ACME challenge 静态目录。 | +| `GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE` | `/var/lib/genarrative/maintenance/enabled` | 存在即进入维护模式。 | +| `GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO` | `http` | 写入 `X-Forwarded-Proto` 的值。 | +| `GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES` | `67108864` | `/api` 通用路由的 `Content-Length` 上限。 | +| `GENARRATIVE_PINGORA_GATEWAY_LOG` | `info,pingora=info,pingora_gateway=info` | tracing 过滤器。 | +| `GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED` | `false` | 是否启用共享 OpenTelemetry 初始化。 | + +## 当前路由口径 + +| 路由 | 行为 | +| --- | --- | +| `/.well-known/acme-challenge/*` | 从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取静态文件。 | +| `/admin` | 301 到 `/admin/`。 | +| `/admin/api/*` | 转发到 `api-server`。 | +| `/admin/assets/*` | 从 Web 根目录精确读取静态文件。 | +| `/admin/*` | 先读取静态文件或目录 index，失败回退 `/admin/index.html`。 | +| `/assets/*` | 从 Web 根目录精确读取静态文件。 | +| `/api/runtime/puzzle/gallery`、`/api/runtime/custom-world-gallery` | 转发到 `api-server`。 | +| `/api/runtime/puzzle/gallery/{id}`、`/api/runtime/custom-world-gallery/{profile}/{owner}` | 转发到 `api-server`。 | +| `/api`、`/api/*` | 转发到 `api-server`，按配置执行 `Content-Length` 上限检查。 | +| `/v1/database/{db}/subscribe`、`/v1/identity*` | 转发到 SpacetimeDB，保留 WebSocket Upgrade 头。 | +| `/v1/*`、`/generated-*`、`/healthz*`、`/readyz*` | 返回 404，保持生产公网不暴露口径。 | +| 其它路径 | 先读取静态文件或目录 index，失败回退 `/index.html`。 | + +维护模式下，API-like 路由返回 JSON `503`，Web 静态路由优先返回 `maintenance.html`，不存在时返回纯文本 `503`。 + +## 后续替换前验收 + +1. 容器内使用同一份 Web 产物、同一组上游地址跑 Pingora smoke。 +2. 对照 `deploy/nginx/genarrative.conf` 做路由 parity 自动测试，覆盖 API、Admin、SPA fallback、静态资源、维护模式和 SpacetimeDB WebSocket。 +3. 补齐限流、连接数限制、压缩、TLS、ACME 和访问日志字段。 +4. 增加 systemd service 模板与健康巡检，但仍由 Nginx 反向代理到 Pingora 做 canary。 +5. canary 稳定后，再评估是否让 Pingora 直接承接公网入口。 diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index f285faaf3..aa52b22d9 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -35,6 +44,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -64,6 +74,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -73,6 +89,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -216,7 +282,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -227,7 +293,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -297,6 +363,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.21.7" @@ -398,6 +479,17 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -410,6 +502,12 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "byteorder-lite" version = "0.1.0" @@ -462,6 +560,34 @@ dependencies = [ "shlex", ] +[[package]] +name = "cf-rustracing" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f85c3824e4191621dec0551e3cef3d511f329da9a8990bf3e450a85651d97e" +dependencies = [ + "backtrace", + "rand 0.8.6", + "tokio", + "trackable", +] + +[[package]] +name = "cf-rustracing-jaeger" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a5f80d44c257c3300a7f45ada676c211e64bbbac591bbec19344a8f61fbcab" +dependencies = [ + "cf-rustracing", + "hostname", + "local-ip-address", + "percent-encoding", + "rand 0.9.4", + "thrift_codec", + "tokio", + "trackable", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -498,6 +624,61 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "console" version = "0.16.3" @@ -665,7 +846,35 @@ dependencies = [ "openssl-sys", "pkg-config", "vcpkg", - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "daemonize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8bfdaacb3c887a54d41bdf48d3af8873b3f5566469f8ba21b92057509f116e" +dependencies = [ + "libc", +] + +[[package]] +name = "daggy" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70def8d72740e44d9f676d8dab2c933a236663d86dd24319b57a2bed4d694774" +dependencies = [ + "petgraph 0.7.1", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", ] [[package]] @@ -674,8 +883,22 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] @@ -688,7 +911,18 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", ] [[package]] @@ -697,9 +931,9 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -728,6 +962,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -736,7 +981,38 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", ] [[package]] @@ -749,7 +1025,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", ] [[package]] @@ -771,7 +1047,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -816,7 +1092,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -836,7 +1112,7 @@ checksum = "f282cfdfe92516eb26c2af8589c274c7c17681f5ecc03c18255fe741c6aa64eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -852,7 +1128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -900,6 +1176,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -907,6 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", + "libz-ng-sys", "miniz_oxide", ] @@ -922,6 +1205,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1002,7 +1291,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1084,6 +1373,24 @@ dependencies = [ "wasip3", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "glob" version = "0.3.3" @@ -1109,6 +1416,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1121,7 +1447,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1130,7 +1456,9 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", "equivalent", + "foldhash 0.2.0", "rayon", "serde", "serde_core", @@ -1141,6 +1469,11 @@ name = "hashbrown" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -1178,6 +1511,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "0.2.12" @@ -1261,7 +1605,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1342,7 +1686,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "tokio", "tower-service", "tracing", @@ -1578,6 +1922,12 @@ dependencies = [ "serde", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.12.1" @@ -1700,6 +2050,16 @@ dependencies = [ "glob", ] +[[package]] +name = "libz-ng-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879917b256f6317769b9f374b435805a8697013098aacce5a38ac106cd6a9469" +dependencies = [ + "cmake", + "libc", +] + [[package]] name = "libz-sys" version = "1.1.29" @@ -1724,6 +2084,17 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "local-ip-address" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa08fb2b1ec3ea84575e94b489d06d4ce0cbf052d12acd515838f50e3c3d63e3" +dependencies = [ + "libc", + "neli", + "windows-sys 0.61.2", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -1757,6 +2128,15 @@ dependencies = [ "weezl", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1794,6 +2174,15 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2091,6 +2480,47 @@ dependencies = [ "tempfile", ] +[[package]] +name = "neli" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f9786d56d972959e1408b6a93be6af13b9c1392036c5c1fafa08a1b0c6ee87" +dependencies = [ + "bitflags 2.11.1", + "byteorder", + "derive_builder", + "getset", + "libc", + "log", + "neli-proc-macros", + "parking_lot", +] + +[[package]] +name = "neli-proc-macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d8d08c6e98f20a62417478ebf7be8e1425ec9acecc6f63e22da633f6b71609" +dependencies = [ + "either", + "proc-macro2", + "quote", + "serde", + "syn 2.0.117", +] + +[[package]] +name = "nix" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2113,7 +2543,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2150,12 +2580,27 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "openssl" version = "0.10.78" @@ -2179,7 +2624,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2361,7 +2806,17 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap 2.14.0", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap 2.14.0", ] @@ -2382,7 +2837,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2391,6 +2846,230 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pingora" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a75f2ff8e122aa80ab202dc865294fe59cd856c2a5dab2d3df6e122c93b941" +dependencies = [ + "pingora-core", + "pingora-http", + "pingora-proxy", + "pingora-timeout", +] + +[[package]] +name = "pingora-cache" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527735ac204efb9fa3884bfd9224d016c5735fabe1d394ebed145b40e7545b99" +dependencies = [ + "ahash", + "async-trait", + "blake2", + "bstr", + "bytes", + "cf-rustracing", + "cf-rustracing-jaeger", + "hex", + "http 1.4.0", + "httparse", + "httpdate", + "indexmap 1.9.3", + "log", + "lru", + "once_cell", + "parking_lot", + "pingora-core", + "pingora-error", + "pingora-header-serde", + "pingora-http", + "pingora-lru", + "pingora-timeout", + "rand 0.8.6", + "regex", + "rmp", + "rmp-serde", + "serde", + "strum 0.26.3", + "tokio", +] + +[[package]] +name = "pingora-core" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a7ffe2f5acf9f94fd255cfd1438866bc9124f8f0c7d42562bd3f853df2094b7" +dependencies = [ + "ahash", + "async-trait", + "brotli", + "bstr", + "bytes", + "chrono", + "clap", + "daemonize", + "daggy", + "derivative", + "flate2", + "futures", + "h2 0.4.14", + "http 1.4.0", + "httparse", + "httpdate", + "libc", + "log", + "nix", + "once_cell", + "openssl-probe", + "parking_lot", + "percent-encoding", + "pingora-error", + "pingora-http", + "pingora-pool", + "pingora-runtime", + "pingora-timeout", + "prometheus 0.13.4", + "rand 0.8.6", + "regex", + "serde", + "serde_yaml", + "sfv", + "socket2 0.6.3", + "strum 0.26.3", + "strum_macros 0.26.4", + "tokio", + "tokio-stream", + "tokio-test", + "unicase", + "windows-sys 0.59.0", + "zstd", +] + +[[package]] +name = "pingora-error" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b23f7bc013de67e44ed902a82843f6157460b89d11da882bcc6f09f8ae380af1" + +[[package]] +name = "pingora-gateway" +version = "0.1.0" +dependencies = [ + "async-trait", + "bytes", + "mime_guess", + "pingora", + "pingora-http", + "pingora-proxy", + "shared-logging", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "pingora-header-serde" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "828c0e53e74160cbfe8e67dd3a811eb6a253c36acbaf7a39a01d9aacfb9ac139" +dependencies = [ + "bytes", + "http 1.4.0", + "httparse", + "pingora-error", + "pingora-http", + "thread_local", + "zstd", + "zstd-safe", +] + +[[package]] +name = "pingora-http" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d553d310a15ec88107b9388a02885f798efc57764d8e9bdaaf32a76722927a10" +dependencies = [ + "bytes", + "http 1.4.0", + "pingora-error", +] + +[[package]] +name = "pingora-lru" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6705a26ad89d241a989a5395641931ba37076f5ab5fbd19ee92402414a43af32" +dependencies = [ + "arrayvec", + "hashbrown 0.17.0", + "parking_lot", + "rand 0.8.6", +] + +[[package]] +name = "pingora-pool" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb1237893b15a9cf6b371bee8d7e2e1c10742e4be6eb00ed38cfe87fd1363f8" +dependencies = [ + "crossbeam-queue", + "log", + "lru", + "parking_lot", + "pingora-timeout", + "thread_local", + "tokio", +] + +[[package]] +name = "pingora-proxy" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a92ee756ecf6ecb6419864da651cad6cecd933b6d420a26877031efa16bef57" +dependencies = [ + "async-trait", + "bytes", + "clap", + "futures", + "h2 0.4.14", + "http 1.4.0", + "log", + "once_cell", + "pingora-cache", + "pingora-core", + "pingora-error", + "pingora-http", + "rand 0.8.6", + "regex", + "tokio", +] + +[[package]] +name = "pingora-runtime" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41815a13691a3e7d9ad0e34767d4140284132e31b95a4481f5e73ab6f407f834" +dependencies = [ + "once_cell", + "rand 0.8.6", + "thread_local", + "tokio", +] + +[[package]] +name = "pingora-timeout" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3e321452eaa461e0b6c5aaa35b7e42527ee89df33710279f37fae7f066b68e" +dependencies = [ + "once_cell", + "parking_lot", + "pin-project-lite", + "thread_local", + "tokio", +] + [[package]] name = "pkg-config" version = "0.3.33" @@ -2586,7 +3265,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -2598,6 +3299,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf 2.28.0", + "thiserror 1.0.69", +] + [[package]] name = "prometheus" version = "0.14.0" @@ -2609,7 +3325,7 @@ dependencies = [ "lazy_static", "memchr", "parking_lot", - "protobuf", + "protobuf 3.7.2", "thiserror 2.0.18", ] @@ -2633,9 +3349,15 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "protobuf" version = "3.7.2" @@ -2681,7 +3403,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -2718,7 +3440,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.59.0", ] @@ -2855,7 +3577,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2898,7 +3620,7 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", @@ -2987,6 +3709,41 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rmp" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba8be72d372b2c9b35542551678538b562e7cf86c3315773cae48dfbfe7790c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "rmp-serde" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f81bee8c8ef9b577d1681a70ebbc962c232461e397b22c208c43c04b67a155" +dependencies = [ + "rmp", + "serde", +] + +[[package]] +name = "rust_decimal" +version = "1.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c5108e3d4d903e21aac27f12ba5377b6b34f9f44b325e4894c7924169d06995" +dependencies = [ + "arrayvec", + "num-traits", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -3012,7 +3769,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3125,7 +3882,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.117", ] [[package]] @@ -3202,7 +3959,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3213,7 +3970,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3286,10 +4043,34 @@ version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sfv" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa1f336066b758b7c9df34ed049c0e693a426afe2b27ff7d5b14f410ab1a132" +dependencies = [ + "base64 0.22.1", + "indexmap 2.14.0", + "rust_decimal", ] [[package]] @@ -3534,7 +4315,7 @@ dependencies = [ "proc-macro2", "quote", "spacetimedb-primitives", - "syn", + "syn 2.0.117", ] [[package]] @@ -3564,7 +4345,7 @@ dependencies = [ "spacetimedb-lib", "spacetimedb-primitives", "spacetimedb-sats", - "strum", + "strum 0.25.0", "thiserror 1.0.69", ] @@ -3627,7 +4408,7 @@ dependencies = [ "arrayvec", "itertools", "paste", - "prometheus", + "prometheus 0.14.0", ] [[package]] @@ -3702,7 +4483,7 @@ dependencies = [ "itertools", "lazy_static", "lean_string", - "petgraph", + "petgraph 0.6.5", "serde_json", "smallvec", "spacetimedb-data-structures", @@ -3735,7 +4516,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "prometheus", + "prometheus 0.14.0", "rand 0.9.4", "spacetimedb-client-api-messages", "spacetimedb-data-structures", @@ -3788,7 +4569,16 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", +] + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", ] [[package]] @@ -3801,7 +4591,20 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", ] [[package]] @@ -3810,6 +4613,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.117" @@ -3844,7 +4658,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3878,7 +4692,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3920,7 +4734,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3931,7 +4745,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3943,6 +4757,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "thrift_codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d957f535b242b91aa9f47bde08080f9a6fef276477e55b0079979d002759d5" +dependencies = [ + "byteorder", + "trackable", +] + [[package]] name = "time" version = "0.3.47" @@ -4024,7 +4848,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4058,6 +4882,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6d24790a10a7af737693a3e8f1d03faef7e6ca0cc99aae5066f533766de545" +dependencies = [ + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-tungstenite" version = "0.27.0" @@ -4239,7 +5074,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4297,6 +5132,25 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trackable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15bd114abb99ef8cee977e517c8f37aee63f184f2d08e3e6ceca092373369ae" +dependencies = [ + "trackable_derive", +] + +[[package]] +name = "trackable_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -4388,6 +5242,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -4424,6 +5284,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.23.1" @@ -4529,7 +5395,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -4649,7 +5515,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -4673,7 +5539,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4684,7 +5550,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4923,7 +5789,7 @@ dependencies = [ "heck 0.5.0", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -4939,7 +5805,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -5006,7 +5872,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5027,7 +5893,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5047,7 +5913,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -5087,7 +5953,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5125,6 +5991,34 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-core" version = "0.5.1" diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index cdc461bdf..953865a93 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -40,6 +40,7 @@ members = [ "crates/platform-wechat", "crates/platform-speech", "crates/platform-agent", + "crates/pingora-gateway", "crates/shared-contracts", "crates/shared-kernel", "crates/shared-logging", @@ -87,6 +88,7 @@ platform-llm = { path = "crates/platform-llm", default-features = false } platform-oss = { path = "crates/platform-oss", default-features = false } platform-speech = { path = "crates/platform-speech", default-features = false } platform-wechat = { path = "crates/platform-wechat", default-features = false } +pingora-gateway = { path = "crates/pingora-gateway", default-features = false } shared-contracts = { path = "crates/shared-contracts", default-features = false } shared-kernel = { path = "crates/shared-kernel", default-features = false } shared-logging = { path = "crates/shared-logging", default-features = false } @@ -111,6 +113,10 @@ image = { version = "0.25", default-features = false } jsonwebtoken = "9" langchainrust = "0.2.18" log = "0.4" +mime_guess = "2.0.5" +pingora = { version = "0.8.1", default-features = false, features = ["proxy"] } +pingora-http = { version = "0.8.1", default-features = false } +pingora-proxy = { version = "0.8.1", default-features = false } rand_core = "0.6" reqwest = { version = "0.12", default-features = false } ring = "0.17" diff --git a/server-rs/crates/pingora-gateway/Cargo.toml b/server-rs/crates/pingora-gateway/Cargo.toml new file mode 100644 index 000000000..c4602aaf2 --- /dev/null +++ b/server-rs/crates/pingora-gateway/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "pingora-gateway" +edition.workspace = true +version.workspace = true +license.workspace = true + +[dependencies] +async-trait = { workspace = true } +bytes = { workspace = true } +mime_guess = { workspace = true } +pingora = { workspace = true } +pingora-http = { workspace = true } +pingora-proxy = { workspace = true } +shared-logging = { workspace = true } +tokio = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["v4"] } diff --git a/server-rs/crates/pingora-gateway/src/main.rs b/server-rs/crates/pingora-gateway/src/main.rs new file mode 100644 index 000000000..45757fcdc --- /dev/null +++ b/server-rs/crates/pingora-gateway/src/main.rs @@ -0,0 +1,818 @@ +use std::{ + env, io, + net::SocketAddr, + path::{Component, Path, PathBuf}, + sync::Arc, + time::Instant, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use pingora::{ + Result as PingoraResult, + prelude::HttpPeer, + server::{Server, configuration::Opt}, +}; +use pingora_http::{Method, ResponseHeader}; +use pingora_proxy::{ProxyHttp, Session, http_proxy_service}; +use shared_logging::{OtelConfig, init_tracing}; +use tracing::{error, info, warn}; +use uuid::Uuid; + +const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:18081"; +const DEFAULT_API_UPSTREAM: &str = "127.0.0.1:8082"; +const DEFAULT_SPACETIME_UPSTREAM: &str = "127.0.0.1:3101"; +const DEFAULT_WEB_ROOT: &str = "/srv/genarrative/web"; +const DEFAULT_ACME_ROOT: &str = "/var/www/html"; +const DEFAULT_MAINTENANCE_FILE: &str = "/var/lib/genarrative/maintenance/enabled"; +const DEFAULT_MAX_API_BODY_BYTES: u64 = 64 * 1024 * 1024; +const DEFAULT_LOG_FILTER: &str = "info,pingora=info,pingora_gateway=info"; + +#[derive(Clone, Debug)] +struct GatewayConfig { + listen_addr: String, + api_upstream: SocketAddr, + spacetime_upstream: SocketAddr, + web_root: PathBuf, + acme_root: PathBuf, + maintenance_file: PathBuf, + forwarded_proto: String, + max_api_body_bytes: u64, + log_filter: String, + otel_enabled: bool, +} + +impl GatewayConfig { + fn from_env() -> io::Result { + Ok(Self { + listen_addr: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_LISTEN", + DEFAULT_LISTEN_ADDR, + ), + api_upstream: read_socket_addr_env( + "GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM", + DEFAULT_API_UPSTREAM, + )?, + spacetime_upstream: read_socket_addr_env( + "GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM", + DEFAULT_SPACETIME_UPSTREAM, + )?, + web_root: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT", + DEFAULT_WEB_ROOT, + )), + acme_root: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT", + DEFAULT_ACME_ROOT, + )), + maintenance_file: PathBuf::from(read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE", + DEFAULT_MAINTENANCE_FILE, + )), + forwarded_proto: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO", + "http", + ), + max_api_body_bytes: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES", + DEFAULT_MAX_API_BODY_BYTES, + )?, + log_filter: read_env_or_default("GENARRATIVE_PINGORA_GATEWAY_LOG", DEFAULT_LOG_FILTER), + otel_enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED", false), + }) + } +} + +#[derive(Clone)] +struct GenarrativeGateway { + config: Arc, +} + +#[derive(Debug)] +struct RequestContext { + request_id: String, + route: RouteDecision, + started_at: Instant, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ProxyTarget { + Api, + Spacetime, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticRoot { + Web, + Acme, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticMode { + Exact, + SpaFallback, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum LocalResponse { + RedirectPermanent { location: &'static str }, + Static { root: StaticRoot, mode: StaticMode }, + NotFound, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum RouteDecision { + Proxy { + target: ProxyTarget, + body_limit: Option, + }, + Local(LocalResponse), +} + +impl RouteDecision { + fn applies_maintenance_gate(&self) -> bool { + matches!( + self, + RouteDecision::Proxy { .. } + | RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + .. + }) + ) + } + + fn is_api_like(&self) -> bool { + matches!( + self, + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } + ) + } + + fn proxy_target(&self) -> Option { + match self { + RouteDecision::Proxy { target, .. } => Some(*target), + RouteDecision::Local(_) => None, + } + } +} + +fn main() -> std::result::Result<(), Box> { + let config = Arc::new(GatewayConfig::from_env()?); + init_tracing( + &config.log_filter, + OtelConfig { + enabled: config.otel_enabled, + }, + )?; + + info!( + listen = %config.listen_addr, + api_upstream = %config.api_upstream, + spacetime_upstream = %config.spacetime_upstream, + web_root = %config.web_root.display(), + "Pingora shadow gateway 启动" + ); + + let opt = Opt::parse_args(); + let mut server = Server::new(Some(opt))?; + server.bootstrap(); + + let gateway = GenarrativeGateway { config }; + let listen_addr = gateway.config.listen_addr.clone(); + let mut proxy = http_proxy_service(&server.configuration, gateway); + proxy.add_tcp(&listen_addr); + server.add_service(proxy); + + server.run_forever(); +} + +#[async_trait] +impl ProxyHttp for GenarrativeGateway { + type CTX = RequestContext; + + fn new_ctx(&self) -> Self::CTX { + RequestContext { + request_id: Uuid::new_v4().to_string(), + route: RouteDecision::Local(LocalResponse::NotFound), + started_at: Instant::now(), + } + } + + async fn request_filter( + &self, + session: &mut Session, + ctx: &mut Self::CTX, + ) -> PingoraResult + where + Self::CTX: Send + Sync, + { + let path = session.req_header().uri.path(); + ctx.route = classify_path(path); + apply_configured_body_limit(&mut ctx.route, self.config.max_api_body_bytes); + ctx.request_id = resolve_request_id(session); + + if self.is_maintenance_enabled() && ctx.route.applies_maintenance_gate() { + respond_maintenance(session, ctx.route.is_api_like(), &self.config.web_root).await?; + return Ok(true); + } + + if let RouteDecision::Proxy { body_limit, .. } = ctx.route + && let Some(limit) = body_limit + && let Some(content_length) = content_length(session) + && content_length > limit + { + respond_json( + session, + 413, + r#"{"ok":false,"error":{"code":"PAYLOAD_TOO_LARGE","message":"请求体过大"}}"#, + ) + .await?; + return Ok(true); + } + + match &ctx.route { + RouteDecision::Proxy { .. } => Ok(false), + RouteDecision::Local(LocalResponse::RedirectPermanent { location }) => { + respond_redirect(session, location).await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::Static { root, mode }) => { + let root_path = match root { + StaticRoot::Web => &self.config.web_root, + StaticRoot::Acme => &self.config.acme_root, + }; + serve_static(session, root_path, *mode).await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::NotFound) => { + session.respond_error(404).await?; + Ok(true) + } + } + } + + async fn upstream_peer( + &self, + _session: &mut Session, + ctx: &mut Self::CTX, + ) -> PingoraResult> { + let upstream = match ctx.route.proxy_target().unwrap_or(ProxyTarget::Api) { + ProxyTarget::Api => self.config.api_upstream, + ProxyTarget::Spacetime => self.config.spacetime_upstream, + }; + + Ok(Box::new(HttpPeer::new(upstream, false, String::new()))) + } + + async fn upstream_request_filter( + &self, + session: &mut Session, + upstream_request: &mut pingora_http::RequestHeader, + ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + upstream_request.insert_header("X-Request-Id", ctx.request_id.as_str())?; + upstream_request + .insert_header("X-Forwarded-Proto", self.config.forwarded_proto.as_str())?; + + if let Some(host) = session.req_header().headers.get("host").cloned() { + upstream_request.insert_header("Host", host.clone())?; + upstream_request.insert_header("X-Forwarded-Host", host)?; + } + + if let Some(client_ip) = client_ip(session) { + upstream_request.insert_header("X-Real-IP", client_ip.as_str())?; + let forwarded_for = append_forwarded_for(session, &client_ip); + upstream_request.insert_header("X-Forwarded-For", forwarded_for.as_str())?; + } + + // 中文注释：SpacetimeDB 订阅走 WebSocket Upgrade；普通 API 连接头保持干净，贴近当前 Nginx 模板。 + if ctx.route.proxy_target() == Some(ProxyTarget::Spacetime) && is_upgrade_request(session) { + upstream_request.insert_header("Connection", "Upgrade")?; + } else { + upstream_request.remove_header("connection"); + } + + Ok(()) + } + + async fn response_filter( + &self, + _session: &mut Session, + upstream_response: &mut ResponseHeader, + _ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + upstream_response.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + Ok(()) + } + + async fn logging( + &self, + session: &mut Session, + error: Option<&pingora::Error>, + ctx: &mut Self::CTX, + ) where + Self::CTX: Send + Sync, + { + let status = session + .response_written() + .map_or(0, |response| response.status.as_u16()); + let elapsed_ms = ctx.started_at.elapsed().as_millis(); + let method = session.req_header().method.as_str(); + let path = session.req_header().uri.path(); + let route = format!("{:?}", ctx.route); + + if let Some(error) = error { + error!( + request_id = %ctx.request_id, + method, + path, + status, + route, + elapsed_ms, + %error, + "Pingora gateway request failed" + ); + } else { + info!( + request_id = %ctx.request_id, + method, + path, + status, + route, + elapsed_ms, + "Pingora gateway request completed" + ); + } + } +} + +impl GenarrativeGateway { + fn is_maintenance_enabled(&self) -> bool { + self.config.maintenance_file.exists() + } +} + +fn classify_path(path: &str) -> RouteDecision { + if path == "/admin" { + return RouteDecision::Local(LocalResponse::RedirectPermanent { + location: "/admin/", + }); + } + + if path.starts_with("/.well-known/acme-challenge/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Acme, + mode: StaticMode::Exact, + }); + } + + if path.starts_with("/admin/api/") { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if is_gallery_list_path(path) { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if is_gallery_detail_path(path) { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: None, + }; + } + + if path == "/api" || path.starts_with("/api/") { + return RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: Some(DEFAULT_MAX_API_BODY_BYTES), + }; + } + + // 中文注释：公网只转发前端 SDK 必需的 SpacetimeDB subscribe / identity 路由，其它 /v1 继续关闭。 + if is_spacetime_subscribe_path(path) || path.starts_with("/v1/identity") { + return RouteDecision::Proxy { + target: ProxyTarget::Spacetime, + body_limit: None, + }; + } + + if path.starts_with("/v1/") + || path.starts_with("/generated-") + || path.starts_with("/healthz") + || path.starts_with("/readyz") + { + return RouteDecision::Local(LocalResponse::NotFound); + } + + if path.starts_with("/admin/assets/") || path.starts_with("/assets/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::Exact, + }); + } + + if path.starts_with("/admin/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }); + } + + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }) +} + +fn apply_configured_body_limit(route: &mut RouteDecision, max_api_body_bytes: u64) { + if let RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: Some(body_limit), + } = route + { + *body_limit = max_api_body_bytes; + } +} + +fn is_gallery_list_path(path: &str) -> bool { + matches!( + path, + "/api/runtime/puzzle/gallery" | "/api/runtime/custom-world-gallery" + ) +} + +fn is_gallery_detail_path(path: &str) -> bool { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + matches!( + parts.as_slice(), + ["api", "runtime", "puzzle", "gallery", _] + | ["api", "runtime", "custom-world-gallery", _, _] + ) +} + +fn is_spacetime_subscribe_path(path: &str) -> bool { + let parts: Vec<_> = path.trim_matches('/').split('/').collect(); + matches!(parts.as_slice(), ["v1", "database", _, "subscribe"]) +} + +async fn serve_static(session: &mut Session, root: &Path, mode: StaticMode) -> PingoraResult<()> { + let path = session.req_header().uri.path(); + let Some(candidate) = resolve_static_candidate(root, path, mode).await else { + session.respond_error(404).await?; + return Ok(()); + }; + + let body = match tokio::fs::read(&candidate).await { + Ok(bytes) => Bytes::from(bytes), + Err(error) => { + warn!( + path = %path, + file = %candidate.display(), + %error, + "静态文件读取失败" + ); + session.respond_error(404).await?; + return Ok(()); + } + }; + let content_type = mime_guess::from_path(&candidate) + .first_or_octet_stream() + .essence_str() + .to_string(); + + respond_bytes(session, 200, Some(&content_type), body).await +} + +async fn resolve_static_candidate(root: &Path, path: &str, mode: StaticMode) -> Option { + let candidate = sanitize_static_path(root, path)?; + if is_regular_file(&candidate).await { + return Some(candidate); + } + + if is_directory(&candidate).await { + let index = candidate.join("index.html"); + if is_regular_file(&index).await { + return Some(index); + } + } + + match mode { + StaticMode::Exact => None, + StaticMode::SpaFallback if path.starts_with("/admin/") => { + let fallback = root.join("admin").join("index.html"); + is_regular_file(&fallback).await.then_some(fallback) + } + StaticMode::SpaFallback => { + let fallback = root.join("index.html"); + is_regular_file(&fallback).await.then_some(fallback) + } + } +} + +fn sanitize_static_path(root: &Path, path: &str) -> Option { + let mut result = root.to_path_buf(); + for component in Path::new(path.trim_start_matches('/')).components() { + match component { + Component::Normal(value) => result.push(value), + Component::CurDir => {} + Component::Prefix(_) | Component::RootDir | Component::ParentDir => return None, + } + } + Some(result) +} + +async fn is_regular_file(path: &Path) -> bool { + tokio::fs::metadata(path) + .await + .map(|metadata| metadata.is_file()) + .unwrap_or(false) +} + +async fn is_directory(path: &Path) -> bool { + tokio::fs::metadata(path) + .await + .map(|metadata| metadata.is_dir()) + .unwrap_or(false) +} + +async fn respond_maintenance( + session: &mut Session, + api_like: bool, + web_root: &Path, +) -> PingoraResult<()> { + if api_like { + return respond_json( + session, + 503, + r#"{"ok":false,"error":{"code":"MAINTENANCE","message":"服务维护中"}}"#, + ) + .await; + } + + let maintenance_page = web_root.join("maintenance.html"); + if is_regular_file(&maintenance_page).await { + let body = Bytes::from(tokio::fs::read(&maintenance_page).await.unwrap_or_default()); + respond_bytes(session, 503, Some("text/html; charset=utf-8"), body).await + } else { + respond_bytes( + session, + 503, + Some("text/plain; charset=utf-8"), + Bytes::from_static("服务维护中".as_bytes()), + ) + .await + } +} + +async fn respond_redirect(session: &mut Session, location: &str) -> PingoraResult<()> { + let mut header = ResponseHeader::build(301, Some(0))?; + header.insert_header("Location", location)?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_json(session: &mut Session, status: u16, body: &str) -> PingoraResult<()> { + respond_bytes( + session, + status, + Some("application/json; charset=utf-8"), + Bytes::copy_from_slice(body.as_bytes()), + ) + .await +} + +async fn respond_bytes( + session: &mut Session, + status: u16, + content_type: Option<&str>, + body: Bytes, +) -> PingoraResult<()> { + let is_head = session.req_header().method == Method::HEAD; + let mut header = ResponseHeader::build(status, Some(body.len()))?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session + .write_response_header(Box::new(header), is_head || body.is_empty()) + .await?; + if !is_head && !body.is_empty() { + session.write_response_body(Some(body), true).await?; + } + Ok(()) +} + +fn resolve_request_id(session: &Session) -> String { + session + .req_header() + .headers + .get("x-request-id") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .unwrap_or_else(|| Uuid::new_v4().to_string()) +} + +fn content_length(session: &Session) -> Option { + session + .req_header() + .headers + .get("content-length")? + .to_str() + .ok()? + .parse() + .ok() +} + +fn client_ip(session: &Session) -> Option { + session + .as_downstream() + .client_addr() + .map(|addr| addr.ip().to_string()) +} + +fn append_forwarded_for(session: &Session, client_ip: &str) -> String { + session + .req_header() + .headers + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(|existing| format!("{existing}, {client_ip}")) + .unwrap_or_else(|| client_ip.to_string()) +} + +fn is_upgrade_request(session: &Session) -> bool { + session + .req_header() + .headers + .get("upgrade") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("websocket")) +} + +fn read_env_or_default(key: &str, default_value: &str) -> String { + env::var(key) + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| default_value.to_string()) +} + +fn read_bool_env(key: &str, default_value: bool) -> bool { + env::var(key) + .ok() + .and_then(|value| match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => None, + }) + .unwrap_or(default_value) +} + +fn read_u64_env(key: &str, default_value: u64) -> io::Result { + match env::var(key) { + Ok(value) if !value.trim().is_empty() => value.trim().parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效整数：{error}"), + ) + }), + _ => Ok(default_value), + } +} + +fn read_socket_addr_env(key: &str, default_value: &str) -> io::Result { + let value = read_env_or_default(key, default_value); + value.parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效 socket 地址 {value:?}：{error}"), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_nginx_api_routes_to_api_upstream() { + assert_eq!( + classify_path("/admin/api/users").proxy_target(), + Some(ProxyTarget::Api) + ); + assert_eq!( + classify_path("/api/runtime/puzzle/gallery").proxy_target(), + Some(ProxyTarget::Api) + ); + assert_eq!( + classify_path("/api/runtime/custom-world-gallery/profile-a/owner-b").proxy_target(), + Some(ProxyTarget::Api) + ); + assert_eq!( + classify_path("/api/creation-entry/config").proxy_target(), + Some(ProxyTarget::Api) + ); + } + + #[test] + fn applies_configured_body_limit_to_generic_api_routes() { + let mut route = classify_path("/api/creation-entry/config"); + apply_configured_body_limit(&mut route, 1024); + + assert_eq!( + route, + RouteDecision::Proxy { + target: ProxyTarget::Api, + body_limit: Some(1024), + } + ); + } + + #[test] + fn keeps_public_health_and_legacy_generated_paths_closed() { + assert_eq!( + classify_path("/healthz"), + RouteDecision::Local(LocalResponse::NotFound) + ); + assert_eq!( + classify_path("/readyz"), + RouteDecision::Local(LocalResponse::NotFound) + ); + assert_eq!( + classify_path("/healthz/internal"), + RouteDecision::Local(LocalResponse::NotFound) + ); + assert_eq!( + classify_path("/generated-puzzle-assets/foo.webp"), + RouteDecision::Local(LocalResponse::NotFound) + ); + } + + #[test] + fn only_forwards_minimal_spacetime_public_routes() { + assert_eq!( + classify_path("/v1/database/genarrative/subscribe").proxy_target(), + Some(ProxyTarget::Spacetime) + ); + assert_eq!( + classify_path("/v1/identity").proxy_target(), + Some(ProxyTarget::Spacetime) + ); + assert_eq!( + classify_path("/v1/ping"), + RouteDecision::Local(LocalResponse::NotFound) + ); + } + + #[test] + fn classifies_static_shell_routes() { + assert_eq!( + classify_path("/admin"), + RouteDecision::Local(LocalResponse::RedirectPermanent { + location: "/admin/" + }) + ); + assert_eq!( + classify_path("/admin/assets/index.js"), + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::Exact, + }) + ); + assert_eq!( + classify_path("/admin/settings"), + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }) + ); + assert_eq!( + classify_path("/some/spa/path"), + RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Web, + mode: StaticMode::SpaFallback, + }) + ); + } + + #[test] + fn rejects_path_traversal_for_static_files() { + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/app.js").is_some()); + assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/../secret").is_none()); + } +} From a7711d2dc3ae24fed917b2757b5a5f3ca59a20c9 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 20:16:51 +0800 Subject: [PATCH 02/22] =?UTF-8?q?=E5=BC=95=E5=85=A5Pingora=E7=8B=AC?= =?UTF-8?q?=E7=AB=8B=E7=BD=91=E5=85=B3=E6=AD=A3=E5=BC=8F=E5=8C=96=E9=93=BE?= =?UTF-8?q?=E8=B7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 pingora-gateway 独立二进制 crate，覆盖路由、静态资源、压缩、接流保护、TLS 直连和访问日志能力。 新增 Nginx canary、realpath canary、direct preflight、direct live、direct enable 和 rollback 脚本。 新增 Pingora 切流证据包、命令证据、manifest 验真、根目录总审计和 release readiness 聚合门禁。 完善 API release、Jenkins、systemd、health patrol、生产部署和发布包自包含校验。 更新 Pingora 试点文档、Nginx README 与 Hermes 共享记忆。 --- .hermes/shared-memory/decision-log.md | 104 + .hermes/shared-memory/pitfalls.md | 85 + deploy/env/health-patrol.env.example | 19 + deploy/env/pingora-canary-live.env.example | 35 + deploy/env/pingora-direct-live.env.example | 65 + deploy/logrotate/genarrative-pingora-gateway | 10 + deploy/nginx/README.md | 47 + .../snippets/genarrative-pingora-canary.conf | 64 + .../genarrative-pingora-realpath-canary.conf | 90 + deploy/pingora/nginx-route-parity.matrix.json | 375 ++ deploy/pingora/pingora-gateway.env.example | 58 + .../genarrative-database-backup.service | 2 +- ...arrative-pingora-gateway-direct-entry.conf | 17 + .../genarrative-pingora-gateway.service | 31 + ...开发运维】Pingora独立网关试点-2026-06-11.md | 415 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 74 +- jenkins/Jenkinsfile.production-api-build | 30 +- jenkins/Jenkinsfile.production-api-deploy | 19 +- package.json | 31 +- scripts/build-production-release.sh | 108 +- scripts/check-nginx-pingora-canary.mjs | 517 ++ ...pingora-canary-access-log-parity-check.mjs | 493 ++ ...check-pingora-canary-access-log-parity.mjs | 427 ++ scripts/check-pingora-canary-docker.mjs | 981 +++ scripts/check-pingora-canary-live-guard.mjs | 72 + scripts/check-pingora-canary-live.mjs | 360 ++ .../check-pingora-current-release-audit.mjs | 630 ++ ...check-pingora-cutover-command-evidence.mjs | 908 +++ .../check-pingora-cutover-evidence-audit.mjs | 2901 +++++++++ .../check-pingora-cutover-evidence-bundle.mjs | 1876 ++++++ .../check-pingora-cutover-evidence-verify.mjs | 597 ++ .../check-pingora-cutover-status-snapshot.mjs | 939 +++ scripts/check-pingora-direct-enable.mjs | 2339 +++++++ scripts/check-pingora-direct-live-guard.mjs | 111 + scripts/check-pingora-direct-live.mjs | 1353 ++++ .../check-pingora-direct-preflight-guard.mjs | 134 + scripts/check-pingora-direct-preflight.mjs | 901 +++ scripts/check-pingora-direct-rollback.mjs | 2062 ++++++ scripts/check-pingora-gateway-smoke.mjs | 2225 +++++++ ...check-pingora-health-patrol-env-switch.mjs | 801 +++ ...check-pingora-production-release-build.mjs | 173 + .../check-pingora-release-readiness-plan.mjs | 3872 ++++++++++++ scripts/check-pingora-release-readiness.mjs | 1990 ++++++ scripts/check-pingora-route-parity.mjs | 242 + scripts/check-production-api-deploy.mjs | 1302 ++++ scripts/check-production-api-release.mjs | 534 ++ ...eck-production-health-patrol-env-check.mjs | 281 + .../check-production-health-patrol-env.mjs | 313 + scripts/check-production-health-patrol.mjs | 474 ++ scripts/check-production-ops-guardrails.mjs | 5611 ++++++++++++++++- scripts/database-backup-to-oss.mjs | 4 +- scripts/deploy/pingora-direct-enable.sh | 909 +++ scripts/deploy/pingora-direct-rollback.sh | 768 +++ .../pingora-health-patrol-env-switch.mjs | 391 ++ scripts/deploy/production-api-deploy.sh | 321 +- scripts/deploy/production-stdb-publish.sh | 6 +- scripts/jenkins-server-provision.sh | 50 +- scripts/ops/pingora-current-release-audit.mjs | 666 ++ .../ops/pingora-cutover-command-evidence.mjs | 518 ++ .../ops/pingora-cutover-evidence-audit.mjs | 1919 ++++++ .../ops/pingora-cutover-evidence-bundle.mjs | 1146 ++++ .../ops/pingora-cutover-evidence-verify.mjs | 341 + .../ops/pingora-cutover-status-snapshot.mjs | 939 +++ scripts/ops/production-health-patrol.mjs | 312 +- server-rs/Cargo.lock | 275 +- server-rs/Cargo.toml | 3 +- server-rs/crates/pingora-gateway/Cargo.toml | 7 +- server-rs/crates/pingora-gateway/src/main.rs | 3047 ++++++++- 68 files changed, 48450 insertions(+), 270 deletions(-) create mode 100644 deploy/env/health-patrol.env.example create mode 100644 deploy/env/pingora-canary-live.env.example create mode 100644 deploy/env/pingora-direct-live.env.example create mode 100644 deploy/logrotate/genarrative-pingora-gateway create mode 100644 deploy/nginx/snippets/genarrative-pingora-canary.conf create mode 100644 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf create mode 100644 deploy/pingora/nginx-route-parity.matrix.json create mode 100644 deploy/systemd/genarrative-pingora-gateway-direct-entry.conf create mode 100644 deploy/systemd/genarrative-pingora-gateway.service create mode 100644 scripts/check-nginx-pingora-canary.mjs create mode 100644 scripts/check-pingora-canary-access-log-parity-check.mjs create mode 100644 scripts/check-pingora-canary-access-log-parity.mjs create mode 100644 scripts/check-pingora-canary-docker.mjs create mode 100644 scripts/check-pingora-canary-live-guard.mjs create mode 100644 scripts/check-pingora-canary-live.mjs create mode 100644 scripts/check-pingora-current-release-audit.mjs create mode 100644 scripts/check-pingora-cutover-command-evidence.mjs create mode 100644 scripts/check-pingora-cutover-evidence-audit.mjs create mode 100644 scripts/check-pingora-cutover-evidence-bundle.mjs create mode 100644 scripts/check-pingora-cutover-evidence-verify.mjs create mode 100644 scripts/check-pingora-cutover-status-snapshot.mjs create mode 100644 scripts/check-pingora-direct-enable.mjs create mode 100644 scripts/check-pingora-direct-live-guard.mjs create mode 100644 scripts/check-pingora-direct-live.mjs create mode 100644 scripts/check-pingora-direct-preflight-guard.mjs create mode 100644 scripts/check-pingora-direct-preflight.mjs create mode 100644 scripts/check-pingora-direct-rollback.mjs create mode 100644 scripts/check-pingora-gateway-smoke.mjs create mode 100644 scripts/check-pingora-health-patrol-env-switch.mjs create mode 100644 scripts/check-pingora-production-release-build.mjs create mode 100644 scripts/check-pingora-release-readiness-plan.mjs create mode 100644 scripts/check-pingora-release-readiness.mjs create mode 100644 scripts/check-pingora-route-parity.mjs create mode 100644 scripts/check-production-api-deploy.mjs create mode 100644 scripts/check-production-api-release.mjs create mode 100644 scripts/check-production-health-patrol-env-check.mjs create mode 100644 scripts/check-production-health-patrol-env.mjs create mode 100644 scripts/check-production-health-patrol.mjs create mode 100644 scripts/deploy/pingora-direct-enable.sh create mode 100644 scripts/deploy/pingora-direct-rollback.sh create mode 100644 scripts/deploy/pingora-health-patrol-env-switch.mjs create mode 100644 scripts/ops/pingora-current-release-audit.mjs create mode 100644 scripts/ops/pingora-cutover-command-evidence.mjs create mode 100644 scripts/ops/pingora-cutover-evidence-audit.mjs create mode 100644 scripts/ops/pingora-cutover-evidence-bundle.mjs create mode 100644 scripts/ops/pingora-cutover-evidence-verify.mjs create mode 100644 scripts/ops/pingora-cutover-status-snapshot.mjs diff --git a/.hermes/shared-memory/decision-log.md b/.hermes/shared-memory/decision-log.md index 53b8a753d..5deb903b7 100644 --- a/.hermes/shared-memory/decision-log.md +++ b/.hermes/shared-memory/decision-log.md @@ -16,6 +16,87 @@ --- +## 2026-06-13 Pingora 低端口直连只通过显式 systemd drop-in 启用 + +- 背景：`genarrative-pingora-gateway.service` 默认以 `genarrative` 非 root 用户运行，shadow 阶段只监听本机高端口；如果正式评估让 Pingora 直接绑定公网 `80/443`，需要低端口绑定能力，但不能让 Server-Provision 或默认 service 自动改变接流边界。 +- 决策：主 systemd service 保持 shadow 口径，不携带 `CAP_NET_BIND_SERVICE`。仓库提供 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用 drop-in 模板，Server-Provision 只安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 备查和手动覆盖；正式切换窗口从 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 执行时默认读取 current release 随包的 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 `/etc` 参考模板、Jenkins 工作区或源码 checkout。直连切换前先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成 JSON runbook，逐条审阅 Host 与回退巡检入口确认、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply（通过命令证据脚本归档 stdout / stderr / 退出码）、切换后 health patrol 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、rollback apply（通过命令证据脚本归档 stdout / stderr / 退出码）、回退后 health patrol 切回 `nginx` 并恢复切换前 public base URL / Host、回退后 health patrol env Nginx 模式复核；启用前基础门禁不带 `--require-direct`，因为 systemd drop-in 尚未生效，启用后复核必须带 `--require-direct`。正式切换 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 和回退 smoke 分别验证到不同入口；还必须显式传 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，切换前 Nginx 巡检需要 Host 覆盖时再传 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 覆盖现场原有巡检入口。若需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文并把参数传给 rollback dry-run / apply。只有切换窗口通过 `pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名>` 先跑 current release 自审，确认发布包自包含、`pingora-gateway` 可执行且 systemd `ExecStart` 指向随包网关，失败时不安装 drop-in；随后跑 direct preflight，确认当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都可读取证书链 / 私钥，确认 service 模板与 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含本次 `/etc/genarrative/pingora-gateway.env`，并确认 service `ExecStart=` 指向的 current release `pingora-gateway` 存在且可执行，再安装到 `/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf`、执行 `systemctl daemon-reload`、重启 Pingora，并用 `systemctl cat` 核验 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl is-active` 确认服务 active、用 direct live smoke 验证 HTTPS / HTTP redirect / ACME / WSS 101 后，才视为授予低端口能力成功。启用前还必须显式配置 TLS / redirect env 和真实证书，并确认 current release 已落盘可执行 `pingora-gateway`、Nginx 或其它进程已释放 `80/443`；启用后仍必须跑 release readiness 门禁；验证失败时统一执行 `pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 回到 shadow / Nginx 入口，回退脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，再用 `systemctl cat` 确认两条低端口 capability 均已从最终 unit 配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 curl smoke URL 证明 Nginx 入口真实可访问。回退脚本 `--apply` 必须同时带 `--reload-nginx` 和 `--nginx-smoke-url`，避免只撤掉 Pingora 低端口能力却没有证明 Nginx 已重新接流；本机打 `127.0.0.1`、`localhost` 或 `::1` 时，`--apply` 必须带 `--nginx-smoke-host <域名>`，且该值只能是 host 或 `host:port`，避免命中默认 vhost。回退后必须复核 health patrol env 已切回 `nginx` 且 public base URL / Host 恢复为切换前记录值；如果 env 已预先修正，rollback 脚本可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 自动执行这项复核，切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若需证明 Pingora 仍以 shadow 高端口存活，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。 +- 决策补充：health patrol env 的直连/回退切换不再靠人工编辑三行变量；正式 runbook 使用 current release 随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply`，只更新 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST` 并立即调用随包 `check-production-health-patrol-env.mjs` 复核。Pingora direct 使用本机 public base URL 时脚本必须带 `--public-host <域名>`；回退到 Nginx 时根据切换前记录传 `--clear-public-host` 或 `--public-host <切换前Host>`。生产巡检、health patrol env 复核和 env 切换脚本读取的布尔 env 必须严格解析，非法值直接失败，不得静默按 false 继续；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，凡是用 Node 启动项目脚本且要把业务 `--env-file` 传给脚本时，必须写成 `node -- ', + '', + ].join(''), + ); + await writeFile( + path.join(webRoot, 'admin', 'index.html'), + '
admin-shell
', + ); + await writeFile( + path.join(webRoot, 'assets', 'app.js'), + 'console.log("site asset");\n', + ); + await writeFile( + path.join(webRoot, 'assets', 'index-B4dmVw0r.js'), + 'console.log("fingerprinted site asset");\n', + ); + await writeFile( + path.join(webRoot, 'assets', 'large-app.js'), + `console.log("site asset large");\n${'/* asset filler */\n'.repeat(120)}`, + ); + await writeFile( + path.join(webRoot, 'assets', 'large-image.webp'), + Buffer.alloc(4096, 0x52), + ); + await writeFile( + path.join(webRoot, 'admin', 'assets', 'admin.js'), + 'console.log("admin asset");', + ); + await writeFile( + path.join(webRoot, 'admin', 'assets', 'admin-Ca8f3012.css'), + 'body{color:#123456;}', + ); + await writeFile( + path.join(webRoot, 'maintenance.html'), + '
maintenance
', + ); + await writeFile( + path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'), + 'acme-token', + ); +} + +async function createSelfSignedCertificate(tempRoot) { + const tlsDir = path.join(tempRoot, 'tls'); + await mkdir(tlsDir, { recursive: true }); + const certFile = path.join(tlsDir, 'cert.pem'); + const keyFile = path.join(tlsDir, 'key.pem'); + runCommand('openssl', [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-keyout', + keyFile, + '-out', + certFile, + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost,IP:127.0.0.1', + '-days', + '1', + ]); + return { certFile, keyFile }; +} + +async function startApiMock() { + const state = { + requests: [], + releaseHold: undefined, + }; + const server = http.createServer(async (request, response) => { + let body; + try { + body = await readRequestBody(request); + } catch (error) { + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + body: Buffer.alloc(0), + aborted: true, + error: error instanceof Error ? error.message : String(error), + }); + return; + } + + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + body, + }); + + if (request.url?.endsWith('/hold')) { + await new Promise((resolve) => { + state.releaseHold = resolve; + }); + } else if (request.url?.endsWith('/upstream-close')) { + request.socket.destroy(); + return; + } else if (request.url?.endsWith('/slow')) { + await delay(1500); + } + + response.writeHead(200, { + 'Content-Type': 'application/json; charset=utf-8', + 'X-Upstream': 'api', + }); + response.end( + JSON.stringify({ + ok: true, + upstream: 'api', + method: request.method, + url: request.url, + requestId: request.headers['x-request-id'] || '', + forwardedProto: request.headers['x-forwarded-proto'] || '', + forwardedHost: request.headers['x-forwarded-host'] || '', + forwardedFor: request.headers['x-forwarded-for'] || '', + realIp: request.headers['x-real-ip'] || '', + host: request.headers.host || '', + bodyBytes: body.length, + }), + ); + }); + + const port = await listen(server); + return { + port, + state, + }; +} + +async function startSpacetimeMock() { + const state = { + websocketMessages: [], + requests: [], + }; + const server = http.createServer(async (request, response) => { + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + }); + response.writeHead(200, { + 'Content-Type': 'application/json; charset=utf-8', + }); + response.end( + JSON.stringify({ ok: true, upstream: 'spacetime', url: request.url }), + ); + }); + + server.on('upgrade', (request, socket) => { + socket.on('error', () => {}); + state.requests.push({ + method: request.method || '', + url: request.url || '', + headers: request.headers, + }); + const key = request.headers['sec-websocket-key']; + if (!key) { + socket.destroy(); + return; + } + + const accept = createHash('sha1') + .update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`) + .digest('base64'); + socket.write( + [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Protocol: ${SPACETIME_WEBSOCKET_PROTOCOL}`, + `Sec-WebSocket-Accept: ${accept}`, + '', + '', + ].join('\r\n'), + ); + socket.write(encodeWebSocketTextFrame('spacetime-upgrade-ok')); + socket.on('data', (chunk) => { + state.websocketMessages.push(chunk); + }); + }); + + const port = await listen(server); + return { + port, + state, + }; +} + +async function runSmokeCases( + baseUrl, + tlsBaseUrl, + redirectBaseUrl, + probeToken, + maintenanceFile, + accessLogFile, + api, + spacetime, +) { + await expectHttp( + baseUrl, + '/', + 200, + 'site-shell', + '主站根路径返回 index.html', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + baseUrl, + '/some/deep/link', + 200, + 'site-shell', + '主站深链回退 index.html', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp(baseUrl, '/admin', 301, '', '/admin 301 到 /admin/', { + validate: (response) => response.headers.location === '/admin/', + }); + await expectHttp( + baseUrl, + '/admin/settings', + 200, + 'admin-shell', + '后台深链回退 admin/index.html', + ); + const staticAssetResponse = await expectHttp( + baseUrl, + '/assets/app.js', + 200, + 'site asset', + '主站静态资源精确读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + assertStaticValidatorHeaders(staticAssetResponse, '主站静态资源精确读取'); + ensure( + staticAssetResponse.headers['accept-ranges'] === 'bytes', + `主站静态资源精确读取: 缺少 Accept-Ranges=bytes，实际 ${staticAssetResponse.headers['accept-ranges'] || '-'}`, + ); + await expectHttp( + baseUrl, + '/assets/app.js', + 200, + '', + 'HEAD 静态资源只返回响应头', + { + method: 'HEAD', + validate: (response) => + response.body === '' && + response.headers.etag === staticAssetResponse.headers.etag && + response.headers['last-modified'] === + staticAssetResponse.headers['last-modified'] && + Number(response.headers['content-length']) === + Buffer.byteLength('console.log("site asset");\n'), + }, + ); + await expectStaticNotModifiedByEtag( + baseUrl, + '/assets/app.js', + staticAssetResponse, + { + headers: { + 'X-Request-Id': 'static-etag-304-request-id', + }, + }, + ); + await expectStaticNotModifiedByLastModified( + baseUrl, + '/assets/app.js', + staticAssetResponse, + { + headers: { + 'X-Request-Id': 'static-last-modified-304-request-id', + }, + }, + ); + await expectHttp( + baseUrl, + '/assets/app.js', + 405, + '', + '静态资源非读取方法返回 405', + { + method: 'POST', + headers: { + 'X-Request-Id': 'static-method-405-request-id', + }, + body: 'ignored', + validate: (response) => + response.body === '' && + response.headers.allow === 'GET, HEAD' && + response.headers['x-genarrative-gateway'] === 'pingora-shadow', + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + 'console', + 'bytes 0-6/27', + '静态资源 Range 返回 206', + { + headers: { + 'X-Request-Id': 'static-range-206-request-id', + }, + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + 'console', + 'bytes 0-6/27', + '静态资源 If-Range 日期匹配返回 206', + { + headers: { + 'If-Range': staticAssetResponse.headers['last-modified'], + }, + }, + ); + await expectStaticRangeFallback( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 'Tue, 14 Nov 2023 22:13:12 GMT', + '静态资源 If-Range 旧日期回完整文件', + ); + await expectStaticRangeFallback( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + staticAssetResponse.headers.etag, + '静态资源 If-Range 弱 ETag 回完整文件', + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=-8', + 206, + 'sset");\n', + 'bytes 19-26/27', + '静态资源 suffix Range 返回 206', + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=999-1000', + 416, + '', + 'bytes */27', + '静态资源越界 Range 返回 416', + { + headers: { + 'X-Request-Id': 'static-range-416-request-id', + }, + }, + ); + await expectStaticRange( + baseUrl, + '/assets/app.js', + 'bytes=0-6', + 206, + '', + 'bytes 0-6/27', + 'HEAD 静态资源 Range 只返回响应头', + { + method: 'HEAD', + expectedContentLength: 7, + headers: { + 'X-Request-Id': 'static-head-range-206-request-id', + }, + }, + ); + await expectHttp( + baseUrl, + '/assets/index-B4dmVw0r.js', + 200, + 'fingerprinted site asset', + '主站指纹静态资源长缓存', + { + validate: (response) => + response.headers['cache-control'] === + 'public, max-age=31536000, immutable', + }, + ); + await expectNotCompressedResponse( + baseUrl, + '/assets/app.js', + 'site asset', + 'gzip 最小长度不压缩小响应', + ); + await expectGzipResponse(baseUrl, '/assets/large-app.js', 'site asset large'); + await expectGzipResponse(baseUrl, '/assets/large-app.js', 'site asset large', { + acceptEncoding: 'br, gzip', + label: 'gzip-only 压缩算法白名单', + }); + await expectNotCompressedResponse( + baseUrl, + '/assets/large-image.webp', + undefined, + 'gzip types 不压缩图片资源', + ); + await expectHttp( + baseUrl, + '/admin/assets/admin.js', + 200, + 'admin asset', + '后台静态资源精确读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + baseUrl, + '/admin/assets/admin-Ca8f3012.css', + 200, + 'body{color', + '后台指纹静态资源长缓存', + { + validate: (response) => + response.headers['cache-control'] === + 'public, max-age=31536000, immutable', + }, + ); + await expectHttp( + baseUrl, + '/.well-known/acme-challenge/token', + 200, + 'acme-token', + 'ACME challenge 读取', + { + validate: (response) => + response.headers['cache-control'] === 'no-cache', + }, + ); + await expectHttp( + tlsBaseUrl, + '/', + 200, + 'site-shell', + 'TLS 直连接口返回 index.html', + { insecureTls: true }, + ); + await expectHttp( + redirectBaseUrl, + '/api/creation-entry/config?from=smoke', + 301, + '', + 'HTTP 入口 301 到 HTTPS', + { + headers: { Host: 'example.test' }, + validate: (response) => + response.headers.location === + 'https://example.test/api/creation-entry/config?from=smoke', + }, + ); + await expectHttp( + redirectBaseUrl, + '/.well-known/acme-challenge/token', + 200, + 'acme-token', + 'HTTP 重定向入口保留 ACME challenge', + ); + await expectHttp( + baseUrl, + '/generated-puzzle-assets/a.png', + 404, + '', + 'generated 私有资产公网拒绝', + ); + await expectHttp(baseUrl, '/healthz', 404, '', '公网 healthz 拒绝'); + await expectHttp( + baseUrl, + '/v1/ping', + 404, + '', + '非白名单 SpacetimeDB 路由拒绝', + ); + + await expectHttp( + baseUrl, + '/__genarrative_pingora/healthz', + 404, + '', + 'shadow probe 无 token 返回 404', + ); + await expectHttp( + baseUrl, + '/__genarrative_pingora/healthz', + 200, + '"gateway":"pingora-shadow"', + 'shadow probe token 通过', + { + headers: { 'X-Genarrative-Pingora-Probe': probeToken }, + }, + ); + + const apiResponse = await expectHttp( + baseUrl, + '/api/creation-entry/config', + 200, + '"upstream":"api"', + '通用 API 转发', + { + headers: { + 'X-Request-Id': 'smoke-request-id', + 'X-Forwarded-For': '203.0.113.10', + Host: 'example.test', + }, + }, + ); + ensure( + apiResponse.headers['x-genarrative-gateway'] === 'pingora-shadow', + 'API 响应缺少网关标记', + ); + ensure( + apiResponse.headers['x-accel-buffering'] === 'no', + 'API 响应缺少 X-Accel-Buffering=no', + ); + const apiPayload = JSON.parse(apiResponse.body); + ensure( + apiPayload.requestId === 'smoke-request-id', + 'API 上游未收到 X-Request-Id', + ); + ensure( + apiPayload.forwardedProto === 'https', + 'API 上游未收到配置的 X-Forwarded-Proto', + ); + ensure(apiPayload.host === 'example.test', 'API 上游 Host 未透传'); + ensure( + apiPayload.forwardedHost === 'example.test', + 'API 上游未收到 X-Forwarded-Host', + ); + ensure( + apiPayload.forwardedFor.endsWith(', 127.0.0.1'), + `API 上游 X-Forwarded-For 未追加 TCP 对端 IP：${apiPayload.forwardedFor}`, + ); + ensure( + apiPayload.realIp === '127.0.0.1', + `API 上游 X-Real-IP 未使用 TCP 对端 IP：${apiPayload.realIp}`, + ); + + await expectHttp( + baseUrl, + '/api/upload', + 413, + 'PAYLOAD_TOO_LARGE', + 'Content-Length body limit', + { + method: 'POST', + body: '0123456789abcdef', + }, + ); + await expectChunkedLimit(baseUrl, '/api/upload'); + + await expectConcurrencyLimit(baseUrl, api); + + const rateLimitHeaders = { 'X-Forwarded-For': '203.0.113.13' }; + const beforeRateLimitRequestCount = api.state.requests.length; + await expectHttp( + baseUrl, + '/api/rate-limited', + 200, + '"upstream":"api"', + 'API RPS 首次请求放行', + { + headers: rateLimitHeaders, + }, + ); + await expectHttp( + baseUrl, + '/api/rate-limited', + 429, + 'GATEWAY_RATE_LIMITED', + 'API RPS 保护', + { + headers: rateLimitHeaders, + }, + ); + ensure( + api.state.requests.length === beforeRateLimitRequestCount + 1, + 'API RPS 保护的 429 请求不应打到上游', + ); + await expectHttp( + baseUrl, + '/api/upstream-close', + 502, + 'GATEWAY_UPSTREAM_ERROR', + 'API 上游断连返回稳定 JSON', + { + headers: { 'X-Forwarded-For': '203.0.113.14' }, + }, + ); + await expectHttp( + baseUrl, + '/api/slow', + 504, + 'GATEWAY_UPSTREAM_TIMEOUT', + 'API 上游读取超时返回稳定 JSON', + { + headers: { 'X-Forwarded-For': '203.0.113.15' }, + timeoutMs: 5000, + }, + ); + + await expectHttp( + baseUrl, + '/v1/identity', + 200, + '"upstream":"spacetime"', + 'SpacetimeDB identity 转发', + ); + await expectWebSocketUpgrade( + baseUrl, + '/v1/database/genarrative/subscribe', + spacetime, + ); + + await expectAccessLog(accessLogFile); + await expectAccessLogEntries(accessLogFile, [ + { + requestId: 'static-etag-304-request-id', + method: 'GET', + path: '/assets/app.js', + status: 304, + }, + { + requestId: 'static-last-modified-304-request-id', + method: 'GET', + path: '/assets/app.js', + status: 304, + }, + { + requestId: 'static-method-405-request-id', + method: 'POST', + path: '/assets/app.js', + status: 405, + }, + { + requestId: 'static-range-206-request-id', + method: 'GET', + path: '/assets/app.js', + status: 206, + }, + { + requestId: 'static-range-416-request-id', + method: 'GET', + path: '/assets/app.js', + status: 416, + }, + { + requestId: 'static-head-range-206-request-id', + method: 'HEAD', + path: '/assets/app.js', + status: 206, + }, + ]); + + await mkdir(path.dirname(maintenanceFile), { recursive: true }); + await writeFile(maintenanceFile, 'enabled'); + await expectHttp( + baseUrl, + '/api/creation-entry/config', + 503, + 'MAINTENANCE', + '维护模式 API JSON', + ); + await expectHttp(baseUrl, '/', 503, 'maintenance', '维护模式 Web 页面'); + await expectAccessLogContains(accessLogFile, [ + 'status=503', + 'path=/api/creation-entry/config', + ]); +} + +async function expectAccessLog(accessLogFile) { + console.log('[pingora-gateway-smoke] access log 落盘'); + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + return ( + content.includes('request_id=smoke-request-id') && + content.includes('method=GET') && + content.includes('path=/api/creation-entry/config') && + content.includes('status=200') && + content.includes('proxy_target=Api') && + content.includes('protection_class=api') && + content.includes('status=429') && + content.includes('status=502') && + content.includes('status=504') + ); + } catch { + return false; + } + }); +} + +function assertStaticValidatorHeaders(response, label) { + ensure(Boolean(response.headers.etag), `${label}: 缺少 ETag`); + ensure( + /^W\/"[0-9a-f]+-[0-9a-f]+"$/iu.test(String(response.headers.etag || '')), + `${label}: ETag 格式不符合静态弱校验器口径：${response.headers.etag || '-'}`, + ); + ensure(Boolean(response.headers['last-modified']), `${label}: 缺少 Last-Modified`); + ensure( + Number.isFinite(Date.parse(response.headers['last-modified'] || '')), + `${label}: Last-Modified 不是有效 HTTP 日期：${response.headers['last-modified'] || '-'}`, + ); +} + +async function expectStaticNotModifiedByEtag( + baseUrl, + route, + sourceResponse, + options = {}, +) { + await expectHttp( + baseUrl, + route, + 304, + '', + '静态资源 If-None-Match 返回 304', + { + headers: { + 'If-None-Match': sourceResponse.headers.etag, + ...(options.headers || {}), + }, + validate: (response) => + response.body === '' && + response.headers.etag === sourceResponse.headers.etag && + response.headers['cache-control'] === + sourceResponse.headers['cache-control'], + }, + ); +} + +async function expectStaticNotModifiedByLastModified( + baseUrl, + route, + sourceResponse, + options = {}, +) { + await expectHttp( + baseUrl, + route, + 304, + '', + '静态资源 If-Modified-Since 返回 304', + { + headers: { + 'If-Modified-Since': sourceResponse.headers['last-modified'], + ...(options.headers || {}), + }, + validate: (response) => + response.body === '' && + response.headers['last-modified'] === + sourceResponse.headers['last-modified'] && + response.headers['cache-control'] === + sourceResponse.headers['cache-control'], + }, + ); +} + +async function expectStaticRange( + baseUrl, + route, + range, + status, + bodyNeedle, + contentRange, + label, + options = {}, +) { + const headers = { + Range: range, + 'Accept-Encoding': 'gzip', + ...(options.headers || {}), + }; + await expectHttp(baseUrl, route, status, bodyNeedle, label, { + method: options.method, + headers, + validate: (response) => + response.headers['accept-ranges'] === 'bytes' && + response.headers['content-range'] === contentRange && + !response.headers['content-encoding'] && + (options.method === 'HEAD' ? response.body === '' : true) && + (status === 206 + ? Number(response.headers['content-length']) === + (options.expectedContentLength ?? Buffer.byteLength(bodyNeedle)) + : Number(response.headers['content-length']) === 0), + }); +} + +async function expectStaticRangeFallback(baseUrl, route, range, ifRange, label) { + await expectHttp(baseUrl, route, 200, 'site asset', label, { + headers: { + Range: range, + 'If-Range': ifRange, + 'Accept-Encoding': 'gzip', + }, + validate: (response) => + response.headers['accept-ranges'] === 'bytes' && + !response.headers['content-range'] && + Number(response.headers['content-length']) === + Buffer.byteLength('console.log("site asset");\n'), + }); +} + +async function expectGzipResponse(baseUrl, route, bodyNeedle, options = {}) { + return expectCompressedResponse(baseUrl, route, bodyNeedle, { + acceptEncoding: options.acceptEncoding || 'gzip', + contentEncoding: 'gzip', + decode: (bodyBuffer) => gunzipSync(bodyBuffer).toString('utf8'), + label: options.label || 'gzip 压缩响应', + }); +} + +async function expectNotCompressedResponse(baseUrl, route, bodyNeedle, label) { + console.log(`[pingora-gateway-smoke] ${label}`); + const response = await requestHttp(`${baseUrl}${route}`, { + headers: { + 'Accept-Encoding': 'gzip', + }, + rawBody: true, + }); + + if (response.status !== 200) { + failures.push(`${label}: 期望 HTTP 200，实际 ${response.status}`); + return; + } + if (response.headers['content-encoding']) { + failures.push( + `${label}: 不应返回 Content-Encoding，实际 ${response.headers['content-encoding']}`, + ); + } + const body = response.bodyBuffer.toString('utf8'); + if (bodyNeedle && !body.includes(bodyNeedle)) { + failures.push(`${label}: 响应体缺少 ${bodyNeedle}`); + } +} + +async function expectCompressedResponse(baseUrl, route, bodyNeedle, options) { + const label = options.label; + console.log(`[pingora-gateway-smoke] ${label}`); + const response = await requestHttp(`${baseUrl}${route}`, { + headers: { + 'Accept-Encoding': options.acceptEncoding, + }, + rawBody: true, + }); + + if (response.status !== 200) { + failures.push(`${label}: 期望 HTTP 200，实际 ${response.status}`); + return; + } + if (response.headers['content-encoding'] !== options.contentEncoding) { + failures.push( + `${label}: 缺少 Content-Encoding=${options.contentEncoding}，实际 ${response.headers['content-encoding'] || '-'}`, + ); + } + if ( + !String(response.headers.vary || '') + .toLowerCase() + .includes('accept-encoding') + ) { + failures.push( + `${label}: 缺少 Vary: Accept-Encoding，实际 ${response.headers.vary || '-'}`, + ); + } + + let decoded = ''; + try { + decoded = options.decode(response.bodyBuffer); + } catch (error) { + failures.push( + `${label}: 解压失败：${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + if (!decoded.includes(bodyNeedle)) { + failures.push(`${label}: 解压后内容缺少 ${bodyNeedle}`); + } +} + +async function expectAccessLogContains(accessLogFile, needles) { + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + return needles.every((needle) => content.includes(needle)); + } catch { + return false; + } + }); +} + +async function expectAccessLogEntries(accessLogFile, expectedEntries) { + console.log('[pingora-gateway-smoke] access log 静态边界状态对账'); + await waitForCondition(async () => { + try { + const content = await readFile(accessLogFile, 'utf8'); + const entries = content.split(/\r?\n/u).filter(Boolean).map(parseAccessLogLine); + return expectedEntries.every((expected) => { + const entry = entries.find((item) => item.request_id === expected.requestId); + return ( + entry && + entry.method === expected.method && + entry.path === expected.path && + Number.parseInt(entry.status || '', 10) === expected.status && + entry.proxy_target === 'Local' + ); + }); + } catch { + return false; + } + }); +} + +function parseAccessLogLine(line) { + const fields = {}; + for (const part of line.split('\t')) { + const separator = part.indexOf('='); + if (separator <= 0) { + continue; + } + fields[part.slice(0, separator)] = unescapeAccessLogValue( + part.slice(separator + 1), + ); + } + return fields; +} + +function unescapeAccessLogValue(value) { + return value + .replace(/\\\\/gu, '\\') + .replace(/\\t/gu, '\t') + .replace(/\\n/gu, '\n') + .replace(/\\r/gu, '\r'); +} + +async function expectHttp( + baseUrl, + route, + status, + bodyNeedle, + label, + options = {}, +) { + console.log(`[pingora-gateway-smoke] ${label}`); + const pending = options.beforeRequest + ? await options.beforeRequest() + : undefined; + let response; + try { + response = await requestHttp(`${baseUrl}${route}`, { + method: options.method, + headers: options.headers, + body: options.body, + insecureTls: options.insecureTls, + }); + } catch (error) { + failures.push( + `${label}: ${error instanceof Error ? error.message : String(error)}`, + ); + } finally { + if (options.afterRequest) { + try { + await options.afterRequest(pending); + } catch (error) { + failures.push(error instanceof Error ? error.message : String(error)); + } + } + } + + if (!response) { + return { status: 0, headers: {}, body: '' }; + } + + if (config.verbose) { + console.log(`[pingora-gateway-smoke] ${label} -> HTTP ${response.status}`); + } + + if (response.status !== status) { + failures.push(`${label}: 期望 HTTP ${status}，实际 ${response.status}`); + } + + if (bodyNeedle && !response.body.includes(bodyNeedle)) { + failures.push(`${label}: 响应体缺少 ${bodyNeedle}`); + } + + if (options.validate && !options.validate(response)) { + failures.push(`${label}: 自定义响应校验未通过`); + } + + return response; +} + +async function expectChunkedLimit(baseUrl, route) { + console.log('[pingora-gateway-smoke] chunked body limit'); + const response = await new Promise((resolve, reject) => { + const request = http.request( + `${baseUrl}${route}`, + { + method: 'POST', + headers: { + 'Transfer-Encoding': 'chunked', + 'X-Forwarded-For': '203.0.113.11', + }, + }, + (incoming) => { + collectIncoming(incoming).then(resolve, reject); + }, + ); + request.on('error', reject); + request.write('012345'); + request.write('6789abcdef'); + request.end(); + }); + + if (response.status !== 413 || !response.body.includes('PAYLOAD_TOO_LARGE')) { + failures.push( + `chunked body limit: 期望 413 PAYLOAD_TOO_LARGE，实际 ${response.status} ${response.body}`, + ); + } +} + +async function expectConcurrencyLimit(baseUrl, api) { + console.log('[pingora-gateway-smoke] API 并发保护'); + const hold = await openRawHttpRequest(`${baseUrl}/admin/api/hold`, { + 'X-Forwarded-For': '203.0.113.12', + }); + + try { + await waitForCondition(() => typeof api.state.releaseHold === 'function'); + const beforeLimitedRequestCount = api.state.requests.length; + const response = await rawHttpRequest(`${baseUrl}/admin/api/users`, { + 'X-Forwarded-For': '203.0.113.12', + }); + + if (response.status !== 429) { + failures.push(`API 并发保护: 期望 HTTP 429，实际 ${response.status}`); + } + if (!response.body.includes('GATEWAY_CONCURRENCY_LIMITED')) { + failures.push('API 并发保护: 响应体缺少 GATEWAY_CONCURRENCY_LIMITED'); + } + ensure( + api.state.requests.length === beforeLimitedRequestCount, + `API 并发保护的 429 请求不应打到上游，上游请求为 ${describeRequests(api.state.requests.slice(beforeLimitedRequestCount))}`, + ); + } finally { + api.state.releaseHold?.(); + hold.socket.end(); + const holdResponse = await hold.done.catch((error) => ({ error })); + if (holdResponse?.error) { + failures.push( + `API 并发保护: hold 请求失败：${holdResponse.error.message}`, + ); + } + } +} + +async function expectWebSocketUpgrade(baseUrl, route, spacetime) { + console.log('[pingora-gateway-smoke] SpacetimeDB WebSocket Upgrade'); + const url = new URL(route, baseUrl); + const key = randomBytes(16).toString('base64'); + const response = await new Promise((resolve, reject) => { + const socket = net.connect( + { host: url.hostname, port: Number(url.port) }, + () => { + socket.write( + [ + `GET ${url.pathname} HTTP/1.1`, + `Host: ${url.host}`, + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Key: ${key}`, + 'Sec-WebSocket-Version: 13', + '', + '', + ].join('\r\n'), + ); + }, + ); + + let raw = Buffer.alloc(0); + let settled = false; + const timeout = setTimeout(() => { + settled = true; + socket.destroy(); + reject(new Error('WebSocket Upgrade 超时')); + }, 5000); + + socket.on('data', (chunk) => { + raw = Buffer.concat([raw, chunk]); + const text = raw.toString('latin1'); + if (text.includes('\r\n\r\n') && text.includes('spacetime-upgrade-ok')) { + settled = true; + clearTimeout(timeout); + socket.destroy(); + resolve(raw); + } + }); + socket.on('error', (error) => { + settled = true; + clearTimeout(timeout); + reject(error); + }); + socket.on('close', () => { + clearTimeout(timeout); + if (!settled) { + reject(new Error('WebSocket Upgrade 连接提前关闭')); + } + }); + }); + + const text = response.toString('latin1'); + ensure( + text.startsWith('HTTP/1.1 101'), + `WebSocket Upgrade 未返回 101：${text.split('\r\n')[0] || text}`, + ); + ensure( + text.includes('spacetime-upgrade-ok'), + 'WebSocket tunnel 未收到上游 welcome frame', + ); + const upgradeRequest = spacetime.state.requests.find( + (request) => + request.url === route && request.headers.upgrade === 'websocket', + ); + ensure( + Boolean(upgradeRequest), + 'SpacetimeDB mock 未收到 WebSocket Upgrade 请求', + ); +} + +function requestHttp(url, options = {}) { + return new Promise((resolve, reject) => { + const headers = { ...(options.headers || {}) }; + if ( + options.body !== undefined && + !hasHeader(headers, 'content-length') && + !hasHeader(headers, 'transfer-encoding') + ) { + headers['Content-Length'] = Buffer.byteLength(String(options.body)); + } + const client = url.startsWith('https:') ? https : http; + const request = client.request( + url, + { + method: options.method || 'GET', + headers, + agent: false, + rejectUnauthorized: options.insecureTls ? false : undefined, + }, + (incoming) => { + collectIncoming(incoming).then(resolve, reject); + }, + ); + const timeout = setTimeout(() => { + request.destroy(new Error(`HTTP 请求超时：${url}`)); + }, options.timeoutMs || 5000); + request.on('error', reject); + request.on('close', () => { + clearTimeout(timeout); + }); + if (options.body !== undefined) { + request.write(options.body); + } + request.end(); + }); +} + +function openRawHttpRequest(url, headers = {}) { + return new Promise((resolve, reject) => { + const target = new URL(url); + const socket = net.connect( + { host: target.hostname, port: Number(target.port) }, + () => { + socket.write(buildRawHttpRequest(target, headers)); + resolve({ + socket, + done: collectRawHttpResponse(socket, url, 35000), + }); + }, + ); + socket.on('error', reject); + }); +} + +async function rawHttpRequest(url, headers = {}) { + const request = await openRawHttpRequest(url, headers); + return request.done.finally(() => { + request.socket.destroy(); + }); +} + +function buildRawHttpRequest(target, headers) { + const mergedHeaders = { + Host: target.host, + Connection: 'close', + ...headers, + }; + const headerLines = Object.entries(mergedHeaders).map( + ([name, value]) => `${name}: ${value}`, + ); + return [ + `GET ${target.pathname}${target.search} HTTP/1.1`, + ...headerLines, + '', + '', + ].join('\r\n'); +} + +function collectRawHttpResponse(socket, url, timeoutMs) { + return new Promise((resolve, reject) => { + let raw = Buffer.alloc(0); + let settled = false; + const timer = setTimeout(() => { + settled = true; + socket.destroy(); + reject(new Error(`HTTP 请求超时：${url}`)); + }, timeoutMs); + + socket.on('data', (chunk) => { + raw = Buffer.concat([raw, chunk]); + let parsed; + try { + parsed = tryParseRawHttpResponse(raw); + } catch (error) { + settled = true; + clearTimeout(timer); + socket.destroy(); + reject(error); + return; + } + if (parsed) { + settled = true; + clearTimeout(timer); + resolve(parsed); + } + }); + socket.on('error', (error) => { + if (!settled) { + settled = true; + clearTimeout(timer); + reject(error); + } + }); + socket.on('close', () => { + if (!settled) { + settled = true; + clearTimeout(timer); + try { + const parsed = tryParseRawHttpResponse(raw, { final: true }); + if (parsed) { + resolve(parsed); + } else { + reject(new Error(`HTTP 响应提前关闭：${url}`)); + } + } catch (error) { + reject(error); + } + } + }); + }); +} + +function tryParseRawHttpResponse(raw, options = {}) { + const headerEnd = raw.indexOf('\r\n\r\n'); + if (headerEnd < 0) { + return null; + } + + const headerText = raw.slice(0, headerEnd).toString('latin1'); + const [statusLine, ...headerLines] = headerText.split('\r\n'); + const status = Number.parseInt(statusLine.split(' ')[1] || '0', 10); + const headers = {}; + for (const line of headerLines) { + const separator = line.indexOf(':'); + if (separator <= 0) { + continue; + } + headers[line.slice(0, separator).trim().toLowerCase()] = line + .slice(separator + 1) + .trim(); + } + + const bodyStart = headerEnd + 4; + const body = raw.slice(bodyStart); + if (hasToken(headers['transfer-encoding'], 'chunked')) { + const decoded = decodeChunkedBody(body); + if (!decoded.complete) { + return null; + } + + return { + status, + headers, + body: decoded.body.toString('utf8'), + }; + } + + const contentLength = Number.parseInt(headers['content-length'] || '', 10); + if (Number.isFinite(contentLength) && body.length < contentLength) { + return null; + } + if (Number.isFinite(contentLength)) { + return { + status, + headers, + body: body.slice(0, contentLength).toString('utf8'), + }; + } + + if (!options.final) { + return null; + } + + return { + status, + headers, + body: body.toString('utf8'), + }; +} + +function hasToken(value, expected) { + if (!value) { + return false; + } + const normalized = expected.toLowerCase(); + return value + .split(',') + .map((token) => token.trim().toLowerCase()) + .includes(normalized); +} + +function decodeChunkedBody(body) { + let offset = 0; + const chunks = []; + + while (offset < body.length) { + const lineEnd = body.indexOf('\r\n', offset); + if (lineEnd < 0) { + return { complete: false, body: Buffer.alloc(0) }; + } + + const sizeLine = body.slice(offset, lineEnd).toString('latin1'); + const sizeText = sizeLine.split(';')[0]?.trim() || ''; + const size = Number.parseInt(sizeText, 16); + if (!Number.isFinite(size)) { + throw new Error(`无法解析 chunked 响应大小：${sizeLine}`); + } + + offset = lineEnd + 2; + if (size === 0) { + const trailerEnd = body.indexOf('\r\n\r\n', offset); + if (trailerEnd >= 0) { + return { complete: true, body: Buffer.concat(chunks) }; + } + if (body.slice(offset, offset + 2).toString('latin1') === '\r\n') { + return { complete: true, body: Buffer.concat(chunks) }; + } + return { complete: false, body: Buffer.alloc(0) }; + } + + const chunkStart = offset; + const chunkEnd = chunkStart + size; + const nextChunkStart = chunkEnd + 2; + if (body.length < nextChunkStart) { + return { complete: false, body: Buffer.alloc(0) }; + } + if (body.slice(chunkEnd, nextChunkStart).toString('latin1') !== '\r\n') { + throw new Error('chunked 响应缺少 chunk 结束换行'); + } + + chunks.push(body.slice(chunkStart, chunkEnd)); + offset = nextChunkStart; + } + + return { complete: false, body: Buffer.alloc(0) }; +} + +function hasHeader(headers, name) { + const expected = name.toLowerCase(); + return Object.keys(headers).some((key) => key.toLowerCase() === expected); +} + +function collectIncoming(incoming, options = {}) { + return new Promise((resolve, reject) => { + const chunks = []; + incoming.on('data', (chunk) => chunks.push(chunk)); + incoming.on('error', reject); + incoming.on('end', () => { + const bodyBuffer = Buffer.concat(chunks); + resolve({ + status: incoming.statusCode || 0, + headers: incoming.headers, + body: options.rawBody ? '' : bodyBuffer.toString('utf8'), + bodyBuffer, + }); + }); + }); +} + +function readRequestBody(request) { + return new Promise((resolve, reject) => { + const chunks = []; + request.on('data', (chunk) => chunks.push(chunk)); + request.on('error', reject); + request.on('end', () => resolve(Buffer.concat(chunks))); + }); +} + +function listen(server) { + return new Promise((resolve, reject) => { + server.on('error', reject); + server.on('connection', (socket) => { + sockets.add(socket); + socket.on('close', () => { + sockets.delete(socket); + }); + }); + server.listen(0, '127.0.0.1', () => { + servers.push(server); + const address = server.address(); + if (!address || typeof address === 'string') { + reject(new Error('无法读取 mock server 端口')); + return; + } + resolve(address.port); + }); + }); +} + +function getFreePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address(); + if (!address || typeof address === 'string') { + server.close(); + reject(new Error('无法分配临时端口')); + return; + } + const { port } = address; + server.close(() => resolve(port)); + }); + }); +} + +async function waitForHttp(url, expectedStatus) { + const startedAt = Date.now(); + let lastError = ''; + while (Date.now() - startedAt < 10000) { + try { + const response = await requestHttp(url); + if (response.status === expectedStatus) { + return; + } + lastError = `HTTP ${response.status}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + await delay(100); + } + throw new Error(`等待 Pingora 网关就绪超时：${lastError}`); +} + +function runCommand(command, args) { + console.log( + `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`, + ); + const result = spawnSync(command, args, { + cwd: repoRoot, + env: smokeEnv(), + shell: false, + stdio: 'inherit', + }); + + if (result.error) { + throw new Error(`${command} 启动失败：${result.error.message}`); + } + if (result.signal) { + throw new Error(`${command} 被信号终止：${result.signal}`); + } + if ((result.status ?? 0) !== 0) { + throw new Error( + `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${result.status}`, + ); + } +} + +async function runDirectLiveSmoke(tlsBaseUrl, redirectBaseUrl, probeToken, accessLogFile) { + const output = await runCommandAsync('node', [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + tlsBaseUrl, + '--http-base-url', + redirectBaseUrl, + '--host', + new URL(tlsBaseUrl).host, + '--probe-token', + probeToken, + '--pingora-access-log', + accessLogFile, + '--redirect-host', + new URL(tlsBaseUrl).host, + '--spacetime-database', + 'genarrative', + '--require-wss-upgrade', + '--insecure-tls', + '--json', + ]); + assertDirectLiveAccessLogEvidence(output.stdout); +} + +function runCommandAsync(command, args) { + console.log( + `[pingora-gateway-smoke] ${command} ${redactSecretArgs(args).join(' ')}`, + ); + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + const child = spawn(command, args, { + cwd: repoRoot, + env: smokeEnv(), + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout?.on('data', (chunk) => { + const text = String(chunk); + stdout += text; + process.stdout.write(text); + }); + child.stderr?.on('data', (chunk) => { + const text = String(chunk); + stderr += text; + process.stderr.write(text); + }); + child.on('error', (error) => { + reject(new Error(`${command} 启动失败：${error.message}`)); + }); + child.on('exit', (status, signal) => { + if (signal) { + reject(new Error(`${command} 被信号终止：${signal}`)); + return; + } + if ((status ?? 0) !== 0) { + reject( + new Error( + `${command} ${redactSecretArgs(args).join(' ')} 退出码 ${status}\nstdout:\n${stdout}\nstderr:\n${stderr}`, + ), + ); + return; + } + resolve({ stdout, stderr }); + }); + }); +} + +function assertDirectLiveAccessLogEvidence(stdout) { + const payload = extractJsonObject(stdout, stdout.indexOf('{')); + if (!payload) { + failures.push('direct live JSON 输出缺失。'); + return; + } + let result; + try { + result = JSON.parse(payload); + } catch (error) { + failures.push( + `direct live JSON 输出无法解析：${error instanceof Error ? error.message : String(error)}`, + ); + return; + } + const accessLogCheck = result.results?.find( + (item) => item.name === 'direct-access-log', + ); + const staticAssetCheck = result.results?.find( + (item) => item.name === 'https-static-asset', + ); + ensure( + staticAssetCheck?.fingerprinted?.statusCode === 200, + 'direct live 必须验证首页引用的指纹静态资源 GET 200。', + ); + ensure( + staticAssetCheck?.fingerprinted?.head?.statusCode === 200, + 'direct live 必须验证首页引用的指纹静态资源 HEAD 200。', + ); + ensure( + staticAssetCheck?.fingerprinted?.range?.statusCode === 206, + 'direct live 必须验证首页引用的指纹静态资源 Range 206。', + ); + ensure( + staticAssetCheck?.headers?.['cache-control'] === 'no-cache', + 'direct live JSON 必须保留普通静态资源 Cache-Control 证据。', + ); + ensure( + Boolean(staticAssetCheck?.headers?.etag), + 'direct live JSON 必须保留普通静态资源 ETag 证据。', + ); + ensure( + staticAssetCheck?.range?.headers?.['content-range']?.startsWith('bytes 0-0/'), + 'direct live JSON 必须保留普通静态资源 Content-Range 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.headers?.['cache-control'] === + 'public, max-age=31536000, immutable', + 'direct live JSON 必须保留指纹静态资源 immutable Cache-Control 证据。', + ); + ensure( + Boolean(staticAssetCheck?.fingerprinted?.headers?.etag), + 'direct live JSON 必须保留指纹静态资源 ETag 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.range?.headers?.['content-range']?.startsWith( + 'bytes 0-0/', + ), + 'direct live JSON 必须保留指纹静态资源 Content-Range 证据。', + ); + ensure( + staticAssetCheck?.notModified?.etag?.statusCode === 304, + 'direct live 必须验证首页引用的静态资源 ETag 304。', + ); + ensure( + staticAssetCheck?.notModified?.lastModified?.statusCode === 304, + 'direct live 必须验证首页引用的静态资源 Last-Modified 304。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.etag?.statusCode === 304, + 'direct live 必须验证首页引用的指纹静态资源 ETag 304。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.lastModified?.statusCode === 304, + 'direct live 必须验证首页引用的指纹静态资源 Last-Modified 304。', + ); + ensure( + staticAssetCheck?.notModified?.etag?.headers?.['cache-control'] === + staticAssetCheck?.headers?.['cache-control'], + 'direct live JSON 必须保留普通静态 304 Cache-Control 证据。', + ); + ensure( + staticAssetCheck?.fingerprinted?.notModified?.etag?.headers?.['cache-control'] === + staticAssetCheck?.fingerprinted?.headers?.['cache-control'], + 'direct live JSON 必须保留指纹静态 304 Cache-Control 证据。', + ); + if (!accessLogCheck) { + failures.push('direct live JSON 缺少 direct-access-log 检查结果。'); + return; + } + ensure(accessLogCheck.checked >= 10, 'direct live access log 检查请求数不足。'); + ensure( + accessLogCheck.matchedCount === accessLogCheck.checked, + 'direct live access log matchedCount 必须等于 checked。', + ); + ensure( + accessLogCheck.missingCount === 0, + 'direct live access log 不应缺少 request_id。', + ); + ensure( + accessLogCheck.mismatchCount === 0, + 'direct live access log 不应出现 method/path/status 漂移。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'wss-spacetime-subscribe' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 101 && + item.actualStatusCode === 101, + ), + 'direct live access log 必须保留 WSS 101 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-asset-head' && + item.expectedMethod === 'HEAD' && + item.actualMethod === 'HEAD' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留静态 HEAD 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留指纹静态 GET 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-head' && + item.expectedMethod === 'HEAD' && + item.actualMethod === 'HEAD' && + item.expectedStatusCode === 200 && + item.actualStatusCode === 200, + ), + 'direct live access log 必须保留指纹静态 HEAD 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-range' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 206 && + item.actualStatusCode === 206, + ), + 'direct live access log 必须保留指纹静态 Range 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-asset-etag-304' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 304 && + item.actualStatusCode === 304, + ), + 'direct live access log 必须保留静态 ETag 304 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.matched) && + accessLogCheck.matched.some( + (item) => + item.name === 'https-static-fingerprinted-asset-etag-304' && + item.expectedMethod === 'GET' && + item.actualMethod === 'GET' && + item.expectedStatusCode === 304 && + item.actualStatusCode === 304, + ), + 'direct live access log 必须保留指纹静态 ETag 304 的 method/path/status 明细。', + ); + ensure( + Array.isArray(accessLogCheck.missing) && + accessLogCheck.missing.length === 0, + 'direct live access log missing 明细必须为空数组。', + ); + ensure( + Array.isArray(accessLogCheck.mismatches) && + accessLogCheck.mismatches.length === 0, + 'direct live access log mismatches 明细必须为空数组。', + ); +} + +function extractJsonObject(text, startIndex) { + if (startIndex < 0) { + return ''; + } + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + + return ''; +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : arg, + ); +} + +function resolveGatewayBinary() { + const explicit = process.env.GENARRATIVE_PINGORA_GATEWAY_BINARY; + if (explicit) { + return explicit; + } + + const candidates = [ + path.join(repoRoot, 'server-rs', 'target', 'debug', 'pingora-gateway'), + path.join(repoRoot, 'target', 'debug', 'pingora-gateway'), + ]; + const found = candidates.find((candidate) => existsSync(candidate)); + if (!found) { + throw new Error( + `未找到 pingora-gateway debug 二进制，请先构建或设置 GENARRATIVE_PINGORA_GATEWAY_BINARY。候选：${candidates.join(', ')}`, + ); + } + return found; +} + +function smokeEnv() { + return { + ...process.env, + PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`, + }; +} + +function collectProcessLogs(child, name) { + child.stdout?.on('data', (chunk) => { + if (config.verbose) { + process.stdout.write(`[${name}] ${chunk}`); + } + }); + child.stderr?.on('data', (chunk) => { + if (config.verbose) { + process.stderr.write(`[${name}] ${chunk}`); + } + }); +} + +function encodeWebSocketTextFrame(text) { + const payload = Buffer.from(text, 'utf8'); + if (payload.length > 125) { + throw new Error('smoke WebSocket frame payload too large'); + } + return Buffer.concat([Buffer.from([0x81, payload.length]), payload]); +} + +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitForCondition(predicate, timeoutMs = 2000) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + if (await predicate()) { + return; + } + await delay(20); + } + throw new Error('等待 smoke 条件达成超时'); +} + +function ensure(condition, message) { + if (!condition) { + failures.push(message); + } +} + +function describeRequests(requests) { + return JSON.stringify( + requests.map((request) => ({ + method: request.method, + url: request.url, + xForwardedFor: request.headers?.['x-forwarded-for'], + aborted: Boolean(request.aborted), + })), + ); +} + +async function cleanup() { + for (const { child, name } of childProcesses.reverse()) { + await stopChild(child, name); + } + + for (const socket of sockets) { + socket.destroy(); + } + sockets.clear(); + + await Promise.all( + servers.reverse().map( + (server) => + new Promise((resolve) => { + const timer = setTimeout(resolve, 1000); + server.close(() => { + clearTimeout(timer); + resolve(); + }); + }), + ), + ); + + await Promise.all( + tempDirs.reverse().map((dir) => rm(dir, { recursive: true, force: true })), + ); +} + +async function stopChild(child, name) { + if (child.exitCode !== null || child.signalCode !== null) { + return; + } + + child.kill('SIGTERM'); + const exited = await waitForExit(child, 3000); + if (!exited) { + child.kill('SIGKILL'); + const killed = await waitForExit(child, 3000); + if (!killed) { + failures.push(`${name} 未能退出`); + } + } +} + +function waitForExit(child, timeoutMs) { + return new Promise((resolve) => { + if (child.exitCode !== null || child.signalCode !== null) { + resolve(true); + return; + } + + const timer = setTimeout(() => { + child.off('exit', onExit); + resolve(false); + }, timeoutMs); + + function onExit() { + clearTimeout(timer); + resolve(true); + } + + child.once('exit', onExit); + }); +} diff --git a/scripts/check-pingora-health-patrol-env-switch.mjs b/scripts/check-pingora-health-patrol-env-switch.mjs new file mode 100644 index 000000000..922664aeb --- /dev/null +++ b/scripts/check-pingora-health-patrol-env-switch.mjs @@ -0,0 +1,801 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + chmodSync, + copyFileSync, + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const SWITCH_SCRIPT = 'scripts/deploy/pingora-health-patrol-env-switch.mjs'; +const ENV_CHECK_SCRIPT = path.join( + process.cwd(), + 'scripts/check-production-health-patrol-env.mjs', +); +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-health-patrol-switch-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-health-patrol-env-switch] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-health-patrol-env-switch] OK'); + +function main() { + assertScriptShape(); + assertDryRunDoesNotModifyEnv(); + assertDryRunEnvFileModeIsPrivate(); + assertApplyPreservesEnvFileModeAndOwnership(); + assertApplySwitchesToPingoraDirectAndPreservesOtherKeys(); + assertApplySwitchesBackToNginxAndClearsHost(); + assertApplyDoesNotWriteWhenVerificationFails(); + assertApplyRejectsSymlinkEnvFileBeforeWrite(); + assertMissingManagedKeysAreAppended(); + assertRejectsDirectLoopbackWithoutHost(); + assertRejectsInvalidAndConflictingHostOptions(); + assertRejectsRelativePaths(); + assertRejectsFilesystemRootPaths(); + assertRejectsControlCharacterInputsBeforeWrite(); + assertRejectsInvalidBoolEnv(); + assertReleaseLayoutUsesBundledCheckScript(); +} + +function assertScriptShape() { + const content = readFileSync(SWITCH_SCRIPT, 'utf8'); + assertIncludes(content, '--apply', '切换脚本必须显式要求 --apply 才写 env。'); + assertIncludes( + content, + '当前是 dry-run', + '切换脚本必须在 dry-run 中明确不会写 env。', + ); + assertIncludes( + content, + 'check-production-health-patrol-env.mjs', + '切换脚本必须复用生产 health patrol env 复核脚本。', + ); + assertIncludes( + content, + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host', + '切换脚本必须阻断 direct loopback 缺少正式 Host。', + ); + assertIncludes( + content, + 'health patrol env 中存在重复配置', + '切换脚本必须拒绝重复目标键，避免 env 解析歧义。', + ); + assertIncludes( + content, + 'runHealthPatrolEnvCheck(checkFile);', + '切换脚本必须先用临时目标 env 复核。', + ); + assertIncludes( + content, + 'writeEnvFileAtomically(config.envFile, nextText);', + '切换脚本通过复核后才可原子写入真实 env。', + ); + assertIncludes( + content, + '--env-file 不能是符号链接', + 'apply 写入真实 env 前必须拒绝符号链接目标。', + ); + assertIncludes( + content, + 'DRY_RUN_ENV_FILE_MODE = 0o600', + '临时复核 env 文件权限必须固定为 0600。', + ); + assertIncludes( + content, + 'chownSync(tempFile, currentStat.uid, currentStat.gid);', + '真实 env 原子替换必须保留原文件 owner/group。', + ); +} + +function assertDryRunDoesNotModifyEnv() { + const envFile = writeEnv('dry-run.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ]); + + assertStatus(result, 0, 'dry-run 应成功并通过临时 env 复核。'); + assertIncludes(result.stdout, 'apply=false', 'dry-run 输出必须显示 apply=false。'); + assertIncludes( + result.stdout, + '[check:production-health-patrol-env] OK', + 'dry-run 必须用临时 env 跑复核脚本。', + ); + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('dry-run 不应修改 health patrol env 文件。'); + } +} + +function assertDryRunEnvFileModeIsPrivate() { + const envFile = writeEnv('dry-run-mode.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const modeCheckScript = path.join(tmpRoot, 'check-dry-run-env-mode.mjs'); + writeFileSync( + modeCheckScript, + [ + '#!/usr/bin/env node', + "import { statSync } from 'node:fs';", + "const envFileIndex = process.argv.indexOf('--env-file');", + 'const envFile = process.argv[envFileIndex + 1];', + 'const mode = statSync(envFile).mode & 0o777;', + 'if (mode !== 0o600) {', + ' console.error(`dry-run env mode must be 0600, got ${mode.toString(8)}`);', + ' process.exit(1);', + '}', + 'console.log(`[fake-health-patrol-env-check] dry-run env mode ${mode.toString(8)}`);', + '', + ].join('\n'), + 'utf8', + ); + chmodSync(modeCheckScript, 0o755); + + const result = runSwitch([ + '--env-file', + envFile, + '--check-script', + modeCheckScript, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ]); + + assertStatus(result, 0, 'dry-run 临时复核 env 文件权限应为 0600。'); + assertIncludes( + result.stdout, + 'dry-run env mode 600', + 'dry-run 必须把 0600 临时 env 交给复核脚本。', + ); +} + +function assertApplyPreservesEnvFileModeAndOwnership() { + const envFile = writeEnv('apply-mode-owner.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + chmodSync(envFile, 0o640); + const beforeStat = statSync(envFile); + + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 0, 'apply 原子替换应成功。'); + const afterStat = statSync(envFile); + const beforeMode = beforeStat.mode & 0o777; + const afterMode = afterStat.mode & 0o777; + if (afterMode !== beforeMode) { + failures.push( + `apply 原子替换必须保留 env 文件权限，实际 ${afterMode.toString(8)}，预期 ${beforeMode.toString(8)}。`, + ); + } + if (afterStat.uid !== beforeStat.uid || afterStat.gid !== beforeStat.gid) { + failures.push('apply 原子替换必须保留 env 文件 owner/group。'); + } +} + +function assertApplySwitchesToPingoraDirectAndPreservesOtherKeys() { + const envFile = writeEnv('direct.env', { + GENARRATIVE_HEALTH_PATROL_API_BASE_URL: 'http://127.0.0.1:8082', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS: '5000', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 0, 'apply 切到 pingora-direct 应成功。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct', + 'apply 必须写入 pingora-direct gateway mode。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://127.0.0.1', + 'apply 必须写入直连 public base URL。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=example.com', + 'apply 必须写入正式 public Host。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000', + 'apply 必须保留其它 health patrol env 项。', + ); +} + +function assertApplySwitchesBackToNginxAndClearsHost() { + const envFile = writeEnv('nginx.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'example.com', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + + assertStatus(result, 0, 'apply 切回 nginx 并清空 Host 应成功。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + '回退 apply 必须写回 nginx gateway mode。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1', + '回退 apply 必须恢复 Nginx public base URL。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + '回退 apply 必须清空 public Host。', + ); +} + +function assertApplyDoesNotWriteWhenVerificationFails() { + const envFile = writeEnv('apply-verify-fails.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + const failingCheckScript = path.join( + tmpRoot, + 'failing-health-patrol-env-check.mjs', + ); + writeFileSync( + failingCheckScript, + [ + '#!/usr/bin/env node', + 'console.error("[fake-health-patrol-env-check] forced failure");', + 'process.exit(1);', + '', + ].join('\n'), + 'utf8', + ); + + const result = runSwitch([ + '--env-file', + envFile, + '--check-script', + failingCheckScript, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 1, '复核失败时 apply 必须失败。'); + assertIncludes( + result.stderr, + '[fake-health-patrol-env-check] forced failure', + '复核失败时必须透出复核脚本错误。', + ); + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('复核失败时 apply 不应写入真实 health patrol env 文件。'); + } +} + +function assertApplyRejectsSymlinkEnvFileBeforeWrite() { + const targetEnvFile = writeEnv('symlink-target.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const symlinkEnvFile = path.join(tmpRoot, 'symlink-health-patrol.env'); + symlinkSync(targetEnvFile, symlinkEnvFile); + const before = readFileSync(targetEnvFile, 'utf8'); + + const result = runSwitch([ + '--env-file', + symlinkEnvFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + '--apply', + ]); + + assertStatus(result, 1, 'apply 传入符号链接 env 文件时必须失败。'); + assertIncludes( + result.stderr, + '--env-file 不能是符号链接', + '符号链接 env 文件失败时必须给出明确错误。', + ); + const after = readFileSync(targetEnvFile, 'utf8'); + if (after !== before) { + failures.push('符号链接 env 文件被拒绝时不应写入真实目标文件。'); + } + const linkText = readFileSync(symlinkEnvFile, 'utf8'); + if (linkText !== before) { + failures.push('符号链接 env 文件被拒绝时不应替换或改写链接内容。'); + } +} + +function assertMissingManagedKeysAreAppended() { + const envFile = path.join(tmpRoot, 'append.env'); + writeFileSync( + envFile, + [ + '# existing env', + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000', + '', + ].join('\n'), + 'utf8', + ); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + + assertStatus(result, 0, '缺少目标键时 apply 应追加并通过复核。'); + const content = readFileSync(envFile, 'utf8'); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + '缺少 gateway mode 时必须追加。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1', + '缺少 public base URL 时必须追加。', + ); + assertIncludes( + content, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + '缺少 public Host 时必须追加。', + ); +} + +function assertRejectsDirectLoopbackWithoutHost() { + const envFile = writeEnv('direct-missing-host.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--clear-public-host', + ]); + + assertStatus(result, 1, 'direct loopback 缺少 Host 必须失败。'); + assertIncludes( + result.stderr, + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host', + 'direct loopback 缺少 Host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidAndConflictingHostOptions() { + const envFile = writeEnv('invalid-host.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const invalidHost = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--public-host', + 'https://example.com', + ]); + assertStatus(invalidHost, 1, '非法 Host 必须失败。'); + assertIncludes( + invalidHost.stderr, + '--public-host 只能是 host 或 host:port', + '非法 Host 必须给出明确错误。', + ); + + const conflictingHost = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--public-host', + 'example.com', + '--clear-public-host', + ]); + assertStatus(conflictingHost, 1, 'Host 与清空 Host 同时传入必须失败。'); + assertIncludes( + conflictingHost.stderr, + '--public-host 和 --clear-public-host 不能同时使用', + 'Host 选项冲突必须给出明确错误。', + ); +} + +function assertRejectsRelativePaths() { + const relativeEnv = runSwitch([ + '--env-file', + 'relative.env', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(relativeEnv, 1, '相对 env 路径必须失败。'); + assertIncludes( + relativeEnv.stderr, + '--env-file 必须是绝对路径', + '相对 env 路径必须给出明确错误。', + ); + + const envFile = writeEnv('relative-check.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const relativeCheck = runSwitch([ + '--env-file', + envFile, + '--check-script', + 'scripts/check-production-health-patrol-env.mjs', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(relativeCheck, 1, '相对 check script 路径必须失败。'); + assertIncludes( + relativeCheck.stderr, + '--check-script 必须是绝对路径', + '相对 check script 路径必须给出明确错误。', + ); +} + +function assertRejectsFilesystemRootPaths() { + const rootEnv = runSwitch([ + '--env-file', + '/', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(rootEnv, 1, '文件系统根目录 env 路径必须失败。'); + assertIncludes( + rootEnv.stderr, + '--env-file 不能是文件系统根目录', + '文件系统根目录 env 路径必须给出明确错误。', + ); + + const envFile = writeEnv('root-check-script.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const rootCheckScript = runSwitch([ + '--env-file', + envFile, + '--check-script', + '/', + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + ]); + assertStatus(rootCheckScript, 1, '文件系统根目录 check script 路径必须失败。'); + assertIncludes( + rootCheckScript.stderr, + '--check-script 不能是文件系统根目录', + '文件系统根目录 check script 路径必须给出明确错误。', + ); +} + +function assertRejectsControlCharacterInputsBeforeWrite() { + const envFile = writeEnv('control-character.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const before = readFileSync(envFile, 'utf8'); + + const envFileResult = runSwitch([ + '--env-file', + `${envFile}\nspoofed`, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + assertStatus(envFileResult, 1, '带控制字符的 env 文件路径必须失败。'); + assertIncludes( + envFileResult.stderr, + '--env-file 不能包含换行或 NUL 字符', + '带控制字符的 env 文件路径必须给出明确错误。', + ); + + const checkScriptResult = runSwitch([ + '--env-file', + envFile, + '--check-script', + `${ENV_CHECK_SCRIPT}\nspoofed`, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + '--clear-public-host', + '--apply', + ]); + assertStatus(checkScriptResult, 1, '带控制字符的 check script 路径必须失败。'); + assertIncludes( + checkScriptResult.stderr, + '--check-script 不能包含换行或 NUL 字符', + '带控制字符的 check script 路径必须给出明确错误。', + ); + + const baseUrlResult = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1\nspoofed', + '--clear-public-host', + '--apply', + ]); + assertStatus(baseUrlResult, 1, '带控制字符的 public base URL 必须失败。'); + assertIncludes( + baseUrlResult.stderr, + '--public-base-url 不能包含换行或 NUL 字符', + '带控制字符的 public base URL 必须给出明确错误。', + ); + + const hostResult = runSwitch([ + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com\nspoofed', + '--apply', + ]); + assertStatus(hostResult, 1, '带控制字符的 public Host 必须失败。'); + assertIncludes( + hostResult.stderr, + '--public-host 不能包含换行或 NUL 字符', + '带控制字符的 public Host 必须给出明确错误。', + ); + + const after = readFileSync(envFile, 'utf8'); + if (after !== before) { + failures.push('带控制字符的切换参数被拒绝时不应写入真实 env 文件。'); + } +} + +function assertRejectsInvalidBoolEnv() { + const envFile = writeEnv('invalid-bool-env.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runSwitch( + [ + '--env-file', + envFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + 'http://127.0.0.1', + ], + { + GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST: 'maybe', + }, + ); + assertStatus(result, 1, 'health patrol env 切换脚本必须拒绝非法布尔 env。'); + assertIncludes( + result.stderr, + 'GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST 必须是布尔值', + 'health patrol env 切换脚本非法布尔 env 必须给出明确错误。', + ); +} + +function assertReleaseLayoutUsesBundledCheckScript() { + const releaseRoot = path.join(tmpRoot, 'current-release'); + const releaseScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ); + const releaseCheckScript = path.join( + releaseRoot, + 'scripts/check-production-health-patrol-env.mjs', + ); + const envFile = writeEnv('release-layout.env', { + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + + mkdirSync(path.dirname(releaseScript), { recursive: true }); + mkdirSync(path.dirname(releaseCheckScript), { recursive: true }); + copyFileSync(SWITCH_SCRIPT, releaseScript); + copyFileSync('scripts/check-production-health-patrol-env.mjs', releaseCheckScript); + chmodSync(releaseScript, 0o755); + + const result = spawnSync( + 'node', + [ + '--', + releaseScript, + '--env-file', + envFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + 'https://127.0.0.1', + '--public-host', + 'example.com', + ], + { cwd: tmpRoot, encoding: 'utf8', env: cleanEnv() }, + ); + + assertStatus( + result, + 0, + 'current release 中的切换脚本 dry-run 应能使用随包 env 复核脚本。', + ); + assertIncludes( + result.stdout, + realpathSync(releaseCheckScript), + 'current release 中的切换脚本必须默认调用随包 env 复核脚本。', + ); + if (!existsSync(releaseCheckScript)) { + failures.push('release layout 测试必须保留随包 env 复核脚本。'); + } +} + +function writeEnv(name, values) { + const filePath = path.join(tmpRoot, name); + const body = Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join('\n'); + writeFileSync(filePath, `${body}\n`, 'utf8'); + return filePath; +} + +function runSwitch(args, extraEnv = {}) { + return spawnSync( + 'node', + ['--', SWITCH_SCRIPT, '--check-script', ENV_CHECK_SCRIPT, ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...cleanEnv(), + ...extraEnv, + }, + }, + ); +} + +function cleanEnv() { + return { + ...process.env, + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: '', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: '', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_ENV_CHECK_SCRIPT: '', + }; +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} diff --git a/scripts/check-pingora-production-release-build.mjs b/scripts/check-pingora-production-release-build.mjs new file mode 100644 index 000000000..73da296c2 --- /dev/null +++ b/scripts/check-pingora-production-release-build.mjs @@ -0,0 +1,173 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const BUILD_SCRIPT = 'scripts/build-production-release.sh'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-production-release-build-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-production-release-build] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-production-release-build] OK'); + +function main() { + const fixture = prepareFixture(); + const result = runBuild(fixture); + assertStatus(result, 0, 'Pingora production release 真实构建烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'api-server'), + '发布包必须包含 api-server 占位 release binary。', + ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + '发布包必须包含 api-server checksum。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '真实 include Pingora 时发布包必须包含 pingora-gateway。', + ); + assertExecutable( + path.join(releaseDir, 'pingora-gateway'), + 'pingora-gateway 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '真实 include Pingora 时发布包必须包含 pingora-gateway checksum。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + '真实 include Pingora 的发布包仍必须携带 direct live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), + '真实 include Pingora 的发布包仍必须携带 current release 自审脚本。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/nginx-route-parity.matrix.json'), + '真实 include Pingora 的发布包仍必须携带 Nginx/Pingora 路由矩阵。', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (manifest.component_type !== 'api-server') { + failures.push( + `release manifest component_type 应为 api-server，实际 ${manifest.component_type}`, + ); + } + if (!manifest.artifacts?.some((item) => item.path === 'api-server')) { + failures.push('release manifest 必须登记 api-server artifact。'); + } + if (!manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push( + '真实 include Pingora 时 release manifest 必须登记 pingora-gateway artifact。', + ); + } + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function prepareFixture() { + const cargoTargetDir = path.join(tmpRoot, 'cargo-target'); + const binaryDir = path.join( + cargoTargetDir, + 'x86_64-unknown-linux-gnu/release', + ); + const version = `check-pingora-production-release-build-${process.pid}-${Date.now()}`; + mkdirSync(binaryDir, { recursive: true }); + + const apiBinary = path.join(binaryDir, 'api-server'); + writeFileSync(apiBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', apiBinary], { encoding: 'utf8' }); + + return { cargoTargetDir, version }; +} + +function runBuild(fixture) { + return spawnSync( + 'bash', + [ + BUILD_SCRIPT, + '--component', + 'api-server', + '--name', + fixture.version, + '--skip-api-build', + '--include-pingora-gateway', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + CARGO_TARGET_DIR: fixture.cargoTargetDir, + PATH: `${path.join(process.env.HOME || '', '.local', 'bin')}:${process.env.PATH || ''}`, + SOURCE_BRANCH: 'test-branch', + SOURCE_COMMIT: 'test-commit', + }, + }, + ); +} + +function readJson(filePath) { + try { + return JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + failures.push(`${filePath} 不是合法 JSON: ${error.message}`); + return {}; + } +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function assertExecutable(filePath, reason) { + if (!existsSync(filePath)) { + return; + } + if ((statSync(filePath).mode & 0o111) === 0) { + failures.push(`${reason} 文件不可执行: ${filePath}`); + } +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} diff --git a/scripts/check-pingora-release-readiness-plan.mjs b/scripts/check-pingora-release-readiness-plan.mjs new file mode 100644 index 000000000..47e5eff9f --- /dev/null +++ b/scripts/check-pingora-release-readiness-plan.mjs @@ -0,0 +1,3872 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; + +const requireDirectBaseArgs = [ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', +]; + +const requireLiveBaseArgs = [ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + '/var/log/nginx/genarrative.access.log', + '--live-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--live-access-log-since-lines', + '3000', +]; + +const requireRealpathLiveBaseArgs = [ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--realpath-live-host', + 'example.com', + '--realpath-live-nginx-access-log', + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + '--realpath-live-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--realpath-live-access-log-since-lines', + '3000', +]; + +main(); + +if (failures.length > 0) { + console.error('[check:pingora-release-readiness-plan] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-release-readiness-plan] OK'); + +function main() { + assertDefaultPlanIncludesStaticDirectPreflight(); + assertDefaultPlanIncludesProductionApiReleaseSmoke(); + assertDefaultPlanIncludesProductionApiDeploySmoke(); + assertDefaultPlanIncludesHealthPatrolEnvSmoke(); + assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke(); + assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke(); + assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke(); + assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceVerifySmoke(); + assertDefaultPlanIncludesPingoraCutoverEvidenceAuditSmoke(); + assertDefaultPlanIncludesCanaryAccessLogParitySmoke(); + assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke(); + assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts(); + assertReleaseRuntimeOnlyRejectsSourceOnlyFlags(); + assertDryRunCutoverPlanIncludesDirectEnableAndRollbackRunbook(); + assertDryRunCutoverDefaultRollbackBodyIgnoresProcessEnv(); + assertDryRunCutoverPostEnableReadinessRedactsDirectProbeToken(); + assertDryRunCutoverPlanIncludesHostConsistencyConfirmation(); + assertDryRunCutoverAllowsSameHostnameWithDifferentPorts(); + assertDryRunCutoverCanRestoreRollbackHealthPatrolPublicHost(); + assertDryRunCutoverPlanCanIncludeRollbackShadowProbe(); + assertDryRunCutoverPlanCanOverrideEvidenceTimelineMaxSpan(); + assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches(); + assertTechnicalDocCutoverAuditExamplesRequireCommandExecutables(); + assertDryRunCutoverPlanCarriesCutoverRunId(); + assertDryRunCutoverPlanGeneratesDefaultCutoverRunId(); + assertRequireLiveForcesHost(); + assertRequireRealpathLiveForcesHostAndRealpathParity(); + assertRequireLiveRejectsRelativeAccessLog(); + assertRequireLiveRejectsFilesystemRootAccessLog(); + assertRequireLiveRejectsInvalidAccessLogSinceLines(); + assertRequireRealpathLiveRejectsMissingHost(); + assertRequireRealpathLiveRejectsRelativeAccessLog(); + assertLiveSmokeScriptsRejectInvalidTimeout(); + assertDirectLiveRejectsInvalidBoolEnv(); + assertDirectLiveRejectsFilesystemRootAccessLog(); + assertReleaseReadinessRejectsInvalidDirectBoolEnv(); + assertDirectPreflightRejectsInvalidBoolEnv(); + assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection(); + assertCutoverEvidenceScriptsRejectInvalidBoolEnv(); + assertRequireDirectForcesWssUpgrade(); + assertRequireDirectForcesHttpBaseUrl(); + assertRequireDirectForcesHostSni(); + assertRequireDirectForcesRedirectHost(); + assertRequireDirectForcesPingoraAccessLog(); + assertRequireDirectForcesPreflightSystemdCertReadableAndPortsFree(); + assertRequireDirectForcesExplicitSpacetimeDatabase(); + assertRequireDirectForcesHealthPatrolEnvCheck(); + assertDryRunPlanRedactsDirectProbeToken(); + assertRunStepLogRedactsDirectProbeToken(); + assertRequireLiveRejectsMissingHost(); + assertRequireLiveRejectsInvalidHost(); + assertRequireDirectRejectsMissingHttpBaseUrl(); + assertRequireDirectRejectsMissingHost(); + assertRequireDirectRejectsInvalidHost(); + assertRequireDirectRejectsMissingRedirectHost(); + assertRequireDirectRejectsInvalidRedirectHost(); + assertRequireDirectRejectsMissingPingoraAccessLog(); + assertRequireDirectRejectsFilesystemRootPingoraAccessLog(); + assertRequireDirectRejectsMissingSystemdPreflight(); + assertRequireDirectRejectsMissingCertReadablePreflight(); + assertRequireDirectRejectsMissingServiceEnvFilePreflight(); + assertRequireDirectRejectsMissingServiceUserCertReadablePreflight(); + assertRequireDirectRejectsMissingServiceBinaryExecutablePreflight(); + assertRequireDirectRejectsMissingPortsFreePreflight(); + assertRequireDirectRejectsMissingSpacetimeDatabase(); + assertRequireDirectRejectsFilesystemRootPreflightEnvFile(); + assertRequireDirectRejectsMissingHealthPatrolEnvFile(); + assertRequireDirectRejectsRelativeHealthPatrolEnvFile(); + assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile(); + assertRequireDirectRejectsSkipWss(); + assertRequireDirectRejectsInsecureTls(); + assertDryRunCutoverRejectsMissingRequireDirect(); + assertDryRunCutoverRejectsRelativeEvidenceOutputRoot(); + assertDryRunCutoverRejectsFilesystemRootReleaseRoot(); + assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot(); + assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot(); + assertUsageExamplesIncludeDirectPingoraAccessLog(); + assertDryRunCutoverRejectsMissingRollbackHealthPatrolPublicBaseUrl(); + assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicBaseUrl(); + assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicHost(); + assertDryRunCutoverRejectsRedirectHostMismatch(); + assertDryRunCutoverRejectsRollbackHostMismatch(); + assertDryRunCutoverRejectsIncompleteRollbackShadowProbe(); + assertDryRunCutoverRejectsInvalidRollbackShadowProbeUrl(); + assertDryRunCutoverRejectsInvalidEvidenceTimelineMaxSpan(); + assertDryRunCutoverRejectsInvalidCutoverRunId(); +} + +function assertDefaultPlanIncludesStaticDirectPreflight() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora direct entry 静态预检'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 direct-entry 静态预检。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-direct-preflight', + 'direct-entry 静态预检必须调用 check:pingora-direct-preflight。', + ); +} + +function assertDefaultPlanIncludesProductionApiDeploySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'API deploy release 烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 API deploy release 烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-api-deploy', + 'API deploy release 烟测必须调用 check:production-api-deploy。', + ); +} + +function assertDefaultPlanIncludesHealthPatrolEnvSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, '生产健康巡检 env 复核烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含生产健康巡检 env 复核烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-health-patrol-env', + '生产健康巡检 env 复核烟测必须调用 check:production-health-patrol-env。', + ); +} + +function assertDefaultPlanIncludesPingoraHealthPatrolEnvSwitchSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora health patrol env 切换烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora health patrol env 切换烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-health-patrol-env-switch', + 'Pingora health patrol env 切换烟测必须调用 check:pingora-health-patrol-env-switch。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverStatusSnapshotSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 状态快照烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 状态快照烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-status-snapshot', + 'Pingora cutover 状态快照烟测必须调用 check:pingora-cutover-status-snapshot。', + ); +} + +function assertDefaultPlanIncludesPingoraCurrentReleaseAuditSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora current release 自审烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora current release 自审烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-current-release-audit', + 'Pingora current release 自审烟测必须调用 check:pingora-current-release-audit。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceBundleSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据包烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据包烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-bundle', + 'Pingora cutover 证据包烟测必须调用 check:pingora-cutover-evidence-bundle。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverCommandEvidenceSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 命令证据烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 命令证据烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-command-evidence', + 'Pingora cutover 命令证据烟测必须调用 check:pingora-cutover-command-evidence。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceVerifySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据 manifest 验真烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据 manifest 验真烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-verify', + 'Pingora cutover 证据 manifest 验真烟测必须调用 check:pingora-cutover-evidence-verify。', + ); +} + +function assertDefaultPlanIncludesPingoraCutoverEvidenceAuditSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora cutover 证据根目录审计烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora cutover 证据根目录审计烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-cutover-evidence-audit', + 'Pingora cutover 证据根目录审计烟测必须调用 check:pingora-cutover-evidence-audit。', + ); +} + +function assertDefaultPlanIncludesCanaryAccessLogParitySmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora canary access log 对账烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora canary access log 对账烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-canary-access-log-parity', + 'Pingora canary access log 对账烟测必须调用 check:pingora-canary-access-log-parity。', + ); +} + +function assertDefaultPlanIncludesProductionApiReleaseSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'API release build 烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 API release build 烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:production-api-release', + 'API release build 烟测必须调用 check:production-api-release。', + ); +} + +function assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora production release 真实构建烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora production release 真实构建烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-production-release-build', + 'Pingora production release 真实构建烟测必须调用 check:pingora-production-release-build。', + ); +} + +function assertDryRunCutoverPlanIncludesDirectEnableAndRollbackRunbook() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const preflight = findStep( + plan, + '切换前 current release 直连 env / 证书 / 端口预检', + ); + const releaseAudit = findStep(plan, '切换前 current release 自包含自审'); + if (!releaseAudit) { + failures.push('dry-run cutover 必须包含切换前 current release 自包含自审。'); + } else { + assertIncludes( + releaseAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + '切换前 current release 自审必须使用 current release 随包脚本。', + ); + assertIncludes( + releaseAudit.args, + '--require-pingora-gateway', + '正式直连 runbook 的 current release 自审必须要求 pingora-gateway 已随包存在。', + ); + assertIncludes( + releaseAudit.args, + '--systemd-show', + '正式直连 runbook 的 current release 自审必须核验 systemd ExecStart。', + ); + } + const preSnapshot = findStep(plan, '切换前状态快照证据包'); + const preVerify = findStep(plan, '切换前证据 manifest 只读验真'); + if (!preSnapshot) { + failures.push('dry-run cutover 必须包含切换前状态快照证据包。'); + } else { + assertIncludes( + preSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '切换前状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + preSnapshot.args, + '--snapshot-script', + '切换前状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + preSnapshot.args, + '--output-root', + '切换前状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + preSnapshot.args, + '/var/log/genarrative/pingora-cutover-evidence', + '切换前状态快照证据包必须默认写入固定证据根目录。', + ); + assertIncludes( + preSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '切换前状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + preSnapshot.args, + '--expected-gateway-mode', + '切换前状态快照证据包必须显式传 expected gateway mode。', + ); + assertIncludes( + preSnapshot.args, + 'nginx', + '切换前状态快照证据包必须确认仍是 Nginx 巡检口径。', + ); + assertIncludes( + preSnapshot.args, + '--run-health-patrol', + '切换前状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + preSnapshot.args, + '--require-pingora-gateway', + '切换前状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + assertIncludes( + preSnapshot.args, + '--fail-on-critical', + '切换前状态快照证据包出现 CRITICAL 必须阻断切换。', + ); + } + if (!preVerify) { + failures.push('dry-run cutover 必须包含切换前证据 manifest 只读验真。'); + } else { + assertIncludes( + preVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '切换前证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + preVerify.args, + '', + '切换前证据验真必须提示替换刚生成的 pre-cutover bundleDir。', + ); + assertIncludes( + preVerify.args, + '--require-summary-ok', + '切换前证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!preflight) { + failures.push('dry-run cutover 必须包含启用前 current release 预检。'); + } else { + assertIncludes( + preflight.args, + '/opt/genarrative/current/scripts/check-pingora-direct-preflight.mjs', + 'dry-run cutover 启用前预检必须使用 current release 随包脚本。', + ); + assertIncludes( + preflight.args, + '--check-service-user-cert-readable', + 'dry-run cutover 启用前预检必须检查 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + preflight.args, + '--check-service-env-file', + 'dry-run cutover 启用前预检必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + preflight.args, + '--check-service-binary-executable', + 'dry-run cutover 启用前预检必须检查 current release Pingora 二进制可执行。', + ); + assertIncludes( + preflight.args, + '--check-ports-free', + 'dry-run cutover 启用前预检必须检查 80/443 端口释放。', + ); + assertIncludes( + preflight.args, + '--systemd-cat', + 'dry-run cutover 启用前预检必须检查当前 systemd direct-entry 状态。', + ); + } + + const baseReadiness = findStep(plan, '切换前 release readiness 基础门禁'); + if (!baseReadiness) { + failures.push( + 'dry-run cutover 必须包含启用前 release readiness 基础门禁。', + ); + } else if (baseReadiness.args.includes('--require-direct')) { + failures.push( + 'dry-run cutover 启用前基础门禁不能带 --require-direct；systemd drop-in 尚未生效。', + ); + } else { + assertIncludes( + baseReadiness.args, + '/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs', + 'dry-run cutover 启用前基础门禁必须使用 current release 随包 release readiness 聚合门禁脚本。', + ); + assertIncludes( + baseReadiness.args, + '--release-runtime-only', + 'dry-run cutover 启用前基础门禁必须使用 current release 包内运行时复核模式。', + ); + } + + const enableDryRun = findStep(plan, 'Pingora direct enable dry-run'); + if (!enableDryRun) { + failures.push('dry-run cutover 必须包含 direct enable dry-run。'); + } else { + assertIncludes( + enableDryRun.command, + '/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'dry-run cutover enable dry-run 必须使用 current release 随包启用脚本。', + ); + if (enableDryRun.args.includes('--apply')) { + failures.push('dry-run cutover enable dry-run 不能带 --apply。'); + } + assertIncludes( + enableDryRun.args, + '--current-release-audit-script', + 'dry-run cutover enable dry-run 必须显式固定 current release 自审脚本。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + 'dry-run cutover enable dry-run 必须使用 current release 随包自审脚本。', + ); + assertIncludes( + enableDryRun.args, + '--current-release-root', + 'dry-run cutover enable dry-run 必须显式固定 current release 根目录。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current', + 'dry-run cutover enable dry-run 必须把 current release 根目录传给启用脚本。', + ); + assertIncludes( + enableDryRun.args, + '/opt/genarrative/current/scripts/check-pingora-direct-live.mjs', + 'dry-run cutover enable dry-run 必须指定 current release 随包 direct live smoke。', + ); + } + + const enableApply = findStep(plan, 'Pingora direct enable apply'); + const enableApplyVerify = findStep(plan, '启用命令证据 manifest 只读验真'); + if (!enableApply) { + failures.push('dry-run cutover 必须包含 direct enable apply。'); + } else { + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover enable apply 必须通过 current release 随包命令证据脚本归档执行输出。', + ); + assertIncludes( + enableApply.args, + '--command-name', + 'dry-run cutover enable apply 必须显式记录命令证据名称。', + ); + assertIncludes( + enableApply.args, + 'pingora-direct-enable-apply', + 'dry-run cutover enable apply 必须记录固定命令证据名称。', + ); + assertIncludes( + enableApply.args, + '--output-root', + 'dry-run cutover enable apply 必须显式传证据输出根目录。', + ); + assertIncludes( + enableApply.args, + '/var/log/genarrative/pingora-cutover-evidence', + 'dry-run cutover enable apply 必须使用固定证据根目录。', + ); + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'dry-run cutover enable apply 必须在命令证据脚本后执行 current release 随包启用脚本。', + ); + assertIncludes( + enableApply.args, + '--expected-executable', + 'dry-run cutover enable apply 必须把预期真实命令绑定到命令证据。', + ); + assertIncludes( + enableApply.args, + '--require-arg', + 'dry-run cutover enable apply 必须在命令证据脚本中要求真实命令参数。', + ); + assertIncludes( + enableApply.args, + '--apply', + 'dry-run cutover enable apply 必须显式带 --apply。', + ); + assertIncludes( + enableApply.args, + '--current-release-audit-script', + 'dry-run cutover enable apply 必须显式固定 current release 自审脚本。', + ); + assertIncludes( + enableApply.args, + '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + 'dry-run cutover enable apply 必须使用 current release 随包自审脚本。', + ); + assertIncludes( + enableApply.args, + '--current-release-root', + 'dry-run cutover enable apply 必须显式固定 current release 根目录。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-user-cert-readable', + 'dry-run cutover enable apply 必须验证 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-env-file', + 'dry-run cutover enable apply 必须验证 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + enableApply.args, + '--preflight-check-service-binary-executable', + 'dry-run cutover enable apply 必须验证 current release Pingora 二进制可执行。', + ); + assertIncludes( + enableApply.args, + '--direct-spacetime-database', + 'dry-run cutover enable apply 必须验证显式 SpacetimeDB 数据库名。', + ); + } + if (!enableApplyVerify) { + failures.push('dry-run cutover 必须包含启用命令证据 manifest 只读验真。'); + } else { + assertIncludes( + enableApplyVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '启用命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + enableApplyVerify.args, + '', + '启用命令证据验真必须提示替换刚生成的 enable apply bundleDir。', + ); + assertIncludes( + enableApplyVerify.args, + '--require-summary-ok', + '启用命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + + const postReadiness = findStep(plan, '启用后 release readiness 直连复核'); + const postEnableSnapshot = findStep(plan, '启用后状态快照证据包'); + const postEnableVerify = findStep(plan, '启用后证据 manifest 只读验真'); + if (!postEnableSnapshot) { + failures.push('dry-run cutover 必须包含启用后状态快照证据包。'); + } else { + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '启用后状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '--snapshot-script', + '启用后状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '--output-root', + '启用后状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '启用后状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + 'pingora-direct', + '启用后状态快照证据包必须确认 pingora-direct 巡检口径。', + ); + assertIncludes( + postEnableSnapshot.args, + '--expected-public-host', + '启用后状态快照证据包必须确认正式 Host 覆盖。', + ); + assertIncludes( + postEnableSnapshot.args, + '--run-health-patrol', + '启用后状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-live-script', + '启用后状态快照证据包必须显式透传 direct live smoke 脚本。', + ); + assertIncludes( + postEnableSnapshot.args, + '/opt/genarrative/current/scripts/check-pingora-direct-live.mjs', + '启用后状态快照证据包必须使用 current release 随包 direct live smoke。', + ); + assertIncludes( + postEnableSnapshot.args, + '--run-direct-live', + '启用后状态快照证据包必须归档 direct live smoke 证据。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-pingora-access-log', + '启用后状态快照证据包必须归档 direct access log request_id 证据。', + ); + assertIncludes( + postEnableSnapshot.args, + '/var/log/genarrative/pingora-gateway.access.log', + '启用后状态快照证据包必须使用显式 Pingora access log 路径。', + ); + assertIncludes( + postEnableSnapshot.args, + '--direct-spacetime-database', + '启用后状态快照证据包必须传递显式 SpacetimeDB 数据库名。', + ); + assertIncludes( + postEnableSnapshot.args, + '--require-pingora-gateway', + '启用后状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + } + if (!postEnableVerify) { + failures.push('dry-run cutover 必须包含启用后证据 manifest 只读验真。'); + } else { + assertIncludes( + postEnableVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '启用后证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + postEnableVerify.args, + '', + '启用后证据验真必须提示替换刚生成的 post-enable bundleDir。', + ); + assertIncludes( + postEnableVerify.args, + '--require-summary-ok', + '启用后证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!postReadiness) { + failures.push( + 'dry-run cutover 必须包含启用后 release readiness 直连复核。', + ); + } else { + assertIncludes( + postReadiness.args, + '/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs', + '启用后 release readiness 直连复核必须使用 current release 随包 release readiness 聚合门禁脚本。', + ); + assertIncludes( + postReadiness.args, + '--release-runtime-only', + '启用后 release readiness 直连复核必须使用 current release 包内运行时复核模式。', + ); + assertIncludes( + postReadiness.args, + '--require-direct', + 'dry-run cutover 启用后复核必须带 --require-direct。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-user-cert-readable', + 'dry-run cutover 启用后复核必须检查 systemd 服务用户可读证书和私钥。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-env-file', + 'dry-run cutover 启用后复核必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-check-service-binary-executable', + 'dry-run cutover 启用后复核必须检查 Pingora service 二进制可执行。', + ); + assertIncludes( + postReadiness.args, + '--direct-preflight-systemd', + 'dry-run cutover 启用后复核必须检查 systemd drop-in 生效。', + ); + } + + const healthPatrolEnvCheck = findStep( + plan, + '启用后 health patrol env 直连复核', + ); + if (!healthPatrolEnvCheck) { + failures.push('dry-run cutover 必须包含启用后 health patrol env 复核。'); + } else { + assertIncludes( + healthPatrolEnvCheck.args, + '/opt/genarrative/current/scripts/check-production-health-patrol-env.mjs', + 'dry-run cutover health patrol env 复核必须使用 current release 随包脚本。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover health patrol env 复核必须检查目标 env 文件。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-gateway-mode', + 'dry-run cutover health patrol env 复核必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + 'pingora-direct', + 'dry-run cutover health patrol env 复核必须要求 pingora-direct。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-public-base-url', + 'dry-run cutover health patrol env 复核必须校验 public base URL。', + ); + assertIncludes( + healthPatrolEnvCheck.args, + '--expected-public-host', + 'dry-run cutover health patrol env 复核必须校验正式 Host。', + ); + } + + const healthPatrolDirect = findStep( + plan, + '切换后 health patrol 切到 Pingora direct', + ); + if (!healthPatrolDirect) { + failures.push( + 'dry-run cutover 必须包含切换后 health patrol direct 模式脚本。', + ); + } else { + assertIncludes( + healthPatrolDirect.args, + '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'dry-run cutover 切换后必须使用 current release 随包 health patrol env 切换脚本。', + ); + assertIncludes( + healthPatrolDirect.args, + '--apply', + 'dry-run cutover 切换后 health patrol env 切换必须显式带 --apply。', + ); + assertIncludes( + healthPatrolDirect.args, + '--env-file', + 'dry-run cutover 切换后 health patrol env 切换必须显式传 env 文件。', + ); + assertIncludes( + healthPatrolDirect.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover 切换后 health patrol env 切换必须使用显式 env 文件。', + ); + assertIncludes( + healthPatrolDirect.args, + '--gateway-mode', + 'dry-run cutover 切换后 health patrol env 切换必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolDirect.args, + 'pingora-direct', + 'dry-run cutover 切换后 health patrol env 切换必须改为 pingora-direct。', + ); + assertIncludes( + healthPatrolDirect.args, + '--public-base-url', + 'dry-run cutover 切换后 health patrol env 切换必须写 direct public base URL。', + ); + assertIncludes( + healthPatrolDirect.args, + 'https://127.0.0.1', + 'dry-run cutover 切换后 health patrol env 切换必须写 direct HTTPS base URL。', + ); + assertIncludes( + healthPatrolDirect.args, + '--public-host', + 'dry-run cutover 切换后 health patrol env 切换必须写正式 Host。', + ); + assertIncludes( + healthPatrolDirect.args, + 'example.com', + 'dry-run cutover 切换后 health patrol env 切换必须使用正式 Host。', + ); + } + assertStepBefore( + plan, + 'Pingora direct enable apply', + '启用命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 enable apply 命令证据，再验真该命令证据。', + ); + assertStepBefore( + plan, + '启用命令证据 manifest 只读验真', + '切换后 health patrol 切到 Pingora direct', + 'dry-run cutover 必须先验真 enable apply 命令证据，再切换 health patrol 到 pingora-direct。', + ); + + const rollbackDryRun = findStep(plan, 'Pingora direct rollback dry-run'); + if (!rollbackDryRun) { + failures.push('dry-run cutover 必须包含 rollback dry-run。'); + } else { + assertIncludes( + rollbackDryRun.command, + '/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + 'dry-run cutover rollback dry-run 必须使用 current release 随包回退脚本。', + ); + if (rollbackDryRun.args.includes('--apply')) { + failures.push('dry-run cutover rollback dry-run 不能带 --apply。'); + } + assertIncludes( + rollbackDryRun.args, + '--reload-nginx', + 'dry-run cutover rollback dry-run 必须包含 Nginx reload。', + ); + assertIncludes( + rollbackDryRun.args, + '--nginx-smoke-host', + 'dry-run cutover rollback dry-run 使用 loopback smoke URL 时必须带正式 Host。', + ); + assertIncludes( + rollbackDryRun.args, + '--nginx-smoke-expect-body', + 'dry-run cutover rollback dry-run 必须校验 Nginx smoke 响应体片段。', + ); + assertIncludes( + rollbackDryRun.args, + '"ok":true', + 'dry-run cutover rollback dry-run 必须默认校验 healthz ok body。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-env-file', + 'dry-run cutover rollback dry-run 必须把 health patrol env 复核交给回退脚本。', + ); + assertIncludes( + rollbackDryRun.args, + '/etc/genarrative/health-patrol.env', + 'dry-run cutover rollback dry-run 必须使用显式 health patrol env 文件。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-expected-public-base-url', + 'dry-run cutover rollback dry-run 必须校验回退 health patrol public base URL。', + ); + assertIncludes( + rollbackDryRun.args, + 'http://127.0.0.1', + 'dry-run cutover rollback dry-run 必须传递显式回退 public base URL。', + ); + assertIncludes( + rollbackDryRun.args, + '--health-patrol-require-empty-public-host', + 'dry-run cutover rollback dry-run 默认必须要求回退 public Host 为空。', + ); + } + + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + const rollbackApplyVerify = findStep(plan, '回退命令证据 manifest 只读验真'); + if (!rollbackApply) { + failures.push('dry-run cutover 必须包含 rollback apply。'); + } else { + assertIncludes( + rollbackApply.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-command-evidence.mjs', + 'dry-run cutover rollback apply 必须通过 current release 随包命令证据脚本归档执行输出。', + ); + assertIncludes( + rollbackApply.args, + '--command-name', + 'dry-run cutover rollback apply 必须显式记录命令证据名称。', + ); + assertIncludes( + rollbackApply.args, + 'pingora-direct-rollback-apply', + 'dry-run cutover rollback apply 必须记录固定命令证据名称。', + ); + assertIncludes( + rollbackApply.args, + '/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + 'dry-run cutover rollback apply 必须在命令证据脚本后执行 current release 随包回退脚本。', + ); + assertIncludes( + rollbackApply.args, + '--expected-executable', + 'dry-run cutover rollback apply 必须把预期真实命令绑定到命令证据。', + ); + assertIncludes( + rollbackApply.args, + '--require-arg', + 'dry-run cutover rollback apply 必须在命令证据脚本中要求真实命令参数。', + ); + assertIncludes( + rollbackApply.args, + '--apply', + 'dry-run cutover rollback apply 必须显式带 --apply。', + ); + assertIncludes( + rollbackApply.args, + 'http://127.0.0.1/healthz', + 'dry-run cutover rollback apply 必须包含 Nginx smoke URL。', + ); + assertIncludes( + rollbackApply.args, + 'example.com', + 'dry-run cutover rollback apply 默认必须复用 direct Host 作为 Nginx smoke Host。', + ); + assertIncludes( + rollbackApply.args, + '--nginx-smoke-host', + 'dry-run cutover rollback apply 使用 loopback smoke URL 时必须带正式 Host。', + ); + assertIncludes( + rollbackApply.args, + '--nginx-smoke-expect-body', + 'dry-run cutover rollback apply 必须校验 Nginx smoke 响应体片段。', + ); + assertIncludes( + rollbackApply.args, + '"ok":true', + 'dry-run cutover rollback apply 必须默认校验 healthz ok body。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-env-file', + 'dry-run cutover rollback apply 必须把 health patrol env 复核交给回退脚本。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-expected-public-base-url', + 'dry-run cutover rollback apply 必须校验回退 health patrol public base URL。', + ); + assertIncludes( + rollbackApply.args, + 'http://127.0.0.1', + 'dry-run cutover rollback apply 必须传递显式回退 public base URL。', + ); + assertIncludes( + rollbackApply.args, + '--health-patrol-require-empty-public-host', + 'dry-run cutover rollback apply 默认必须要求回退 public Host 为空。', + ); + } + if (!rollbackApplyVerify) { + failures.push('dry-run cutover 必须包含回退命令证据 manifest 只读验真。'); + } else { + assertIncludes( + rollbackApplyVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退命令证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + rollbackApplyVerify.args, + '', + '回退命令证据验真必须提示替换刚生成的 rollback apply bundleDir。', + ); + assertIncludes( + rollbackApplyVerify.args, + '--require-summary-ok', + '回退命令证据验真必须要求 manifest.summary.status=OK。', + ); + } + + const healthPatrolNginx = findStep(plan, '回退前 health patrol 预置回 Nginx'); + if (!healthPatrolNginx) { + failures.push( + 'dry-run cutover 必须包含回退前 health patrol Nginx 模式预置脚本。', + ); + } else { + assertIncludes( + healthPatrolNginx.args, + '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'dry-run cutover 回退前必须使用 current release 随包 health patrol env 切换脚本。', + ); + assertIncludes( + healthPatrolNginx.args, + '--apply', + 'dry-run cutover 回退前 health patrol env 切换必须显式带 --apply。', + ); + assertIncludes( + healthPatrolNginx.args, + '--gateway-mode', + 'dry-run cutover 回退前 health patrol env 切换必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolNginx.args, + 'nginx', + 'dry-run cutover 回退前 health patrol env 切换必须改回 nginx。', + ); + assertIncludes( + healthPatrolNginx.args, + '--public-base-url', + 'dry-run cutover 回退前 health patrol env 切换必须恢复 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginx.args, + 'http://127.0.0.1', + 'dry-run cutover 回退前 health patrol env 切换必须恢复切换前 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginx.args, + '--clear-public-host', + 'dry-run cutover 回退前 health patrol env 切换必须清理 public Host 覆盖。', + ); + } + + const healthPatrolIndex = plan.findIndex( + (step) => step.name === '回退前 health patrol 预置回 Nginx', + ); + const rollbackApplyIndex = plan.findIndex( + (step) => step.name === 'Pingora direct rollback apply', + ); + if ( + healthPatrolIndex >= 0 && + rollbackApplyIndex >= 0 && + healthPatrolIndex > rollbackApplyIndex + ) { + failures.push( + 'dry-run cutover 必须先预置 health patrol env，再执行 rollback apply。', + ); + } + assertStepBefore( + plan, + 'Pingora direct rollback apply', + '回退命令证据 manifest 只读验真', + 'dry-run cutover 必须先生成 rollback apply 命令证据，再验真该命令证据。', + ); + assertStepBefore( + plan, + '回退命令证据 manifest 只读验真', + '回退后 health patrol env Nginx 模式复核', + 'dry-run cutover 必须先验真 rollback apply 命令证据，再执行回退后 env 复核。', + ); + + const healthPatrolNginxCheck = findStep( + plan, + '回退后 health patrol env Nginx 模式复核', + ); + if (!healthPatrolNginxCheck) { + failures.push( + 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核。', + ); + } else { + assertIncludes( + healthPatrolNginxCheck.args, + '/opt/genarrative/current/scripts/check-production-health-patrol-env.mjs', + 'dry-run cutover 回退后 health patrol env 复核必须使用 current release 随包脚本。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-gateway-mode', + 'dry-run cutover 回退后 health patrol env 复核必须显式传 gateway mode。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'nginx', + 'dry-run cutover 回退后 health patrol env 复核必须要求 nginx。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-public-base-url', + 'dry-run cutover 回退后 health patrol env 复核必须校验 Nginx public base URL。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'http://127.0.0.1', + 'dry-run cutover 回退后 health patrol env 复核必须使用显式回退 public base URL。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + '--require-empty-public-host', + 'dry-run cutover 回退后 health patrol env 复核必须要求清空 public Host。', + ); + } + + const postRollbackSnapshot = findStep(plan, '回退后状态快照证据包'); + const postRollbackVerify = findStep(plan, '回退后证据 manifest 只读验真'); + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!postRollbackSnapshot) { + failures.push('dry-run cutover 必须包含回退后状态快照证据包。'); + } else { + assertIncludes( + postRollbackSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + '回退后状态快照证据包必须使用 current release 随包证据包脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--snapshot-script', + '回退后状态快照证据包必须显式透传 snapshot 脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--output-root', + '回退后状态快照证据包必须显式传证据输出根目录。', + ); + assertIncludes( + postRollbackSnapshot.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + '回退后状态快照证据包必须使用 current release 随包状态快照脚本。', + ); + assertIncludes( + postRollbackSnapshot.args, + 'nginx', + '回退后状态快照证据包必须确认 nginx 巡检口径。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--require-empty-public-host', + '默认回退后状态快照证据包必须要求 public Host 为空。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--run-health-patrol', + '回退后状态快照证据包必须执行生产健康巡检。', + ); + assertIncludes( + postRollbackSnapshot.args, + '--require-pingora-gateway', + '回退后状态快照证据包必须收录 Pingora 发布物 checksum / manifest 自审。', + ); + } + if (!postRollbackVerify) { + failures.push('dry-run cutover 必须包含回退后证据 manifest 只读验真。'); + } else { + assertIncludes( + postRollbackVerify.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '回退后证据验真必须使用 current release 随包 verifier。', + ); + assertIncludes( + postRollbackVerify.args, + '', + '回退后证据验真必须提示替换刚生成的 post-rollback bundleDir。', + ); + assertIncludes( + postRollbackVerify.args, + '--require-summary-ok', + '回退后证据验真必须要求 manifest.summary.status=OK。', + ); + } + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + } else { + assertIncludes( + evidenceRootAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-audit.mjs', + '三阶段总审计必须使用 current release 随包 evidence audit 脚本。', + ); + assertIncludes( + evidenceRootAudit.args, + '--evidence-root', + '三阶段总审计必须显式传证据根目录。', + ); + assertIncludes( + evidenceRootAudit.args, + '/var/log/genarrative/pingora-cutover-evidence', + '三阶段总审计必须使用固定证据根目录。', + ); + assertIncludes( + evidenceRootAudit.args, + '--verify-script', + '三阶段总审计必须显式固定 verifier 脚本。', + ); + assertIncludes( + evidenceRootAudit.args, + '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs', + '三阶段总审计必须复用 current release 随包 verifier。', + ); + for (const phase of ['pre-cutover', 'post-enable', 'post-rollback']) { + assertIncludes( + evidenceRootAudit.args, + phase, + `三阶段总审计必须要求 ${phase} 证据。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-phase-direct-live-access-log', + '三阶段总审计必须要求 post-enable 阶段带 direct live access log 对账摘要。', + ); + assertIncludes( + evidenceRootAudit.args, + 'post-enable', + '三阶段总审计必须把 direct live access log 摘要要求绑定到 post-enable 阶段。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-phase-direct-live-static-headers', + '三阶段总审计必须要求 post-enable 阶段带 direct live 静态响应头摘要。', + ); + assertIncludes( + evidenceRootAudit.args, + 'post-enable', + '三阶段总审计必须把 direct live 静态响应头摘要要求绑定到 post-enable 阶段。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-command', + '三阶段总审计必须显式要求真实切换命令证据。', + ); + for (const command of [ + 'enable-apply:pingora-direct-enable-apply', + 'rollback-apply:pingora-direct-rollback-apply', + ]) { + assertIncludes( + evidenceRootAudit.args, + command, + `三阶段总审计必须要求 ${command} 命令证据。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-command-executable', + '三阶段总审计必须显式要求真实切换脚本身份。', + ); + for (const commandExecutable of [ + 'enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]) { + assertIncludes( + evidenceRootAudit.args, + commandExecutable, + `三阶段总审计必须要求 ${commandExecutable} 命令脚本身份。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--require-command-arg', + '三阶段总审计必须显式要求真实切换命令必需参数。', + ); + for (const commandArg of [ + 'enable-apply:pingora-direct-enable-apply:--apply', + 'rollback-apply:pingora-direct-rollback-apply:--apply', + ]) { + assertIncludes( + evidenceRootAudit.args, + commandArg, + `三阶段总审计必须要求 ${commandArg} 命令参数。`, + ); + } + assertIncludes( + evidenceRootAudit.args, + '--timeline-max-span-ms', + '三阶段总审计必须显式传标准切换时间线最大跨度。', + ); + assertIncludes( + evidenceRootAudit.args, + '86400000', + '三阶段总审计默认必须把标准切换时间线最大跨度固定为 24 小时。', + ); + assertIncludes( + evidenceRootAudit.args, + '--require-cutover-run-id', + '三阶段总审计必须要求同一 cutoverRunId。', + ); + } + +} + +function assertDryRunCutoverPlanCanOverrideEvidenceTimelineMaxSpan() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--cutover-evidence-timeline-max-span-ms', + '172800000', + ]); + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + '--timeline-max-span-ms', + '三阶段总审计必须传递自定义标准切换时间线最大跨度参数。', + ); + assertIncludes( + evidenceRootAudit.args, + '172800000', + '三阶段总审计必须传递自定义标准切换时间线最大跨度值。', + ); +} + +function assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const forbiddenArgs = [ + '--warn-only', + '--allow-extra-files', + '--allow-extra-root-entries', + ]; + + for (const step of plan) { + for (const forbiddenArg of forbiddenArgs) { + assertNotIncludes( + step.args || [], + forbiddenArg, + `正式 dry-run cutover 计划的 ${step.name} 不能携带人工排障放行参数 ${forbiddenArg}。`, + ); + } + } +} + +function assertTechnicalDocCutoverAuditExamplesRequireCommandExecutables() { + const docPath = + 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md'; + const content = readFileSync(docPath, 'utf8'); + const executableArgs = [ + '--require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + '--require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]; + const auditExamples = [ + ...content.matchAll( + /\/opt\/genarrative\/current\/scripts\/ops\/pingora-cutover-evidence-audit\.mjs[\s\S]*?(?=。|` 生成总审计 JSON|`，由脚本|`。)/gu, + ), + ] + .map((match) => match[0]) + .filter( + (example) => + example.includes( + '--require-command enable-apply:pingora-direct-enable-apply', + ) || + example.includes( + '--require-command rollback-apply:pingora-direct-rollback-apply', + ), + ); + + if (auditExamples.length === 0) { + failures.push('Pingora 技术文档必须包含最终证据根目录总审计命令示例。'); + return; + } + for (const example of auditExamples) { + for (const executableArg of executableArgs) { + assertIncludes( + example, + executableArg, + 'Pingora 技术文档里的最终证据根目录总审计命令示例必须要求真实脚本身份。', + ); + } + } +} + +function assertDryRunCutoverPlanIncludesHostConsistencyConfirmation() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const step = findStep(plan, '切换 Host 一致性确认'); + if (!step) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + return; + } + if ( + !step.args.some((arg) => String(arg).startsWith('cutover-run-id=')) + ) { + failures.push('dry-run cutover Host 一致性确认必须展示 cutoverRunId。'); + } + assertIncludes( + step.args, + 'direct-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 direct Host。', + ); + assertIncludes( + step.args, + 'direct-redirect-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 redirect Host。', + ); + assertIncludes( + step.args, + 'health-patrol-public-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 health patrol Host。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-host=example.com', + 'dry-run cutover Host 一致性确认必须展示 rollback smoke Host。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-expect-body="ok":true', + 'dry-run cutover Host 一致性确认必须展示 rollback smoke body 证据片段。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public base URL。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-host=', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public Host 为空。', + ); +} + +function assertDryRunCutoverPlanCarriesCutoverRunId() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-run-id', + 'cutover-20260617T010000Z', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + } else { + assertIncludes( + hostConsistency.args, + 'cutover-run-id=cutover-20260617T010000Z', + 'Host 一致性确认必须展示显式 cutoverRunId。', + ); + } + + for (const stepName of [ + '切换前状态快照证据包', + 'Pingora direct enable apply', + '启用后状态快照证据包', + 'Pingora direct rollback apply', + '回退后状态快照证据包', + ]) { + const step = findStep(plan, stepName); + if (!step) { + failures.push(`dry-run cutover 必须包含 ${stepName}。`); + continue; + } + assertIncludes( + step.args, + '--cutover-run-id', + `${stepName} 必须传递 cutoverRunId。`, + ); + assertIncludes( + step.args, + 'cutover-20260617T010000Z', + `${stepName} 必须使用同一个 cutoverRunId。`, + ); + } + + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + '--require-cutover-run-id', + '三阶段总审计必须要求同一 cutoverRunId。', + ); + assertIncludes( + evidenceRootAudit.args, + 'cutover-20260617T010000Z', + '三阶段总审计必须使用同一个 cutoverRunId。', + ); +} + +function assertDryRunCutoverPlanGeneratesDefaultCutoverRunId() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + return; + } + const runArg = hostConsistency.args.find((arg) => + String(arg).startsWith('cutover-run-id='), + ); + if (!runArg) { + failures.push('dry-run cutover 未显式提供 run id 时必须自动生成 cutoverRunId。'); + return; + } + const runId = runArg.slice('cutover-run-id='.length); + if (!/^pingora-direct-\d{8}T\d{6}Z$/u.test(runId)) { + failures.push(`自动生成的 cutoverRunId 格式不正确: ${runId}`); + } + + const evidenceRootAudit = findStep(plan, '切换证据根目录三阶段总审计'); + if (!evidenceRootAudit) { + failures.push('dry-run cutover 必须包含切换证据根目录三阶段总审计。'); + return; + } + assertIncludes( + evidenceRootAudit.args, + runId, + '自动生成的 cutoverRunId 必须传给三阶段总审计。', + ); +} + +function assertDryRunCutoverDefaultRollbackBodyIgnoresProcessEnv() { + const previous = + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY; + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY = + 'polluted-body'; + try { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + if (!rollbackApply) { + failures.push('dry-run cutover 必须包含 rollback apply。'); + return; + } + assertIncludes( + rollbackApply.args, + '"ok":true', + 'dry-run cutover 默认 rollback body 断言不能被本机 env 污染。', + ); + if (rollbackApply.args.includes('polluted-body')) { + failures.push( + 'readinessPlanEnv 必须清理 GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY，避免 plan 自测受本机 env 污染。', + ); + } + } finally { + if (previous === undefined) { + delete process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY; + } else { + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY = + previous; + } + } +} + +function assertDryRunCutoverPostEnableReadinessRedactsDirectProbeToken() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--direct-probe-token', + 'direct-secret-token', + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + const step = findStep(plan, '启用后 release readiness 直连复核'); + if (!step) { + failures.push('带 direct probe token 的 cutover runbook 必须包含启用后 readiness 复核。'); + return; + } + assertIncludes( + step.args, + '--direct-probe-token', + '启用后 release readiness 直连复核必须继续传递 direct probe token 参数。', + ); + assertIncludes( + step.args, + '', + '启用后 release readiness 直连复核必须隐藏 direct probe token 值。', + ); + if (JSON.stringify(plan).includes('direct-secret-token')) { + failures.push('dry-run cutover JSON 不能泄露 direct probe token 原文。'); + } +} + +function assertDryRunCutoverAllowsSameHostnameWithDifferentPorts() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--direct-redirect-host', + 'example.com:443', + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-nginx-smoke-host', + 'example.com:80', + ]); + const step = findStep(plan, '切换 Host 一致性确认'); + if (!step) { + failures.push('dry-run cutover 同 hostname 不同端口时仍必须能生成计划。'); + return; + } + assertIncludes( + step.args, + 'direct-redirect-host=example.com:443', + 'dry-run cutover 应允许 redirect Host 使用同 hostname 的显式端口。', + ); + assertIncludes( + step.args, + 'rollback-nginx-smoke-host=example.com:80', + 'dry-run cutover 应允许 rollback smoke Host 使用同 hostname 的显式端口。', + ); + assertIncludes( + step.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover 同 hostname 不同端口时仍必须展示回退 health patrol public base URL。', + ); +} + +function assertDryRunCutoverCanRestoreRollbackHealthPatrolPublicHost() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-health-patrol-public-host', + 'nginx.example.com', + ]); + const healthPatrolNginx = findStep(plan, '回退前 health patrol 预置回 Nginx'); + const hostConsistency = findStep(plan, '切换 Host 一致性确认'); + if (!hostConsistency) { + failures.push('dry-run cutover 必须包含 Host 一致性确认步骤。'); + } else { + assertIncludes( + hostConsistency.args, + 'rollback-health-patrol-public-base-url=http://127.0.0.1', + 'dry-run cutover Host 一致性确认必须展示回退 health patrol public base URL。', + ); + assertIncludes( + hostConsistency.args, + 'rollback-health-patrol-public-host=nginx.example.com', + 'dry-run cutover Host 一致性确认必须展示显式回退 health patrol public Host。', + ); + } + + if (!healthPatrolNginx) { + failures.push( + 'dry-run cutover 必须包含回退前 health patrol Nginx 模式预置脚本。', + ); + } else { + assertIncludes( + healthPatrolNginx.args, + '--public-host', + 'dry-run cutover 回退前必须能恢复切换前 Nginx public Host。', + ); + assertIncludes( + healthPatrolNginx.args, + 'nginx.example.com', + 'dry-run cutover 回退前必须使用显式回退 public Host。', + ); + } + + const healthPatrolNginxCheck = findStep( + plan, + '回退后 health patrol env Nginx 模式复核', + ); + if (!healthPatrolNginxCheck) { + failures.push( + 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核。', + ); + return; + } + assertIncludes( + healthPatrolNginxCheck.args, + '--expected-public-host', + 'dry-run cutover 回退后 health patrol env 复核必须能校验 Nginx public Host。', + ); + assertIncludes( + healthPatrolNginxCheck.args, + 'nginx.example.com', + 'dry-run cutover 回退后 health patrol env 复核必须使用显式回退 public Host。', + ); + if (healthPatrolNginxCheck.args.includes('--require-empty-public-host')) { + failures.push( + '显式提供回退 public Host 时，回退后 health patrol env 复核不能同时要求 Host 为空。', + ); + } + + for (const stepName of [ + 'Pingora direct rollback dry-run', + 'Pingora direct rollback apply', + ]) { + const step = findStep(plan, stepName); + if (!step) { + failures.push(`dry-run cutover 必须包含 ${stepName}。`); + continue; + } + assertIncludes( + step.args, + '--health-patrol-expected-public-host', + `dry-run cutover ${stepName} 必须把显式回退 public Host 传给回退脚本。`, + ); + assertIncludes( + step.args, + 'nginx.example.com', + `dry-run cutover ${stepName} 必须使用显式回退 public Host。`, + ); + if (step.args.includes('--health-patrol-require-empty-public-host')) { + failures.push( + `显式提供回退 public Host 时，${stepName} 不能同时要求 Host 为空。`, + ); + } + } +} + +function assertDryRunCutoverPlanCanIncludeRollbackShadowProbe() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-release-root', + '/opt/genarrative/current', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-pingora-shadow-probe-url', + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + '--rollback-pingora-shadow-probe-token', + 'probe-secret-token', + ]); + const rollbackDryRun = findStep(plan, 'Pingora direct rollback dry-run'); + const rollbackApply = findStep(plan, 'Pingora direct rollback apply'); + for (const step of [rollbackDryRun, rollbackApply]) { + if (!step) { + failures.push('带 shadow probe 的 dry-run cutover 必须包含 rollback 步骤。'); + continue; + } + assertIncludes( + step.args, + '--pingora-shadow-probe-url', + 'dry-run cutover rollback 步骤必须传递 shadow probe URL。', + ); + assertIncludes( + step.args, + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + 'dry-run cutover rollback 步骤必须保留 shadow probe URL。', + ); + assertIncludes( + step.args, + '--pingora-shadow-probe-token', + 'dry-run cutover rollback 步骤必须传递 shadow probe token 参数。', + ); + assertIncludes( + step.args, + '', + 'dry-run cutover rollback 步骤必须隐藏 shadow probe token 值。', + ); + if (step.args.includes('probe-secret-token')) { + failures.push('dry-run cutover JSON 不能泄露 shadow probe token 原文。'); + } + } +} + +function assertRequireLiveForcesHost() { + const plan = readPlan([...requireLiveBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Nginx live canary smoke'); + if (!step) { + failures.push('--require-live 计划必须包含 live canary smoke。'); + return; + } + assertIncludes( + step.args, + '--host', + '--require-live 必须把正式域名 Host 传给 live canary smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-live live canary smoke 必须使用显式传入的正式域名 Host。', + ); + + const parity = findStep(plan, '目标 Nginx live canary access log 对账'); + if (!parity) { + failures.push('--require-live 计划必须在 live smoke 后包含 access log 对账。'); + return; + } + assertIncludes( + parity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + '--require-live 必须调用 canary access log 对账脚本。', + ); + assertIncludes( + parity.args, + '--nginx-log-file', + '--require-live access log 对账必须显式传 Nginx access log。', + ); + assertIncludes( + parity.args, + '/var/log/nginx/genarrative.access.log', + '--require-live access log 对账必须使用显式 Nginx access log 路径。', + ); + assertIncludes( + parity.args, + '--pingora-log-file', + '--require-live access log 对账必须显式传 Pingora access log。', + ); + assertIncludes( + parity.args, + '/var/log/genarrative/pingora-gateway.access.log', + '--require-live access log 对账必须使用显式 Pingora access log 路径。', + ); + assertIncludes( + parity.args, + '--since-lines', + '--require-live access log 对账必须显式传 tail 行数。', + ); + assertIncludes( + parity.args, + '3000', + '--require-live access log 对账必须使用显式 tail 行数。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_canary/healthz', + '--require-live access log 对账必须覆盖 canary healthz。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_canary/api/creation-entry/config', + '--require-live access log 对账必须覆盖代表性 API canary 路径。', + ); +} + +function assertRequireRealpathLiveForcesHostAndRealpathParity() { + const plan = readPlan([...requireRealpathLiveBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Nginx realpath canary smoke'); + if (!step) { + failures.push('--require-realpath-live 计划必须包含 realpath canary smoke。'); + return; + } + assertIncludes( + step.args, + '--realpath', + '--require-realpath-live 必须把 live smoke 切到真实路径 canary 模式。', + ); + assertIncludes( + step.args, + '--host', + '--require-realpath-live 必须把正式域名 Host 传给 realpath live canary smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-realpath-live realpath smoke 必须使用显式传入的正式域名 Host。', + ); + + const parity = findStep(plan, '目标 Nginx realpath canary access log 对账'); + if (!parity) { + failures.push( + '--require-realpath-live 计划必须在 realpath smoke 后包含 access log 对账。', + ); + return; + } + assertIncludes( + parity.args, + '--realpath', + '--require-realpath-live access log 对账必须使用真实路径模式。', + ); + assertIncludes( + parity.args, + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + '--require-realpath-live access log 对账必须使用独立 Nginx realpath canary access log。', + ); + assertIncludes( + parity.args, + '/__genarrative_pingora_realpath_canary/healthz', + '--require-realpath-live access log 对账必须覆盖 realpath healthz。', + ); + assertIncludes( + parity.args, + '/api/creation-entry/config', + '--require-realpath-live access log 对账必须覆盖真实 API 路径。', + ); + assertIncludes( + parity.args, + '/v1/identity', + '--require-realpath-live access log 对账必须覆盖真实 SpacetimeDB identity 路径。', + ); + assertIncludes( + parity.args, + '/assets/app.js', + '--require-realpath-live access log 对账必须覆盖真实静态资源路径。', + ); +} + +function assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts() { + const plan = readPlan([ + ...requireLiveBaseArgs, + ...requireRealpathLiveBaseArgs, + ...requireDirectBaseArgs, + '--release-runtime-only', + '--dry-run-plan', + ]); + const sourceOnlyNames = new Set([ + 'Rust 路由与保护单测', + 'Pingora mock 上游 smoke', + 'Nginx/Pingora 路由矩阵 parity', + 'Pingora production release 真实构建烟测', + 'API deploy release 烟测', + ]); + for (const name of sourceOnlyNames) { + if (findStep(plan, name)) { + failures.push(`release runtime-only 计划不能包含源码全量门禁: ${name}。`); + } + } + + const audit = findStep(plan, 'current release 自包含自审'); + if (!audit) { + failures.push('release runtime-only 计划必须先包含 current release 自包含自审。'); + } else { + assertAbsoluteScriptArg( + audit.args, + 'scripts/ops/pingora-current-release-audit.mjs', + 'release runtime-only 自审必须使用随当前脚本定位的包内自审脚本。', + ); + assertIncludes( + audit.args, + '--release-root', + 'release runtime-only 自审必须显式传 current release 根目录。', + ); + assertIncludes( + audit.args, + '--require-pingora-gateway', + 'release runtime-only direct 复核必须要求 Pingora 发布物存在。', + ); + assertIncludes( + audit.args, + '--systemd-show', + 'release runtime-only direct 复核必须核验 systemd ExecStart。', + ); + } + + const live = findStep(plan, '目标 Nginx live canary smoke'); + if (!live) { + failures.push('release runtime-only 计划必须包含 live canary smoke。'); + } else { + assertAbsoluteScriptArg( + live.args, + 'scripts/check-pingora-canary-live.mjs', + 'release runtime-only live canary smoke 必须使用随当前脚本定位的包内脚本。', + ); + } + + const liveParity = findStep(plan, '目标 Nginx live canary access log 对账'); + if (!liveParity) { + failures.push('release runtime-only 计划必须包含 live access log 对账。'); + } else { + assertAbsoluteScriptArg( + liveParity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'release runtime-only live access log 对账必须使用随当前脚本定位的包内脚本。', + ); + } + + const realpathLive = findStep(plan, '目标 Nginx realpath canary smoke'); + if (!realpathLive) { + failures.push('release runtime-only 计划必须包含 realpath live canary smoke。'); + } else { + assertAbsoluteScriptArg( + realpathLive.args, + 'scripts/check-pingora-canary-live.mjs', + 'release runtime-only realpath live canary smoke 必须使用随当前脚本定位的包内脚本。', + ); + assertIncludes( + realpathLive.args, + '--realpath', + 'release runtime-only realpath live canary smoke 必须传 --realpath。', + ); + } + + const realpathParity = findStep( + plan, + '目标 Nginx realpath canary access log 对账', + ); + if (!realpathParity) { + failures.push('release runtime-only 计划必须包含 realpath live access log 对账。'); + } else { + assertAbsoluteScriptArg( + realpathParity.args, + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'release runtime-only realpath access log 对账必须使用随当前脚本定位的包内脚本。', + ); + assertIncludes( + realpathParity.args, + '--realpath', + 'release runtime-only realpath access log 对账必须传 --realpath。', + ); + } + + const preflight = findStep(plan, '目标 Pingora direct entry preflight'); + if (!preflight) { + failures.push('release runtime-only 计划必须包含 direct preflight。'); + } else { + assertAbsoluteScriptArg( + preflight.args, + 'scripts/check-pingora-direct-preflight.mjs', + 'release runtime-only direct preflight 必须使用随当前脚本定位的包内脚本。', + ); + } + + const healthPatrol = findStep(plan, '目标 health patrol env 直连模式复核'); + if (!healthPatrol) { + failures.push('release runtime-only 计划必须包含 health patrol env 复核。'); + } else { + assertAbsoluteScriptArg( + healthPatrol.args, + 'scripts/check-production-health-patrol-env.mjs', + 'release runtime-only health patrol env 复核必须使用随当前脚本定位的包内脚本。', + ); + } + + const direct = findStep(plan, '目标 Pingora direct live smoke'); + if (!direct) { + failures.push('release runtime-only 计划必须包含 direct live smoke。'); + } else { + assertAbsoluteScriptArg( + direct.args, + 'scripts/check-pingora-direct-live.mjs', + 'release runtime-only direct live smoke 必须使用随当前脚本定位的包内脚本。', + ); + } +} + +function assertReleaseRuntimeOnlyRejectsSourceOnlyFlags() { + for (const flag of ['--require-docker', '--pull-docker', '--require-nginx']) { + const result = runReadinessExpectFailure([ + '--release-runtime-only', + flag, + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--release-runtime-only 只执行 current release 包内运行时复核', + `release runtime-only 不能同时接受源码或本机依赖参数 ${flag}。`, + ); + } +} + +function assertRequireDirectForcesWssUpgrade() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--require-wss-upgrade', + '--require-direct 必须自动强制 WSS subscribe 返回 101。', + ); +} + +function assertRequireDirectForcesHttpBaseUrl() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--http-base-url', + '--require-direct 必须把 HTTP redirect / ACME 入口纳入 direct live smoke。', + ); +} + +function assertRequireDirectForcesHostSni() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--host', + '--require-direct 必须把正式域名 Host/SNI 传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct direct live smoke 必须使用显式传入的正式域名 Host/SNI。', + ); +} + +function assertRequireDirectForcesRedirectHost() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--redirect-host', + '--require-direct 必须把显式 redirect Location host 传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct direct live smoke 必须使用显式传入的 redirect Location host。', + ); +} + +function assertRequireDirectForcesPingoraAccessLog() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--pingora-access-log', + '--require-direct 必须把 Pingora access log 落盘校验传给 direct live smoke。', + ); + assertIncludes( + step.args, + '/var/log/genarrative/pingora-gateway.access.log', + '--require-direct direct live smoke 必须使用显式 Pingora access log 路径。', + ); +} + +function assertRequireDirectForcesPreflightSystemdCertReadableAndPortsFree() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct entry preflight'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct entry preflight。'); + return; + } + assertIncludes( + step.args, + '--systemd-cat', + '--require-direct 必须检查 systemd drop-in 最终生效配置。', + ); + assertIncludes( + step.args, + '--check-cert-readable', + '--require-direct 必须检查当前用户可读 TLS 证书和私钥。', + ); + assertIncludes( + step.args, + '--check-service-env-file', + '--require-direct 必须检查 service EnvironmentFile 包含本次 env。', + ); + assertIncludes( + step.args, + '--check-service-user-cert-readable', + '--require-direct 必须检查 systemd 服务用户可读 TLS 证书和私钥。', + ); + assertIncludes( + step.args, + '--check-service-binary-executable', + '--require-direct 必须检查 Pingora service ExecStart 二进制可执行。', + ); + assertIncludes( + step.args, + '--check-ports-free', + '--require-direct 必须检查 TLS/HTTP redirect 端口已释放。', + ); +} + +function assertRequireDirectForcesExplicitSpacetimeDatabase() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('--require-direct 计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--spacetime-database', + '--require-direct 必须把显式 SpacetimeDB 数据库名传给 direct live smoke。', + ); + assertIncludes( + step.args, + 'genarrative-prod', + '--require-direct direct live smoke 必须使用显式传入的 SpacetimeDB 数据库名。', + ); +} + +function assertRequireDirectForcesHealthPatrolEnvCheck() { + const plan = readPlan([...requireDirectBaseArgs, '--dry-run-plan']); + const step = findStep(plan, '目标 health patrol env 直连模式复核'); + if (!step) { + failures.push('--require-direct 计划必须包含 health patrol env 复核。'); + return; + } + assertIncludes( + step.args, + 'scripts/check-production-health-patrol-env.mjs', + '--require-direct 必须复核 health patrol env。', + ); + assertIncludes( + step.args, + '/etc/genarrative/health-patrol.env', + '--require-direct health patrol env 复核必须使用显式 env 文件。', + ); + assertIncludes( + step.args, + 'pingora-direct', + '--require-direct health patrol env 复核必须要求 pingora-direct。', + ); + assertIncludes( + step.args, + 'https://127.0.0.1', + '--require-direct health patrol env 复核必须校验 direct HTTPS base URL。', + ); + assertIncludes( + step.args, + 'example.com', + '--require-direct health patrol env 复核必须校验正式 Host。', + ); +} + +function assertDryRunPlanRedactsDirectProbeToken() { + const plan = readPlan([ + ...requireDirectBaseArgs, + '--direct-probe-token', + 'direct-secret-token', + '--dry-run-plan', + ]); + const step = findStep(plan, '目标 Pingora direct live smoke'); + if (!step) { + failures.push('带 direct probe token 的计划必须包含 direct live smoke。'); + return; + } + assertIncludes( + step.args, + '--probe-token', + 'dry-run plan direct live smoke 必须传递 probe token 参数。', + ); + assertIncludes( + step.args, + '', + 'dry-run plan direct live smoke 必须隐藏 probe token 值。', + ); + if (step.args.includes('direct-secret-token')) { + failures.push('dry-run plan JSON 不能泄露 direct probe token 原文。'); + } +} + +function assertRunStepLogRedactsDirectProbeToken() { + const output = readFileSync( + 'scripts/check-pingora-release-readiness.mjs', + 'utf8', + ); + assertIncludes( + output, + 'redactSecretArgs(args).join', + '真实执行日志里的命令展示必须使用脱敏参数。', + ); +} + +function assertRequireLiveRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-live 时必须提供 --live-host 或 GENARRATIVE_PINGORA_CANARY_HOST', + '--require-live 缺少 live Host 时必须失败。', + ); +} + +function assertRequireLiveRejectsInvalidHost() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'https://example.com', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-host 只能是 host 或 host:port', + '--require-live 的 live Host 不能接受 URL 或路径。', + ); +} + +function assertRequireLiveRejectsRelativeAccessLog() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + 'nginx.access.log', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-nginx-access-log 必须是绝对路径', + '--require-live 的 Nginx access log 不能接受相对路径。', + ); +} + +function assertRequireLiveRejectsFilesystemRootAccessLog() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-nginx-access-log', + '/', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-nginx-access-log 不能是文件系统根目录', + '--require-live 的 Nginx access log 不能指向文件系统根目录。', + ); +} + +function assertRequireLiveRejectsInvalidAccessLogSinceLines() { + const result = runReadinessExpectFailure([ + '--require-live', + '--live-base-url', + 'http://127.0.0.1', + '--live-host', + 'example.com', + '--live-access-log-since-lines', + '0', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--live-access-log-since-lines 必须是正整数', + '--require-live 的 access log tail 行数必须是正整数。', + ); +} + +function assertRequireRealpathLiveRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-realpath-live 时必须提供 --realpath-live-host 或 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST', + '--require-realpath-live 缺少 realpath live Host 时必须失败。', + ); +} + +function assertRequireRealpathLiveRejectsRelativeAccessLog() { + const result = runReadinessExpectFailure([ + '--require-realpath-live', + '--realpath-live-base-url', + 'http://127.0.0.1:18083', + '--realpath-live-host', + 'example.com', + '--realpath-live-nginx-access-log', + 'nginx.realpath.access.log', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--realpath-live-nginx-access-log 必须是绝对路径', + '--require-realpath-live 的 Nginx access log 不能接受相对路径。', + ); +} + +function assertRequireDirectRejectsMissingHttpBaseUrl() { + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-release-readiness.mjs', + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('--require-direct 缺少 direct HTTP base URL 时必须失败。'); + return; + } + const output = `${result.stdout}\n${result.stderr}`; + if ( + !output.includes( + '启用 --require-direct 时必须提供 --direct-http-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + ) + ) { + failures.push( + '--require-direct 缺少 direct HTTP base URL 时必须给出明确错误。', + ); + } +} + +function assertRequireDirectRejectsMissingHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-host 或 GENARRATIVE_PINGORA_DIRECT_HOST', + '--require-direct 缺少 direct Host/SNI 时必须失败。', + ); +} + +function assertRequireDirectRejectsInvalidHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com/path', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-host 只能是 host 或 host:port', + '--require-direct 的 direct Host/SNI 不能接受 URL 或路径。', + ); +} + +function assertRequireDirectRejectsMissingRedirectHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-redirect-host 或 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + '--require-direct 缺少 direct redirect Location host 时必须失败。', + ); +} + +function assertRequireDirectRejectsInvalidRedirectHost() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'https://example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-redirect-host 只能是 host 或 host:port', + '--require-direct 的 redirect Location host 不能接受 URL 或路径。', + ); +} + +function assertRequireDirectRejectsMissingPingoraAccessLog() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-pingora-access-log 或 GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + '--require-direct 缺少 direct Pingora access log 时必须失败。', + ); +} + +function assertRequireDirectRejectsFilesystemRootPingoraAccessLog() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-pingora-access-log 不能是文件系统根目录', + '--require-direct 的 Pingora access log 不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsMissingSystemdPreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-systemd 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true', + '--require-direct 缺少 systemd preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingCertReadablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true', + '--require-direct 缺少证书可读 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceEnvFilePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true', + '--require-direct 缺少 service EnvironmentFile preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceUserCertReadablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true', + '--require-direct 缺少服务用户证书可读 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingServiceBinaryExecutablePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true', + '--require-direct 缺少 service 二进制可执行 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingPortsFreePreflight() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-preflight-check-ports-free 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE=true', + '--require-direct 缺少端口释放 preflight 时必须失败。', + ); +} + +function assertRequireDirectRejectsMissingSpacetimeDatabase() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-spacetime-database 或 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + '--require-direct 缺少 SpacetimeDB 数据库名时必须失败。', + ); +} + +function assertRequireDirectRejectsFilesystemRootPreflightEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-preflight-env-file 不能是文件系统根目录', + '--require-direct 的 Pingora preflight env 文件不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsMissingHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时必须提供 --direct-health-patrol-env-file 或 GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE', + '--require-direct 缺少 health patrol env 文件时必须失败。', + ); +} + +function assertRequireDirectRejectsRelativeHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + 'health-patrol.env', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-health-patrol-env-file 必须是绝对路径', + '--require-direct 的 health patrol env 文件不能接受相对路径。', + ); +} + +function assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile() { + const result = runReadinessExpectFailure([ + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-health-patrol-env-file', + '/', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '--direct-health-patrol-env-file 不能是文件系统根目录', + '--require-direct 的 health patrol env 文件不能指向文件系统根目录。', + ); +} + +function assertRequireDirectRejectsSkipWss() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--direct-skip-wss', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时不能使用 --direct-skip-wss 或 GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=true', + '--require-direct 不能允许跳过 WSS subscribe。', + ); +} + +function assertRequireDirectRejectsInsecureTls() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--direct-insecure-tls', + '--dry-run-plan', + ]); + assertFailureIncludes( + result, + '启用 --require-direct 时不能使用 --direct-insecure-tls 或 GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=true', + '--require-direct 不能允许 insecure TLS。', + ); +} + +function assertDryRunCutoverRejectsMissingRequireDirect() { + const result = runReadinessExpectFailure([ + '--dry-run-cutover', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-preflight-env-file', + '/tmp/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--direct-spacetime-database', + 'genarrative-prod', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须同时提供 --require-direct', + 'dry-run cutover 缺少 --require-direct 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRelativeEvidenceOutputRoot() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--cutover-evidence-output-root', + 'relative-evidence', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-output-root 必须是绝对路径', + 'dry-run cutover 证据输出根目录使用相对路径时必须失败。', + ); +} + +function assertDryRunCutoverRejectsFilesystemRootReleaseRoot() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--cutover-release-root', + '/', + ]); + assertFailureIncludes( + result, + '--cutover-release-root 不能是文件系统根目录', + 'dry-run cutover current release 根目录使用 / 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--cutover-evidence-output-root', + '/', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-output-root 不能是文件系统根目录', + 'dry-run cutover 证据输出根目录使用 / 时必须失败。', + ); +} + +function assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot() { + const usage = readUsage(); + assertIncludes( + usage, + '--cutover-evidence-output-root /var/log/genarrative/pingora-cutover-evidence', + '只生成直连切换 runbook 的 usage 示例必须显式传证据输出根目录。', + ); +} + +function assertUsageExamplesIncludeDirectPingoraAccessLog() { + const usage = readUsage(); + assertIncludes( + usage, + 'Pingora 直连入口切换窗口追加：', + 'usage 必须包含正式直连入口示例。', + ); + assertIncludes( + usage, + '只生成直连切换 runbook：', + 'usage 必须包含只生成直连切换 runbook 示例。', + ); + const directExample = usage.slice( + usage.indexOf('Pingora 直连入口切换窗口追加：'), + usage.indexOf('只生成直连切换 runbook：'), + ); + const cutoverExample = usage.slice( + usage.indexOf('只生成直连切换 runbook：'), + ); + assertIncludes( + directExample, + '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + '正式直连入口 usage 示例必须显式传 Pingora access log 路径。', + ); + assertIncludes( + cutoverExample, + '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + '只生成直连切换 runbook 的 usage 示例必须显式传 Pingora access log 路径。', + ); +} + +function assertDryRunCutoverRejectsMissingRollbackHealthPatrolPublicBaseUrl() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + ]); + assertFailureIncludes( + result, + '启用 --dry-run-cutover 时必须提供 --rollback-health-patrol-public-base-url', + 'dry-run cutover 缺少回退后 health patrol public base URL 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicBaseUrl() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'ftp://127.0.0.1', + ]); + assertFailureIncludes( + result, + '--rollback-health-patrol-public-base-url 必须是 http(s) URL', + 'dry-run cutover 回退 health patrol public base URL 非 http(s) 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackHealthPatrolPublicHost() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-health-patrol-public-host', + 'https://example.com', + ]); + assertFailureIncludes( + result, + '--rollback-health-patrol-public-host 只能是 host 或 host:port', + 'dry-run cutover 回退 health patrol public Host 非 host 格式时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRedirectHostMismatch() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--direct-redirect-host', + 'redirect.example.com', + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + assertFailureIncludes( + result, + '--direct-redirect-host 必须与 --direct-host 使用同一正式 hostname', + 'dry-run cutover redirect Host 与 direct Host 不一致时必须失败。', + ); +} + +function assertDryRunCutoverRejectsRollbackHostMismatch() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-nginx-smoke-host', + 'nginx.example.com', + ]); + assertFailureIncludes( + result, + '--rollback-nginx-smoke-host 必须与 --direct-host 使用同一正式 hostname', + 'dry-run cutover rollback smoke Host 与 direct Host 不一致时必须失败。', + ); +} + +function assertDryRunCutoverRejectsIncompleteRollbackShadowProbe() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-pingora-shadow-probe-url', + 'http://127.0.0.1:18081/__genarrative_pingora/healthz', + ]); + assertFailureIncludes( + result, + '--rollback-pingora-shadow-probe-url 必须同时提供 --rollback-pingora-shadow-probe-token', + 'dry-run cutover rollback shadow probe URL 缺少 token 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidRollbackShadowProbeUrl() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--rollback-pingora-shadow-probe-url', + 'ftp://127.0.0.1/probe', + '--rollback-pingora-shadow-probe-token', + 'probe-secret-token', + ]); + assertFailureIncludes( + result, + '--rollback-pingora-shadow-probe-url 必须是 http(s) URL', + 'dry-run cutover rollback shadow probe URL 非 http(s) 时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidEvidenceTimelineMaxSpan() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + '--cutover-evidence-timeline-max-span-ms', + '0', + ]); + assertFailureIncludes( + result, + '--cutover-evidence-timeline-max-span-ms 必须是正整数', + 'dry-run cutover 标准切换时间线最大跨度非正整数时必须失败。', + ); +} + +function assertDryRunCutoverRejectsInvalidCutoverRunId() { + const result = runReadinessExpectFailure([ + ...requireDirectBaseArgs, + '--dry-run-cutover', + '--cutover-run-id', + 'cutover 1', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ]); + assertFailureIncludes( + result, + '--cutover-run-id 只能包含 ASCII 字母、数字、点、下划线或短横线', + 'dry-run cutover 必须拒绝不安全 cutoverRunId。', + ); +} + +function runReadinessExpectFailure(args) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('release readiness 缺少硬门禁参数时必须失败。'); + } + return result; +} + +function assertLiveSmokeScriptsRejectInvalidTimeout() { + const cases = [ + { + script: 'scripts/check-pingora-canary-live.mjs', + args: ['--base-url', 'http://127.0.0.1', '--timeout-ms', '0'], + expected: '--timeout-ms 必须是正整数', + reason: 'canary live smoke 必须拒绝非正数 --timeout-ms。', + }, + { + script: 'scripts/check-pingora-canary-live.mjs', + args: ['--base-url', 'http://127.0.0.1'], + env: { + GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS 必须是正整数', + reason: 'canary live smoke 必须拒绝非法 env timeout。', + }, + { + script: 'scripts/check-pingora-direct-live.mjs', + args: ['--https-base-url', 'https://127.0.0.1', '--timeout-ms', '0'], + expected: '--timeout-ms 必须是正整数', + reason: 'direct live smoke 必须拒绝非正数 --timeout-ms。', + }, + { + script: 'scripts/check-pingora-direct-live.mjs', + args: ['--https-base-url', 'https://127.0.0.1'], + env: { + GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS 必须是正整数', + reason: 'direct live smoke 必须拒绝非法 env timeout。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync('node', [testCase.script, ...testCase.args], { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...(testCase.env || {}), + }, + }); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectLiveRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: 'ture', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE 必须是布尔值', + reason: 'direct live smoke 必须拒绝拼写错误的 require WSS env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_SKIP_WSS: 'maybe', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_SKIP_WSS 必须是布尔值', + reason: 'direct live smoke 必须拒绝非法 skip WSS env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS: 'enabled', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS 必须是布尔值', + reason: 'direct live smoke 必须拒绝非法 insecure TLS env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-direct-live.mjs', '--https-base-url', 'https://127.0.0.1'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectLiveRejectsFilesystemRootAccessLog() { + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + 'https://127.0.0.1', + '--pingora-access-log', + '/', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('direct live smoke 必须拒绝 Pingora access log 指向文件系统根目录。'); + } + assertFailureIncludes( + result, + '--pingora-access-log 不能是文件系统根目录', + 'direct live smoke 的 Pingora access log 不能指向文件系统根目录。', + ); +} + +function assertReleaseReadinessRejectsInvalidDirectBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT 必须是布尔值', + reason: 'release readiness 必须拒绝拼写错误的 preflight systemd env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: 'definitely', + }, + expected: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE 必须是布尔值', + reason: 'release readiness 必须拒绝非法 require WSS env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', '--dry-run-plan'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectPreflightRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_REQUIRE_LIVE_ENV: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_REQUIRE_LIVE_ENV 必须是布尔值', + reason: 'direct preflight 必须拒绝拼写错误的 require live env。', + }, + { + env: { + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE: 'maybe', + }, + expected: + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE 必须是布尔值', + reason: 'direct preflight 必须拒绝非法 check ports free env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-direct-preflight.mjs'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection() { + const root = mkdtempSync(path.join(tmpdir(), 'genarrative-direct-preflight-')); + try { + const envFile = path.join(root, 'pingora-gateway.env'); + writeFileSync( + envFile, + [ + 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:18443', + 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:18080', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/tmp/cert.pem', + 'GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/tmp/key.pem', + 'GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO=https', + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=2', + 'GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false', + '', + ].join('\n'), + ); + + const result = spawnSync( + 'node', + [ + 'scripts/check-pingora-direct-preflight.mjs', + '--env-file', + envFile, + '--require-live-env', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) === 0) { + failures.push( + 'direct preflight 必须拒绝未确认共享保护的多实例 Pingora 配置。', + ); + } + assertFailureIncludes( + result, + 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + 'direct preflight 多实例保护失败必须给出明确错误。', + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +function assertCutoverEvidenceScriptsRejectInvalidBoolEnv() { + const cases = [ + { + script: 'scripts/ops/pingora-current-release-audit.mjs', + env: { + GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY: 'ture', + }, + expected: + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY 必须是布尔值', + reason: + 'current release 自审 smoke 必须覆盖非法 require gateway 布尔 env。', + }, + { + script: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + env: { + GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL: 'maybe', + }, + expected: + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL 必须是布尔值', + reason: + 'cutover 状态快照 smoke 必须覆盖非法 run health patrol 布尔 env。', + }, + { + script: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + env: { + GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL: 'enabled', + }, + expected: + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL 必须是布尔值', + reason: + 'cutover 证据包 smoke 必须覆盖非法 fail on critical 布尔 env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync('node', [testCase.script], { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...readinessPlanEnv(), + ...testCase.env, + }, + }); + if ((result.status ?? 0) === 0) { + failures.push( + `${testCase.reason} 脚本本体必须失败。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } + assertFailureIncludes( + result, + testCase.expected, + `${testCase.reason} 必须能在输出中看到明确错误。`, + ); + } +} + +function assertFailureIncludes(result, expected, reason) { + const output = `${result.stdout}\n${result.stderr}`; + if (!output.includes(expected)) { + failures.push(reason); + } +} + +function readPlan(args) { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `读取 release readiness dry-run plan 失败，退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return []; + } + try { + return JSON.parse(result.stdout); + } catch (error) { + failures.push( + `release readiness dry-run plan 不是合法 JSON: ${error.message}`, + ); + return []; + } +} + +function readUsage() { + const result = spawnSync( + 'node', + ['scripts/check-pingora-release-readiness.mjs', '--help'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `读取 release readiness usage 失败，退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return ''; + } + return result.stdout; +} + +function readinessPlanEnv() { + return { + ...process.env, + GENARRATIVE_PINGORA_CANARY_HOST: '', + GENARRATIVE_PINGORA_CANARY_BASE_URL: '', + GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL: '', + GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL: '', + GENARRATIVE_PINGORA_DIRECT_HOST: '', + GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE: '', + GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE: '', + GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE: '', + GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE: '', + GENARRATIVE_PINGORA_DIRECT_SKIP_WSS: '', + GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_HOST: '', + GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY: '', + GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_HOST: '', + GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL: '', + GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN: '', + GENARRATIVE_PINGORA_CUTOVER_RUN_ID: '', + }; +} + +function findStep(plan, name) { + return plan.find((step) => step.name === name); +} + +function assertIncludes(values, expected, reason) { + if (!values.includes(expected)) { + failures.push(`${reason} 缺少: ${expected}`); + } +} + +function assertAbsoluteScriptArg(values, expectedSuffix, reason) { + const normalizedSuffix = expectedSuffix.replace(/\\/g, '/'); + const matched = values.some((value) => { + const text = String(value).replace(/\\/g, '/'); + return text.startsWith('/') && text.endsWith(normalizedSuffix); + }); + if (!matched) { + failures.push(`${reason} 缺少绝对脚本路径后缀: ${expectedSuffix}`); + } +} + +function assertNotIncludes(values, forbidden, reason) { + if (values.includes(forbidden)) { + failures.push(reason); + } +} + +function assertStepBefore(plan, firstName, secondName, reason) { + const firstIndex = plan.findIndex((step) => step.name === firstName); + const secondIndex = plan.findIndex((step) => step.name === secondName); + if (firstIndex < 0 || secondIndex < 0) { + return; + } + if (firstIndex >= secondIndex) { + failures.push(reason); + } +} diff --git a/scripts/check-pingora-release-readiness.mjs b/scripts/check-pingora-release-readiness.mjs new file mode 100644 index 000000000..4909e369d --- /dev/null +++ b/scripts/check-pingora-release-readiness.mjs @@ -0,0 +1,1990 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = process.cwd(); +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const releaseRoot = path.resolve(scriptDir, '..'); +const DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; +const DEFAULT_CUTOVER_RUN_ID_PREFIX = 'pingora-direct-'; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); + +const config = parseArgs(process.argv.slice(2)); +const failures = []; + +if (config.dryRunPlan) { + printDryRunPlan(buildSteps(config)); + process.exit(0); +} +if (config.dryRunCutover) { + printDryRunPlan(buildCutoverPlan(config)); + process.exit(0); +} + +for (const step of buildSteps(config)) { + const ok = await runStep(step); + if (!ok && !config.keepGoing) { + break; + } +} + +if (failures.length > 0) { + console.error('\n[pingora-release-readiness] 未通过：'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('\n[pingora-release-readiness] 通过'); + +function parseArgs(argv) { + const result = { + requireDocker: false, + pullDocker: false, + requireNginx: false, + requireLive: false, + requireRealpathLive: false, + requireDirect: false, + liveBaseUrl: process.env.GENARRATIVE_PINGORA_CANARY_BASE_URL || '', + liveHost: process.env.GENARRATIVE_PINGORA_CANARY_HOST || '', + liveAccessLogNginx: + process.env.GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG || + '/var/log/nginx/genarrative.access.log', + liveAccessLogPingora: + process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG || + '/var/log/genarrative/pingora-gateway.access.log', + liveAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES || '2000', + realpathLiveBaseUrl: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL || '', + realpathLiveHost: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_HOST || '', + realpathLiveAccessLogNginx: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_NGINX_ACCESS_LOG || + '/var/log/nginx/genarrative-pingora-realpath-canary.access.log', + realpathLiveAccessLogPingora: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_PINGORA_ACCESS_LOG || + process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG || + '/var/log/genarrative/pingora-gateway.access.log', + realpathLiveAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_ACCESS_LOG_SINCE_LINES || + process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES || + '2000', + directHttpsBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL || '', + directHttpBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '', + directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', + directRedirectHost: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', + directPingoraAccessLog: + process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '', + directAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES || '2000', + directPreflightEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || '', + directPreflightSystemd: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT', + ), + directPreflightCheckCertReadable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE', + ), + directPreflightCheckServiceEnvFile: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE', + ), + directPreflightCheckServiceUserCertReadable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE', + ), + directPreflightCheckServiceBinaryExecutable: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE', + ), + directPreflightCheckPortsFree: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE', + ), + directSpacetimeDatabase: + process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '', + directHealthPatrolEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE || + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '', + directRequireWssUpgrade: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE', + ), + directSkipWss: readBoolEnv('GENARRATIVE_PINGORA_DIRECT_SKIP_WSS'), + directInsecureTls: readBoolEnv( + 'GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS', + ), + cutoverReleaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + cutoverEvidenceOutputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + cutoverEvidenceTimelineMaxSpanMs: parsePositiveIntEnv( + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS', + DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS, + ), + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + rollbackNginxSmokeUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL || + 'http://127.0.0.1/healthz', + rollbackNginxSmokeHost: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_HOST || '', + rollbackNginxSmokeExpectBody: + process.env.GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_EXPECT_BODY || + '"ok":true', + rollbackHealthPatrolPublicBaseUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL || + '', + rollbackHealthPatrolPublicHost: + process.env.GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_HOST || '', + rollbackPingoraShadowProbeUrl: + process.env.GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL || '', + rollbackPingoraShadowProbeToken: + process.env.GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN || '', + dryRunPlan: false, + dryRunCutover: false, + releaseRuntimeOnly: readBoolEnv( + 'GENARRATIVE_PINGORA_RELEASE_RUNTIME_ONLY', + ), + keepGoing: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--require-docker': + result.requireDocker = true; + break; + case '--pull-docker': + result.pullDocker = true; + break; + case '--require-nginx': + result.requireNginx = true; + break; + case '--require-live': + result.requireLive = true; + break; + case '--require-realpath-live': + result.requireRealpathLive = true; + break; + case '--require-direct': + result.requireDirect = true; + break; + case '--live-base-url': + result.liveBaseUrl = requireValue(argv, ++index, arg); + break; + case '--live-host': + result.liveHost = requireValue(argv, ++index, arg); + break; + case '--live-nginx-access-log': + result.liveAccessLogNginx = requireValue(argv, ++index, arg); + break; + case '--live-pingora-access-log': + result.liveAccessLogPingora = requireValue(argv, ++index, arg); + break; + case '--live-access-log-since-lines': + result.liveAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--realpath-live-base-url': + result.realpathLiveBaseUrl = requireValue(argv, ++index, arg); + break; + case '--realpath-live-host': + result.realpathLiveHost = requireValue(argv, ++index, arg); + break; + case '--realpath-live-nginx-access-log': + result.realpathLiveAccessLogNginx = requireValue(argv, ++index, arg); + break; + case '--realpath-live-pingora-access-log': + result.realpathLiveAccessLogPingora = requireValue(argv, ++index, arg); + break; + case '--realpath-live-access-log-since-lines': + result.realpathLiveAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--direct-https-base-url': + result.directHttpsBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-http-base-url': + result.directHttpBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-host': + result.directHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-host': + result.directRedirectHost = requireValue(argv, ++index, arg); + break; + case '--direct-probe-token': + result.directProbeToken = requireValue(argv, ++index, arg); + break; + case '--direct-pingora-access-log': + result.directPingoraAccessLog = requireValue(argv, ++index, arg); + break; + case '--direct-access-log-since-lines': + result.directAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--direct-preflight-env-file': + result.directPreflightEnvFile = requireValue(argv, ++index, arg); + break; + case '--direct-preflight-systemd': + result.directPreflightSystemd = true; + break; + case '--direct-preflight-check-cert-readable': + result.directPreflightCheckCertReadable = true; + break; + case '--direct-preflight-check-service-env-file': + result.directPreflightCheckServiceEnvFile = true; + break; + case '--direct-preflight-check-service-user-cert-readable': + result.directPreflightCheckServiceUserCertReadable = true; + break; + case '--direct-preflight-check-service-binary-executable': + result.directPreflightCheckServiceBinaryExecutable = true; + break; + case '--direct-preflight-check-ports-free': + result.directPreflightCheckPortsFree = true; + break; + case '--direct-spacetime-database': + result.directSpacetimeDatabase = requireValue(argv, ++index, arg); + break; + case '--direct-health-patrol-env-file': + result.directHealthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--direct-require-wss-upgrade': + result.directRequireWssUpgrade = true; + break; + case '--direct-skip-wss': + result.directSkipWss = true; + break; + case '--direct-insecure-tls': + result.directInsecureTls = true; + break; + case '--cutover-release-root': + result.cutoverReleaseRoot = requireValue(argv, ++index, arg); + break; + case '--cutover-evidence-output-root': + result.cutoverEvidenceOutputRoot = requireValue(argv, ++index, arg); + break; + case '--cutover-evidence-timeline-max-span-ms': + result.cutoverEvidenceTimelineMaxSpanMs = parsePositiveInt( + requireValue(argv, ++index, arg), + '--cutover-evidence-timeline-max-span-ms', + ); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-url': + result.rollbackNginxSmokeUrl = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-host': + result.rollbackNginxSmokeHost = requireValue(argv, ++index, arg); + break; + case '--rollback-nginx-smoke-expect-body': + result.rollbackNginxSmokeExpectBody = requireValue(argv, ++index, arg); + break; + case '--rollback-health-patrol-public-base-url': + result.rollbackHealthPatrolPublicBaseUrl = requireValue( + argv, + ++index, + arg, + ); + break; + case '--rollback-health-patrol-public-host': + result.rollbackHealthPatrolPublicHost = requireValue( + argv, + ++index, + arg, + ); + break; + case '--rollback-pingora-shadow-probe-url': + result.rollbackPingoraShadowProbeUrl = requireValue(argv, ++index, arg); + break; + case '--rollback-pingora-shadow-probe-token': + result.rollbackPingoraShadowProbeToken = requireValue( + argv, + ++index, + arg, + ); + break; + case '--dry-run-plan': + result.dryRunPlan = true; + break; + case '--dry-run-cutover': + result.dryRunCutover = true; + break; + case '--release-runtime-only': + result.releaseRuntimeOnly = true; + break; + case '--keep-going': + result.keepGoing = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (result.requireLive && !result.liveBaseUrl) { + throw new Error( + '启用 --require-live 时必须提供 --live-base-url 或 GENARRATIVE_PINGORA_CANARY_BASE_URL', + ); + } + if (result.requireLive && !result.liveHost) { + throw new Error( + '启用 --require-live 时必须提供 --live-host 或 GENARRATIVE_PINGORA_CANARY_HOST', + ); + } + if (result.liveHost) { + validateHostOption(result.liveHost, '--live-host'); + } + if (result.requireLive) { + for (const [label, filePath] of [ + ['--live-nginx-access-log', result.liveAccessLogNginx], + ['--live-pingora-access-log', result.liveAccessLogPingora], + ]) { + validateSafeAbsoluteFilePath(filePath, label); + } + if (!isPositiveIntegerString(result.liveAccessLogSinceLines)) { + throw new Error('--live-access-log-since-lines 必须是正整数。'); + } + } + if (result.requireRealpathLive && !result.realpathLiveBaseUrl) { + throw new Error( + '启用 --require-realpath-live 时必须提供 --realpath-live-base-url 或 GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL', + ); + } + if (result.requireRealpathLive && !result.realpathLiveHost) { + throw new Error( + '启用 --require-realpath-live 时必须提供 --realpath-live-host 或 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST', + ); + } + if (result.realpathLiveHost) { + validateHostOption(result.realpathLiveHost, '--realpath-live-host'); + } + if (result.requireRealpathLive || result.realpathLiveBaseUrl) { + for (const [label, filePath] of [ + ['--realpath-live-nginx-access-log', result.realpathLiveAccessLogNginx], + ['--realpath-live-pingora-access-log', result.realpathLiveAccessLogPingora], + ]) { + validateSafeAbsoluteFilePath(filePath, label); + } + if (!isPositiveIntegerString(result.realpathLiveAccessLogSinceLines)) { + throw new Error('--realpath-live-access-log-since-lines 必须是正整数。'); + } + } + if (result.requireDirect && !result.directHttpsBaseUrl) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-https-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL', + ); + } + if (result.requireDirect && !result.directHttpBaseUrl) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-http-base-url 或 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + ); + } + if (result.requireDirect && !result.directHost) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-host 或 GENARRATIVE_PINGORA_DIRECT_HOST', + ); + } + if (result.directHost) { + validateHostOption(result.directHost, '--direct-host'); + } + if (result.requireDirect && !result.directRedirectHost) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-redirect-host 或 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + ); + } + if (result.directRedirectHost) { + validateHostOption(result.directRedirectHost, '--direct-redirect-host'); + } + if (result.requireDirect && !result.directPingoraAccessLog) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-pingora-access-log 或 GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + ); + } + if (result.directPingoraAccessLog) { + validateSafeAbsoluteFilePath( + result.directPingoraAccessLog, + '--direct-pingora-access-log', + ); + } + if ( + result.directPingoraAccessLog && + !isPositiveIntegerString(result.directAccessLogSinceLines) + ) { + throw new Error('--direct-access-log-since-lines 必须是正整数。'); + } + if (result.requireDirect && !result.directPreflightEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE', + ); + } + if (result.directPreflightEnvFile) { + validateSafeAbsoluteFilePath( + result.directPreflightEnvFile, + '--direct-preflight-env-file', + ); + } + if (result.requireDirect && !result.directPreflightSystemd) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-systemd 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true', + ); + } + if (result.requireDirect && !result.directPreflightCheckCertReadable) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true', + ); + } + if (result.requireDirect && !result.directPreflightCheckServiceEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true', + ); + } + if ( + result.requireDirect && + !result.directPreflightCheckServiceUserCertReadable + ) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true', + ); + } + if ( + result.requireDirect && + !result.directPreflightCheckServiceBinaryExecutable + ) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true', + ); + } + if (result.requireDirect && !result.directPreflightCheckPortsFree) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-preflight-check-ports-free 或 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE=true', + ); + } + if (result.requireDirect && result.directSkipWss) { + throw new Error( + '启用 --require-direct 时不能使用 --direct-skip-wss 或 GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=true', + ); + } + if (result.requireDirect && !result.directSpacetimeDatabase) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-spacetime-database 或 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + ); + } + if (result.requireDirect && !result.directHealthPatrolEnvFile) { + throw new Error( + '启用 --require-direct 时必须提供 --direct-health-patrol-env-file 或 GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE', + ); + } + if (result.directHealthPatrolEnvFile) { + validateSafeAbsoluteFilePath( + result.directHealthPatrolEnvFile, + '--direct-health-patrol-env-file', + ); + } + if (result.requireDirect && result.directInsecureTls) { + throw new Error( + '启用 --require-direct 时不能使用 --direct-insecure-tls 或 GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=true', + ); + } + if (result.requireDirect && !result.directSkipWss) { + result.directRequireWssUpgrade = true; + } + if (result.dryRunCutover && !result.requireDirect) { + throw new Error( + '启用 --dry-run-cutover 时必须同时提供 --require-direct，避免生成缺少直连硬门禁的切换计划。', + ); + } + if (result.dryRunCutover && !path.isAbsolute(result.cutoverReleaseRoot)) { + throw new Error('--cutover-release-root 必须是绝对路径。'); + } + if ( + result.dryRunCutover && + isFilesystemRootPath(result.cutoverReleaseRoot) + ) { + throw new Error('--cutover-release-root 不能是文件系统根目录。'); + } + if ( + result.dryRunCutover && + !path.isAbsolute(result.cutoverEvidenceOutputRoot) + ) { + throw new Error('--cutover-evidence-output-root 必须是绝对路径。'); + } + if ( + result.dryRunCutover && + isFilesystemRootPath(result.cutoverEvidenceOutputRoot) + ) { + throw new Error('--cutover-evidence-output-root 不能是文件系统根目录。'); + } + if (result.dryRunCutover && !result.rollbackNginxSmokeUrl) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --rollback-nginx-smoke-url 或 GENARRATIVE_PINGORA_ROLLBACK_NGINX_SMOKE_URL。', + ); + } + if (result.dryRunCutover && !result.rollbackNginxSmokeHost) { + result.rollbackNginxSmokeHost = result.directHost || result.liveHost; + } + if (result.rollbackNginxSmokeHost) { + validateHostOption( + result.rollbackNginxSmokeHost, + '--rollback-nginx-smoke-host', + ); + } + if ( + result.dryRunCutover && + !result.rollbackHealthPatrolPublicBaseUrl + ) { + throw new Error( + '启用 --dry-run-cutover 时必须提供 --rollback-health-patrol-public-base-url 或 GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_PUBLIC_BASE_URL，避免回退 runbook 覆盖现场原有 Nginx 巡检入口。', + ); + } + if (result.rollbackHealthPatrolPublicBaseUrl) { + validateHttpUrl( + result.rollbackHealthPatrolPublicBaseUrl, + '--rollback-health-patrol-public-base-url', + ); + } + if (result.rollbackHealthPatrolPublicHost) { + validateHostOption( + result.rollbackHealthPatrolPublicHost, + '--rollback-health-patrol-public-host', + ); + } + if ( + result.rollbackPingoraShadowProbeUrl && + !result.rollbackPingoraShadowProbeToken + ) { + throw new Error( + '--rollback-pingora-shadow-probe-url 必须同时提供 --rollback-pingora-shadow-probe-token。', + ); + } + if ( + result.rollbackPingoraShadowProbeToken && + !result.rollbackPingoraShadowProbeUrl + ) { + throw new Error( + '--rollback-pingora-shadow-probe-token 必须同时提供 --rollback-pingora-shadow-probe-url。', + ); + } + if (result.rollbackPingoraShadowProbeUrl) { + validateHttpUrl( + result.rollbackPingoraShadowProbeUrl, + '--rollback-pingora-shadow-probe-url', + ); + } + if (result.dryRunCutover) { + assertSameCutoverHostname( + result.directRedirectHost, + result.directHost, + '--direct-redirect-host', + '--direct-host', + ); + assertSameCutoverHostname( + result.rollbackNginxSmokeHost, + result.directHost, + '--rollback-nginx-smoke-host', + '--direct-host', + ); + if (result.liveHost) { + assertSameCutoverHostname( + result.liveHost, + result.directHost, + '--live-host', + '--direct-host', + ); + } + } + if (result.dryRunCutover && !result.cutoverRunId) { + result.cutoverRunId = generateCutoverRunId(); + } + if (result.cutoverRunId) { + validateSafeName(result.cutoverRunId, '--cutover-run-id'); + } + if ( + result.releaseRuntimeOnly && + (result.requireDocker || result.pullDocker || result.requireNginx) + ) { + throw new Error( + '--release-runtime-only 只执行 current release 包内运行时复核，不能同时使用 --require-docker / --pull-docker / --require-nginx；源码全量门禁请在构建环境运行默认模式。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/check-pingora-release-readiness.mjs [options] + +Options: + --require-docker Docker Nginx handoff smoke 必须执行通过，不允许跳过。 + --pull-docker Docker 镜像缺失时允许拉取；通常和 --require-docker 一起用于 CI。 + --require-nginx 本机 Nginx snippet 校验必须执行 nginx -t。 + --require-live 目标 Nginx 已启用 canary 后，强制执行 live canary smoke。 + --require-realpath-live + 目标 Nginx 已启用真实路径 canary 后，强制执行 realpath live canary smoke。 + --require-direct 目标 Pingora 已配置直连 HTTPS / HTTP redirect 后，强制执行 direct preflight 和 direct live smoke。 + --live-base-url live canary base URL；也可用 GENARRATIVE_PINGORA_CANARY_BASE_URL。 + --live-host live canary Host header；也可用 GENARRATIVE_PINGORA_CANARY_HOST。--require-live 时必须显式提供。 + --live-nginx-access-log + live canary 后 Nginx access log 对账路径，默认 /var/log/nginx/genarrative.access.log。 + --live-pingora-access-log + live canary 后 Pingora access log 对账路径，默认 /var/log/genarrative/pingora-gateway.access.log。 + --live-access-log-since-lines + live canary 后只读取日志尾部行数，默认 2000。 + --realpath-live-base-url + 真实路径 canary base URL；也可用 GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL。 + --realpath-live-host + 真实路径 canary Host header；也可用 GENARRATIVE_PINGORA_REALPATH_CANARY_HOST。--require-realpath-live 时必须显式提供。 + --realpath-live-nginx-access-log + 真实路径 canary 独立 Nginx access log，默认 /var/log/nginx/genarrative-pingora-realpath-canary.access.log。 + --realpath-live-pingora-access-log + 真实路径 canary 后 Pingora access log 对账路径，默认 /var/log/genarrative/pingora-gateway.access.log。 + --realpath-live-access-log-since-lines + 真实路径 canary 后只读取日志尾部行数，默认 2000。 + --direct-https-base-url direct HTTPS base URL；也可用 GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL。 + --direct-http-base-url direct HTTP redirect base URL；也可用 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL。 + --direct-host direct live Host header 和 TLS SNI；也可用 GENARRATIVE_PINGORA_DIRECT_HOST。--require-direct 时必须显式提供。 + --direct-redirect-host direct redirect Location host；也可用 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST。--require-direct 时必须显式提供，cutover runbook 中必须和 --direct-host 使用同一 hostname。 + --direct-probe-token direct live shadow probe token；也可用 GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN。 + --direct-pingora-access-log + direct live 后 Pingora access log 落盘校验路径；--require-direct 时必须显式提供。 + --direct-access-log-since-lines + direct live 后只读取 Pingora access log 尾部行数，默认 2000。 + --direct-preflight-env-file + direct entry 预检使用的 pingora-gateway.env；也可用 GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE。 + --direct-preflight-systemd direct entry 预检要求 systemctl cat 已包含低端口 capability drop-in。 + --direct-preflight-check-cert-readable + direct entry 预检要求当前用户可读 cert/key。 + --direct-preflight-check-service-env-file + direct entry 预检要求 service EnvironmentFile 包含本次 env 文件；--require-direct 时必须显式提供。 + --direct-preflight-check-service-user-cert-readable + direct entry 预检要求 systemd 服务用户可读 cert/key；--require-direct 时必须显式提供。 + --direct-preflight-check-service-binary-executable + direct entry 预检要求 service ExecStart 指向的 pingora-gateway 已存在且可执行；--require-direct 时必须显式提供。 + --direct-preflight-check-ports-free + direct entry 预检要求 TLS/HTTP redirect 端口当前可绑定；--require-direct 时必须显式提供。 + --direct-spacetime-database + direct live WSS subscribe 使用的数据库名；也可用 GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE。--require-direct 时必须显式提供。 + --direct-health-patrol-env-file + 目标机 health-patrol env 文件；--require-direct 时必须显式提供，并复核 gateway mode / public base URL / Host。 + --direct-require-wss-upgrade + 要求 direct live WSS subscribe 必须返回 101；--require-direct 时会自动启用。 + --direct-skip-wss 跳过 direct live WSS subscribe 握手；只允许单独 direct live 临时排障，--require-direct 会拒绝。 + --direct-insecure-tls direct live 允许自签证书；仅本机 smoke 使用。 + --cutover-release-root + 生成切换 runbook 时使用的 current release 根目录，默认 /opt/genarrative/current。 + --cutover-evidence-output-root + 生成切换 runbook 时传给证据包脚本的输出根目录，默认 /var/log/genarrative/pingora-cutover-evidence。 + --cutover-evidence-timeline-max-span-ms + 生成切换 runbook 时传给证据根目录总审计的标准五段时间线最大跨度，默认 ${DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS}ms。 + --cutover-run-id 生成切换 runbook 时写入所有证据 manifest 的本次切换批次 ID；不传则自动生成 ${DEFAULT_CUTOVER_RUN_ID_PREFIX}。 + --rollback-nginx-smoke-url + 生成切换 runbook 时传给回退脚本的 Nginx smoke URL，默认 http://127.0.0.1/healthz。 + --rollback-nginx-smoke-host + 生成切换 runbook 时传给回退脚本的 Host header；默认复用 --direct-host，显式传入时必须和 --direct-host 使用同一 hostname。 + --rollback-nginx-smoke-expect-body + 生成切换 runbook 时传给回退脚本的 Nginx smoke 响应体预期片段，默认 "ok":true。 + --rollback-health-patrol-public-base-url + 生成切换 runbook 时回退后恢复的 Nginx public base URL；应填切换前 health-patrol env 的原值。 + --rollback-health-patrol-public-host + 生成切换 runbook 时回退后恢复的 public Host；不传则要求清空 Host 覆盖。 + --rollback-pingora-shadow-probe-url + 生成切换 runbook 时传给回退脚本的可选 Pingora shadow 探针 URL；必须和 token 成对出现。 + --rollback-pingora-shadow-probe-token + 生成切换 runbook 时传给回退脚本的可选 Pingora shadow 探针 token；JSON 输出会隐藏 token 值。 + --dry-run-plan 只打印将执行的检查计划，不启动检查命令。 + --dry-run-cutover 只打印直连切换 runbook，不启动检查或修改系统。 + --release-runtime-only + 只执行 current release 包内运行时复核：current release 自审、live canary、access log 对账、direct preflight、health patrol env 和 direct live；不运行 npm/cargo 源码门禁。 + --keep-going 单项失败后继续执行后续检查。 + +默认模式用于本机提交前检查；正式切换窗口建议： + node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> + +Pingora 直连入口切换窗口追加： + node scripts/check-pingora-release-readiness.mjs --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free + +只生成直连切换 runbook： + node scripts/check-pingora-release-readiness.mjs --dry-run-cutover --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --cutover-evidence-output-root /var/log/genarrative/pingora-cutover-evidence --rollback-nginx-smoke-url http://127.0.0.1/healthz --rollback-health-patrol-public-base-url http://127.0.0.1 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name, fallback = false) { + const raw = process.env[name]; + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + const normalized = String(raw).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parsePositiveIntEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parsePositiveInt(raw, name); +} + +function parsePositiveInt(value, label) { + const text = String(value || '').trim(); + if (!/^[1-9]\d*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label} 超出 JavaScript 安全整数范围。`); + } + return parsed; +} + +function validateHostOption(value, flag) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${flag} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${flag} 不是合法的 host 或 host:port`); + } +} + +function validateHttpUrl(value, flag) { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${flag} 必须是 http(s) URL: ${value}`); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`${flag} 必须是 http(s) URL: ${value}`); + } +} + +function isPositiveIntegerString(value) { + return /^[1-9]\d*$/u.test(String(value)); +} + +function validateSafeAbsoluteFilePath(value, flag) { + if (!path.isAbsolute(value)) { + throw new Error(`${flag} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${flag} 不能是文件系统根目录。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function hostnameForIdentity(value, flag) { + validateHostOption(value, flag); + const parsed = new URL(`https://${String(value).trim()}`); + return parsed.hostname.replace(/\.$/u, '').toLowerCase(); +} + +function assertSameCutoverHostname(actual, expected, actualFlag, expectedFlag) { + const actualHostname = hostnameForIdentity(actual, actualFlag); + const expectedHostname = hostnameForIdentity(expected, expectedFlag); + if (actualHostname !== expectedHostname) { + throw new Error( + `${actualFlag} 必须与 ${expectedFlag} 使用同一正式 hostname；当前 ${actualHostname} != ${expectedHostname}`, + ); + } +} + +function validateSafeName(value, flag) { + if (!/^[0-9A-Za-z._-]+$/u.test(String(value || ''))) { + throw new Error( + `${flag} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function generateCutoverRunId(now = new Date()) { + const timestamp = now + .toISOString() + .replace(/[-:.]/gu, '') + .replace(/\d{3}Z$/u, 'Z'); + return `${DEFAULT_CUTOVER_RUN_ID_PREFIX}${timestamp}`; +} + +function printDryRunPlan(steps) { + console.log( + JSON.stringify( + steps.map((step) => { + const output = { + name: step.name, + command: step.command, + args: redactSecretArgs(step.args), + }; + if (step.phase) { + output.phase = step.phase; + } + if (step.when) { + output.when = step.when; + } + return output; + }), + null, + 2, + ), + ); +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : arg, + ); +} + +function buildCutoverPlan(config) { + const releaseRoot = config.cutoverReleaseRoot.replace(/\/+$/u, ''); + const enableScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-direct-enable.sh', + ); + const rollbackScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-direct-rollback.sh', + ); + const healthPatrolEnvSwitchScript = path.join( + releaseRoot, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ); + const statusSnapshotScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ); + const evidenceBundleScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ); + const commandEvidenceScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ); + const evidenceVerifyScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ); + const evidenceAuditScript = path.join( + releaseRoot, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ); + const currentReleaseAuditScript = path.join( + releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + const releaseReadinessScript = path.join( + releaseRoot, + 'scripts/check-pingora-release-readiness.mjs', + ); + const releasePreflightScript = path.join( + releaseRoot, + 'scripts/check-pingora-direct-preflight.mjs', + ); + const releaseLiveScript = path.join( + releaseRoot, + 'scripts/check-pingora-direct-live.mjs', + ); + const preEnablePreflightArgs = [ + releasePreflightScript, + '--env-file', + config.directPreflightEnvFile, + '--require-live-env', + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + '--check-cert-readable', + '--check-service-env-file', + '--check-service-user-cert-readable', + '--check-service-binary-executable', + '--check-ports-free', + ]; + const enableBaseArgs = [ + '--current-release-audit-script', + currentReleaseAuditScript, + '--current-release-root', + releaseRoot, + '--preflight-script', + releasePreflightScript, + '--preflight-env-file', + config.directPreflightEnvFile, + '--preflight-check-cert-readable', + '--preflight-check-service-env-file', + '--preflight-check-service-user-cert-readable', + '--preflight-check-service-binary-executable', + '--preflight-check-ports-free', + '--direct-live-script', + releaseLiveScript, + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--no-status', + ]; + if (config.directProbeToken) { + enableBaseArgs.push('--direct-probe-token', config.directProbeToken); + } + const rollbackBaseArgs = [ + '--reload-nginx', + '--nginx-smoke-url', + config.rollbackNginxSmokeUrl, + '--nginx-smoke-expect-body', + config.rollbackNginxSmokeExpectBody, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--health-patrol-expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ]; + if (config.rollbackNginxSmokeHost) { + rollbackBaseArgs.push('--nginx-smoke-host', config.rollbackNginxSmokeHost); + } + if (config.rollbackHealthPatrolPublicHost) { + rollbackBaseArgs.push( + '--health-patrol-expected-public-host', + config.rollbackHealthPatrolPublicHost, + ); + } else { + rollbackBaseArgs.push('--health-patrol-require-empty-public-host'); + } + if (config.rollbackPingoraShadowProbeUrl) { + rollbackBaseArgs.push( + '--pingora-shadow-probe-url', + config.rollbackPingoraShadowProbeUrl, + '--pingora-shadow-probe-token', + config.rollbackPingoraShadowProbeToken, + ); + } + + return [ + { + phase: 'preflight', + name: '切换 Host 一致性确认', + command: 'operator-action', + args: [ + 'confirm', + `cutover-run-id=${config.cutoverRunId}`, + `direct-host=${config.directHost}`, + `direct-redirect-host=${config.directRedirectHost}`, + `health-patrol-public-host=${config.directHost}`, + `rollback-nginx-smoke-host=${config.rollbackNginxSmokeHost}`, + `rollback-nginx-smoke-expect-body=${config.rollbackNginxSmokeExpectBody}`, + `rollback-health-patrol-public-base-url=${config.rollbackHealthPatrolPublicBaseUrl}`, + `rollback-health-patrol-public-host=${config.rollbackHealthPatrolPublicHost}`, + ], + when: 'runbook 生成时已拒绝 hostname 漂移；切换窗口继续人工确认直连、回退 smoke 和回退巡检入口都指向预期目标。', + }, + { + phase: 'preflight', + name: '切换前 current release 自包含自审', + command: 'node', + args: [ + '--', + currentReleaseAuditScript, + '--release-root', + releaseRoot, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--systemd-show', + ], + when: '正式启用前执行，只读确认 current release 已携带 Pingora 切换依赖；直连切换时要求 pingora-gateway 存在且 systemd ExecStart 指向 current release。', + }, + { + phase: 'preflight', + name: '切换前状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'pre-cutover', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'nginx', + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--expected-public-host', config.rollbackHealthPatrolPublicHost] + : ['--require-empty-public-host']), + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--fail-on-critical', + ], + when: '正式启用前执行，只读采集 current release、health patrol env 和 systemd 最终配置，并把 JSON / stdout / stderr / manifest 归档为 Nginx 接流基线证据。', + }, + { + phase: 'preflight', + name: '切换前证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '切换前状态快照证据包生成后，将上一步 stdout 中的 bundleDir 替换占位符执行，确认 manifest.files 中的 sizeBytes / sha256 与实际归档文件一致，且 manifest.summary.status 为 OK。', + }, + { + phase: 'preflight', + name: '切换前 current release 直连 env / 证书 / 端口预检', + command: 'node', + args: ['--', ...preEnablePreflightArgs], + }, + { + phase: 'preflight', + name: '切换前 release readiness 基础门禁', + command: 'node', + args: [ + '--', + releaseReadinessScript, + ...readinessArgs(config, { includeDirect: false }), + ], + }, + { + phase: 'enable-dry-run', + name: 'Pingora direct enable dry-run', + command: enableScript, + args: enableBaseArgs, + }, + { + phase: 'enable-apply', + name: 'Pingora direct enable apply', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'enable-apply', + '--command-name', + 'pingora-direct-enable-apply', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + enableScript, + '--require-arg', + '--apply', + '--', + enableScript, + '--apply', + ...enableBaseArgs, + ], + when: '执行真实 direct enable apply，并把 stdout / stderr / 退出码和脱敏命令记录归档到切换证据目录。', + }, + { + phase: 'enable-apply', + name: '启用命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'enable apply 命令证据生成后，将上一步 stdout 中的 bundleDir 替换占位符执行，确认命令 stdout / stderr / command-record 与 manifest 元数据一致，且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-enable', + name: '切换后 health patrol 切到 Pingora direct', + command: 'node', + args: [ + '--', + healthPatrolEnvSwitchScript, + '--apply', + '--env-file', + config.directHealthPatrolEnvFile, + '--gateway-mode', + 'pingora-direct', + '--public-base-url', + config.directHttpsBaseUrl, + '--public-host', + config.directHost, + ], + when: 'direct enable apply 成功后执行，避免巡检继续要求 nginx.service active。', + }, + { + phase: 'post-enable', + name: '启用后 health patrol env 直连复核', + command: 'node', + args: [ + '--', + path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + }, + { + phase: 'post-enable', + name: '启用后状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'post-enable', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--direct-live-script', + releaseLiveScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--run-direct-live', + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + ...(config.directProbeToken + ? ['--direct-probe-token', config.directProbeToken] + : []), + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--fail-on-critical', + ], + when: 'health patrol env 已切到 pingora-direct 且 env 复核通过后执行，归档 direct 接流状态、direct live 和 Pingora access log request_id 证据包。', + }, + { + phase: 'post-enable', + name: '启用后证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '启用后状态快照证据包生成后，将上一步 stdout 中的 bundleDir 替换占位符执行，确认 snapshot / direct live / stdout / stderr / 命令记录未在归档过程中漂移，且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-enable', + name: '启用后 release readiness 直连复核', + command: 'node', + args: [ + '--', + releaseReadinessScript, + ...readinessArgs(config, { includeDirect: true }), + ], + }, + { + phase: 'rollback-dry-run', + name: 'Pingora direct rollback dry-run', + command: rollbackScript, + args: rollbackBaseArgs, + when: '启用失败、direct live smoke 失败或业务验证失败时先执行。', + }, + { + phase: 'rollback-prep', + name: '回退前 health patrol 预置回 Nginx', + command: 'node', + args: [ + '--', + healthPatrolEnvSwitchScript, + '--apply', + '--env-file', + config.directHealthPatrolEnvFile, + '--gateway-mode', + 'nginx', + '--public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--public-host', config.rollbackHealthPatrolPublicHost] + : ['--clear-public-host']), + ], + when: 'rollback apply 前执行，确保回退脚本内置 health patrol env 复核能校验切换前 public base URL / Host。', + }, + { + phase: 'rollback-apply', + name: 'Pingora direct rollback apply', + command: 'node', + args: [ + '--', + commandEvidenceScript, + '--phase', + 'rollback-apply', + '--command-name', + 'pingora-direct-rollback-apply', + '--cutover-run-id', + config.cutoverRunId, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--expected-executable', + rollbackScript, + '--require-arg', + '--apply', + '--', + rollbackScript, + '--apply', + ...rollbackBaseArgs, + ], + when: 'health patrol env 预置回 Nginx 且 dry-run 确认后执行，用于回到 Nginx / shadow 入口，并把回退命令 stdout / stderr / 退出码归档。', + }, + { + phase: 'rollback-apply', + name: '回退命令证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: 'rollback apply 命令证据生成后，将上一步 stdout 中的 bundleDir 替换占位符执行，确认回退命令 stdout / stderr / command-record 与 manifest 元数据一致，且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-rollback', + name: '回退后 health patrol env Nginx 模式复核', + command: 'node', + args: [ + '--', + path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'nginx', + ...(config.rollbackHealthPatrolPublicBaseUrl + ? [ + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ] + : []), + ...(config.rollbackHealthPatrolPublicHost + ? [ + '--expected-public-host', + config.rollbackHealthPatrolPublicHost, + ] + : ['--require-empty-public-host']), + ], + when: 'health patrol env 已切回 nginx 后执行，阻断 pingora-direct 模式或回退目标漂移。', + }, + { + phase: 'post-rollback', + name: '回退后状态快照证据包', + command: 'node', + args: [ + '--', + evidenceBundleScript, + '--phase', + 'post-rollback', + '--cutover-run-id', + config.cutoverRunId, + '--release-root', + releaseRoot, + '--output-root', + config.cutoverEvidenceOutputRoot, + '--snapshot-script', + statusSnapshotScript, + '--health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--pingora-env-file', + config.directPreflightEnvFile, + '--expected-gateway-mode', + 'nginx', + '--expected-public-base-url', + config.rollbackHealthPatrolPublicBaseUrl, + ...(config.rollbackHealthPatrolPublicHost + ? ['--expected-public-host', config.rollbackHealthPatrolPublicHost] + : ['--require-empty-public-host']), + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + '--run-health-patrol', + '--fail-on-critical', + ], + when: 'rollback apply 和回退后 env 复核通过后执行，归档 Nginx 接流状态证据包。', + }, + { + phase: 'post-rollback', + name: '回退后证据 manifest 只读验真', + command: 'node', + args: [ + '--', + evidenceVerifyScript, + '--bundle-dir', + '', + '--require-summary-ok', + ], + when: '回退后状态快照证据包生成后，将上一步 stdout 中的 bundleDir 替换占位符执行，确认回退证据 manifest 和文件内容一致，且 manifest.summary.status 为 OK。', + }, + { + phase: 'post-rollback', + name: '切换证据根目录三阶段总审计', + command: 'node', + args: [ + '--', + evidenceAuditScript, + '--evidence-root', + config.cutoverEvidenceOutputRoot, + '--verify-script', + evidenceVerifyScript, + '--require-phase', + 'pre-cutover', + '--require-phase', + 'post-enable', + '--require-phase', + 'post-rollback', + '--require-phase-direct-live-access-log', + 'post-enable', + '--require-phase-direct-live-static-headers', + 'post-enable', + '--require-command', + 'enable-apply:pingora-direct-enable-apply', + '--require-command', + 'rollback-apply:pingora-direct-rollback-apply', + '--require-command-executable', + `enable-apply:pingora-direct-enable-apply:${enableScript}`, + '--require-command-executable', + `rollback-apply:pingora-direct-rollback-apply:${rollbackScript}`, + '--require-command-arg', + 'enable-apply:pingora-direct-enable-apply:--apply', + '--require-command-arg', + 'rollback-apply:pingora-direct-rollback-apply:--apply', + '--require-cutover-run-id', + config.cutoverRunId, + '--timeline-max-span-ms', + String(config.cutoverEvidenceTimelineMaxSpanMs), + ], + when: '三阶段证据包和 enable / rollback apply 命令证据都生成并分别验真后执行，自动找每个 phase 与命令的最新 bundleDir 并再次做只读 manifest 验真，且要求 post-enable manifest.summary.directLiveAccessLog 和 directLiveStaticHeaders 可直接复盘 request_id 对账、静态缓存、校验器、Range 和 304 证据。', + }, + ]; +} + +function readinessArgs(config, { includeDirect }) { + return [ + '--release-runtime-only', + ...(config.requireLive + ? [ + '--require-live', + '--live-base-url', + config.liveBaseUrl, + '--live-host', + config.liveHost, + ] + : []), + ...(config.requireRealpathLive + ? [ + '--require-realpath-live', + '--realpath-live-base-url', + config.realpathLiveBaseUrl, + '--realpath-live-host', + config.realpathLiveHost, + '--realpath-live-nginx-access-log', + config.realpathLiveAccessLogNginx, + '--realpath-live-pingora-access-log', + config.realpathLiveAccessLogPingora, + '--realpath-live-access-log-since-lines', + String(config.realpathLiveAccessLogSinceLines), + ] + : []), + ...(includeDirect + ? [ + '--require-direct', + '--direct-https-base-url', + config.directHttpsBaseUrl, + '--direct-http-base-url', + config.directHttpBaseUrl, + '--direct-host', + config.directHost, + '--direct-redirect-host', + config.directRedirectHost, + ...(config.directProbeToken + ? ['--direct-probe-token', config.directProbeToken] + : []), + '--direct-pingora-access-log', + config.directPingoraAccessLog, + '--direct-access-log-since-lines', + config.directAccessLogSinceLines, + '--direct-spacetime-database', + config.directSpacetimeDatabase, + '--direct-health-patrol-env-file', + config.directHealthPatrolEnvFile, + '--direct-preflight-env-file', + config.directPreflightEnvFile, + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + ] + : []), + ]; +} + +function buildSteps(config) { + if (config.releaseRuntimeOnly) { + return buildReleaseRuntimeSteps(config); + } + + const steps = [ + { + name: 'Rust 路由与保护单测', + command: 'cargo', + args: [ + 'test', + '-p', + 'pingora-gateway', + '--manifest-path', + 'server-rs/Cargo.toml', + ], + }, + { + name: 'Pingora mock 上游 smoke', + command: 'npm', + args: ['run', 'check:pingora-gateway-smoke'], + }, + { + name: 'Nginx/Pingora 路由矩阵 parity', + command: 'npm', + args: ['run', 'check:pingora-route-parity'], + }, + { + name: 'Nginx canary snippet 校验', + command: 'node', + args: [ + 'scripts/check-nginx-pingora-canary.mjs', + ...(config.requireNginx ? ['--require-nginx'] : []), + ], + }, + { + name: 'Docker Nginx handoff smoke', + command: 'node', + args: [ + 'scripts/check-pingora-canary-docker.mjs', + ...(config.requireDocker ? ['--require-docker'] : []), + ...(config.pullDocker ? ['--pull'] : []), + ], + }, + { + name: 'Pingora canary access log 对账烟测', + command: 'npm', + args: ['run', 'check:pingora-canary-access-log-parity'], + }, + { + name: 'Pingora canary live 参数护栏', + command: 'node', + args: ['scripts/check-pingora-canary-live-guard.mjs'], + }, + { + name: 'Pingora direct entry 静态预检', + command: 'npm', + args: ['run', 'check:pingora-direct-preflight'], + }, + { + name: 'Pingora direct live 参数护栏', + command: 'node', + args: ['scripts/check-pingora-direct-live-guard.mjs'], + }, + { + name: 'Pingora direct rollback dry-run', + command: 'npm', + args: ['run', 'check:pingora-direct-rollback'], + }, + { + name: 'Pingora direct enable dry-run', + command: 'npm', + args: ['run', 'check:pingora-direct-enable'], + }, + { + name: 'Pingora release readiness plan', + command: 'npm', + args: ['run', 'check:pingora-release-readiness-plan'], + }, + { + name: '生产运维护栏', + command: 'npm', + args: ['run', 'check:production-ops'], + }, + { + name: '生产健康巡检烟测', + command: 'npm', + args: ['run', 'check:production-health-patrol'], + }, + { + name: '生产健康巡检 env 复核烟测', + command: 'npm', + args: ['run', 'check:production-health-patrol-env'], + }, + { + name: 'Pingora health patrol env 切换烟测', + command: 'npm', + args: ['run', 'check:pingora-health-patrol-env-switch'], + }, + { + name: 'Pingora current release 自审烟测', + command: 'npm', + args: ['run', 'check:pingora-current-release-audit'], + }, + { + name: 'Pingora cutover 状态快照烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-status-snapshot'], + }, + { + name: 'Pingora cutover 证据包烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-bundle'], + }, + { + name: 'Pingora cutover 命令证据烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-command-evidence'], + }, + { + name: 'Pingora cutover 证据 manifest 验真烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-verify'], + }, + { + name: 'Pingora cutover 证据根目录审计烟测', + command: 'npm', + args: ['run', 'check:pingora-cutover-evidence-audit'], + }, + { + name: 'API release build 烟测', + command: 'npm', + args: ['run', 'check:production-api-release'], + }, + { + name: 'Pingora production release 真实构建烟测', + command: 'npm', + args: ['run', 'check:pingora-production-release-build'], + }, + { + name: 'API deploy release 烟测', + command: 'npm', + args: ['run', 'check:production-api-deploy'], + }, + ]; + + if (config.requireLive || config.liveBaseUrl) { + appendTargetLiveSteps(steps, config, scriptPath); + } + appendTargetRealpathLiveSteps(steps, config, scriptPath); + + if (config.requireDirect || config.directHttpsBaseUrl) { + steps.push({ + name: '目标 Pingora direct entry preflight', + command: 'node', + args: [ + '--', + 'scripts/check-pingora-direct-preflight.mjs', + ...(config.directPreflightEnvFile + ? ['--env-file', config.directPreflightEnvFile, '--require-live-env'] + : []), + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + ...(config.directPreflightCheckCertReadable + ? ['--check-cert-readable'] + : []), + ...(config.directPreflightCheckServiceEnvFile + ? ['--check-service-env-file'] + : []), + ...(config.directPreflightCheckServiceUserCertReadable + ? ['--check-service-user-cert-readable'] + : []), + ...(config.directPreflightCheckServiceBinaryExecutable + ? ['--check-service-binary-executable'] + : []), + ...(config.directPreflightCheckPortsFree ? ['--check-ports-free'] : []), + ], + }); + if (config.requireDirect || config.directHealthPatrolEnvFile) { + steps.push({ + name: '目标 health patrol env 直连模式复核', + command: 'node', + args: [ + '--', + 'scripts/check-production-health-patrol-env.mjs', + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + }); + } + steps.push({ + name: '目标 Pingora direct live smoke', + command: 'node', + args: [ + 'scripts/check-pingora-direct-live.mjs', + '--https-base-url', + config.directHttpsBaseUrl, + ...(config.directHttpBaseUrl + ? ['--http-base-url', config.directHttpBaseUrl] + : []), + ...(config.directHost ? ['--host', config.directHost] : []), + ...(config.directRedirectHost + ? ['--redirect-host', config.directRedirectHost] + : []), + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + ...(config.directPingoraAccessLog + ? [ + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + ] + : []), + ...(config.directSpacetimeDatabase + ? ['--spacetime-database', config.directSpacetimeDatabase] + : []), + ...(config.directRequireWssUpgrade ? ['--require-wss-upgrade'] : []), + ...(config.directSkipWss ? ['--skip-wss'] : []), + ...(config.directInsecureTls ? ['--insecure-tls'] : []), + ], + }); + } + + return steps; +} + +function buildReleaseRuntimeSteps(config) { + const steps = [ + { + name: 'current release 自包含自审', + command: 'node', + args: [ + '--', + releaseScriptPath('scripts/ops/pingora-current-release-audit.mjs'), + '--release-root', + releaseRoot, + ...(config.requireDirect ? ['--require-pingora-gateway'] : []), + ...(config.requireDirect ? ['--systemd-show'] : []), + ], + cwd: releaseRoot, + }, + ]; + + appendTargetLiveSteps(steps, config, releaseScriptPath); + appendTargetRealpathLiveSteps(steps, config, releaseScriptPath); + appendTargetDirectSteps(steps, config, releaseScriptPath); + + return steps; +} + +function appendTargetLiveSteps(steps, config, scriptPath) { + if (!config.requireLive && !config.liveBaseUrl) { + return; + } + steps.push({ + name: '目标 Nginx live canary smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-live.mjs'), + '--base-url', + config.liveBaseUrl, + ...(config.liveHost ? ['--host', config.liveHost] : []), + ], + cwd: releaseRoot, + }); + steps.push({ + name: '目标 Nginx live canary access log 对账', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-access-log-parity.mjs'), + '--nginx-log-file', + config.liveAccessLogNginx, + '--pingora-log-file', + config.liveAccessLogPingora, + '--since-lines', + String(config.liveAccessLogSinceLines), + '--path', + '/__genarrative_pingora_canary/healthz', + '--path', + '/__genarrative_pingora_canary/api/creation-entry/config', + ], + cwd: releaseRoot, + }); +} + +function appendTargetRealpathLiveSteps(steps, config, scriptPath) { + if (!config.requireRealpathLive && !config.realpathLiveBaseUrl) { + return; + } + steps.push({ + name: '目标 Nginx realpath canary smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-live.mjs'), + '--realpath', + '--base-url', + config.realpathLiveBaseUrl, + ...(config.realpathLiveHost ? ['--host', config.realpathLiveHost] : []), + ], + cwd: releaseRoot, + }); + steps.push({ + name: '目标 Nginx realpath canary access log 对账', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-canary-access-log-parity.mjs'), + '--realpath', + '--nginx-log-file', + config.realpathLiveAccessLogNginx, + '--pingora-log-file', + config.realpathLiveAccessLogPingora, + '--since-lines', + String(config.realpathLiveAccessLogSinceLines), + '--path', + '/__genarrative_pingora_realpath_canary/healthz', + '--path', + '/api/creation-entry/config', + '--path', + '/v1/identity', + '--path', + '/assets/app.js', + ], + cwd: releaseRoot, + }); +} + +function appendTargetDirectSteps(steps, config, scriptPath) { + if (!config.requireDirect && !config.directHttpsBaseUrl) { + return; + } + steps.push({ + name: '目标 Pingora direct entry preflight', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-direct-preflight.mjs'), + ...(config.directPreflightEnvFile + ? ['--env-file', config.directPreflightEnvFile, '--require-live-env'] + : []), + ...(config.directPreflightSystemd ? ['--systemd-cat'] : []), + ...(config.directPreflightCheckCertReadable + ? ['--check-cert-readable'] + : []), + ...(config.directPreflightCheckServiceEnvFile + ? ['--check-service-env-file'] + : []), + ...(config.directPreflightCheckServiceUserCertReadable + ? ['--check-service-user-cert-readable'] + : []), + ...(config.directPreflightCheckServiceBinaryExecutable + ? ['--check-service-binary-executable'] + : []), + ...(config.directPreflightCheckPortsFree ? ['--check-ports-free'] : []), + ], + cwd: releaseRoot, + }); + if (config.requireDirect || config.directHealthPatrolEnvFile) { + steps.push({ + name: '目标 health patrol env 直连模式复核', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-production-health-patrol-env.mjs'), + '--env-file', + config.directHealthPatrolEnvFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + config.directHttpsBaseUrl, + '--expected-public-host', + config.directHost, + ], + cwd: releaseRoot, + }); + } + steps.push({ + name: '目标 Pingora direct live smoke', + command: 'node', + args: [ + '--', + scriptPath('scripts/check-pingora-direct-live.mjs'), + '--https-base-url', + config.directHttpsBaseUrl, + ...(config.directHttpBaseUrl + ? ['--http-base-url', config.directHttpBaseUrl] + : []), + ...(config.directHost ? ['--host', config.directHost] : []), + ...(config.directRedirectHost + ? ['--redirect-host', config.directRedirectHost] + : []), + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + ...(config.directPingoraAccessLog + ? [ + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + ] + : []), + ...(config.directSpacetimeDatabase + ? ['--spacetime-database', config.directSpacetimeDatabase] + : []), + ...(config.directRequireWssUpgrade ? ['--require-wss-upgrade'] : []), + ...(config.directSkipWss ? ['--skip-wss'] : []), + ...(config.directInsecureTls ? ['--insecure-tls'] : []), + ], + cwd: releaseRoot, + }); +} + +function releaseScriptPath(relativePath) { + return path.join(releaseRoot, relativePath); +} + +function scriptPath(relativePath) { + return relativePath; +} + +function runStep(step) { + const args = commandArgsForSpawn(step); + console.log(`\n[pingora-release-readiness] ${step.name}`); + console.log( + `[pingora-release-readiness] ${step.command} ${redactSecretArgs(args).join(' ')}`, + ); + + return new Promise((resolve) => { + const child = spawn(step.command, args, { + cwd: step.cwd || repoRoot, + env: smokeEnv(), + shell: false, + stdio: 'inherit', + }); + + child.on('error', (error) => { + failures.push(`${step.name}: ${step.command} 启动失败：${error.message}`); + resolve(false); + }); + child.on('exit', (status, signal) => { + if (signal) { + failures.push(`${step.name}: 被信号终止：${signal}`); + resolve(false); + return; + } + if ((status ?? 0) !== 0) { + failures.push(`${step.name}: 退出码 ${status}`); + resolve(false); + return; + } + resolve(true); + }); + }); +} + +function commandArgsForSpawn(step) { + if (step.command === 'node' && step.args[0] !== '--') { + return ['--', ...step.args]; + } + return step.args; +} + +function smokeEnv() { + return { + ...process.env, + PATH: `${path.join(os.homedir(), '.local', 'bin')}:${process.env.PATH || ''}`, + }; +} diff --git a/scripts/check-pingora-route-parity.mjs b/scripts/check-pingora-route-parity.mjs new file mode 100644 index 000000000..78372e43b --- /dev/null +++ b/scripts/check-pingora-route-parity.mjs @@ -0,0 +1,242 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs'; + +const MATRIX_PATH = 'deploy/pingora/nginx-route-parity.matrix.json'; +const PRODUCTION_NGINX_PATH = 'deploy/nginx/genarrative.conf'; +const DEVELOPMENT_NGINX_PATH = 'deploy/nginx/genarrative-dev-http.conf'; +const PINGORA_DOC_PATH = + 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md'; +const PINGORA_GATEWAY_SOURCE = 'server-rs/crates/pingora-gateway/src/main.rs'; + +const VALID_KINDS = new Set([ + 'proxy', + 'static', + 'redirect_permanent', + 'shadow_probe', + 'not_found', +]); +const VALID_PROXY_TARGETS = new Set(['api', 'spacetime']); +const VALID_STATIC_ROOTS = new Set(['web', 'acme']); +const VALID_STATIC_MODES = new Set(['exact', 'spa_fallback']); +const VALID_PROTECTION_CLASSES = new Set([ + 'admin_api', + 'gallery_list', + 'gallery_detail', + 'api', + 'spacetime', +]); +const REQUIRED_ROUTE_IDS = [ + 'acme_challenge', + 'shadow_probe', + 'admin_redirect', + 'admin_api_proxy', + 'admin_assets', + 'admin_spa_fallback', + 'web_assets', + 'puzzle_gallery_list', + 'custom_world_gallery_list', + 'puzzle_gallery_detail', + 'custom_world_gallery_detail', + 'generic_api_proxy', + 'spacetime_subscribe', + 'spacetime_identity', + 'v1_forbidden', + 'healthz_forbidden', + 'readyz_forbidden', + 'generated_assets_forbidden', + 'web_spa_fallback', +]; + +const files = { + production: readFileSync(PRODUCTION_NGINX_PATH, 'utf8'), + development: readFileSync(DEVELOPMENT_NGINX_PATH, 'utf8'), +}; +const docs = readFileSync(PINGORA_DOC_PATH, 'utf8'); +const pingoraGatewaySource = readFileSync(PINGORA_GATEWAY_SOURCE, 'utf8'); +const matrix = JSON.parse(readFileSync(MATRIX_PATH, 'utf8')); +const failures = []; + +function fail(message) { + failures.push(message); +} + +function hasOwn(object, key) { + return Object.prototype.hasOwnProperty.call(object, key); +} + +function requireString(value, context) { + if (typeof value !== 'string' || value.trim() === '') { + fail(`${context} 必须是非空字符串。`); + return false; + } + return true; +} + +function validateExpectation(route) { + const context = `${MATRIX_PATH} route ${route.id}`; + const expect = route.expect; + if (!expect || typeof expect !== 'object' || Array.isArray(expect)) { + fail(`${context} 缺少 expect 对象。`); + return; + } + + if (!VALID_KINDS.has(expect.kind)) { + fail(`${context} expect.kind 不支持: ${expect.kind}`); + return; + } + + if (expect.kind === 'proxy') { + if (!VALID_PROXY_TARGETS.has(expect.target)) { + fail(`${context} proxy target 不支持: ${expect.target}`); + } + if ( + hasOwn(expect, 'bodyLimit') && + expect.bodyLimit !== null && + expect.bodyLimit !== 'default' && + (!Number.isInteger(expect.bodyLimit) || expect.bodyLimit < 1) + ) { + fail(`${context} bodyLimit 必须是 null、default 或正整数。`); + } + if (!VALID_PROTECTION_CLASSES.has(expect.protectionClass)) { + fail( + `${context} proxy protectionClass 不支持: ${expect.protectionClass}`, + ); + } + return; + } + + if (hasOwn(expect, 'protectionClass')) { + fail(`${context} 非 proxy 路由不能配置 protectionClass。`); + } + + if (expect.kind === 'static') { + if (!VALID_STATIC_ROOTS.has(expect.root)) { + fail(`${context} static root 不支持: ${expect.root}`); + } + if (!VALID_STATIC_MODES.has(expect.mode)) { + fail(`${context} static mode 不支持: ${expect.mode}`); + } + } + + if ( + expect.kind === 'redirect_permanent' && + !requireString(expect.location, `${context} redirect location`) + ) { + fail(`${context} redirect_permanent 必须配置 location。`); + } +} + +function validateNginxFragments(route) { + for (const environment of ['production', 'development']) { + const fragments = route.nginx?.[environment]; + if (fragments === undefined) { + if (environment === 'production' && route.id !== 'shadow_probe') { + fail(`${MATRIX_PATH} route ${route.id} 缺少 production Nginx 片段。`); + } + continue; + } + if (!Array.isArray(fragments) || fragments.length === 0) { + fail( + `${MATRIX_PATH} route ${route.id} 的 ${environment} Nginx 片段不能为空。`, + ); + continue; + } + + for (const fragment of fragments) { + if (!requireString(fragment, `${route.id} ${environment} Nginx 片段`)) { + continue; + } + if (!files[environment].includes(fragment)) { + fail( + `${environment} Nginx 模板缺少 route ${route.id} 片段: ${fragment}`, + ); + } + } + } +} + +function validateDocFragments(route) { + if (!Array.isArray(route.docs) || route.docs.length === 0) { + fail(`${MATRIX_PATH} route ${route.id} 缺少 docs 片段。`); + return; + } + + for (const fragment of route.docs) { + if (!requireString(fragment, `${route.id} docs 片段`)) { + continue; + } + if (!docs.includes(fragment)) { + fail(`Pingora 试点文档缺少 route ${route.id} 片段: ${fragment}`); + } + } +} + +function validateMatrixShape() { + if (matrix.version !== 1) { + fail(`${MATRIX_PATH} version 必须为 1。`); + } + if (!Array.isArray(matrix.routes) || matrix.routes.length === 0) { + fail(`${MATRIX_PATH} routes 不能为空。`); + return; + } + + const ids = new Set(); + const samplePaths = new Set(); + for (const route of matrix.routes) { + if (!requireString(route.id, `${MATRIX_PATH} route.id`)) { + continue; + } + if (ids.has(route.id)) { + fail(`${MATRIX_PATH} route id 重复: ${route.id}`); + } + ids.add(route.id); + + if (!requireString(route.samplePath, `${route.id} samplePath`)) { + continue; + } + if (!route.samplePath.startsWith('/')) { + fail(`${MATRIX_PATH} route ${route.id} samplePath 必须以 / 开头。`); + } + if (samplePaths.has(route.samplePath)) { + fail(`${MATRIX_PATH} samplePath 重复: ${route.samplePath}`); + } + samplePaths.add(route.samplePath); + + validateExpectation(route); + validateNginxFragments(route); + validateDocFragments(route); + } + + for (const routeId of REQUIRED_ROUTE_IDS) { + if (!ids.has(routeId)) { + fail(`${MATRIX_PATH} 缺少必需 route id: ${routeId}`); + } + } +} + +function validateRustTestUsesMatrix() { + for (const fragment of [ + 'include_str!("../../../../deploy/pingora/nginx-route-parity.matrix.json")', + 'serde_json::from_str(ROUTE_PARITY_MATRIX_JSON)', + 'protection_class_for_route(&route, &case.sample_path)', + 'fn matches_nginx_route_parity_matrix()', + ]) { + if (!pingoraGatewaySource.includes(fragment)) { + fail(`Pingora Rust 路由 parity 测试缺少矩阵接入片段: ${fragment}`); + } + } +} + +validateMatrixShape(); +validateRustTestUsesMatrix(); + +if (failures.length > 0) { + console.error('[check:pingora-route-parity] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log(`[check:pingora-route-parity] OK (${matrix.routes.length} routes)`); diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs new file mode 100644 index 000000000..9e7e76044 --- /dev/null +++ b/scripts/check-production-api-deploy.mjs @@ -0,0 +1,1302 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-production-api-deploy-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-api-deploy] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-api-deploy] OK'); + +function main() { + assertDeployCopiesPingoraDirectReleaseDependencies(); + assertDeployRestartsActivePingoraWhenArtifactIncluded(); + assertDeploySkipsInactivePingoraWhenArtifactIncluded(); + assertDeployRejectsPingoraArtifactMissingManifestEntry(); + assertMissingReleaseManifestFails(); + assertReleaseManifestMissingApiArtifactFails(); + assertDeployRejectsDotVersion(); + assertDeployRejectsDotDotVersion(); + assertDeployRejectsVersionStartingWithDot(); + assertDeployRejectsExistingReleaseDirectory(); + assertDeployRejectsDirectoryCurrentLink(); + assertDeployRejectsRelativeReleaseRoot(); + assertDeployRejectsRelativeCurrentLink(); + assertDeployRejectsRelativeApiEnvFile(); + assertDeployCleansStagingReleaseOnFailure(); + assertDeployRejectsFinalReleaseRaceAndCleansStaging(); + assertMissingBackupScriptFails(); + assertMissingHealthPatrolScriptFails(); + assertMissingPingoraCurrentReleaseAuditFails(); + assertMissingPingoraCutoverStatusSnapshotFails(); + assertMissingPingoraCutoverEvidenceBundleFails(); + assertMissingPingoraCutoverCommandEvidenceFails(); + assertMissingPingoraCutoverEvidenceVerifyFails(); + assertMissingPingoraCutoverEvidenceAuditFails(); + assertMissingHealthPatrolEnvCheckFails(); + assertMissingPingoraReleaseReadinessFails(); + assertMissingPingoraHealthPatrolEnvSwitchFails(); + assertMissingEnvExamplesFails(); + assertMissingPingoraDirectCheckFails(); + assertMissingPingoraCanaryLiveFails(); + assertMissingPingoraCanaryAccessLogParityFails(); +} + +function assertDeployCopiesPingoraDirectReleaseDependencies() { + const fixture = prepareFixture('with-direct-checks'); + const result = runDeploy(fixture); + + assertStatus(result, 0, '完整 fixture 应部署成功。'); + if (result.status !== 0) { + return; + } + assertIncludes( + result.stdout, + '[production-api-deploy] 完成:', + '部署成功时必须输出完成信息。', + ); + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + const currentTarget = readlinkSync(fixture.currentLink); + if (currentTarget !== releaseDir) { + failures.push( + `current link 应指向新 release。实际 ${currentTarget}，预期 ${releaseDir}`, + ); + } + + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'current release 必须包含 Pingora 直连启用脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'current release 必须包含 Pingora 直连回退脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'current release 必须包含 Pingora health patrol env 切换脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'current release 必须包含 Pingora 直连预检脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), + 'current release 必须包含健康巡检 env 复核脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), + 'current release 必须包含 Pingora release readiness 聚合门禁脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), + 'current release 必须包含 Pingora current release 自审脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), + 'current release 必须包含 Pingora 直连切换状态快照脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), + 'current release 必须包含 Pingora 直连切换证据包脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), + 'current release 必须包含 Pingora 直连切换命令证据脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), + 'current release 必须包含 Pingora 直连切换证据验真脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), + 'current release 必须包含 Pingora 直连切换证据根目录审计脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + 'current release 必须包含 Pingora 直连 live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), + 'current release 必须包含 Pingora canary live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), + 'current release 必须包含 Pingora canary access log 对账脚本。', + ); + assertFileExists( + path.join(releaseDir, 'release-manifest.api-server.json'), + 'current release 必须包含 API release manifest 副本。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/systemd/genarrative-pingora-gateway.service'), + 'current release 必须包含 systemd 主 service 模板。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + 'current release 必须包含 Pingora 直连 drop-in 模板。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'current release 必须包含 Pingora env 示例。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + 'current release 必须包含 Pingora 真实路径 canary Nginx snippet。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/health-patrol.env.example'), + 'current release 必须包含健康巡检 env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), + 'current release 必须包含 Pingora direct live env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), + 'current release 必须包含 Pingora canary live env 示例。', + ); + assertPingoraEnvProductionDefaults( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'current release 内 Pingora env 示例', + ); + + const copiedPreflight = readFileSync( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'utf8', + ); + assertIncludes( + copiedPreflight, + 'deploy/pingora/pingora-gateway.env.example', + '复制后的 preflight 脚本必须仍能从 release root 推导 Pingora 配置。', + ); + + const apiEnv = readFileSync(fixture.apiEnvFile, 'utf8'); + assertIncludes( + apiEnv, + 'GENARRATIVE_SPACETIME_DATABASE=genarrative-prod', + '部署脚本必须写入 SpacetimeDB database。', + ); + assertIncludes( + apiEnv, + 'GENARRATIVE_SPACETIME_SERVER_URL=http://127.0.0.1:3101', + '部署脚本必须写入 SpacetimeDB server URL。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl restart genarrative-api.service', + '部署脚本必须重启 API service。', + ); + assertIncludes( + commandsLog, + 'curl -fsS http://127.0.0.1:18082/readyz', + '部署脚本必须执行 readiness curl。', + ); + + if (existsSync(fixture.maintenanceFile)) { + failures.push('部署成功后应退出维护模式。'); + } +} + +function assertDeployRestartsActivePingoraWhenArtifactIncluded() { + const fixture = prepareFixture('with-active-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture); + + assertStatus( + result, + 0, + '包含 Pingora 且 shadow service active 时应部署成功。', + ); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '包含 Pingora 的发布包部署后 current release 必须包含网关二进制。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl is-active --quiet genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须先检查 shadow service 是否 active。', + ); + assertIncludes( + commandsLog, + 'systemctl try-restart genarrative-pingora-gateway.service', + 'Pingora shadow service 已 active 时必须随 current release 切换 try-restart。', + ); + assertIncludes( + result.stdout, + '发布包包含 Pingora，重启已运行的影子服务', + 'Pingora shadow service 自动重启时必须输出明确提示。', + ); +} + +function assertDeploySkipsInactivePingoraWhenArtifactIncluded() { + const fixture = prepareFixture('with-inactive-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture, { pingoraActive: false }); + + assertStatus( + result, + 0, + '包含 Pingora 但 shadow service inactive 时应部署成功。', + ); + if (result.status !== 0) { + return; + } + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl is-active --quiet genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须检查 shadow service 是否 active。', + ); + if ( + commandsLog.includes( + 'systemctl try-restart genarrative-pingora-gateway.service', + ) + ) { + failures.push('Pingora shadow service inactive 时不能主动 try-restart。'); + } + assertIncludes( + result.stdout, + 'Pingora 影子服务未处于 active，跳过自动重启', + 'Pingora shadow service inactive 时必须明确说明不会主动拉起。', + ); +} + +function assertDeployRejectsPingoraArtifactMissingManifestEntry() { + const fixture = prepareFixture('pingora-artifact-missing-manifest-entry'); + addPingoraGatewayArtifact(fixture, { registerInManifest: false }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布包包含 Pingora 但 manifest 未登记时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'release-manifest.json 缺少 pingora-gateway artifact', + 'manifest 未登记 Pingora 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('manifest 未登记 Pingora 导致部署失败时必须保持维护模式。'); + } +} + +function assertMissingReleaseManifestFails() { + const fixture = prepareFixture('missing-release-manifest'); + rmSync(path.join(fixture.sourceDir, 'release-manifest.json')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 release-manifest.json 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 release-manifest.json', + '缺少 release-manifest.json 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('缺少 release-manifest.json 导致部署失败时必须保持维护模式。'); + } +} + +function assertReleaseManifestMissingApiArtifactFails() { + const fixture = prepareFixture('release-manifest-missing-api-artifact'); + writeFileSync( + path.join(fixture.sourceDir, 'release-manifest.json'), + `${JSON.stringify({ version: fixture.version, artifacts: [] }, null, 2)}\n`, + 'utf8', + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('release manifest 缺少 api-server artifact 时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'release-manifest.json 缺少 api-server artifact', + 'manifest 缺少 api-server artifact 时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('manifest 缺少 api-server artifact 导致部署失败时必须保持维护模式。'); + } +} + +function assertDeployRejectsDotVersion() { + const fixture = prepareFixture('dot-version'); + const result = runDeploy(fixture, { version: '.' }); + + if (result.status === 0) { + failures.push('--version=. 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version=. 时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version=. 且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsDotDotVersion() { + const fixture = prepareFixture('dot-dot-version'); + const result = runDeploy(fixture, { version: '..' }); + + if (result.status === 0) { + failures.push('--version=.. 时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version=.. 时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version=.. 且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsVersionStartingWithDot() { + const fixture = prepareFixture('dot-prefix-version'); + const result = runDeploy(fixture, { version: '.hidden-release' }); + + if (result.status === 0) { + failures.push('--version 以点开头时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--version 必须以数字或字母开头', + '--version 以点开头时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--version 以点开头且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsExistingReleaseDirectory() { + const fixture = prepareFixture('existing-release-directory'); + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + mkdirSync(releaseDir, { recursive: true }); + writeFileSync(path.join(releaseDir, 'old-file'), 'old\n', 'utf8'); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('目标 release 目录已存在时部署必须失败。'); + } + assertIncludes( + result.stderr, + '目标 release 已存在，拒绝覆盖或合并旧文件', + '目标 release 目录已存在时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('目标 release 目录已存在且未进入部署阶段时不应开启维护模式。'); + } + assertFileExists( + path.join(releaseDir, 'old-file'), + '拒绝覆盖既有 release 时必须保留原目录。', + ); +} + +function assertDeployRejectsDirectoryCurrentLink() { + const fixture = prepareFixture('directory-current-link'); + mkdirSync(fixture.currentLink, { recursive: true }); + writeFileSync(path.join(fixture.currentLink, 'old-file'), 'old\n', 'utf8'); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('current 路径已存在但不是符号链接时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'current 链接路径已存在但不是符号链接，拒绝覆盖', + 'current 路径不是符号链接时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('current 路径不是符号链接且未进入部署阶段时不应开启维护模式。'); + } + assertFileExists( + path.join(fixture.currentLink, 'old-file'), + '拒绝覆盖目录型 current 时必须保留原目录内容。', + ); +} + +function assertDeployRejectsRelativeReleaseRoot() { + const fixture = prepareFixture('relative-release-root'); + const result = runDeploy(fixture, { releaseRoot: 'relative/releases' }); + + if (result.status === 0) { + failures.push('--release-root 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--release-root 必须使用绝对路径', + '--release-root 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--release-root 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsRelativeCurrentLink() { + const fixture = prepareFixture('relative-current-link'); + const result = runDeploy(fixture, { currentLink: 'relative/current' }); + + if (result.status === 0) { + failures.push('--current-link 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--current-link 必须使用绝对路径', + '--current-link 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--current-link 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployRejectsRelativeApiEnvFile() { + const fixture = prepareFixture('relative-api-env-file'); + const result = runDeploy(fixture, { apiEnvFile: 'relative/api-server.env' }); + + if (result.status === 0) { + failures.push('--api-env-file 使用相对路径时部署必须失败。'); + } + assertIncludes( + result.stderr, + '--api-env-file 必须使用绝对路径', + '--api-env-file 使用相对路径时必须给出明确错误。', + ); + if (existsSync(fixture.maintenanceFile)) { + failures.push('--api-env-file 相对路径且未进入部署阶段时不应开启维护模式。'); + } +} + +function assertDeployCleansStagingReleaseOnFailure() { + const fixture = prepareFixture('cleans-staging-on-failure'); + rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('缺少数据库备份脚本时部署必须失败。'); + } + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + if (existsSync(releaseDir)) { + failures.push('部署失败时不应留下正式 release 目录。'); + } + const entries = existsSync(fixture.releaseRoot) + ? readDirNames(fixture.releaseRoot) + : []; + const stagingEntries = entries.filter((entry) => + entry.includes(`${fixture.version}.staging`), + ); + if (stagingEntries.length > 0) { + failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { + const fixture = prepareFixture('final-release-race'); + const result = runDeploy(fixture, { createReleaseDuringCopy: true }); + + if (result.status === 0) { + failures.push('最终提升前目标 release 目录被外部创建时部署必须失败。'); + } + assertIncludes( + result.stderr, + '目标 release 在发布过程中出现，拒绝合并 staging', + '最终提升前目标 release 目录被外部创建时必须给出明确错误。', + ); + + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + assertFileExists( + path.join(releaseDir, 'raced-file'), + '部署脚本拒绝竞态目标 release 时必须保留外部创建的目录。', + ); + if (existsSync(path.join(releaseDir, 'api-server'))) { + failures.push('目标 release 竞态出现后不应把 staging 内容合并进去。'); + } + const entries = existsSync(fixture.releaseRoot) + ? readDirNames(fixture.releaseRoot) + : []; + const stagingEntries = entries.filter((entry) => + entry.includes(`${fixture.version}.staging`), + ); + if (stagingEntries.length > 0) { + failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('目标 release 竞态失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraDirectCheckFails() { + const fixture = prepareFixture('missing-direct-live'); + rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 direct live smoke 脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连 live smoke 脚本', + '缺少 direct live smoke 脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCanaryLiveFails() { + const fixture = prepareFixture('missing-canary-live'); + rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-canary-live.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 canary live smoke 脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora canary live smoke 脚本', + '缺少 canary live smoke 脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCanaryAccessLogParityFails() { + const fixture = prepareFixture('missing-canary-log-parity'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/check-pingora-canary-access-log-parity.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 canary access log 对账脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora canary access log 对账脚本', + '缺少 canary access log 对账脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingBackupScriptFails() { + const fixture = prepareFixture('missing-backup-script'); + rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少数据库备份脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少数据库备份脚本', + '缺少数据库备份脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingHealthPatrolScriptFails() { + const fixture = prepareFixture('missing-health-patrol-script'); + rmSync( + path.join(fixture.sourceDir, 'scripts/ops/production-health-patrol.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少生产健康巡检脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少生产健康巡检脚本', + '缺少生产健康巡检脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCurrentReleaseAuditFails() { + const fixture = prepareFixture('missing-pingora-current-release-audit'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-current-release-audit.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora current release 自审脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora current release 自审脚本', + '缺少 Pingora current release 自审脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverStatusSnapshotFails() { + const fixture = prepareFixture('missing-pingora-cutover-status-snapshot'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换状态快照脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换状态快照脚本', + '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceBundleFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-bundle'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换证据包脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据包脚本', + '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverCommandEvidenceFails() { + const fixture = prepareFixture('missing-pingora-cutover-command-evidence'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换命令证据脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换命令证据脚本', + '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceVerifyFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-verify'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora 直连切换证据验真脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据验真脚本', + '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraCutoverEvidenceAuditFails() { + const fixture = prepareFixture('missing-pingora-cutover-evidence-audit'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push( + '发布产物缺少 Pingora 直连切换证据根目录审计脚本时部署必须失败。', + ); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora 直连切换证据根目录审计脚本', + '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingHealthPatrolEnvCheckFails() { + const fixture = prepareFixture('missing-health-patrol-env-check'); + rmSync( + path.join(fixture.sourceDir, 'scripts/check-production-health-patrol-env.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少生产健康巡检 env 复核脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少生产健康巡检 env 复核脚本', + '缺少生产健康巡检 env 复核脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraReleaseReadinessFails() { + const fixture = prepareFixture('missing-pingora-release-readiness'); + rmSync( + path.join(fixture.sourceDir, 'scripts/check-pingora-release-readiness.mjs'), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push( + '发布产物缺少 Pingora release readiness 聚合门禁脚本时部署必须失败。', + ); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora release readiness 聚合门禁脚本', + '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraHealthPatrolEnvSwitchFails() { + const fixture = prepareFixture('missing-pingora-health-patrol-env-switch'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora health patrol env 切换脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora health patrol env 切换脚本', + '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingEnvExamplesFails() { + const fixture = prepareFixture('missing-env-examples'); + rmSync(path.join(fixture.sourceDir, 'deploy/env'), { + recursive: true, + force: true, + }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少环境变量示例目录时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少环境变量示例目录', + '缺少环境变量示例目录时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function prepareFixture(name) { + const root = path.join(tmpRoot, name); + const sourceDir = path.join(root, 'source'); + const releaseRoot = path.join(root, 'releases'); + const currentLink = path.join(root, 'current'); + const apiEnvFile = path.join(root, 'etc', 'api-server.env'); + const maintenanceFile = path.join(root, 'maintenance', 'enabled'); + const fakeBin = path.join(root, 'bin'); + const commandsLog = path.join(root, 'commands.log'); + const version = `20260614-${name}`; + + mkdirSync(sourceDir, { recursive: true }); + mkdirSync(fakeBin, { recursive: true }); + mkdirSync(path.dirname(apiEnvFile), { recursive: true }); + mkdirSync(path.join(sourceDir, 'scripts/deploy'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'scripts/ops'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/systemd'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/pingora'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/env'), { recursive: true }); + mkdirSync(path.join(sourceDir, 'deploy/nginx/snippets'), { recursive: true }); + + writeFileSync( + path.join(sourceDir, 'api-server'), + '#!/usr/bin/env bash\n', + 'utf8', + ); + writeFileSync( + apiEnvFile, + [ + 'GENARRATIVE_TRACKING_OUTBOX_ENABLED=false', + 'GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS=5000', + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(path.join(sourceDir, 'api-server')); + writeSha256(sourceDir, 'api-server'); + writeFileSync( + path.join(sourceDir, 'release-manifest.json'), + `${JSON.stringify( + { + version, + artifacts: [ + { + component: 'api-server', + path: 'api-server', + checksum_path: 'api-server.sha256', + }, + ], + }, + null, + 2, + )}\n`, + 'utf8', + ); + writeFileSync( + path.join(sourceDir, 'scripts/database-backup-to-oss.mjs'), + 'console.log("backup");\n', + 'utf8', + ); + writeFileSync( + path.join(sourceDir, 'scripts/ops/production-health-patrol.mjs'), + 'console.log("patrol");\n', + 'utf8', + ); + copyFile( + 'scripts/ops/pingora-current-release-audit.mjs', + path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-command-evidence.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), + ); + copyFile( + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), + ); + copyFile( + 'scripts/deploy/production-api-deploy.sh', + path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), + ); + copyFile( + 'scripts/deploy/maintenance-on.sh', + path.join(sourceDir, 'scripts/deploy/maintenance-on.sh'), + ); + copyFile( + 'scripts/deploy/maintenance-off.sh', + path.join(sourceDir, 'scripts/deploy/maintenance-off.sh'), + ); + copyFile( + 'scripts/deploy/pingora-direct-enable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-direct-enable.sh'), + ); + copyFile( + 'scripts/deploy/pingora-direct-rollback.sh', + path.join(sourceDir, 'scripts/deploy/pingora-direct-rollback.sh'), + ); + copyFile( + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), + ); + copyFile( + 'scripts/check-production-health-patrol-env.mjs', + path.join(sourceDir, 'scripts/check-production-health-patrol-env.mjs'), + ); + copyFile( + 'scripts/check-pingora-release-readiness.mjs', + path.join(sourceDir, 'scripts/check-pingora-release-readiness.mjs'), + ); + copyFile( + 'scripts/check-pingora-direct-preflight.mjs', + path.join(sourceDir, 'scripts/check-pingora-direct-preflight.mjs'), + ); + copyFile( + 'scripts/check-pingora-direct-live.mjs', + path.join(sourceDir, 'scripts/check-pingora-direct-live.mjs'), + ); + copyFile( + 'scripts/check-pingora-canary-live.mjs', + path.join(sourceDir, 'scripts/check-pingora-canary-live.mjs'), + ); + copyFile( + 'scripts/check-pingora-canary-access-log-parity.mjs', + path.join(sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), + ); + copyFile( + 'deploy/systemd/genarrative-pingora-gateway.service', + path.join(sourceDir, 'deploy/systemd/genarrative-pingora-gateway.service'), + ); + copyFile( + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + path.join( + sourceDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + ); + copyFile( + 'deploy/pingora/pingora-gateway.env.example', + path.join(sourceDir, 'deploy/pingora/pingora-gateway.env.example'), + ); + copyFile( + 'deploy/env/health-patrol.env.example', + path.join(sourceDir, 'deploy/env/health-patrol.env.example'), + ); + copyFile( + 'deploy/env/pingora-direct-live.env.example', + path.join(sourceDir, 'deploy/env/pingora-direct-live.env.example'), + ); + copyFile( + 'deploy/env/pingora-canary-live.env.example', + path.join(sourceDir, 'deploy/env/pingora-canary-live.env.example'), + ); + copyFile( + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + path.join( + sourceDir, + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + ), + ); + copyFile( + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + path.join( + sourceDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + ); + chmodExecutable( + path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), + ); + chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-on.sh')); + chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-off.sh')); + + writeFileSync( + path.join(fakeBin, 'systemctl'), + [ + '#!/usr/bin/env bash', + `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'if [[ "$1 $2 $3" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then', + ' exit 3', + 'fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'curl'), + [ + '#!/usr/bin/env bash', + `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'cp'), + [ + '#!/usr/bin/env bash', + 'set -euo pipefail', + '/usr/bin/cp "$@"', + 'if [[ "${FAKE_CREATE_RELEASE_DURING_COPY:-false}" == "true" ]]; then', + ' marker="${FAKE_RELEASE_ROOT}/.${FAKE_RELEASE_VERSION}.race-created"', + ' if [[ ! -e "${marker}" ]]; then', + ' mkdir -p "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}"', + ' printf "race\\n" > "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}/raced-file"', + ' printf "created\\n" > "${marker}"', + ' fi', + 'fi', + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(path.join(fakeBin, 'systemctl')); + chmodExecutable(path.join(fakeBin, 'curl')); + chmodExecutable(path.join(fakeBin, 'cp')); + + return { + root, + sourceDir, + releaseRoot, + currentLink, + apiEnvFile, + maintenanceFile, + fakeBin, + commandsLog, + version, + }; +} + +function addPingoraGatewayArtifact(fixture, options = {}) { + writeFileSync( + path.join(fixture.sourceDir, 'pingora-gateway'), + '#!/usr/bin/env bash\n', + 'utf8', + ); + chmodExecutable(path.join(fixture.sourceDir, 'pingora-gateway')); + writeSha256(fixture.sourceDir, 'pingora-gateway'); + if (options.registerInManifest === false) { + return; + } + const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); + manifest.artifacts = Array.isArray(manifest.artifacts) + ? manifest.artifacts + : []; + manifest.artifacts.push({ + component: 'pingora-gateway', + path: 'pingora-gateway', + checksum_path: 'pingora-gateway.sha256', + }); + writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); +} + +function runDeploy(fixture, options = {}) { + const deployScript = path.join( + fixture.sourceDir, + 'scripts/deploy/production-api-deploy.sh', + ); + return spawnSync( + 'bash', + [ + deployScript, + '--source-dir', + fixture.sourceDir, + '--version', + options.version ?? fixture.version, + '--release-root', + options.releaseRoot ?? fixture.releaseRoot, + '--current-link', + options.currentLink ?? fixture.currentLink, + '--service', + 'genarrative-api.service', + '--health-url', + 'http://127.0.0.1:18082/readyz', + '--api-env-file', + options.apiEnvFile ?? fixture.apiEnvFile, + '--database', + 'genarrative-prod', + '--spacetime-server-url', + 'http://127.0.0.1:3101', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, + GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile, + FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true', + FAKE_CREATE_RELEASE_DURING_COPY: + options.createReleaseDuringCopy === true ? 'true' : 'false', + FAKE_RELEASE_ROOT: fixture.releaseRoot, + FAKE_RELEASE_VERSION: fixture.version, + }, + }, + ); +} + +function copyFile(source, target) { + writeFileSync(target, readFileSync(source, 'utf8'), 'utf8'); +} + +function writeSha256(directory, fileName) { + const result = spawnSync('sha256sum', [fileName], { + cwd: directory, + encoding: 'utf8', + }); + if (result.status !== 0) { + throw new Error(result.stderr || `sha256sum failed for ${fileName}`); + } + writeFileSync( + path.join(directory, `${fileName}.sha256`), + result.stdout, + 'utf8', + ); +} + +function chmodExecutable(filePath) { + spawnSync('chmod', ['0755', filePath], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function readDirNames(directory) { + return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], { + cwd: process.cwd(), + encoding: 'utf8', + }) + .stdout.split('\n') + .filter(Boolean); +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertPingoraEnvProductionDefaults(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const content = readFileSync(filePath, 'utf8'); + const requiredLines = [ + [ + 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + '压缩算法必须保持 gzip-only，避免 Brotli 在未验证前进入 current release。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', + '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', + '前置代理信任确认开关必须默认关闭。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + '接流保护必须默认开启。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', + '内部探针 token 示例必须保持空值，避免 current release 夹带真实 token。', + ], + ]; + for (const [line, reason] of requiredLines) { + assertIncludes(content, line, `${label} ${reason}`); + } +} diff --git a/scripts/check-production-api-release.mjs b/scripts/check-production-api-release.mjs new file mode 100644 index 000000000..27647404c --- /dev/null +++ b/scripts/check-production-api-release.mjs @@ -0,0 +1,534 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const BUILD_SCRIPT = 'scripts/build-production-release.sh'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-production-api-release-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-api-release] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-api-release] OK'); + +function main() { + assertApiReleaseContainsPingoraDirectDependencies(); + assertApiReleaseCanIncludePingoraGatewayArtifact(); +} + +function assertApiReleaseContainsPingoraDirectDependencies() { + const fixture = prepareFixture('api-direct-dependencies'); + const result = runBuild(fixture); + assertStatus(result, 0, 'API release build 烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'api-server'), + 'API release 必须包含 api-server。', + ); + assertExecutable( + path.join(releaseDir, 'api-server'), + 'api-server 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + 'API release 必须包含 api-server checksum。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'), + 'API release 必须包含随产物执行的 API deploy 脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'), + '随产物执行的 API deploy 脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'), + 'API release 必须在 deploy 脚本同目录包含 maintenance-on.sh。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'), + 'deploy 同目录 maintenance-on.sh 必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'), + 'API release 必须在 deploy 脚本同目录包含 maintenance-off.sh。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'), + 'deploy 同目录 maintenance-off.sh 必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'API release 必须包含 Pingora 直连启用脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), + 'Pingora 直连启用脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'API release 必须包含 Pingora 直连回退脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), + 'Pingora 直连回退脚本必须可执行。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'API release 必须包含 Pingora health patrol env 切换脚本。', + ); + assertExecutable( + path.join( + releaseDir, + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + ), + 'Pingora health patrol env 切换脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), + 'API release 必须包含健康巡检 env 复核脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), + 'API release 必须包含 Pingora release readiness 聚合门禁脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-current-release-audit.mjs', + ), + 'API release 必须包含 Pingora current release 自审脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ), + 'API release 必须包含 Pingora 直连切换状态快照脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据包脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-command-evidence.mjs', + ), + 'API release 必须包含 Pingora 直连切换命令证据脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据验真脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + ), + 'API release 必须包含 Pingora 直连切换证据根目录审计脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), + 'API release 必须包含 Pingora 直连预检脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), + 'API release 必须包含 Pingora 直连 live smoke 脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), + 'API release 必须包含 Pingora canary live smoke 脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'scripts/check-pingora-canary-access-log-parity.mjs', + ), + 'API release 必须包含 Pingora canary access log 对账脚本。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway.service', + ), + 'API release 必须包含 Pingora systemd 主 service 模板。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + ), + 'API release 必须包含 Pingora 直连 drop-in 模板。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'API release 必须包含 Pingora env 示例。', + ); + assertFileExists( + path.join( + releaseDir, + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + ), + 'API release 必须包含 Pingora 真实路径 canary Nginx snippet。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/health-patrol.env.example'), + 'API release 必须包含健康巡检 env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), + 'API release 必须包含 Pingora direct live env 示例。', + ); + assertFileExists( + path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), + 'API release 必须包含 Pingora canary live env 示例。', + ); + assertPingoraEnvProductionDefaults( + path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), + 'API release 内 Pingora env 示例', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (manifest.component_type !== 'api-server') { + failures.push( + `release manifest component_type 应为 api-server，实际 ${manifest.component_type}`, + ); + } + if (!manifest.artifacts?.some((item) => item.path === 'api-server')) { + failures.push('release manifest 必须登记 api-server artifact。'); + } + if (manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push('默认 API release 不应登记 pingora-gateway artifact。'); + } + + const releaseReadme = readFileSync( + path.join(releaseDir, 'README.md'), + 'utf8', + ); + assertIncludes( + releaseReadme, + 'Pingora release readiness 聚合门禁、直连启用 / 回退 / health patrol env 切换 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', + 'API release README 必须说明随包携带 Pingora 直连脚本。', + ); + assertIncludes( + releaseReadme, + '状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', + 'API release README 必须说明随包携带 Pingora 切换证据脚本。', + ); + for (const executableArg of requiredCutoverAuditExecutableArgs()) { + assertIncludes( + releaseReadme, + `--require-command-executable ${executableArg}`, + 'API release README 必须记录最终证据根目录总审计绑定 current release 切换脚本。', + ); + } + assertIncludes( + releaseReadme, + 'scripts/deploy/production-api-deploy.sh', + 'API release README 必须说明随包携带 API Deploy 执行入口。', + ); + assertIncludes( + releaseReadme, + '同目录的 `maintenance-on.sh` / `maintenance-off.sh` 必须来自同一发布包', + 'API release README 必须说明 deploy 同目录维护脚本来自同一发布包。', + ); + assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir); + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function assertApiReleaseCanIncludePingoraGatewayArtifact() { + const fixture = prepareFixture('api-with-pingora-artifact', { + includePingoraGateway: true, + }); + const result = runBuild(fixture, { + includePingoraGateway: true, + skipPingoraGatewayBuild: true, + }); + assertStatus(result, 0, '包含 Pingora 的 API release build 烟测应成功。'); + if (result.status !== 0) { + return; + } + + const releaseDir = path.join(process.cwd(), 'build', fixture.version); + try { + assertFileExists( + path.join(releaseDir, 'pingora-gateway'), + '显式 include Pingora 时 API release 必须包含 pingora-gateway。', + ); + assertExecutable( + path.join(releaseDir, 'pingora-gateway'), + 'pingora-gateway 必须保留可执行权限。', + ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '显式 include Pingora 时 API release 必须包含 pingora-gateway checksum。', + ); + + const manifest = readJson(path.join(releaseDir, 'release-manifest.json')); + if (!manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) { + failures.push( + '显式 include Pingora 时 release manifest 必须登记 pingora-gateway artifact。', + ); + } + } finally { + rmSync(releaseDir, { recursive: true, force: true }); + } +} + +function prepareFixture(name, options = {}) { + const cargoTargetDir = path.join(tmpRoot, name, 'cargo-target'); + const binaryDir = path.join( + cargoTargetDir, + 'x86_64-unknown-linux-gnu/release', + ); + const version = `check-production-api-release-${process.pid}-${Date.now()}`; + mkdirSync(binaryDir, { recursive: true }); + const apiBinary = path.join(binaryDir, 'api-server'); + writeFileSync(apiBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', apiBinary], { encoding: 'utf8' }); + if (options.includePingoraGateway) { + const pingoraBinary = path.join(binaryDir, 'pingora-gateway'); + writeFileSync(pingoraBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); + spawnSync('chmod', ['0755', pingoraBinary], { encoding: 'utf8' }); + } + return { cargoTargetDir, version }; +} + +function runBuild(fixture, options = {}) { + const args = [ + BUILD_SCRIPT, + '--component', + 'api-server', + '--name', + fixture.version, + '--skip-api-build', + ]; + if (options.includePingoraGateway) { + args.push('--include-pingora-gateway'); + } + if (options.skipPingoraGatewayBuild) { + args.push('--skip-pingora-gateway-build'); + } + + return spawnSync('bash', args, { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + CARGO_TARGET_DIR: fixture.cargoTargetDir, + SOURCE_BRANCH: 'test-branch', + SOURCE_COMMIT: 'test-commit', + }, + }); +} + +function readJson(filePath) { + try { + return JSON.parse(readFileSync(filePath, 'utf8')); + } catch (error) { + failures.push(`${filePath} 不是合法 JSON: ${error.message}`); + return {}; + } +} + +function assertFileExists(filePath, reason) { + if (!existsSync(filePath)) { + failures.push(`${reason} 缺少: ${filePath}`); + } +} + +function assertExecutable(filePath, reason) { + if (!existsSync(filePath)) { + return; + } + if ((statSync(filePath).mode & 0o111) === 0) { + failures.push(`${reason} 文件不可执行: ${filePath}`); + } +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir) { + const readinessScript = path.join( + releaseDir, + 'scripts/check-pingora-release-readiness.mjs', + ); + const result = spawnSync( + 'node', + [ + readinessScript, + '--dry-run-cutover', + '--require-direct', + '--direct-https-base-url', + 'https://127.0.0.1', + '--direct-http-base-url', + 'http://127.0.0.1', + '--direct-host', + 'example.com', + '--direct-redirect-host', + 'example.com', + '--direct-spacetime-database', + 'genarrative-prod', + '--direct-pingora-access-log', + '/var/log/genarrative/pingora-gateway.access.log', + '--direct-health-patrol-env-file', + '/etc/genarrative/health-patrol.env', + '--direct-preflight-env-file', + '/etc/genarrative/pingora-gateway.env', + '--direct-preflight-systemd', + '--direct-preflight-check-cert-readable', + '--direct-preflight-check-service-env-file', + '--direct-preflight-check-service-user-cert-readable', + '--direct-preflight-check-service-binary-executable', + '--direct-preflight-check-ports-free', + '--cutover-run-id', + 'check-production-api-release', + '--rollback-nginx-smoke-url', + 'http://127.0.0.1/healthz', + '--rollback-health-patrol-public-base-url', + 'http://127.0.0.1', + ], + { + cwd: releaseDir, + encoding: 'utf8', + env: { + ...process.env, + GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT: '', + }, + }, + ); + assertStatus( + result, + 0, + 'API release 随包 readiness 脚本必须能生成直连切换 dry-run runbook。', + ); + if (result.status !== 0) { + return; + } + + let plan; + try { + plan = JSON.parse(result.stdout); + } catch (error) { + failures.push( + `API release 随包 readiness dry-run cutover 输出不是合法 JSON: ${error.message}`, + ); + return; + } + + const auditStep = Array.isArray(plan) + ? plan.find((step) => step?.name === '切换证据根目录三阶段总审计') + : null; + if (!auditStep) { + failures.push( + 'API release 随包 readiness dry-run cutover 必须包含切换证据根目录三阶段总审计步骤。', + ); + return; + } + + if (!auditStep.args?.includes('--require-command-executable')) { + failures.push( + 'API release 随包 readiness 总审计步骤必须携带 --require-command-executable。', + ); + } + for (const executableArg of requiredCutoverAuditExecutableArgs()) { + if (!auditStep.args?.includes(executableArg)) { + failures.push( + `API release 随包 readiness 总审计步骤必须绑定 current release 切换脚本，缺少: ${executableArg}`, + ); + } + } +} + +function requiredCutoverAuditExecutableArgs() { + return [ + 'enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh', + 'rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh', + ]; +} + +function assertPingoraEnvProductionDefaults(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const content = readFileSync(filePath, 'utf8'); + const requiredLines = [ + [ + 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + '压缩算法必须保持 gzip-only，避免 Brotli 在未验证前进入直连发布包。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', + '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', + '前置代理信任确认开关必须默认关闭。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', + '接流保护必须默认开启。', + ], + [ + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', + '内部探针 token 示例必须保持空值，避免发布包夹带真实 token。', + ], + ]; + for (const [line, reason] of requiredLines) { + assertIncludes(content, line, `${label} ${reason}`); + } +} diff --git a/scripts/check-production-health-patrol-env-check.mjs b/scripts/check-production-health-patrol-env-check.mjs new file mode 100644 index 000000000..e8c1c003b --- /dev/null +++ b/scripts/check-production-health-patrol-env-check.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-env-check-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-health-patrol-env] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol-env] OK'); + +function main() { + assertAcceptsPingoraDirectLoopbackWithHost(); + assertRejectsPingoraDirectLoopbackWithoutHost(); + assertRejectsWrongGatewayMode(); + assertAcceptsNginxWithEmptyPublicHost(); + assertRejectsNginxWithStalePublicHost(); + assertRejectsInvalidPublicHost(); + assertRejectsInvalidEnvFilePaths(); + assertRejectsInvalidBoolEnv(); + assertExampleDocumentsDirectMode(); +} + +function assertAcceptsPingoraDirectLoopbackWithHost() { + const envFile = writeEnv('pingora-direct-ok.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + 'https://127.0.0.1', + '--expected-public-host', + 'genarrative.example', + ]); + assertStatus(result, 0, 'Pingora direct loopback 配正式 Host 应通过。'); +} + +function assertRejectsPingoraDirectLoopbackWithoutHost() { + const envFile = writeEnv('pingora-direct-missing-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + '--expected-public-base-url', + 'https://127.0.0.1', + ]); + assertStatus( + result, + 1, + 'Pingora direct loopback 缺 Host 必须失败。', + ); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + 'Pingora direct loopback 缺 Host 必须给出明确错误。', + ); +} + +function assertRejectsWrongGatewayMode() { + const envFile = writeEnv('wrong-mode.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + ]); + assertStatus(result, 1, 'gateway mode 与期望不一致必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 pingora-direct，实际 nginx', + 'gateway mode mismatch 必须给出明确错误。', + ); +} + +function assertAcceptsNginxWithEmptyPublicHost() { + const envFile = writeEnv('nginx-empty-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + '--expected-public-base-url', + 'http://127.0.0.1', + '--require-empty-public-host', + ]); + assertStatus(result, 0, 'Nginx 模式回退且 Host 为空应通过。'); +} + +function assertRejectsNginxWithStalePublicHost() { + const envFile = writeEnv('nginx-stale-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + '--require-empty-public-host', + ]); + assertStatus(result, 1, 'Nginx 回退后残留 Host 覆盖必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空', + 'Nginx 回退后残留 Host 覆盖必须给出明确错误。', + ); +} + +function assertRejectsInvalidPublicHost() { + const envFile = writeEnv('invalid-host.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'pingora-direct', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'https://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: 'https://genarrative.example', + }); + const result = runCheck([ + '--env-file', + envFile, + '--expected-gateway-mode', + 'pingora-direct', + ]); + assertStatus(result, 1, '非法 public Host 必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 只能是 host 或 host:port', + '非法 public Host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidEnvFilePaths() { + const relativePath = runCheck([ + '--env-file', + 'health-patrol.env', + '--expected-gateway-mode', + 'nginx', + ]); + assertStatus(relativePath, 1, 'health patrol env 复核必须拒绝相对 env 文件路径。'); + assertIncludes( + `${relativePath.stdout}\n${relativePath.stderr}`, + '--env-file 必须是绝对路径', + '相对 env 文件路径必须给出明确错误。', + ); + + const rootPath = runCheck([ + '--env-file', + '/', + '--expected-gateway-mode', + 'nginx', + ]); + assertStatus(rootPath, 1, 'health patrol env 复核必须拒绝文件系统根目录。'); + assertIncludes( + `${rootPath.stdout}\n${rootPath.stderr}`, + '--env-file 不能是文件系统根目录', + '文件系统根目录 env 文件路径必须给出明确错误。', + ); +} + +function assertRejectsInvalidBoolEnv() { + const envFile = writeEnv('invalid-bool-env.env', { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: 'nginx', + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: 'http://127.0.0.1', + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: '', + }); + const result = runCheck( + [ + '--env-file', + envFile, + '--expected-gateway-mode', + 'nginx', + ], + { + GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST: 'enabled', + }, + ); + assertStatus(result, 1, 'health patrol env 复核必须拒绝非法布尔 env。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST 必须是布尔值', + 'health patrol env 复核非法布尔 env 必须给出明确错误。', + ); +} + +function assertExampleDocumentsDirectMode() { + const example = readFileSync('deploy/env/health-patrol.env.example', 'utf8'); + assertIncludes( + example, + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + 'health patrol env 示例必须保留默认 nginx 模式。', + ); + assertIncludes( + example, + 'Pingora 直连切换后改为 pingora-direct', + 'health patrol env 示例必须说明 Pingora direct 切换口径。', + ); +} + +function writeEnv(name, values) { + const filePath = path.join(tmpRoot, name); + mkdirSync(path.dirname(filePath), { recursive: true }); + const body = Object.entries(values) + .map(([key, value]) => `${key}=${value}`) + .join('\n'); + writeFileSync(filePath, `${body}\n`, 'utf8'); + return filePath; +} + +function runCheck(args, extraEnv = {}) { + return spawnSync( + process.execPath, + ['--', 'scripts/check-production-health-patrol-env.mjs', ...args], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + GENARRATIVE_HEALTH_PATROL_ENV_FILE: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL: '', + GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST: '', + GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST: '', + ...extraEnv, + }, + }, + ); +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} diff --git a/scripts/check-production-health-patrol-env.mjs b/scripts/check-production-health-patrol-env.mjs new file mode 100644 index 000000000..309908df0 --- /dev/null +++ b/scripts/check-production-health-patrol-env.mjs @@ -0,0 +1,313 @@ +#!/usr/bin/env node + +import { readFileSync } from 'node:fs'; +import path from 'node:path'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); + +const config = parseArgs(process.argv.slice(2)); +const failures = []; +const envValues = readEnvFile(config.envFile); + +checkGatewayMode(); +checkPublicBaseUrl(); +checkPublicHost(); + +if (failures.length > 0) { + console.error('[check:production-health-patrol-env] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol-env] OK'); + +function parseArgs(argv) { + const result = { + envFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: + readBoolEnv('GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST'), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--env-file': + result.envFile = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!result.expectedGatewayMode) { + throw new Error( + '必须提供 --expected-gateway-mode 或 GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE。', + ); + } + if (!path.isAbsolute(result.envFile)) { + throw new Error(`--env-file 必须是绝对路径: ${result.envFile}`); + } + if (isFilesystemRootPath(result.envFile)) { + throw new Error('--env-file 不能是文件系统根目录。'); + } + if (!GATEWAY_MODES.has(result.expectedGatewayMode)) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`, + ); + } + if (result.expectedPublicBaseUrl) { + normalizeBaseUrl(result.expectedPublicBaseUrl, '--expected-public-base-url'); + } + if (result.expectedPublicHost !== null) { + validateHostOption(result.expectedPublicHost, '--expected-public-host'); + } + if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node -- scripts/check-production-health-patrol-env.mjs --expected-gateway-mode [options] + +Options: + --env-file health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。 + --expected-gateway-mode nginx 或 pingora-direct。 + --expected-public-base-url 可选，要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 与该 URL 一致。 + --expected-public-host 可选，要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 与该 Host 一致。 + --require-empty-public-host 要求 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 为空，适合回退到 Nginx 后复核。 +`); +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readEnvFile(filePath) { + let text; + try { + text = readFileSync(filePath, 'utf8'); + } catch (error) { + throw new Error(`无法读取 health patrol env 文件 ${filePath}: ${error.message}`); + } + + const values = new Map(); + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + failures.push(`第 ${index + 1} 行不是 KEY=VALUE 格式: ${rawLine}`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + const value = stripQuotes(line.slice(equalsIndex + 1).trim()); + values.set(key, value); + } + return values; +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function envValue(name) { + return envValues.get(name) ?? ''; +} + +function checkGatewayMode() { + const actual = envValue('GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE'); + if (!actual) { + failures.push( + 'health patrol env 缺少 GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE。', + ); + return; + } + if (actual !== config.expectedGatewayMode) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${config.expectedGatewayMode}，实际 ${actual}。`, + ); + } +} + +function checkPublicBaseUrl() { + if (!config.expectedPublicBaseUrl) { + return; + } + + const actual = envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL'); + if (!actual) { + failures.push( + 'health patrol env 缺少 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL。', + ); + return; + } + const expected = normalizeBaseUrl( + config.expectedPublicBaseUrl, + '--expected-public-base-url', + ); + const normalizedActual = normalizeBaseUrl( + actual, + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL', + ); + if (normalizedActual !== expected) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${expected}，实际 ${normalizedActual}。`, + ); + } +} + +function checkPublicHost() { + const publicHost = envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST'); + const baseUrl = + config.expectedPublicBaseUrl || + envValue('GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL'); + + if (publicHost) { + try { + validateHostOption(publicHost, 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST'); + } catch (error) { + failures.push(error.message); + } + } + + if (config.expectedPublicHost !== null && publicHost !== config.expectedPublicHost) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${config.expectedPublicHost}，实际 ${publicHost || '(空)'}。`, + ); + } + + if (config.requireEmptyPublicHost && publicHost) { + failures.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空，实际 ${publicHost}。`, + ); + } + + if ( + config.expectedGatewayMode === 'pingora-direct' && + baseUrl && + isLoopbackUrl(baseUrl) && + !publicHost + ) { + failures.push( + 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST，避免巡检命中默认 vhost。', + ); + } +} + +function normalizeBaseUrl(value, label) { + try { + const parsed = new URL(value); + if (!['http:', 'https:'].includes(parsed.protocol)) { + throw new Error('unsupported protocol'); + } + return parsed.toString().replace(/\/+$/u, ''); + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符。`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function isLoopbackUrl(value) { + try { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === 'localhost' || + hostname === '::1' || + hostname === '[::1]' || + hostname.startsWith('127.') + ); + } catch { + return false; + } +} diff --git a/scripts/check-production-health-patrol.mjs b/scripts/check-production-health-patrol.mjs new file mode 100644 index 000000000..abc2fe718 --- /dev/null +++ b/scripts/check-production-health-patrol.mjs @@ -0,0 +1,474 @@ +#!/usr/bin/env node + +import { spawn, spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-check-'), +); + +try { + await main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:production-health-patrol] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:production-health-patrol] OK'); + +async function main() { + assertPublicBaseUrlDefaultsToGatewayEntry(); + await assertNginxModeChecksNginxService(); + await assertPingoraDirectModeChecksPingoraServiceAndPublicHost(); + assertRejectsInvalidGatewayMode(); + assertRejectsInvalidPublicHost(); + assertRejectsInvalidTimeoutAndSlowThreshold(); + assertRejectsInvalidBoolEnv(); +} + +function assertPublicBaseUrlDefaultsToGatewayEntry() { + const script = readFileSync( + 'scripts/ops/production-health-patrol.mjs', + 'utf8', + ); + assertIncludes( + script, + "process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n 'http://127.0.0.1'", + 'publicBaseUrl 默认必须指向本机网关入口，不能回落到 API 直连端口。', + ); + if ( + script.includes( + 'process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL', + ) + ) { + failures.push('publicBaseUrl 默认不应回落到 API base URL。'); + } +} + +async function assertNginxModeChecksNginxService() { + const fixture = await prepareFixture('nginx-mode'); + const result = await runPatrol(fixture, ['--gateway-mode', 'nginx']); + + assertStatus(result, 0, 'nginx gateway mode 巡检应成功。'); + if (result.status !== 0) { + return; + } + + const payload = JSON.parse(result.stdout); + assertEqual( + payload.gatewayMode, + 'nginx', + '巡检 JSON 必须记录 gatewayMode=nginx。', + ); + + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl is-active nginx.service', + 'nginx gateway mode 必须检查 nginx.service。', + ); + if (commandsLog.includes('genarrative-pingora-gateway.service')) { + failures.push( + 'nginx gateway mode 不应要求 Pingora gateway service active。', + ); + } +} + +async function assertPingoraDirectModeChecksPingoraServiceAndPublicHost() { + const fixture = await prepareFixture('pingora-direct-mode'); + const result = await runPatrol(fixture, [ + '--gateway-mode', + 'pingora-direct', + '--public-host', + 'genarrative.example', + ]); + + assertStatus(result, 0, 'pingora-direct gateway mode 巡检应成功。'); + if (result.status !== 0) { + return; + } + + const payload = JSON.parse(result.stdout); + assertEqual( + payload.gatewayMode, + 'pingora-direct', + '巡检 JSON 必须记录 gatewayMode=pingora-direct。', + ); + + const commandsLog = readTextFile(fixture.commandsLog, 'systemctl 命令日志'); + assertIncludes( + commandsLog, + 'systemctl is-active genarrative-pingora-gateway.service', + 'pingora-direct gateway mode 必须检查 Pingora gateway service。', + ); + if (commandsLog.includes('systemctl is-active nginx.service')) { + failures.push( + 'pingora-direct gateway mode 不应要求 nginx.service active。', + ); + } + + const requestsLog = readTextFile( + fixture.requestsLog, + 'public probe 请求日志', + ); + assertIncludes( + requestsLog, + 'host=genarrative.example path=/api/creation-entry/config', + 'public probe 必须带正式域名 Host header。', + ); +} + +function assertRejectsInvalidGatewayMode() { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--gateway-mode', + 'direct', + '--skip-journal', + '--json', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push('非法 gateway mode 必须失败。'); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--gateway-mode 只支持 nginx 或 pingora-direct', + '非法 gateway mode 必须给出明确错误。', + ); +} + +function assertRejectsInvalidPublicHost() { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--public-host', + 'https://genarrative.example', + '--skip-journal', + '--json', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push('非法 public host 必须失败。'); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--public-host 只接受域名或 host:port', + '非法 public host 必须给出明确错误。', + ); +} + +function assertRejectsInvalidTimeoutAndSlowThreshold() { + const cases = [ + { + args: ['--timeout-ms', '0'], + env: {}, + expected: '--timeout-ms 必须是正整数', + reason: '非法 CLI timeout 必须失败。', + }, + { + args: ['--slow-ms', 'abc'], + env: {}, + expected: '--slow-ms 必须是正整数', + reason: '非法 CLI slow threshold 必须失败。', + }, + { + args: [], + env: { + GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS: 'abc', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS 必须是正整数', + reason: '非法 env timeout 必须失败。', + }, + { + args: [], + env: { + GENARRATIVE_HEALTH_PATROL_SLOW_MS: '0', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_SLOW_MS 必须是正整数', + reason: '非法 env slow threshold 必须失败。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--skip-journal', + '--json', + ...testCase.args, + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + ...testCase.env, + }, + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +function assertRejectsInvalidBoolEnv() { + const cases = [ + { + env: { + GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING: 'ture', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING 必须是布尔值', + reason: '生产健康巡检必须拒绝拼写错误的 fail-on-warning env。', + }, + { + env: { + GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL: 'maybe', + }, + expected: 'GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL 必须是布尔值', + reason: '生产健康巡检必须拒绝非法 skip-journal env。', + }, + ]; + + for (const testCase of cases) { + const result = spawnSync( + 'node', + ['scripts/ops/production-health-patrol.mjs', '--json'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: { + ...process.env, + ...testCase.env, + }, + }, + ); + + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + testCase.expected, + `${testCase.reason} 必须给出明确错误。`, + ); + } +} + +async function prepareFixture(name) { + const root = path.join(tmpRoot, name); + const fakeBin = path.join(root, 'bin'); + const commandsLog = path.join(root, 'commands.log'); + const requestsLog = path.join(root, 'requests.log'); + + mkdirSync(fakeBin, { recursive: true }); + + writeFileSync( + path.join(fakeBin, 'systemctl'), + [ + '#!/usr/bin/env bash', + `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'echo active', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + path.join(fakeBin, 'journalctl'), + [ + '#!/usr/bin/env bash', + `printf 'journalctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'echo "-- No entries --"', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(path.join(fakeBin, 'systemctl')); + chmodExecutable(path.join(fakeBin, 'journalctl')); + + const server = http.createServer((request, response) => { + writeFileSync( + requestsLog, + `host=${request.headers.host || ''} path=${request.url || ''}\n`, + { + encoding: 'utf8', + flag: 'a', + }, + ); + response.writeHead(200, { + 'Content-Type': 'application/json', + }); + response.end(JSON.stringify({ ok: true })); + }); + await listen(server, '127.0.0.1', 0); + + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('测试 HTTP server 未返回 TCP 端口。'); + } + + return { + root, + fakeBin, + commandsLog, + requestsLog, + server, + baseUrl: `http://127.0.0.1:${address.port}`, + }; +} + +async function runPatrol(fixture, args) { + try { + return await spawnProcess( + 'node', + [ + 'scripts/ops/production-health-patrol.mjs', + '--api-base-url', + fixture.baseUrl, + '--spacetime-base-url', + fixture.baseUrl, + '--public-base-url', + fixture.baseUrl, + '--skip-journal', + '--json', + ...args, + ], + { + cwd: process.cwd(), + env: { + ...process.env, + PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, + }, + }, + ); + } finally { + await closeServer(fixture.server); + } +} + +function spawnProcess(command, args, options) { + return new Promise((resolve) => { + const child = spawn(command, args, { + ...options, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.on('error', (error) => { + resolve({ + status: 1, + stdout, + stderr: `${stderr}${error.message}`, + }); + }); + child.on('close', (code, signal) => { + resolve({ + status: code ?? 1, + signal, + stdout, + stderr, + }); + }); + }); +} + +function listen(server, host, port) { + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, host, resolve); + }); +} + +function closeServer(server) { + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +function readTextFile(filePath, label) { + if (!existsSync(filePath)) { + failures.push(`${label} 未生成：${filePath}`); + return ''; + } + return readFileSync(filePath, 'utf8'); +} + +function chmodExecutable(filePath) { + spawnSync('chmod', ['0755', filePath], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, "'\\''")}'`; +} + +function assertStatus(result, expected, reason) { + const actual = result.status ?? 0; + if (actual !== expected) { + failures.push( + `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, reason) { + if (!content.includes(needle)) { + failures.push(`${reason} 缺少: ${needle}`); + } +} + +function assertEqual(actual, expected, reason) { + if (actual !== expected) { + failures.push(`${reason} 实际 ${actual}，预期 ${expected}。`); + } +} diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 6a8537f0a..b9ed0a8d0 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import {readFileSync} from 'node:fs'; +import { readFileSync } from 'node:fs'; const checks = [ { @@ -8,53 +8,5660 @@ const checks = [ includes: '--restart-service-after genarrative-api.service', reason: '生产冷备份恢复 SpacetimeDB 后必须显式拉起依赖它的 API 服务。', }, + { + file: 'deploy/systemd/genarrative-database-backup.service', + includes: 'ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file', + reason: + '生产冷备份 service 必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', + }, { file: 'deploy/systemd/genarrative-health-patrol.service', includes: 'scripts/ops/production-health-patrol.mjs', reason: '健康巡检 systemd service 必须调用随 API release 发布的巡检脚本。', }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL', + reason: '健康巡检环境示例必须保留 Pingora shadow 可选巡检配置。', + }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + reason: '健康巡检环境示例必须显式保留默认 Nginx 网关模式。', + }, + { + file: 'deploy/env/health-patrol.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + reason: '健康巡检环境示例必须暴露公网 Host header 配置。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'pingora-direct', + reason: + '健康巡检必须支持 Pingora 直连模式，避免直连切换后误要求 Nginx active。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连巡检模式必须检查 Pingora gateway service。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + reason: '健康巡检必须支持给 public probe 传正式域名 Host header。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: 'parseRequiredPositiveInt', + reason: + '健康巡检 timeout 和慢请求阈值必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'current release 自包含能力', + reason: 'Pingora current release 自审脚本必须说明它只检查发布包自包含能力。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: 'Pingora current release 自审必须确认 release readiness 聚合门禁脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: 'Pingora current release 自审必须确认 canary live smoke 脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '--systemd-show', + reason: 'Pingora current release 自审必须能核验 systemd ExecStart 指向 current release。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'pingora-gateway.sha256', + reason: 'Pingora current release 自审必须校验 Pingora 网关 checksum。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'release-manifest.api-server.json', + reason: 'Pingora current release 自审必须读取部署后的 API release manifest。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'release manifest 缺少 pingora-gateway artifact', + reason: 'Pingora current release 自审必须阻断 manifest 未登记 Pingora 产物。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora current release 自审 timeout 参数必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora current release 自审布尔 env 必须严格校验，避免拼写错误静默降级。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora current release 自审必须显式拒绝把 release root 指到 /。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora current release 自审必须在执行 systemctl 前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora current release 自审必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: '完整 release 自审必须确认 canary live smoke 脚本存在', + reason: 'Pingora current release 自审自测必须覆盖 canary live smoke 脚本随包存在。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertChecksumMismatchFails', + reason: 'Pingora current release 自审自测必须覆盖 checksum 不匹配负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertManifestMustRegisterIncludedPingoraGateway', + reason: 'Pingora current release 自审自测必须覆盖 manifest 未登记 Pingora 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora current release 自审自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora current release 自审自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsFilesystemRootReleaseRoot', + reason: 'Pingora current release 自审自测必须覆盖 release root 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-current-release-audit.mjs', + includes: 'assertRejectsPathArgsWithControlCharacters', + reason: + 'Pingora current release 自审自测必须覆盖带控制字符路径参数在执行 systemctl 前失败。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: 'Pingora 切换状态快照必须把 current release 自审脚本纳入随包依赖采集。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--fail-on-critical', + reason: 'Pingora 切换状态快照必须能作为只读硬门禁运行。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--run-health-patrol', + reason: 'Pingora 切换状态快照必须复用生产健康巡检作为证据。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'current-release-audit', + reason: 'Pingora 切换状态快照必须执行 current release 自审。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'releaseManifest', + reason: 'Pingora 切换状态快照必须收录 manifest 自审结果。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'collectSecretValuesFromEnv', + reason: 'Pingora 切换状态快照必须从 env 中收集敏感值用于后续输出脱敏。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'delete healthPatrolEnv.secretValues', + reason: 'Pingora 切换状态快照 JSON 不能输出 health patrol env 敏感值缓存。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'delete pingoraEnv.secretValues', + reason: 'Pingora 切换状态快照 JSON 不能输出 Pingora env 敏感值缓存。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'redactSecrets', + reason: 'Pingora 切换状态快照必须在收录子命令 stdout/stderr 前脱敏敏感值。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora 切换状态快照 timeout 参数必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora 切换状态快照布尔 env 必须严格校验，避免 run/fail/require 开关拼写错误静默降级。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora 切换状态快照必须显式拒绝把 release root 指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 切换状态快照必须在执行子命令前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 切换状态快照必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--health-patrol-env-file 不能是文件系统根目录', + reason: 'Pingora 切换状态快照必须拒绝把 health patrol env 文件指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: '--pingora-env-file 不能是文件系统根目录', + reason: 'Pingora 切换状态快照必须拒绝把 Pingora env 文件指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'environmentFileMatchesPingoraEnvFile', + reason: + 'Pingora 切换状态快照必须确认 systemd EnvironmentFile 指向本次 pingora env。', + }, + { + file: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + includes: 'EnvironmentFile 未包含本次 --pingora-env-file', + reason: + 'Pingora 切换状态快照必须在 systemd EnvironmentFile 漂移时给出明确诊断。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertFailOnCriticalRejectsReleaseAuditFailure', + reason: 'Pingora 状态快照自测必须覆盖 current release 自审失败负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertFailOnCriticalRejectsSystemdPingoraEnvDrift', + reason: 'Pingora 状态快照自测必须覆盖 systemd 读取了另一份 Pingora env 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertSnapshotRedactsEnvProbeTokens', + reason: 'Pingora 状态快照自测必须确认 env probe token 不进入快照输出。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora 状态快照自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora 状态快照自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsFilesystemRootReleaseRoot', + reason: 'Pingora 状态快照自测必须覆盖 release root 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsPathArgsWithControlCharacters', + reason: + 'Pingora 状态快照自测必须覆盖带控制字符路径参数在执行 systemctl 前失败。', + }, + { + file: 'scripts/check-pingora-cutover-status-snapshot.mjs', + includes: 'assertRejectsFilesystemRootEnvFiles', + reason: 'Pingora 状态快照自测必须覆盖 env 文件参数指到 / 的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'snapshot.stdout.txt', + reason: 'Pingora 切换证据包必须保存状态快照 stdout 便于复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.stdout.txt', + reason: + 'Pingora 切换证据包必须保存 direct live stdout，保留 request_id / access log 复核证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'parseDirectLiveJson', + reason: + 'Pingora 切换证据包必须能从 direct live stdout 中提取 JSON 结果。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'snapshotParseError', + reason: + 'Pingora 切换证据包必须把 snapshot 解析失败文件纳入 manifest 和 stdout 索引。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveParseError', + reason: + 'Pingora 切换证据包必须把 direct live 解析失败文件纳入 manifest 和 stdout 索引。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--run-direct-live', + reason: + 'Pingora 切换证据包必须提供显式 direct live 归档开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-pingora-access-log', + reason: + 'Pingora 切换证据包 direct live 归档必须接入 access log request_id 校验路径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'SECRET_VALUE_FLAGS', + reason: 'Pingora 切换证据包命令记录必须集中维护敏感参数脱敏列表。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '', + reason: 'Pingora 切换证据包命令记录不能落盘 probe token 原文。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'args: redactSecretArgs', + reason: 'Pingora 切换证据包命令记录必须保存结构化脱敏参数数组。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateEvidenceCommandArgs', + reason: + 'Pingora 切换证据包必须在执行状态快照或 direct live 前拒绝带控制字符的子命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateSafeAbsoluteFilePath', + reason: + 'Pingora 切换证据包 direct live 显式文件路径必须统一拒绝相对路径、根目录和控制字符。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-spacetime-database', + reason: + 'Pingora 切换证据包必须在配置层拒绝带控制字符的 direct SpacetimeDB 数据库名。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--direct-probe-token', + reason: + 'Pingora 切换证据包必须在配置层拒绝带控制字符的 direct live probe token。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 切换证据包必须明确拒绝带换行或 NUL 的状态快照 / direct live 参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: "createHash('sha256')", + reason: 'Pingora 切换证据包 manifest 必须记录证据文件 sha256，便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'sizeBytes', + reason: 'Pingora 切换证据包 manifest 必须记录证据文件大小，便于归档后复核。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'snapshot-command.json 结构化参数数组必须保留快照参数', + reason: 'Pingora 切换证据包自测必须覆盖结构化命令参数记录。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertManifestFileMetadata', + reason: 'Pingora 切换证据包自测必须覆盖 manifest 文件大小与 sha256 元数据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertBundleCanArchiveDirectLiveEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live 证据归档成功路径。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveFailureStillWritesEvidenceAndFails', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live 失败时保留证据并失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveParseFailureWritesParseErrorEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live stdout 解析失败时保留 parse error 证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertSnapshotParseFailureWritesParseErrorEvidence', + reason: + 'Pingora 切换证据包自测必须覆盖 snapshot stdout 解析失败时保留 parse error 证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsSnapshotArgsWithControlCharacters', + reason: + 'Pingora 切换证据包自测必须覆盖带控制字符的状态快照参数在执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsDirectLiveArgsWithControlCharacters', + reason: + 'Pingora 切换证据包自测必须覆盖带控制字符的 direct live 参数在执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: '--direct-pingora-access-log 不能是文件系统根目录', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live access log 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: '--direct-access-log-since-lines 不能包含换行或 NUL 字符', + reason: + 'Pingora 切换证据包自测必须覆盖 direct live access log tail 行数带控制字符的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'BUNDLE_DIR_MODE = 0o750', + reason: 'Pingora 切换证据包目录必须显式限制为 owner/group 可读。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'EVIDENCE_FILE_MODE = 0o640', + reason: 'Pingora 切换证据文件必须显式禁止 world-readable。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: "flag: 'wx'", + reason: 'Pingora 切换证据包写文件必须避免覆盖既有证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validatePhase(config.phase);', + reason: 'Pingora 切换证据包必须拒绝不安全 phase，避免目录名和 manifest 阶段漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'validateOutputRootForWriting(outputRoot);', + reason: + 'Pingora 切换证据包写入前必须复核 output-root 路径，避免写入软链或非目录。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--output-root 不能是符号链接', + reason: 'Pingora 切换证据包必须拒绝符号链接 output-root。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--output-root 已存在但不是目录', + reason: 'Pingora 切换证据包必须拒绝已存在但不是目录的 output-root。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '不能是文件系统根目录', + reason: 'Pingora 切换证据包必须拒绝把显式路径参数指到 /。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'parseRequiredPositiveInt', + reason: 'Pingora 切换证据包 timeout 参数必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora 切换证据包布尔 env 必须严格校验，避免证据生成开关拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsUnsafePhase', + reason: 'Pingora 切换证据包自测必须覆盖不安全 phase 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertBundleRedactsProbeTokensFromArtifacts', + reason: 'Pingora 切换证据包自测必须确认所有证据文件不落盘 probe token 原文。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertMode', + reason: 'Pingora 切换证据包自测必须断言目录和文件权限。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsInvalidTimeout', + reason: 'Pingora 切换证据包自测必须覆盖非法 timeout 参数和 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: 'Pingora 切换证据包自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsSymlinkOutputRootBeforeSnapshot', + reason: + 'Pingora 切换证据包自测必须覆盖符号链接 output-root，且失败时不执行状态快照。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsFileOutputRootBeforeSnapshot', + reason: + 'Pingora 切换证据包自测必须覆盖非目录 output-root，且失败时不执行状态快照。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertRejectsFilesystemRootPaths', + reason: + 'Pingora 切换证据包自测必须覆盖显式路径参数指向文件系统根目录的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '--require-pingora-gateway', + reason: 'Pingora 切换证据包必须能透传发布物自审要求。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: '只写 --output-root 下的新证据目录', + reason: 'Pingora 切换证据包必须明确不会修改 /etc、systemd、Nginx 或 Pingora。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command.stdout.txt', + reason: 'Pingora 切换命令证据脚本必须保存真实命令 stdout。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command.stderr.txt', + reason: 'Pingora 切换命令证据脚本必须保存真实命令 stderr。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'command-record.json', + reason: 'Pingora 切换命令证据脚本必须保存结构化命令记录。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "createHash('sha256')", + reason: 'Pingora 切换命令证据 manifest 必须记录证据文件 sha256，便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'sizeBytes', + reason: 'Pingora 切换命令证据 manifest 必须记录证据文件大小，便于归档后复核。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'redactSecrets(text', + reason: 'Pingora 切换命令证据脚本必须脱敏 stdout/stderr 中的 probe token。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'validateCommandPath(config.command);', + reason: 'Pingora 切换命令证据脚本必须拒绝非绝对路径真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: 'path.resolve(command) === path.parse(path.resolve(command)).root', + reason: 'Pingora 切换命令证据脚本必须拒绝文件系统根目录真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '/[\\0\\r\\n]/u.test(command)', + reason: 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令参数不能包含换行或 NUL 字符', + reason: 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的真实命令参数。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '--expected-executable', + reason: 'Pingora 切换命令证据脚本必须支持绑定预期真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "validateSafeExecutablePath(config.expectedExecutable, '--expected-executable')", + reason: 'Pingora 切换命令证据生成端必须拒绝不安全 expected executable 路径。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '--require-arg', + reason: 'Pingora 切换命令证据脚本必须支持执行前校验真实命令必需参数。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令与 --expected-executable 不一致', + reason: 'Pingora 切换命令证据脚本必须拒绝挂错真实执行脚本。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: '真实命令参数缺少 --require-arg 要求', + reason: 'Pingora 切换命令证据脚本必须在执行前拒绝缺少必需参数的真实命令。', + }, + { + file: 'scripts/ops/pingora-cutover-command-evidence.mjs', + includes: "validateNoControlCharacters(config.outputRoot, '--output-root')", + reason: + 'Pingora 切换命令证据脚本必须拒绝带换行或 NUL 的 output-root。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertCommandFailureStillWritesEvidenceAndFails', + reason: 'Pingora 切换命令证据自测必须覆盖真实命令失败时仍保留证据。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsMissingRequiredArgBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖缺少 --apply 时执行前失败。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsBareCommandName', + reason: 'Pingora 切换命令证据自测必须覆盖 PATH 裸命令名被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsFilesystemRootCommand', + reason: 'Pingora 切换命令证据自测必须覆盖文件系统根目录真实命令被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsOutputRootWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖 output-root 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsCommandWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖带控制字符真实命令被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsCommandArgWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖带控制字符真实命令参数被拒绝。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsFilesystemRootExpectedExecutable', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsExpectedExecutableWithControlCharacters', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertCommandEvidenceRedactsProbeTokensFromArtifacts', + reason: 'Pingora 切换命令证据自测必须覆盖 token 不落盘。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertManifestFileMetadata', + reason: 'Pingora 切换命令证据自测必须覆盖 manifest 文件大小与 sha256 元数据。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsSymlinkOutputRootBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖符号链接 output-root 且失败时不执行真实命令。', + }, + { + file: 'scripts/check-pingora-cutover-command-evidence.mjs', + includes: 'assertRejectsMismatchedExpectedExecutableBeforeCommand', + reason: 'Pingora 切换命令证据自测必须覆盖 expected executable 不匹配且失败时不执行真实命令。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertPingoraDirectModeChecksPingoraServiceAndPublicHost', + reason: + '生产健康巡检动态烟测必须覆盖 Pingora 直连模式和 public Host header。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertRejectsInvalidTimeoutAndSlowThreshold', + reason: + '生产健康巡检动态烟测必须覆盖非法 timeout 和慢请求阈值负例。', + }, + { + file: 'scripts/ops/production-health-patrol.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + '生产健康巡检布尔 env 必须严格校验，避免 fail-on-warning / skip-journal 拼写错误静默降级。', + }, + { + file: 'scripts/check-production-health-patrol.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: '生产健康巡检动态烟测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: 'Pingora direct 模式使用本机 public base URL 时必须配置 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', + reason: + '生产健康巡检 env 复核必须阻断直连后本机探测缺少正式 Host 的配置。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + '生产健康巡检 env 复核布尔 env 必须严格校验，避免 require-empty-public-host 拼写错误静默降级。', + }, + { + file: 'scripts/check-production-health-patrol-env.mjs', + includes: 'isFilesystemRootPath', + reason: '生产健康巡检 env 复核必须显式拒绝把 env 文件指到 /。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsPingoraDirectLoopbackWithoutHost', + reason: + '生产健康巡检 env 自测必须覆盖 Pingora direct loopback 缺 Host 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: '生产健康巡检 env 自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/check-production-health-patrol-env-check.mjs', + includes: 'assertRejectsInvalidEnvFilePaths', + reason: '生产健康巡检 env 自测必须覆盖相对路径和 / 路径负例。', + }, + { + file: 'deploy/logrotate/genarrative-pingora-gateway', + includes: '/var/log/genarrative/pingora-gateway.access.log', + reason: + 'Pingora access log 必须有 logrotate 模板，避免 canary 日志无限增长。', + }, { file: 'deploy/systemd/genarrative-health-patrol.timer', includes: 'genarrative-health-patrol.service', reason: '健康巡检 timer 必须绑定巡检 service。', }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + includes: '/var/log/genarrative', + reason: 'Pingora systemd 沙箱必须允许写入 access log 目录。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + includes: '影子网关只监听本机高端口', + reason: + 'Pingora 主 service 必须保持默认 shadow 口径，不能默认声明公网直连。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway.service', + excludes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: + 'Pingora 主 service 不能默认带低端口 capability，直连只能通过显式 drop-in 启用。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连 80/443 时必须通过显式 drop-in 授予低端口绑定能力。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连 drop-in 必须把 capability 边界限制为低端口绑定。', + }, + { + file: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + includes: + '/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf', + reason: 'Pingora 直连 drop-in 必须明确人工启用位置，不能被误认为默认生效。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + includes: 'X-Genarrative-Nginx-Handoff', + reason: + 'Pingora canary snippet 必须带明确响应头，方便验证 Nginx 已经切到 Pingora handoff。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + includes: 'deny all', + reason: + 'Pingora canary snippet 默认必须限制访问来源，避免误 include 后公开接流。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'listen 127.0.0.1:18083;', + reason: + 'Pingora 真实路径 canary 必须默认只监听本机端口，避免误 include 后公开接流。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'genarrative-pingora-realpath-canary.access.log', + reason: + 'Pingora 真实路径 canary 必须使用独立 Nginx access log，避免和生产真实路径请求混淆。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'pingora-realpath-canary', + reason: + 'Pingora 真实路径 canary 必须带独立 handoff 标识，方便区分前缀 canary。', + }, + { + file: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + includes: 'location = /api/creation-entry/config', + reason: + 'Pingora 真实路径 canary 必须覆盖代表性 API 真实路径。', + }, { file: 'scripts/jenkins-server-provision.sh', includes: 'genarrative-health-patrol.timer', reason: 'Server-Provision 必须安装并启用健康巡检 timer。', }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-canary.conf', + reason: + 'Server-Provision 必须安装 Pingora canary snippet，供人工启用 Nginx handoff。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-realpath-canary.conf', + reason: + 'Server-Provision 必须安装 Pingora 真实路径 canary snippet，供人工启用直连前中间验证。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: 'genarrative-pingora-gateway', + reason: 'Server-Provision 必须安装 Pingora access log 的 logrotate 配置。', + }, + { + file: 'scripts/jenkins-server-provision.sh', + includes: + '/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf', + reason: + 'Server-Provision 必须只把 Pingora 直连 drop-in 安装为人工启用模板。', + }, { file: 'scripts/build-production-release.sh', includes: 'production-health-patrol.mjs', reason: '生产 API release 必须携带健康巡检脚本。', }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: '生产 API release 必须携带 Pingora current release 自审脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: '生产 API release 必须携带 Pingora 切换状态快照脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: '生产 API release 必须携带 Pingora 切换证据包脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: '生产 API release 必须携带 Pingora 切换命令证据脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-production-health-patrol-env.mjs', + reason: '生产 API release 必须携带健康巡检 env 复核脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-release-readiness.mjs', + reason: + '生产 API release 必须携带 Pingora release readiness 聚合门禁脚本，确保正式 runbook 不依赖源码 checkout。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-canary-live.mjs', + reason: '生产 API release 必须携带 Pingora canary live smoke 脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'check-pingora-canary-access-log-parity.mjs', + reason: '生产 API release 必须携带 Pingora canary access log 对账脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: '--include-pingora-gateway', + reason: 'Pingora 影子网关必须保持显式可选产物打包入口。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_pingora_build_toolchain', + reason: '显式构建 Pingora 影子网关时必须先检查 cmake 和 C/C++ 编译器。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_command cmake', + reason: 'Pingora release 构建必须对 cmake 缺失 fail-fast。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'require_any_command "C++ 编译器"', + reason: 'Pingora release 构建必须对 C++ 编译器缺失 fail-fast。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: '生产 API release 必须携带随产物执行的 API deploy 脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: + '同目录的 \\`maintenance-on.sh\\` / \\`maintenance-off.sh\\` 必须来自同一发布包', + reason: + '生产 API release README 必须说明 deploy 同目录维护脚本来自同一发布包。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/maintenance-on.sh', + reason: '生产 API release 必须在 deploy 脚本同目录携带 maintenance-on.sh。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/maintenance-off.sh', + reason: + '生产 API release 必须在 deploy 脚本同目录携带 maintenance-off.sh。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: '生产 API release 必须携带 Pingora 直连回退脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: '生产 API release 必须携带 Pingora health patrol env 切换脚本。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-expected-public-base-url', + reason: + 'Pingora 直连回退脚本必须能复核 health patrol public base URL 已恢复为切换前记录值。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-expected-public-host', + reason: + 'Pingora 直连回退脚本必须能复核切换前存在的 health patrol public Host 覆盖。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: '生产 API release 必须携带 Pingora 直连启用脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: + '生产 API release 必须携带 Pingora 直连预检脚本，确保 current release 可独立执行直连启用。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + '生产 API release 必须携带 Pingora 直连 live smoke 脚本，确保启用脚本 apply 后可独立验收。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'deploy/pingora', + reason: + '生产 API release 必须携带 Pingora 配置目录，供直连预检读取 env 示例和路由矩阵。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'deploy/env', + reason: + '生产 API release 必须携带 env 示例目录，供健康巡检和 Pingora live smoke 复制配置。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ENV_DEPLOY_DIR_SOURCE', + reason: 'API deploy 必须从发布产物复制 env 示例目录到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CANARY_LIVE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora canary live smoke 脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora current release 自审脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换状态快照脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换证据包脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora 切换命令证据脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/env/health-patrol.env.example', + reason: 'API release 动态烟测必须断言健康巡检 env 示例随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: 'API release 动态烟测必须断言 Pingora current release 自审脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换状态快照脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换证据包脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API release 动态烟测必须断言 Pingora 切换命令证据脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: 'API release 动态烟测必须断言健康巡检 env 复核脚本随包发布。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/env/health-patrol.env.example', + reason: + 'API deploy 动态烟测必须断言健康巡检 env 示例复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-current-release-audit.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora current release 自审脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换状态快照脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换证据包脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: + 'API deploy 动态烟测必须断言 Pingora 切换命令证据脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraCutoverCommandEvidenceFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora 切换命令证据脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolEnvCheckFails', + reason: + 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingEnvExamplesFails', + reason: 'API deploy 动态烟测必须覆盖缺少 env 示例目录时 fail-fast。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-direct-enable', + reason: 'Pingora 直连启用必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-direct-rollback', + reason: 'Pingora 直连回退必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-gateway-smoke', + reason: 'Pingora 网关必须保留可重复本地 smoke 入口。', + }, + { + file: 'package.json', + includes: 'check:nginx-pingora-canary', + reason: 'Pingora Nginx canary snippet 必须保留可重复本地校验入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-route-parity', + reason: 'Pingora 与 Nginx 路由矩阵必须保留可重复本地校验入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-live', + reason: 'Pingora Nginx 前缀 canary 必须保留启用后的真实入口 smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-live', + reason: 'Pingora 直连 HTTPS / HTTP redirect 必须保留目标机 live smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-live-guard', + reason: + 'Pingora direct live 参数护栏必须保留单独 npm 入口，方便正式切换前聚合门禁和本地 focused 验证复用。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-preflight', + reason: 'Pingora 直连切换前必须保留目标机 env / systemd / 端口预检入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-enable', + reason: 'Pingora 直连启用必须保留可重复 dry-run 行为检查入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-direct-rollback', + reason: 'Pingora 直连回退必须保留可重复 dry-run 行为检查入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-docker', + reason: 'Pingora Nginx handoff 必须保留本机 Docker 可复现 smoke。', + }, + { + file: 'package.json', + includes: 'check:pingora-production-release-build', + reason: 'Pingora release 真实构建烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-release-readiness', + reason: 'Pingora 正式切换前必须保留聚合门禁入口。', + }, + { + file: 'package.json', + includes: 'plan:pingora-direct-cutover', + reason: 'Pingora 直连切换必须保留可审阅 runbook 计划入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-release-readiness-plan', + reason: 'Pingora 聚合门禁计划必须有可重复自检入口。', + }, + { + file: 'package.json', + includes: 'check:production-api-release', + reason: 'API release build 动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:production-health-patrol', + reason: '生产健康巡检动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:production-health-patrol-env', + reason: '生产健康巡检 env 复核动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-health-patrol-env-switch', + reason: 'Pingora health patrol env 切换脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-current-release-audit', + reason: 'Pingora current release 自审必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-status-snapshot', + reason: 'Pingora 切换状态快照脚本必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-evidence-bundle', + reason: 'Pingora 切换证据包必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-cutover-command-evidence', + reason: 'Pingora 切换命令证据必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-access-log-parity', + reason: 'Pingora canary access log 对账必须保留可重复 npm 自测入口。', + }, + { + file: 'package.json', + includes: 'check:pingora-canary-live-guard', + reason: + 'Pingora canary live 参数护栏必须保留单独 npm 入口，方便正式切换前 focused 验证复用。', + }, + { + file: 'package.json', + includes: 'check:production-api-deploy', + reason: 'API deploy 动态烟测必须保留可重复 npm 入口。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-api-release', + reason: 'Pingora 聚合门禁必须执行 API release build 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-production-release-build', + reason: 'Pingora 聚合门禁必须执行真实 Pingora release 构建烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'Pingora production release 真实构建烟测', + reason: 'Pingora 聚合门禁计划自检必须覆盖真实 release 构建烟测。', + }, + { + file: 'scripts/check-pingora-production-release-build.mjs', + includes: '--include-pingora-gateway', + reason: 'Pingora production release 构建烟测必须走真实 include 打包路径。', + }, + { + file: 'scripts/check-pingora-production-release-build.mjs', + includes: 'CARGO_TARGET_DIR', + reason: 'Pingora production release 构建烟测必须使用临时 target，避免污染本地默认 target。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-api-deploy', + reason: 'Pingora 聚合门禁必须执行 API deploy 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'redactSecretArgs(args).join', + reason: 'Pingora 聚合门禁真实执行日志必须隐藏 probe token 参数值。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRunStepLogRedactsDirectProbeToken', + reason: 'Pingora 聚合门禁计划自测必须覆盖真实执行日志 token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-health-patrol', + reason: 'Pingora 聚合门禁必须执行生产健康巡检动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:production-health-patrol-env', + reason: 'Pingora 聚合门禁必须执行生产健康巡检 env 动态烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-health-patrol-env-switch', + reason: 'Pingora 聚合门禁必须执行 health patrol env 切换脚本烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-current-release-audit', + reason: 'Pingora 聚合门禁必须执行 current release 自审烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-status-snapshot', + reason: 'Pingora 聚合门禁必须执行切换状态快照烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-evidence-bundle', + reason: 'Pingora 聚合门禁必须执行切换证据包烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-cutover-command-evidence', + reason: 'Pingora 聚合门禁必须执行切换命令证据烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'Pingora cutover runbook apply 步骤必须使用 current release 随包命令证据脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--expected-executable', + reason: 'Pingora cutover runbook apply 步骤必须把命令证据绑定到预期 current release 脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-arg', + reason: 'Pingora cutover runbook apply 步骤必须让命令证据脚本提前校验 --apply 参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'pingora-direct-rollback-apply', + reason: 'Pingora runbook 自检必须确认 rollback apply 被命令证据脚本归档。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '必须在命令证据脚本中要求真实命令参数', + reason: 'Pingora runbook 自检必须锁住命令证据脚本的 --apply 早失败参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'pingora-direct-enable-apply', + reason: 'Pingora runbook 自检必须确认 enable apply 被命令证据脚本归档。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用命令证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在 enable apply 命令证据生成后立即验真 manifest。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 必须提示用 enable apply 命令证据 bundleDir 执行即时验真。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-summary-ok', + reason: 'Pingora cutover runbook 的即时证据验真必须要求 manifest.summary.status 为 OK。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退命令证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在 rollback apply 命令证据生成后立即验真 manifest。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 必须提示用 rollback apply 命令证据 bundleDir 执行即时验真。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用命令证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 enable apply 命令证据即时验真脚本来源。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '回退命令证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 rollback apply 命令证据即时验真脚本来源。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须先验真 enable apply 命令证据，再切换 health patrol 到 pingora-direct', + reason: 'Pingora runbook 自检必须锁住 enable apply 命令证据即时验真的执行顺序。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须先验真 rollback apply 命令证据，再执行回退后 env 复核', + reason: 'Pingora runbook 自检必须锁住 rollback apply 命令证据即时验真的执行顺序。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-canary-access-log-parity', + reason: 'Pingora 聚合门禁必须执行 canary access log 对账烟测。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-canary-live-guard.mjs', + reason: + 'Pingora 聚合门禁必须执行 canary live 参数护栏，避免正式切换前漏掉脚本输入污染负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'Pingora canary access log 对账烟测', + reason: 'Pingora 聚合门禁计划自检必须覆盖 canary access log 对账烟测。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: '复核失败时 apply 不应写入真实 health patrol env 文件', + reason: 'Pingora health patrol env 切换脚本自测必须覆盖复核失败不落盘。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora health patrol env 切换脚本布尔 env 必须严格校验，避免 clear-public-host 拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsInvalidBoolEnv', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖非法布尔 env 负例。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'DRY_RUN_ENV_FILE_MODE = 0o600', + reason: 'Pingora health patrol env 切换脚本的临时复核 env 必须显式限制为 0600。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertDryRunEnvFileModeIsPrivate', + reason: 'Pingora health patrol env 切换脚本自测必须动态验证临时复核 env 权限。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'runHealthPatrolEnvCheck(checkFile);', + reason: 'Pingora health patrol env 切换脚本必须先复核临时目标 env。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'writeEnvFileAtomically(config.envFile, nextText);', + reason: 'Pingora health patrol env 切换脚本只能在复核通过后原子写入真实 env。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'chownSync(tempFile, currentStat.uid, currentStat.gid);', + reason: + 'Pingora health patrol env 切换脚本原子替换真实 env 时必须保留 owner/group。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertApplyPreservesEnvFileModeAndOwnership', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖 apply 保留 mode 与 owner/group。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '--env-file 不能是符号链接', + reason: + 'Pingora health patrol env 切换脚本 apply 必须拒绝符号链接 env，避免替换链接或写入非预期目标。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora health patrol env 切换脚本必须显式拒绝把路径参数指到 /。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: '--check-script 不能是文件系统根目录', + reason: 'Pingora health patrol env 切换脚本必须拒绝把复核脚本路径指到 /。', + }, + { + file: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + includes: 'function validateNoControlCharacters(value, label)', + reason: + 'Pingora health patrol env 切换脚本必须拒绝 env 路径、check script、public URL 和 Host 中的换行 / NUL。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertApplyRejectsSymlinkEnvFileBeforeWrite', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖符号链接 env 被拒绝且不写目标文件。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsFilesystemRootPaths', + reason: 'Pingora health patrol env 切换脚本自测必须覆盖路径参数指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-health-patrol-env-switch.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeWrite', + reason: + 'Pingora health patrol env 切换脚本自测必须覆盖控制字符参数在 apply 写入前失败。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--release-runtime-only', + reason: + 'Pingora release readiness 必须支持 current release 包内运行时复核模式，避免目标机依赖源码 checkout。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'buildReleaseRuntimeSteps', + reason: 'Pingora release runtime-only 模式必须单独构建包内自包含检查步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'releaseScriptPath', + reason: 'Pingora release runtime-only 模式必须从 current release 解析随包脚本绝对路径。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-live', + reason: 'Pingora 正式切换聚合门禁必须支持目标机 live canary 强制验收。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-live 时必须提供 --live-host', + reason: 'Pingora 强制 live canary 门禁不能跳过正式域名 Host header 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '目标 Nginx live canary access log 对账', + reason: 'Pingora 强制 live canary 门禁必须在 live smoke 后执行真实 access log 对账。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--live-nginx-access-log', + reason: 'Pingora 强制 live canary 门禁必须允许显式指定 Nginx access log。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--live-pingora-access-log', + reason: 'Pingora 强制 live canary 门禁必须允许显式指定 Pingora access log。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'validateSafeAbsoluteFilePath(filePath, label);', + reason: + 'Pingora release readiness 的 live/direct 文件路径必须统一拒绝相对路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts', + reason: 'Pingora release readiness plan 自测必须覆盖 runtime-only 使用 current release 随包脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseRuntimeOnlyRejectsSourceOnlyFlags', + reason: 'Pingora release readiness plan 自测必须覆盖 runtime-only 拒绝源码/本机依赖参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 计划必须在 live smoke 后包含 access log 对账', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live smoke 后真实日志对账。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 的 Nginx access log 不能接受相对路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live 日志路径负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireLiveRejectsFilesystemRootAccessLog', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 live 日志路径指向文件系统根目录的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-direct', + reason: 'Pingora 正式切换聚合门禁必须支持目标机直连入口强制验收。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--dry-run-cutover', + reason: 'Pingora 正式直连切换必须支持只打印可审阅 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换 Host 一致性确认', + reason: 'Pingora cutover runbook 必须输出 Host 一致性确认步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'assertSameCutoverHostname', + reason: 'Pingora cutover runbook 必须在生成阶段阻断 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--rollback-pingora-shadow-probe-url', + reason: 'Pingora cutover runbook 必须支持把可选 shadow probe 参数传给 rollback。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--rollback-health-patrol-public-base-url', + reason: + 'Pingora cutover runbook 必须显式声明回退后恢复的 Nginx public base URL。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '避免回退 runbook 覆盖现场原有 Nginx 巡检入口', + reason: + 'Pingora cutover runbook 不能把回退后的 health patrol public base URL 隐式写成仓库默认值。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '', + reason: 'Pingora cutover runbook 输出不能泄露 probe token 原文。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'Pingora direct enable apply', + reason: 'Pingora cutover runbook 必须显式列出 direct enable apply 步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'Pingora direct rollback apply', + reason: 'Pingora cutover runbook 必须显式列出失败回退 apply 步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换后 health patrol 切到 Pingora direct', + reason: + 'Pingora cutover runbook 必须提醒直连启用后把健康巡检切到 pingora-direct。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退前 health patrol 预置回 Nginx', + reason: + 'Pingora cutover runbook 必须在 rollback apply 前提醒把健康巡检预置回 nginx。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'pingora-health-patrol-env-switch.mjs', + reason: + 'Pingora cutover runbook 必须使用随包脚本切换 health patrol env，不能只给手工编辑提示。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前 current release 自包含自审', + reason: 'Pingora cutover runbook 必须先审 current release 自包含能力。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前状态快照', + reason: + 'Pingora cutover runbook 必须在启用前采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换前状态快照证据包', + reason: + 'Pingora cutover runbook 必须在启用前归档状态快照证据包。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-evidence-output-root', + reason: 'Pingora cutover runbook 必须显式固定证据包输出根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用后状态快照', + reason: + 'Pingora cutover runbook 必须在启用后采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--run-direct-live', + reason: + 'Pingora cutover runbook 启用后证据包必须归档 direct live 和 access log request_id 证据。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用后状态快照证据包必须归档 direct live smoke 证据', + reason: + 'Pingora runbook 自检必须锁住 post-enable 证据包的 direct live 归档。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '证据 manifest 只读验真', + reason: 'Pingora cutover runbook 必须在每个证据包阶段后提示只读验真。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '证据验真必须使用 current release 随包 verifier', + reason: 'Pingora runbook 自检必须锁住 current release 随包证据验真脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '证据验真必须要求 manifest.summary.status=OK', + reason: 'Pingora runbook 自检必须锁住即时证据验真的 summary OK 要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverPlanAvoidsManualEvidenceEscapeHatches', + reason: 'Pingora runbook 自检必须确认正式切换计划不携带人工证据放行参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--allow-extra-root-entries', + reason: 'Pingora runbook 自检必须显式覆盖证据根目录额外条目放行参数不得进入正式计划。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'findExtraEvidenceEntries', + reason: 'Pingora 证据验真脚本必须扫描 manifest.files 未登记的额外条目。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '--require-summary-ok', + reason: 'Pingora 证据验真脚本必须支持正式 runbook 的 summary OK 强制模式。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'manifest.summary.status is not OK', + reason: 'Pingora 证据验真脚本必须在 summary 非 OK 时给出结构化失败原因。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra file not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记普通文件。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra directory not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记目录。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'extra symlink not registered in manifest.files', + reason: 'Pingora 证据验真脚本必须默认拒绝未登记符号链接。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '--allow-extra-files', + reason: 'Pingora 证据验真脚本允许额外条目时必须显式使用例外开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '正式切换归档不应使用', + reason: 'Pingora 证据验真脚本必须声明额外条目例外不属于正式归档口径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 证据验真脚本必须拒绝带换行或 NUL 的入口路径和 manifest 文件名。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-verify.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 证据验真脚本必须对控制字符路径给出明确错误。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraFileFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记普通文件默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraDirectoryFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记目录默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertExtraSymlinkFailsByDefault', + reason: 'Pingora 证据验真自测必须覆盖未登记符号链接默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertAllowExtraFilesOverrideSucceeds', + reason: 'Pingora 证据验真自测必须覆盖显式允许额外条目的人工例外。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRequireSummaryOkFailsForCriticalManifest', + reason: 'Pingora 证据验真自测必须覆盖 summary 非 OK 时 --require-summary-ok 失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRequireSummaryOkFailsWhenSummaryMissing', + reason: 'Pingora 证据验真自测必须覆盖缺少 summary 时 --require-summary-ok 失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRejectsEntryPathControlCharacters', + reason: 'Pingora 证据验真自测必须覆盖入口路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-verify.mjs', + includes: 'assertRejectsManifestFilePathControlCharacters', + reason: 'Pingora 证据验真自测必须覆盖 manifest 文件 path 带控制字符的负例。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'manifest.files` 视为闭集', + reason: 'Pingora 试点文档必须说明证据验真默认闭集归档口径。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: 'manifest.files` 视为闭集', + reason: '生产运维文档必须说明证据验真默认闭集归档口径。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: 'manifest.files` 视为闭集', + reason: '团队共享决策必须记录 Pingora 证据验真闭集归档口径。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: 'manifest.files` 视为闭集', + reason: '团队共享踩坑必须记录 Pingora 证据目录不能夹带未登记条目。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '切换证据根目录三阶段总审计', + reason: 'Pingora cutover runbook 必须在三阶段证据后执行根目录总审计。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'enable-apply:pingora-direct-enable-apply', + reason: + 'Pingora cutover runbook 总审计必须要求 direct enable apply 命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollback-apply:pingora-direct-rollback-apply', + reason: + 'Pingora cutover runbook 总审计必须要求 direct rollback apply 命令证据。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须使用 current release 随包 evidence audit 脚本', + reason: 'Pingora runbook 自检必须锁住 current release 随包证据根目录审计脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换命令证据', + reason: 'Pingora runbook 自检必须锁住总审计里的 enable / rollback 命令证据要求。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-command-executable', + reason: 'Pingora cutover runbook 总审计必须要求 enable / rollback 命令证据的真实脚本身份。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换脚本身份', + reason: 'Pingora runbook 自检必须锁住总审计里的 enable / rollback 命令脚本身份要求。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-command-arg', + reason: 'Pingora cutover runbook 总审计必须要求 enable / rollback 命令证据包含 --apply 参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式要求真实切换命令必需参数', + reason: 'Pingora runbook 自检必须锁住总审计里的 enable / rollback 命令参数要求。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-run-id', + reason: 'Pingora cutover runbook 必须公开并传递切换批次 ID。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-cutover-run-id', + reason: 'Pingora cutover runbook 最终总审计必须要求同一切换批次 ID。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverPlanCarriesCutoverRunId', + reason: 'Pingora runbook 自检必须覆盖切换批次 ID 贯穿所有证据步骤。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '不会修改证据目录', + reason: 'Pingora 证据根目录审计脚本必须声明只读边界。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command :', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command-executable ::', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令真实脚本身份。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: + '--require-command-executable executable 不能包含换行或 NUL 字符', + reason: + 'Pingora 证据根目录审计脚本必须拒绝带控制字符的命令脚本身份要求。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora 证据根目录审计脚本必须拒绝带换行或 NUL 的根目录和 verifier 路径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '!/[\\0\\r\\n]/u.test(value)', + reason: + 'Pingora 证据根目录审计脚本必须拒绝 manifest 文件名里的控制字符。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'command executable 带控制字符必须失败', + reason: + 'Pingora 证据根目录审计自测必须覆盖命令脚本身份要求带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根目录带控制字符必须失败', + reason: 'Pingora 证据根目录审计自测必须覆盖 evidence-root 控制字符负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'verifier 路径带控制字符必须失败', + reason: 'Pingora 证据根目录审计自测必须覆盖 verify-script 控制字符负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertManifestFileNameControlCharacterFails', + reason: 'Pingora 证据根目录审计自测必须覆盖 manifest 文件名控制字符负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-command-arg ::', + reason: 'Pingora 证据根目录审计脚本必须支持强制审计切换命令必需参数。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'mergeRequiredCommands', + reason: 'Pingora 命令脚本身份要求必须隐式纳入命令证据审计，不能被静默忽略。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.expectedExecutable 必须是', + reason: 'Pingora 证据根目录审计脚本必须阻断命令证据 expectedExecutable 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'command.executable 必须是', + reason: 'Pingora 证据根目录审计脚本必须阻断命令证据真实 executable 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'readCommandRecord', + reason: 'Pingora 证据根目录审计脚本必须读取独立 command-record.json 做语义复核。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'compareCommandRecords', + reason: 'Pingora 证据根目录审计脚本必须阻断 manifest.command 与 command-record.json 漂移。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'stdoutPath', + reason: 'Pingora 证据根目录审计脚本必须复核命令 stdout 文件引用一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'stderrPath', + reason: 'Pingora 证据根目录审计脚本必须复核命令 stderr 文件引用一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'jsonValuesEqual', + reason: 'Pingora 证据根目录审计脚本必须复核命令参数数组一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'args 不能包含换行或 NUL 字符', + reason: + 'Pingora 证据根目录审计脚本必须拒绝被控制字符污染的命令参数证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'scalarFields', + reason: 'Pingora 证据根目录审计脚本必须复核脱敏命令字符串一致性。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "'command'", + reason: 'Pingora 证据根目录审计脚本必须把脱敏命令字符串纳入一致性比较。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'checkCommandRecordTimes', + reason: 'Pingora 证据根目录审计脚本必须校验命令记录 startedAt / finishedAt / durationMs。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: "'--require-summary-ok'", + reason: 'Pingora 证据根目录审计脚本复用 verifier 时必须启用 strict summary 模式。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.summary.status 必须是 OK', + reason: 'Pingora 证据根目录审计脚本必须阻断 summary 非 OK 的阶段或命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'candidate.commandName === null', + reason: 'Pingora 证据根目录审计脚本必须防止命令证据冒充阶段证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.summary.exitCode 必须是 0', + reason: 'Pingora 证据根目录审计脚本必须阻断退出码非 0 的命令证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'CUTOVER_TIMELINE_ORDER', + reason: 'Pingora 证据根目录审计脚本必须内置标准直连切换时间线顺序。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '疑似混入不同切换窗口证据', + reason: 'Pingora 证据根目录审计脚本必须阻断标准切换时间线倒序证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000', + reason: 'Pingora 证据根目录审计脚本必须默认限制标准切换证据窗口为 24 小时。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--timeline-max-span-ms', + reason: 'Pingora 证据根目录审计脚本必须允许切换窗口显式覆盖最大时间跨度。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--require-cutover-run-id', + reason: 'Pingora 证据根目录审计脚本必须能按切换批次 ID 过滤证据。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'manifest.cutoverRunId', + reason: 'Pingora 证据根目录审计脚本必须读取并校验 manifest.cutoverRunId。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '--allow-extra-root-entries', + reason: 'Pingora 证据根目录审计脚本允许额外根目录条目时必须显式使用例外开关。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '正式切换归档不应使用', + reason: 'Pingora 证据根目录审计脚本必须声明额外根目录条目例外不属于正式归档口径。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '证据根目录只能包含证据目录，发现普通文件', + reason: 'Pingora 证据根目录审计脚本必须默认拒绝根目录普通文件。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '证据根目录只能包含带 manifest.json 的证据目录', + reason: 'Pingora 证据根目录审计脚本必须默认拒绝无 manifest 的根目录子目录。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootFileFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖根目录普通文件默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootDirectoryFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖无 manifest 根目录子目录默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertExtraRootSymlinkFailsByDefault', + reason: 'Pingora 证据根目录审计自测必须覆盖根目录符号链接默认失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertAllowExtraRootEntriesSucceeds', + reason: 'Pingora 证据根目录审计自测必须覆盖显式允许额外根目录条目的人工例外。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandExecutableImplicitlyRequiresCommand', + reason: 'Pingora 证据根目录审计自测必须覆盖命令脚本身份参数隐式要求命令证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandExecutableMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令脚本身份漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandArgImplicitlyRequiresCommand', + reason: 'Pingora 证据根目录审计自测必须覆盖命令必需参数隐式要求命令证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertRequiredCommandArgMissingFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据缺少 --apply 参数负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.command 漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordTimelineMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令记录时间线漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordOutputPathMismatchFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.files 的 stdout/stderr 引用漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordArgsMismatchFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖 command-record.json 与 manifest.command 的参数漂移负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandRecordArgsControlCharacterFails', + reason: + 'Pingora 证据根目录审计自测必须覆盖命令参数证据带控制字符的负例。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: '时间跨度', + reason: 'Pingora 证据根目录审计脚本必须阻断标准切换时间线跨度过大的证据。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCriticalPhaseSummaryFails', + reason: 'Pingora 证据根目录审计自测必须覆盖阶段证据 summary 非 OK 负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCommandBundleCannotSatisfyPhaseRequirement', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据不能满足阶段证据要求。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '必须让 verifier 启用 --require-summary-ok', + reason: 'Pingora 证据根目录审计自测必须覆盖总审计调用 verifier 的 strict summary 模式。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertFailedCommandSummaryFails', + reason: 'Pingora 证据根目录审计自测必须覆盖命令证据退出失败负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertOutOfOrderCutoverTimelineFails', + reason: 'Pingora 证据根目录审计自测必须覆盖标准切换时间线倒序负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertLongCutoverTimelineFails', + reason: 'Pingora 证据根目录审计自测必须覆盖标准切换时间线跨度过大负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: 'assertCutoverRunIdMismatchFails', + reason: 'Pingora 证据根目录审计自测必须覆盖切换批次 ID 不匹配负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--cutover-evidence-timeline-max-span-ms', + reason: 'Pingora cutover runbook 必须公开证据时间线最大跨度参数。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '三阶段总审计必须显式传标准切换时间线最大跨度', + reason: 'Pingora runbook 自检必须锁住总审计的时间线最大跨度参数。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退后状态快照', + reason: + 'Pingora cutover runbook 必须在回退后采集只读状态快照。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '回退后 health patrol env Nginx 模式复核', + reason: + 'Pingora cutover runbook 必须在回退后复核 health patrol env 已回到 Nginx 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包自审脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包 health patrol env 切换脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包切换状态快照脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'Pingora runbook 自检必须覆盖 current release 随包切换证据包脚本。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsRelativeEvidenceOutputRoot', + reason: 'Pingora runbook 自检必须覆盖证据包输出根目录相对路径负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsFilesystemRootReleaseRoot', + reason: 'Pingora runbook 自检必须覆盖 current release 根目录指向 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDryRunCutoverRejectsFilesystemRootEvidenceOutputRoot', + reason: 'Pingora runbook 自检必须覆盖证据包输出根目录指向 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'isFilesystemRootPath', + reason: 'Pingora runbook 参数校验必须显式拒绝文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertUsageCutoverExampleIncludesExplicitEvidenceOutputRoot', + reason: + 'Pingora runbook usage 示例必须显式展示证据包输出根目录，避免值班复制命令时依赖隐藏默认值。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertUsageExamplesIncludeDirectPingoraAccessLog', + reason: + 'Pingora release readiness usage 示例必须显式展示 direct access log 参数，避免值班复制命令后被 --require-direct 拦住。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--clear-public-host', + reason: 'Pingora runbook 自检必须覆盖回退后清空 health patrol public Host。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含回退后 health patrol env Nginx 模式复核', + reason: 'Pingora runbook 自检必须覆盖回退后的健康巡检 env 机器复核步骤。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'cutoverReleaseRoot', + reason: 'Pingora cutover runbook 必须允许固定 current release 根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-health-patrol-env-file', + reason: 'Pingora 强制直连门禁必须允许指定 health patrol env 文件做复核。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '目标 health patrol env 直连模式复核', + reason: 'Pingora 强制直连门禁必须复核 health patrol 已切到 direct 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-health-patrol-env-file', + reason: 'Pingora 强制直连门禁不能跳过 health patrol env 复核。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'Pingora 直连入口门禁必须先执行 direct-entry 预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-env-file', + reason: 'Pingora 直连入口门禁必须允许指定目标机 env 文件做预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-preflight-env-file', + reason: 'Pingora 强制直连门禁不能跳过目标机 env 预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-preflight-systemd', + reason: 'Pingora 强制直连门禁不能跳过 systemd drop-in 生效核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-cert-readable', + reason: 'Pingora 强制直连门禁不能跳过 TLS 证书和私钥可读核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-env-file', + reason: + 'Pingora 强制直连门禁不能跳过 service EnvironmentFile 与 preflight env 一致性核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-user-cert-readable', + reason: + 'Pingora 强制直连门禁不能跳过 systemd 服务用户读取 TLS 证书和私钥核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-service-binary-executable', + reason: + 'Pingora 强制直连门禁不能跳过 current release service 二进制可执行核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: + '启用 --require-direct 时必须提供 --direct-preflight-check-ports-free', + reason: 'Pingora 强制直连门禁不能跳过 80/443 端口释放核验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-http-base-url', + reason: 'Pingora 强制直连门禁不能跳过 HTTP redirect / ACME live 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-host', + reason: + 'Pingora 强制直连门禁不能跳过正式域名 Host header 和 TLS SNI 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-redirect-host', + reason: 'Pingora 强制直连门禁不能跳过 HTTP redirect Location host 验证。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-pingora-access-log', + reason: 'Pingora 强制直连门禁不能跳过 direct access log 落盘校验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: "validateSafeAbsoluteFilePath(\n result.directPingoraAccessLog,\n '--direct-pingora-access-log'", + reason: + 'Pingora 强制直连门禁必须拒绝相对 access log 路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--http-base-url', + reason: + 'Pingora 直连入口门禁必须把 HTTP redirect / ACME base URL 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--host', + reason: + 'Pingora 直连入口门禁必须把正式域名 Host/SNI 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--redirect-host', + reason: + 'Pingora 直连入口门禁必须把 redirect Location host 传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--pingora-access-log', + reason: + 'Pingora 直连入口门禁必须把 access log 路径传给 direct live smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-systemd', + reason: 'Pingora 直连入口门禁必须支持检查 systemd drop-in 是否生效。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-user-cert-readable', + reason: 'Pingora 直连入口门禁必须支持检查 systemd 服务用户可读证书。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-env-file', + reason: + 'Pingora 直连入口门禁必须支持检查 service EnvironmentFile 与 preflight env 一致。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-service-binary-executable', + reason: 'Pingora 直连入口门禁必须支持检查 service ExecStart 二进制可执行。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-preflight-check-ports-free', + reason: 'Pingora 直连入口门禁必须支持检查 TLS/HTTP redirect 端口已释放。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-spacetime-database', + reason: + 'Pingora 直连入口门禁必须允许指定 WSS subscribe 的目标 SpacetimeDB 库。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时必须提供 --direct-spacetime-database', + reason: 'Pingora 强制直连门禁不能让 WSS subscribe 隐式使用默认数据库名。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时不能使用 --direct-skip-wss', + reason: 'Pingora 强制直连门禁不能允许正式直连跳过 WSS subscribe。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '启用 --require-direct 时不能使用 --direct-insecure-tls', + reason: 'Pingora 强制直连门禁不能允许生产直连跳过 TLS 信任校验。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--direct-require-wss-upgrade', + reason: 'Pingora 直连入口门禁必须支持强制要求 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'result.directRequireWssUpgrade = true', + reason: + 'Pingora --require-direct 硬门禁默认必须强制 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-canary-docker.mjs', + reason: 'Pingora 正式切换聚合门禁必须包含 Docker Nginx handoff smoke。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-route-parity', + reason: 'Pingora 正式切换聚合门禁必须包含 Nginx / Pingora 路由 parity。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-rollback', + reason: 'Pingora 正式切换聚合门禁必须包含直连回退 dry-run 行为检查。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-preflight', + reason: 'Pingora 正式切换聚合门禁必须包含直连入口静态预检。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-direct-enable', + reason: 'Pingora 正式切换聚合门禁必须包含直连启用 dry-run 行为检查。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'check:pingora-release-readiness-plan', + reason: 'Pingora 正式切换聚合门禁必须自检关键计划语义。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把 HTTP redirect / ACME 入口纳入 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 HTTP redirect / ACME 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 必须把正式域名 Host 传给 live canary smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-live 的正式域名 Host 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 direct enable apply', + reason: 'Pingora 聚合门禁计划自检必须覆盖直连启用 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 rollback apply', + reason: 'Pingora 聚合门禁计划自检必须覆盖失败回退 runbook。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 缺少 --require-direct 时必须失败', + reason: 'Pingora cutover runbook 不能在缺少直连硬门禁参数时生成。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 必须包含 Host 一致性确认步骤', + reason: 'Pingora cutover runbook 必须显式展示 direct / health patrol / rollback Host 一致性。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover redirect Host 与 direct Host 不一致时必须失败', + reason: 'Pingora cutover runbook 必须阻断 redirect Host 与正式入口 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback smoke Host 与 direct Host 不一致时必须失败', + reason: 'Pingora cutover runbook 必须阻断 rollback smoke Host 与正式入口 Host 漂移。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback 步骤必须传递 shadow probe URL', + reason: 'Pingora cutover runbook 自检必须覆盖可选 rollback shadow probe 传递。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover JSON 不能泄露 shadow probe token 原文', + reason: 'Pingora cutover runbook 自检必须覆盖 shadow probe token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run plan JSON 不能泄露 direct probe token 原文', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct probe token 脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '启用后 release readiness 直连复核必须继续传递 direct probe token 参数', + reason: + 'Pingora cutover runbook 自检必须覆盖启用后 readiness 复核继续检查 direct probe。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover 启用前基础门禁必须使用 current release 包内运行时复核模式', + reason: 'Pingora cutover runbook 自检必须覆盖启用前 readiness 使用 runtime-only 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '启用后 release readiness 直连复核必须使用 current release 包内运行时复核模式', + reason: 'Pingora cutover runbook 自检必须覆盖启用后 readiness 使用 runtime-only 模式。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '启用后 release readiness 直连复核必须使用 current release 随包 release readiness 聚合门禁脚本', + reason: + 'Pingora cutover runbook 自检必须覆盖启用后 readiness 从 current release 随包脚本执行。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + 'dry-run cutover 启用前基础门禁必须使用 current release 随包 release readiness 聚合门禁脚本', + reason: + 'Pingora cutover runbook 自检必须覆盖启用前 readiness 从 current release 随包脚本执行。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover JSON 不能泄露 direct probe token 原文', + reason: + 'Pingora cutover runbook 自检必须覆盖 direct probe token 在切换计划中脱敏。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback shadow probe URL 缺少 token 时必须失败', + reason: 'Pingora cutover runbook 自检必须覆盖 shadow probe 参数成对出现。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须把正式域名 Host/SNI 传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的正式域名 Host/SNI 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把显式 redirect Location host 传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 redirect Location host 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 systemd drop-in 最终生效配置', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 systemd drop-in 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查当前用户可读 TLS 证书和私钥', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的证书可读硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 service EnvironmentFile 包含本次 env', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 service env 一致性硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 systemd 服务用户可读 TLS 证书和私钥', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的服务用户证书可读硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须检查 Pingora service ExecStart 二进制可执行', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 service 二进制可执行硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须检查 TLS/HTTP redirect 端口已释放', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的端口释放硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把显式 SpacetimeDB 数据库名传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的显式 WSS 数据库名。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: + '--require-direct 必须把 Pingora access log 落盘校验传给 direct live smoke', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 Pingora access log 硬要求。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct HTTP base URL 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 direct HTTP base URL 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 缺少 live Host 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 live Host 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-live 的 live Host 不能接受 URL 或路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 live Host 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct Host/SNI 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少正式域名 Host/SNI 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 的 direct Host/SNI 不能接受 URL 或路径', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct Host/SNI 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct redirect Location host 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 redirect Location host 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 的 redirect Location host 不能接受 URL 或路径', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 redirect Location host 格式负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 systemd preflight 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 systemd preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少证书可读 preflight 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 service EnvironmentFile preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 service env 一致性 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少服务用户证书可读 preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少服务用户证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 service 二进制可执行 preflight 时必须失败', + reason: + 'Pingora 聚合门禁计划自检必须覆盖缺少 service 二进制可执行 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少端口释放 preflight 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少端口释放 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 SpacetimeDB 数据库名时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少显式 WSS 数据库名的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 缺少 direct Pingora access log 时必须失败', + reason: 'Pingora 聚合门禁计划自检必须覆盖缺少 direct access log 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 不能允许跳过 WSS subscribe', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct skip WSS 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 不能允许 insecure TLS', + reason: 'Pingora 聚合门禁计划自检必须覆盖 direct insecure TLS 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '--require-direct 必须自动强制 WSS subscribe 返回 101', + reason: + 'Pingora 聚合门禁计划自检必须覆盖 --require-direct 的 WSS 101 硬要求。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--require-docker', + reason: + 'Pingora Docker canary smoke 必须支持在 CI / 目标 agent 上强制执行。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'Pingora Docker canary smoke 必须复用 live canary 断言，避免双套验收漂移。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--realpath', + reason: + 'Pingora Docker canary smoke 必须覆盖真实路径 canary handoff。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'genarrative-pingora-realpath-canary.conf', + reason: + 'Pingora Docker canary smoke 必须加载真实路径 canary snippet。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '--network', + reason: + 'Pingora Docker canary smoke 必须真实启动 Nginx 容器验证 handoff 链路。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'log_format genarrative_upstream', + reason: + 'Pingora Docker canary smoke 必须按生产 Nginx access log 口径写 request_id。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: 'scripts/check-pingora-canary-access-log-parity.mjs', + reason: + 'Pingora Docker canary smoke 必须复用真实 access log 对账脚本验证 handoff。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/api/creation-entry/config', + reason: + 'Pingora Docker canary access log 对账必须覆盖代表性 API canary 路径。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/v1/identity', + reason: + 'Pingora Docker canary access log 对账必须覆盖 SpacetimeDB identity 路径。', + }, + { + file: 'scripts/check-pingora-canary-docker.mjs', + includes: '/__genarrative_pingora_canary/assets/app.js', + reason: 'Pingora Docker canary access log 对账必须覆盖静态资源路径。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: '该脚本只读比较 Nginx canary handoff access log 和 Pingora access log', + reason: 'Pingora canary access log 对账脚本必须说明只读边界。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'request_id', + reason: 'Pingora canary access log 对账必须按 request_id 串联 Nginx 与 Pingora 日志。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'realpath', + reason: 'Pingora canary access log 对账必须支持真实路径 canary 模式。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: '/__genarrative_pingora/healthz', + reason: 'Pingora canary access log 对账必须覆盖 canary healthz 到 shadow healthz 的真实映射。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora canary access log 对账 since-lines 必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'validateSafeAbsoluteFilePath(file, label);', + reason: 'Pingora canary access log 对账必须拒绝相对日志路径和文件系统根目录。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity.mjs', + includes: 'function validateNoControlCharacters(value, label)', + reason: + 'Pingora canary access log 对账必须拒绝日志路径、前缀、必需路径和 since-lines 中的换行 / NUL。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsFilesystemRootLogPaths', + reason: 'Pingora canary access log 对账自测必须覆盖日志路径指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsLogPathControlCharacters', + reason: 'Pingora canary access log 对账自测必须覆盖日志路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsPrefixAndPathControlCharacters', + reason: + 'Pingora canary access log 对账自测必须覆盖 canary prefix 和必需路径带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsParsedLogPathControlCharacters', + reason: + 'Pingora canary access log 对账自测必须覆盖日志行解析出的 path 带控制字符的负例。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: '缺少对应 Pingora access log', + reason: 'Pingora canary access log 对账自测必须覆盖缺少 Pingora 日志的失败场景。', + }, + { + file: 'scripts/check-pingora-canary-access-log-parity-check.mjs', + includes: 'assertRejectsInvalidSinceLines', + reason: + 'Pingora canary access log 对账自测必须覆盖非法 since-lines 参数和 env 负例。', + }, + { + file: 'deploy/env/pingora-canary-live.env.example', + includes: 'GENARRATIVE_PINGORA_CANARY_BASE_URL', + reason: 'Pingora canary live smoke 必须有可复制的环境变量示例。', + }, + { + file: 'deploy/env/pingora-canary-live.env.example', + includes: '不要填 https:// URL', + reason: 'Pingora canary live smoke 环境示例必须提示 Host 只填 host。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL', + reason: 'Pingora direct live smoke 必须有可复制的环境变量示例。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL', + reason: + 'Pingora direct live smoke 环境示例必须暴露 HTTP redirect base URL。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST', + reason: + 'Pingora direct live smoke 环境示例必须暴露 HTTP redirect Location host。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '只填域名或 host:port', + reason: 'Pingora direct live smoke 环境示例必须提示 Host 参数格式。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE', + reason: + 'Pingora direct live smoke 环境示例必须暴露 WSS subscribe 使用的数据库名。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE', + reason: 'Pingora direct live smoke 环境示例必须暴露 WSS 101 强制开关。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct', + reason: 'Pingora direct live env 示例必须提醒直连后切换健康巡检模式。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'current release 自审', + reason: + 'Pingora direct live env 示例必须提醒直连启用 apply 前先做 current release 自审。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-env-file', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含 service EnvironmentFile 一致性预检。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-user-cert-readable', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含服务用户证书可读预检。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: '--preflight-check-service-binary-executable', + reason: + 'Pingora direct live env 示例中的正式启用命令必须包含 service 二进制可执行预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: + 'plan:pingora-direct-cutover` 会在 JSON runbook 中列出 direct / nginx 两个方向的 env 切换脚本和 env 复核步骤', + reason: + 'Nginx / Pingora 运维文档必须说明 cutover runbook 包含健康巡检切换脚本与复核步骤。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--rollback-health-patrol-public-base-url', + reason: + 'Nginx / Pingora 运维文档必须提醒 runbook 显式声明回退后的 Nginx 巡检入口。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-env-file', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含 service EnvironmentFile 一致性预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-user-cert-readable', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含服务用户证书可读预检。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--preflight-check-service-binary-executable', + reason: + 'Nginx / Pingora 运维文档中的正式启用示例必须包含 service 二进制可执行预检。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'X-Genarrative-Nginx-Handoff', + reason: 'Pingora canary live smoke 必须校验 Nginx handoff 响应头。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '/__genarrative_pingora_canary', + reason: 'Pingora canary live smoke 必须默认检查前缀 canary 入口。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '/__genarrative_pingora_realpath_canary/healthz', + reason: 'Pingora canary live smoke 必须支持真实路径 canary healthz 探针。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: '--require-realpath-live', + reason: 'Pingora 聚合门禁必须支持把真实路径 canary 纳入目标机 live 复核。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: '目标 Nginx realpath canary access log 对账', + reason: 'Pingora 聚合门禁计划自检必须覆盖真实路径 canary access log 对账。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'validateHostOption(result.host,', + reason: 'Pingora canary live smoke 必须提前拒绝非法 Host header 参数。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora canary live smoke timeout 参数必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora canary live smoke 必须在发起 canary 请求前拒绝带控制字符的 base URL、prefix、Host、path 和 timeout。', + }, + { + file: 'scripts/check-pingora-canary-live.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora canary live smoke 必须对带换行或 NUL 的现场参数给出明确错误。', + }, + { + file: 'scripts/check-pingora-canary-live-guard.mjs', + includes: 'assertRejectsControlCharacter', + reason: + 'Pingora canary live guard 必须动态覆盖控制字符参数在发起 canary 请求前失败。', + }, + { + file: 'scripts/check-pingora-canary-live-guard.mjs', + includes: '--prefix', + reason: 'Pingora canary live guard 必须覆盖 canary prefix 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'X-Genarrative-Gateway', + reason: 'Pingora direct live smoke 必须确认响应来自 Pingora 网关。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'http-api-redirect', + reason: 'Pingora direct live smoke 必须覆盖 HTTP 到 HTTPS 重定向。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-http2-alpn', + reason: 'Pingora direct live smoke 必须覆盖 HTTPS HTTP/2 ALPN。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'v2.bsatn.spacetimedb', + reason: + 'Pingora direct live smoke 必须覆盖 SpacetimeDB SDK 默认 WSS 子协议透传。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'wss-spacetime-subscribe', + reason: 'Pingora direct live smoke 必须覆盖 WSS subscribe 握手。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '--require-wss-upgrade', + reason: + 'Pingora direct live smoke 必须支持强制要求 WSS subscribe 返回 101。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateHostOption(result.redirectHost,', + reason: + 'Pingora direct live smoke 必须提前拒绝非法 redirect Location host 参数。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'parseRequiredPositiveInt', + reason: + 'Pingora direct live smoke timeout 参数必须严格校验，不能静默回默认值。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora direct live smoke 必须在发起 HTTPS / HTTP / WSS 请求前拒绝带控制字符的 URL、Host、token、路径和日志参数。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora direct live smoke 必须对带换行或 NUL 的现场参数给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateSafeAbsoluteFilePath', + reason: + 'Pingora direct live smoke 的 access log 路径必须统一复核绝对路径、根目录和控制字符。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora direct live smoke 布尔 env 必须严格校验，避免拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: '不能是文件系统根目录', + reason: 'Pingora direct live smoke 必须拒绝 Pingora access log 指向文件系统根目录。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: 'assertRejectsControlCharacter', + reason: + 'Pingora direct live guard 必须动态覆盖控制字符参数在发起 live 请求前失败。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: '--probe-token', + reason: 'Pingora direct live guard 必须覆盖 probe token 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-live-guard.mjs', + includes: '--pingora-access-log', + reason: 'Pingora direct live guard 必须覆盖 access log 路径控制字符负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'scripts/check-pingora-direct-live-guard.mjs', + reason: + 'Pingora 聚合 readiness 必须执行 direct live 参数护栏，避免正式切换前漏掉脚本输入污染负例。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'readBoolEnv(', + reason: + 'Pingora release readiness 直连布尔 env 必须统一严格解析，避免拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectLiveRejectsInvalidBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct live 非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertReleaseReadinessRejectsInvalidDirectBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 release readiness 非法直连布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootPingoraAccessLog', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 Pingora access log 指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootPreflightEnvFile', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 preflight env 文件指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertRequireDirectRejectsFilesystemRootHealthPatrolEnvFile', + reason: + 'Pingora release readiness plan 自测必须覆盖直连 health patrol env 文件指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectLiveRejectsFilesystemRootAccessLog', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct live 单脚本日志路径指到 / 的负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectPreflightRejectsInvalidBoolEnv', + reason: + 'Pingora release readiness plan 自测必须覆盖 direct preflight 非法布尔 env 负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'assertDirectPreflightRejectsUnconfirmedMultiInstanceProtection', + reason: + 'Pingora release readiness plan 自测必须覆盖未确认共享保护的多实例 direct preflight 负例。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora direct preflight 必须检查低端口 capability drop-in。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '必须是布尔值 true/false 或 1/0', + reason: + 'Pingora direct preflight 布尔 env 必须严格校验，避免检查项拼写错误静默降级。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + reason: 'Pingora direct preflight 必须检查直连 TLS env。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-ports-free', + reason: 'Pingora direct preflight 必须支持检查 80/443 是否已释放。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-user-cert-readable', + reason: 'Pingora direct preflight 必须支持检查 systemd 服务用户可读证书。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-env-file', + reason: + 'Pingora direct preflight 必须支持检查 service EnvironmentFile 包含本次 env。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '--check-service-binary-executable', + reason: + 'Pingora direct preflight 必须支持检查 service ExecStart 二进制可执行。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'assertProtectionInstanceBoundary', + reason: + 'Pingora direct preflight 必须检查多实例与进程内接流保护的边界。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: + 'Pingora direct preflight 多实例保护失败必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'ExecStart=', + reason: + 'Pingora direct preflight 必须从 systemd service ExecStart 推导 current release 二进制路径。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'sudo', + reason: + 'Pingora direct preflight 必须能以 systemd 服务用户身份验证 cert/key 可读性。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'fileURLToPath(import.meta.url)', + reason: + 'Pingora direct preflight 默认路径必须从脚本所在 release root 推导，不能依赖 Jenkins 工作区。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'validateNoControlCharacters', + reason: + 'Pingora direct preflight 必须在执行 systemctl 或 sudo 前拒绝带控制字符的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora direct preflight 必须明确拒绝带换行或 NUL 的路径和子命令参数。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'function runCommand(command, args)', + reason: + 'Pingora direct preflight 的 systemctl / sudo 子命令必须经统一执行前参数校验。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: "validateSafeAbsoluteFilePath(result.envFile, '--env-file')", + reason: 'Pingora direct preflight 必须拒绝 --env-file 指向文件系统根目录。', + }, + { + file: 'scripts/check-pingora-direct-preflight-guard.mjs', + includes: 'assertRejectsEnvFileWithControlCharacters', + reason: 'Pingora direct preflight 自测必须覆盖 --env-file 控制字符负例。', + }, + { + file: 'scripts/check-pingora-direct-preflight-guard.mjs', + includes: 'assertRejectsSystemdServiceWithControlCharactersBeforeSystemctl', + reason: + 'Pingora direct preflight 自测必须覆盖 systemd service 控制字符在执行 systemctl 前失败。', + }, + { + file: 'package.json', + includes: 'node scripts/check-pingora-direct-preflight-guard.mjs', + reason: 'Pingora direct preflight npm 自测入口必须执行控制字符负例护栏。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: + 'Pingora direct preflight 必须能从 release root 读取 deploy/pingora 配置。', + }, + { + file: 'scripts/check-pingora-direct-preflight.mjs', + includes: + '公网直连 Pingora 时 GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR 必须保持 false', + reason: + 'Pingora direct preflight 必须阻断公网直连时误信任 X-Forwarded-For。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertDirectPreflightRejectsPublicForwardedForTrust', + reason: + 'Pingora direct preflight 自测必须覆盖公网直连误信任 X-Forwarded-For 的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply', + reason: 'Pingora 直连回退脚本默认必须是 dry-run，只能显式 apply 修改系统。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply 必须同时提供 --reload-nginx', + reason: 'Pingora 直连回退 apply 前必须强制 reload Nginx 并核验 active。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '${NGINX_BINARY} -t', + reason: 'Pingora 直连回退 reload Nginx 前必须先执行 nginx -t。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目录不能是符号链接', + reason: + 'Pingora 直连回退脚本 apply 前必须拒绝符号链接 drop-in 目录，避免删除非预期 systemd 位置。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目标不能是符号链接', + reason: + 'Pingora 直连回退脚本 apply 前必须拒绝符号链接 drop-in 目标，避免误以为已回退真实 drop-in。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'drop-in 目标已存在但不是普通文件', + reason: 'Pingora 直连回退脚本 apply 前必须拒绝非普通文件 drop-in 目标。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--apply 必须同时提供 --nginx-smoke-url', + reason: 'Pingora 直连回退 apply 前必须强制 Nginx 入口 smoke。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host', + reason: + 'Pingora 直连回退本机 smoke 必须显式使用正式 Host，避免命中默认 vhost。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--nginx-smoke-expect-body', + reason: + 'Pingora 直连回退必须支持 Nginx smoke 响应体证据片段，避免 HTTP 200 命中错误入口。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'nginx smoke evidence matched expected body fragment', + reason: 'Pingora 直连回退必须在 Nginx smoke 响应体匹配后输出证据摘要。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'Nginx smoke 响应缺少预期片段', + reason: 'Pingora 直连回退必须拒绝 Nginx smoke 响应体漂移。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '只能是 host 或 host:port', + reason: 'Pingora 直连回退必须拒绝非法 Nginx smoke Host。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--curl-binary', + reason: + 'Pingora 直连回退必须支持覆盖 curl 可执行文件，便于测试和特殊环境。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-env-file', + reason: 'Pingora 直连回退必须支持可选复核 health patrol env 已切回 nginx。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--health-patrol-env-check-script', + reason: + 'Pingora 直连回退 health patrol env 复核必须能固定 current release 随包脚本。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'node -- "${health_patrol_args[@]}"', + reason: + 'Pingora 直连回退 health patrol env 复核必须用 node -- 分隔脚本参数，避免 Node 22 抢占业务 --env-file。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--require-empty-public-host', + reason: 'Pingora 直连回退 health patrol env 复核必须阻断 direct Host 覆盖残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--pingora-shadow-probe-url', + reason: 'Pingora 直连回退必须支持可选复核 shadow 高端口探针。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '--pingora-shadow-probe-token', + reason: 'Pingora 直连回退 shadow 探针复核必须使用 probe token。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'X-Genarrative-Pingora-Probe: ', + reason: 'Pingora 直连回退 shadow 探针命令日志必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run Pingora shadow probe 命令不应输出 probe token 原文', + reason: 'Pingora 直连回退自测必须覆盖 shadow probe token 不进命令日志。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'gateway=pingora-shadow', + reason: 'Pingora 直连回退 shadow 探针复核必须检查响应身份。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '${CURL_BINARY} --fail', + reason: 'Pingora 直连回退 reload Nginx 后必须执行 smoke URL 验证。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl daemon-reload', + reason: 'Pingora 直连回退移除 drop-in 后必须 reload systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连回退脚本必须默认重启 Pingora 网关服务。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'direct-entry.conf', + reason: 'Pingora 直连回退脚本必须移除 direct-entry drop-in。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl cat ${SERVICE_NAME}', + reason: 'Pingora 直连回退 apply 后必须核验 systemd drop-in 最终已移除。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连回退 apply 后必须拒绝 AmbientCapabilities 残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连回退 apply 后必须拒绝 CapabilityBoundingSet 残留。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'systemctl is-active ${NGINX_SERVICE}', + reason: 'Pingora 直连回退 reload Nginx 后必须核验 Nginx active 状态。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'Nginx reload 后状态不是 active', + reason: 'Pingora 直连回退必须拒绝 Nginx reload 后非 active 状态。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply', + reason: 'Pingora 直连启用脚本默认必须是 dry-run，只能显式 apply 修改系统。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl daemon-reload', + reason: 'Pingora 直连启用安装 drop-in 后必须 reload systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'genarrative-pingora-gateway.service', + reason: 'Pingora 直连启用脚本必须默认重启 Pingora 网关服务。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + reason: + 'Pingora 直连启用脚本默认必须读取 current release 随包 direct-entry 模板，不能依赖 /etc 参考模板。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_SCRIPT', + reason: + 'Pingora 直连启用脚本必须支持固定 current release 自审脚本，避免切换窗口依赖源码工作区。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'run_current_release_audit', + reason: + 'Pingora 直连启用 apply 前必须先跑 current release 自包含自审，再修改 systemd。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--require-pingora-gateway', + reason: + 'Pingora 直连启用 apply 前 current release 自审必须要求 Pingora 网关产物存在。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--systemd-show', + reason: + 'Pingora 直连启用 apply 前 current release 自审必须核验 systemd ExecStart。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--service-unit-path', + reason: + 'Pingora 直连启用脚本必须能固定随包主 service 模板作为 ExecStart 真相源。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager', + reason: + 'Pingora 直连启用脚本 apply 后必须核验 systemd ExecStart 指向 current release 网关二进制。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenCurrentReleaseAuditFailsBeforeInstall', + reason: + 'Pingora 直连启用脚本自测必须覆盖 current release 自审失败时不安装 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenSystemdExecStartDiffers', + reason: + 'Pingora 直连启用脚本自测必须覆盖 systemd ExecStart 指向旧 release 的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '该脚本不写入 TLS env', + reason: 'Pingora 直连启用脚本不能隐式修改 TLS/env/证书边界。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-env-file', + reason: 'Pingora 直连启用 apply 前必须强制通过 env 预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-cert-readable', + reason: 'Pingora 直连启用 apply 前必须强制通过证书可读预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-service-env-file', + reason: + 'Pingora 直连启用 apply 前必须强制通过 service EnvironmentFile 一致性预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemd_env_file_matches', + reason: + 'Pingora 直连启用 apply 后必须精确核验 systemctl cat 中的 EnvironmentFile。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: + '--apply 必须同时提供 --preflight-check-service-user-cert-readable', + reason: 'Pingora 直连启用 apply 前必须强制通过服务用户证书可读预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: + '--apply 必须同时提供 --preflight-check-service-binary-executable', + reason: 'Pingora 直连启用 apply 前必须强制通过 service 二进制可执行预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --preflight-check-ports-free', + reason: 'Pingora 直连启用 apply 前必须强制通过端口释放预检。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-https-base-url', + reason: 'Pingora 直连启用 apply 后必须强制验证 HTTPS 直连入口。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-http-base-url', + reason: 'Pingora 直连启用 apply 后必须强制验证 HTTP redirect / ACME 入口。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-host', + reason: 'Pingora 直连启用 apply 后必须用正式域名 Host/SNI 验证。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-redirect-host', + reason: 'Pingora 直连启用 apply 后必须验证 redirect Location host。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-spacetime-database', + reason: 'Pingora 直连启用 apply 后必须验证目标库 WSS subscribe。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl cat ${SERVICE_NAME}', + reason: 'Pingora 直连启用 apply 后必须核验 systemd drop-in 最终生效。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'systemctl is-active ${SERVICE_NAME}', + reason: 'Pingora 直连启用 apply 后必须核验服务已 active。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连启用 apply 后必须核验 AmbientCapabilities。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + reason: 'Pingora 直连启用 apply 后必须核验 CapabilityBoundingSet。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--require-wss-upgrade', + reason: 'Pingora 直连启用 apply 后 direct live smoke 必须强制 WSS 101。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'print_redacted_command node "${direct_live_args[@]}"', + reason: 'Pingora 直连启用脚本展示 direct live 命令时必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 不应安装临时 drop-in 文件', + reason: 'Pingora 直连启用检查必须证明默认 dry-run 不修改 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--probe-token ', + reason: 'Pingora 直连启用检查必须覆盖 direct live 命令 token 脱敏。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示并执行 direct preflight', + reason: 'Pingora 直连启用检查必须证明可在启用前执行 direct preflight。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + 'release current 目录中的 enable dry-run 必须默认读取随包携带的 direct-entry 模板', + reason: + 'Pingora 直连启用检查必须证明从 /opt/genarrative/current 执行时默认使用发布包内 systemd 模板。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-env-file 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 apply 跳过 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-check-cert-readable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + '--apply 缺少 --preflight-check-service-user-cert-readable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过服务用户证书可读 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: + '--apply 缺少 --preflight-check-service-binary-executable 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过 service 二进制可执行 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --preflight-check-ports-free 必须被拒绝', + reason: + 'Pingora 直连启用检查必须覆盖 apply 跳过端口释放 preflight 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示启用后 systemd drop-in 生效核验', + reason: 'Pingora 直连启用检查必须覆盖启用后的 systemd 生效核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'dry-run 必须展示启用后 service active 核验', + reason: 'Pingora 直连启用检查必须覆盖启用后的 service active 核验。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-https-base-url 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct HTTPS 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-http-base-url 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct HTTP 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-host 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 direct Host/SNI 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-redirect-host 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 redirect host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-spacetime-database 必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖缺少 WSS 数据库名的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct live smoke 失败时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 失败负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct preflight 脚本缺失时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct preflight 脚本缺失负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'direct live smoke 脚本缺失时 enable apply 必须失败', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 脚本缺失负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--template-path 必须是绝对路径', + reason: 'Pingora 直连启用检查必须覆盖模板相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--dropin-path 必须是绝对路径', + reason: 'Pingora 直连启用检查必须覆盖 drop-in 相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'drop-in 目录不能是符号链接', + reason: + 'Pingora 直连启用脚本 apply 前必须拒绝符号链接 drop-in 目录，避免写到非预期 systemd 位置。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'drop-in 目标不能是符号链接', + reason: + 'Pingora 直连启用脚本 apply 前必须拒绝符号链接 drop-in 目标，避免覆盖非预期文件。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'reject_control_characters', + reason: + 'Pingora 直连启用脚本必须在修改 systemd 前拒绝带换行或 NUL 的切换参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'reject_filesystem_root_path', + reason: + 'Pingora 直连启用脚本必须在修改 systemd 前拒绝文件系统根目录路径参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 直连启用脚本拒绝控制字符参数时必须给出明确错误。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '不能是文件系统根目录', + reason: + 'Pingora 直连启用脚本拒绝文件系统根目录路径时必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyRejectsSymlinkDropinDirectoryBeforeInstall', + reason: 'Pingora 直连启用检查必须覆盖符号链接 drop-in 目录负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyRejectsSymlinkDropinFileBeforeInstall', + reason: 'Pingora 直连启用检查必须覆盖符号链接 drop-in 目标负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeApply', + reason: + 'Pingora 直连启用检查必须覆盖控制字符参数会在 apply 写 systemd 前失败。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertFilesystemRootPathsRejected', + reason: + 'Pingora 直连启用检查必须覆盖文件系统根目录路径会在 apply 写 systemd 前失败。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--preflight-script 必须是绝对路径', + reason: + 'Pingora 直连启用脚本的 preflight 脚本路径必须固定为绝对路径，避免切换窗口受 cwd 影响。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--direct-live-script 必须是绝对路径', + reason: + 'Pingora 直连启用脚本的 direct live smoke 脚本路径必须固定为绝对路径，避免切换窗口受 cwd 影响。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '相对路径 direct preflight 脚本必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 direct preflight 脚本相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '相对路径 direct live smoke 脚本必须被拒绝', + reason: 'Pingora 直连启用检查必须覆盖 direct live smoke 脚本相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 不应删除临时 drop-in 文件', + reason: 'Pingora 直连回退检查必须证明默认 dry-run 不修改 drop-in。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示回退后 systemd drop-in 移除核验', + reason: 'Pingora 直连回退检查必须覆盖回退后的 systemd 移除核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示回退后 ExecStart 指向核验', + reason: 'Pingora 直连回退检查必须覆盖回退后的 current release ExecStart 核验。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '回退脚本 ExecStart 核验必须以随包主 service 模板为真相源', + reason: 'Pingora 直连回退脚本必须以随包主 service 模板为 ExecStart 真相源。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 reload Nginx 后 active 核验', + reason: 'Pingora 直连回退检查必须覆盖 Nginx reload 后状态核验步骤。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 Nginx 入口 smoke', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke dry-run 命令。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--apply 缺少 --reload-nginx 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 apply 跳过 Nginx reload 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'nginx -t 失败时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx 配置语法错误负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--apply 缺少 --nginx-smoke-url 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖缺少 Nginx smoke URL 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '缺少 --nginx-smoke-host 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 loopback smoke 缺少 Host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '非法 --nginx-smoke-host 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖非法 smoke Host 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Nginx smoke 失败时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke 失败负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Nginx smoke 响应体不匹配时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 Nginx smoke 响应体漂移负例。', + }, + { + file: 'scripts/check-pingora-release-readiness-plan.mjs', + includes: 'dry-run cutover rollback apply 必须默认校验 healthz ok body', + reason: 'Pingora cutover runbook 自检必须覆盖 rollback Nginx smoke body 默认门禁。', + }, + { + file: 'scripts/check-pingora-release-readiness.mjs', + includes: 'rollbackNginxSmokeExpectBody', + reason: 'Pingora cutover runbook 必须把 rollback Nginx smoke body 片段作为显式配置。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 health patrol env 回退后复核命令', + reason: 'Pingora 直连回退检查必须覆盖 health patrol env 复核 dry-run 命令。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --health-patrol-env-file 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 health patrol env 相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'health patrol env 仍停留在 pingora-direct 时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖巡检 env 未切回 nginx 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'dry-run 必须展示 Pingora shadow probe 命令且隐藏 token', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe dry-run 命令并避免泄露 token。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'shadow probe URL 缺少 token 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe 参数成对出现。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'Pingora shadow probe body 错误时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 shadow probe 响应身份错误负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'systemctl show ExecStart 指向旧 release 时 rollback apply 必须失败', + reason: 'Pingora 直连回退检查必须覆盖 systemd ExecStart 漂到旧 release 的负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--service-unit-path 必须是绝对路径', + reason: 'Pingora 直连回退检查必须覆盖主 service 模板相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '--dropin-path 必须是绝对路径', + reason: 'Pingora 直连回退检查必须覆盖相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'require_absolute_path_if_path_like "--nginx-binary"', + reason: + 'Pingora 直连回退脚本覆盖 Nginx 二进制时，路径形式必须固定为绝对路径。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'require_absolute_path_if_path_like "--curl-binary"', + reason: + 'Pingora 直连回退脚本覆盖 curl 二进制时，路径形式必须固定为绝对路径。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --nginx-binary 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 Nginx 二进制相对路径负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '相对路径 --curl-binary 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖 curl 二进制相对路径负例。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'validate_http_url "--nginx-smoke-url"', + reason: 'Pingora 直连回退脚本必须在改 systemd 前拒绝非法 Nginx smoke URL。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'reject_control_characters', + reason: + 'Pingora 直连回退脚本必须在 nginx -t、删除 drop-in 或 reload 前拒绝带换行或 NUL 的回退参数。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: 'reject_filesystem_root_path', + reason: + 'Pingora 直连回退脚本必须在 nginx -t、删除 drop-in 或 reload 前拒绝文件系统根目录路径参数。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '不能包含换行或 NUL 字符', + reason: + 'Pingora 直连回退脚本拒绝控制字符参数时必须给出明确错误。', + }, + { + file: 'scripts/deploy/pingora-direct-rollback.sh', + includes: '不能是文件系统根目录', + reason: + 'Pingora 直连回退脚本拒绝文件系统根目录路径时必须给出明确错误。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: '非法 --nginx-smoke-url 必须被拒绝', + reason: 'Pingora 直连回退检查必须覆盖非法 Nginx smoke URL 负例。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertRejectsControlCharacterInputsBeforeApply', + reason: + 'Pingora 直连回退检查必须覆盖控制字符参数会在 nginx -t 和删除 drop-in 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertRejectsFilesystemRootPathsBeforeApply', + reason: + 'Pingora 直连回退检查必须覆盖文件系统根目录路径会在 nginx -t 和删除 drop-in 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsSymlinkDropinDirectoryBeforeNginxTest', + reason: 'Pingora 直连回退检查必须覆盖符号链接 drop-in 目录会在 nginx -t 前失败。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsSymlinkDropinFileBeforeRemoval', + reason: 'Pingora 直连回退检查必须覆盖符号链接 drop-in 目标不会删除真实目标文件。', + }, + { + file: 'scripts/check-pingora-direct-rollback.mjs', + includes: 'assertApplyRejectsDirectoryDropinTargetBeforeNginxTest', + reason: 'Pingora 直连回退检查必须覆盖目录型 drop-in 目标会在 nginx -t 前失败。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API deploy 必须把 Pingora 直连回退脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API deploy 必须把 Pingora 直连启用脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API deploy 必须把 Pingora 直连预检脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: 'API deploy 必须把 Pingora canary live smoke 脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API deploy 必须把 Pingora 直连 live smoke 脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API deploy 必须把 Pingora release readiness 聚合门禁脚本复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'deploy/pingora', + reason: 'API deploy 必须把 Pingora 配置目录复制到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'deploy/systemd', + reason: + 'API deploy 必须把 systemd 配置目录复制到 current release，供直连预检读取主 service 和 drop-in 模板。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ENV_DEPLOY_DIR_SOURCE', + reason: 'API deploy 必须从发布包内复制 deploy/env 示例目录。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: 'API release build 动态烟测必须覆盖随产物执行的 deploy 脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'API release README 必须说明随包携带 API Deploy 执行入口', + reason: + 'API release build 动态烟测必须覆盖发布包 README 中的 deploy 执行入口说明。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/maintenance-on.sh', + reason: 'API release build 动态烟测必须覆盖 deploy 同目录维护开启脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/maintenance-off.sh', + reason: 'API release build 动态烟测必须覆盖 deploy 同目录维护关闭脚本。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API release build 动态烟测必须覆盖直连启用脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API release build 动态烟测必须覆盖直连回退脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API release build 动态烟测必须覆盖 health patrol env 切换脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API release build 动态烟测必须覆盖直连预检脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: + 'API release build 动态烟测必须覆盖健康巡检 env 复核脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API release build 动态烟测必须覆盖 Pingora release readiness 聚合门禁脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'API release build 动态烟测必须覆盖 canary live smoke 脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API release build 动态烟测必须覆盖直连 live smoke 脚本进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/systemd/genarrative-pingora-gateway.service', + reason: 'API release build 动态烟测必须覆盖 systemd 模板进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: 'API release build 动态烟测必须覆盖 Pingora 配置进入发布包。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: '--skip-api-build', + reason: 'API release build 动态烟测必须避免真实构建 Rust。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/production-api-deploy.sh', + reason: 'API deploy 动态烟测必须从发布产物内执行 deploy 脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/maintenance-on.sh', + reason: 'API deploy 动态烟测必须覆盖 deploy 同目录维护开启脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/maintenance-off.sh', + reason: 'API deploy 动态烟测必须覆盖 deploy 同目录维护关闭脚本。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-canary-live.mjs', + reason: + 'API deploy 动态烟测必须覆盖 Pingora canary live smoke 脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-release-readiness.mjs', + reason: + 'API deploy 动态烟测必须覆盖 Pingora release readiness 聚合门禁脚本进入 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'release-manifest.json 缺少 pingora-gateway artifact', + reason: + 'API deploy 必须阻断带 Pingora 二进制但 manifest 未登记 Pingora artifact 的发布包。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 release-manifest.json', + reason: + 'API deploy 必须强制要求 release manifest，避免 current release 缺少产物证据根。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${RELEASE_ROOT}" "--release-root"', + reason: + 'API deploy 发布根目录必须是绝对路径，避免 Jenkins 工作目录影响 release 位置。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${CURRENT_LINK}" "--current-link"', + reason: + 'API deploy current 链接必须是绝对路径，避免相对路径切错入口。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'require_absolute_path "${API_ENV_FILE}" "--api-env-file"', + reason: + 'API deploy 环境文件必须是绝对路径，避免相对路径写入工作区文件。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '^[0-9A-Za-z][0-9A-Za-z._-]*$', + reason: + 'API deploy release version 必须以数字或字母开头，避免点目录或隐藏 staging 名称污染 release 根目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '--version 不能是点目录', + reason: + 'API deploy 必须显式拒绝 . / .. 这类点目录版本名。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '.staging.$$', + reason: + 'API deploy 必须先写 staging release，避免失败时污染正式 release 目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '目标 release 已存在，拒绝覆盖或合并旧文件', + reason: + 'API deploy 必须拒绝同版本 release 目录复用，避免旧文件混入 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '目标 release 在发布过程中出现，拒绝合并 staging', + reason: + 'API deploy 在最终提升前必须再次拒绝同版本 release 竞态出现，避免把 staging 合并进旧目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'current 链接路径已存在但不是符号链接，拒绝覆盖', + reason: + 'API deploy 必须拒绝目录型 current 路径，避免 ln 把新链接写进旧目录。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}"', + reason: + 'API deploy 切换 current 必须用 -T 固定符号链接替换语义。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}"', + reason: + 'API deploy 必须用非合并语义把 staging release 提升为正式 release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraArtifactMissingManifestEntry', + reason: + 'API deploy 动态烟测必须覆盖 Pingora 二进制与 release manifest 漂移负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingReleaseManifestFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 release manifest 的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertReleaseManifestMissingApiArtifactFails', + reason: + 'API deploy 动态烟测必须覆盖 release manifest 缺少 api-server artifact 的负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDotVersion', + reason: + 'API deploy 动态烟测必须覆盖 . 版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDotDotVersion', + reason: + 'API deploy 动态烟测必须覆盖 .. 版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsVersionStartingWithDot', + reason: + 'API deploy 动态烟测必须覆盖以点开头版本名负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsExistingReleaseDirectory', + reason: + 'API deploy 动态烟测必须覆盖同版本 release 目录已存在时拒绝覆盖。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsDirectoryCurrentLink', + reason: + 'API deploy 动态烟测必须覆盖 current 路径不是符号链接时拒绝覆盖。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeReleaseRoot', + reason: + 'API deploy 动态烟测必须覆盖 release root 相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeCurrentLink', + reason: + 'API deploy 动态烟测必须覆盖 current link 相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsRelativeApiEnvFile', + reason: + 'API deploy 动态烟测必须覆盖 api env 文件相对路径负例。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployCleansStagingReleaseOnFailure', + reason: + 'API deploy 动态烟测必须覆盖失败时清理 staging release 且不留下正式 release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsFinalReleaseRaceAndCleansStaging', + reason: + 'API deploy 动态烟测必须覆盖最终提升前 release 目录竞态出现时拒绝合并并清理 staging。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingBackupScriptFails', + reason: 'API deploy 动态烟测必须覆盖缺少数据库备份脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolScriptFails', + reason: 'API deploy 动态烟测必须覆盖缺少健康巡检脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingHealthPatrolEnvCheckFails', + reason: + 'API deploy 动态烟测必须覆盖缺少健康巡检 env 复核脚本的失败维护模式。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API deploy 动态烟测必须覆盖直连预检脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-production-health-patrol-env.mjs', + reason: + 'API deploy 动态烟测必须覆盖健康巡检 env 复核脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API deploy 动态烟测必须覆盖 health patrol env 切换脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: + 'API deploy 动态烟测必须覆盖直连 live smoke 脚本进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/systemd/genarrative-pingora-gateway.service', + reason: 'API deploy 动态烟测必须覆盖 systemd 模板进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'deploy/pingora/pingora-gateway.env.example', + reason: 'API deploy 动态烟测必须覆盖 Pingora 配置进入 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'fakeBin', + reason: + 'API deploy 动态烟测必须使用 fake systemctl/curl，禁止触碰真实服务。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API Build 归档必须包含 Pingora 直连回退脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API Build 归档必须包含 Pingora 直连启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API Build 归档必须包含 Pingora 直连预检脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: 'API Build 归档必须包含 Pingora 直连 live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'deploy/pingora/**', + reason: 'API Build 归档必须包含 Pingora 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'deploy/systemd/**', + reason: 'API Build 归档必须包含 systemd 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/deploy/pingora-direct-rollback.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连回退脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/deploy/pingora-direct-enable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/check-pingora-direct-preflight.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连预检脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'scripts/check-pingora-direct-live.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 直连 live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'deploy/pingora/**', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 配置目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'deploy/systemd/**', + reason: 'API Deploy 复制上游产物时必须包含 systemd 配置目录。', + }, + { + file: 'deploy/pingora/nginx-route-parity.matrix.json', + includes: 'spacetime_subscribe', + reason: 'Pingora 路由矩阵必须覆盖 SpacetimeDB WebSocket subscribe。', + }, + { + file: 'scripts/check-pingora-route-parity.mjs', + includes: 'deploy/nginx/genarrative.conf', + reason: 'Pingora 路由矩阵检查必须覆盖生产 Nginx 模板。', + }, + { + file: 'scripts/check-pingora-route-parity.mjs', + includes: 'deploy/nginx/genarrative-dev-http.conf', + reason: 'Pingora 路由矩阵检查必须覆盖开发服 Nginx 模板。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'ROUTE_PARITY_MATRIX_JSON', + reason: 'Pingora Rust 路由单测必须读取共享路由矩阵。', + }, + { + file: 'scripts/check-nginx-pingora-canary.mjs', + includes: '--require-nginx', + reason: + 'Pingora canary 校验脚本必须支持在有 Nginx 的环境强制执行 nginx -t。', + }, + { + file: 'scripts/check-nginx-pingora-canary.mjs', + includes: 'proxy_set_header Upgrade $http_upgrade;', + reason: + 'Pingora canary 校验脚本必须覆盖 SpacetimeDB WebSocket Upgrade 设置。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'SpacetimeDB WebSocket Upgrade', + reason: 'Pingora 网关 smoke 必须覆盖 SpacetimeDB WebSocket 代理。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'v2.bsatn.spacetimedb', + reason: 'Pingora 网关 smoke 必须覆盖 SpacetimeDB WSS 子协议透传。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip 压缩响应', + reason: 'Pingora 网关 smoke 必须覆盖 gzip 响应压缩。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip-only 压缩算法白名单', + reason: 'Pingora 网关 smoke 必须覆盖压缩算法白名单。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip 最小长度不压缩小响应', + reason: 'Pingora 网关 smoke 必须覆盖 Nginx gzip_min_length parity。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'gzip types 不压缩图片资源', + reason: 'Pingora 网关 smoke 必须覆盖 Nginx gzip_types parity。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '主站指纹静态资源长缓存', + reason: 'Pingora 网关 smoke 必须覆盖 Vite 指纹资源 immutable 缓存头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: "response.headers['cache-control'] === 'no-cache'", + reason: 'Pingora 网关 smoke 必须覆盖 HTML 和普通静态资源 no-cache 口径。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-None-Match 返回 304', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 ETag 协商缓存。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Modified-Since 返回 304', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 Last-Modified 协商缓存。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源非读取方法返回 405', + reason: 'Pingora 网关 smoke 必须覆盖静态资源只允许 GET / HEAD。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GET, HEAD', + reason: 'Pingora 静态响应必须对非读取方法返回 Allow: GET, HEAD。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 Range 返回 206', + reason: 'Pingora 网关 smoke 必须覆盖静态资源单 Range 局部读取。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Range 日期匹配返回 206', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 If-Range 匹配时继续返回局部内容。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源 If-Range 旧日期回完整文件', + reason: 'Pingora 网关 smoke 必须覆盖静态资源 If-Range 不匹配时回完整内容。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '静态资源越界 Range 返回 416', + reason: 'Pingora 网关 smoke 必须覆盖静态资源越界 Range 响应。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'access log 静态边界状态对账', + reason: + 'Pingora 网关 smoke 必须按 request_id 对账静态 304 / 405 / 206 / 416 access log，避免直连证据链漏掉本地响应状态。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'entry.method === expected.method', + reason: + 'Pingora 网关 smoke 必须按 request_id 对账静态 access log 的 method，避免 HEAD / POST 边界被状态码巧合掩盖。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Accept-Ranges', + reason: 'Pingora 静态响应必须声明 bytes Range 能力。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Content-Range', + reason: 'Pingora 静态 Range 响应必须写入 Content-Range。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'Last-Modified', + reason: 'Pingora 静态响应必须写入 Last-Modified，避免直连后协商缓存能力退化。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'if-none-match', + reason: 'Pingora 静态响应必须支持 ETag 条件请求返回 304。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL', + reason: 'Pingora 网关必须暴露指纹静态资源 Cache-Control 配置。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'has_fingerprinted_file_name', + reason: 'Pingora 网关必须按 Vite 指纹文件名区分长期缓存静态资源。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: '不能包含换行或 NUL 字符', + reason: 'Pingora 静态缓存头配置必须拒绝响应头注入字符。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL', + reason: 'Pingora 网关 smoke 必须覆盖 gzip 配置校验。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS', + reason: 'Pingora 网关 smoke 必须覆盖压缩算法配置校验。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_CONCURRENCY_LIMITED', + reason: 'Pingora 网关 smoke 必须覆盖接流并发保护。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'PAYLOAD_TOO_LARGE', + reason: 'Pingora 网关 smoke 必须覆盖请求体大小限制。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'access log 落盘', + reason: 'Pingora 网关 smoke 必须覆盖 access log 文件写入。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '配置错误启动失败', + reason: 'Pingora 网关 smoke 必须覆盖不安全配置 fail-fast。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'SECRET_VALUE_FLAGS', + reason: 'Pingora 网关 smoke 必须集中维护敏感参数脱敏列表。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'redactSecretArgs(args).join', + reason: 'Pingora 网关 smoke 打印 direct live 命令时必须隐藏 probe token。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_UPSTREAM_ERROR', + reason: 'Pingora 网关 smoke 必须覆盖上游断连时的稳定 JSON 错误。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'GATEWAY_UPSTREAM_TIMEOUT', + reason: 'Pingora 网关 smoke 必须覆盖上游超时时的稳定 JSON 错误。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'TLS 直连接口返回 index.html', + reason: 'Pingora 网关 smoke 必须覆盖可选 TLS 直连接口。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'HTTP 入口 301 到 HTTPS', + reason: 'Pingora 网关 smoke 必须覆盖可选 HTTP 到 HTTPS 重定向入口。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'direct-access-log', + reason: 'Pingora direct live smoke 必须覆盖 Pingora access log 落盘校验。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset', + reason: + 'Pingora direct live smoke 必须自动发现并探测首页引用的静态资源，避免直连只验证 API / WSS 路径。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-range', + reason: + 'Pingora direct live smoke 必须验证首页静态资产 Range: bytes=0-0，避免直连后媒体/浏览器探测能力退化。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-head', + reason: + 'Pingora direct live smoke 必须验证首页静态资产 HEAD 行为，覆盖 curl -I、监控探针和 CDN 预检。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset', + reason: + 'Pingora direct live smoke 必须额外探测首页引用的 Vite 指纹静态资源。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'public, max-age=31536000, immutable', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源长期缓存头，避免直连后旧 tab chunk 缓存口径退化。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'pickEvidenceHeaders', + reason: + 'Pingora direct live smoke 必须把静态响应头白名单写入 JSON 证据，便于切换证据包复盘。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: "'content-range'", + reason: + 'Pingora direct live smoke 的静态响应头 JSON 证据必须包含 Content-Range。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-head', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 HEAD 行为。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-range', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 Range 行为。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-asset-etag-304', + reason: + 'Pingora direct live smoke 必须验证首页静态资源 ETag 条件请求返回 304。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'https-static-fingerprinted-asset-etag-304', + reason: + 'Pingora direct live smoke 必须验证指纹静态资源 ETag 条件请求返回 304。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '', + reason: + 'Pingora 本地 smoke fixture 必须让 direct live 自动发现指纹静态资源。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live 必须验证首页引用的指纹静态资源 GET 200。', + reason: + 'Pingora 本地 smoke 必须从 direct live JSON 结构化确认指纹资源 GET 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live JSON 必须保留指纹静态资源 immutable Cache-Control 证据。', + reason: + 'Pingora 本地 smoke 必须确认 direct live JSON 可复盘指纹资源长期缓存头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live JSON 必须保留指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 本地 smoke 必须确认 direct live JSON 可复盘指纹资源 Range 响应头。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live access log 必须保留指纹静态 Range 的 method/path/status 明细。', + reason: + 'Pingora 本地 smoke 必须确认指纹资源 Range 进入 access log method/path/status 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live 必须验证首页引用的指纹静态资源 ETag 304。', + reason: + 'Pingora 本地 smoke 必须从 direct live JSON 结构化确认指纹资源 304 证据。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'direct live access log 必须保留指纹静态 ETag 304 的 method/path/status 明细。', + reason: + 'Pingora 本地 smoke 必须确认指纹资源 304 进入 access log method/path/status 证据。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'Content-Range 不符合 bytes 0-0/', + reason: + 'Pingora direct live smoke 必须校验静态资产 Range 响应的 Content-Range。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'request_id', + reason: 'Pingora direct live smoke 必须按 request_id 反查 access log。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'matchedCount', + reason: + 'Pingora direct live smoke 的 access log 证据必须记录匹配数量，便于证据包复盘。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'mismatches', + reason: + 'Pingora direct live smoke 的 access log 证据必须保留 method/path/status 漂移明细。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'expectedMethod', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的预期 HTTP 方法。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'actualMethod', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的实际 HTTP 方法。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'expectedStatusCode', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的预期状态码。', + }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'actualStatusCode', + reason: + 'Pingora direct live smoke 的 access log 反查必须记录每个 request_id 的实际状态码。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-access-log 检查必须保留 mismatches 明细数组。', + reason: + 'Pingora 切流证据包自测必须防止 direct live access log JSON 明细退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-access-log 检查必须保留已匹配 request_id 的预期方法。', + reason: + 'Pingora 切流证据包自测必须防止 direct live access log method 明细退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.json 必须保留指纹静态资源 Cache-Control 证据。', + reason: + 'Pingora 切流证据包自测必须防止 direct live 静态响应头证据退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'direct-live.json 必须保留指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 切流证据包自测必须防止 direct live Range 响应头证据退化。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveStaticHeaders', + reason: + 'Pingora 切流证据包 manifest 必须提升 direct live 静态响应头摘要，便于快速复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveStaticHeaders?.ok', + reason: + 'Pingora 切流证据包必须把静态响应头摘要完整性纳入 direct live 成功判定。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'ETag 304 statusCode 应为 304', + reason: + 'Pingora 切流证据包必须诊断静态 ETag 304 证据缺失。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'Range statusCode 应为 206', + reason: + 'Pingora 切流证据包必须诊断静态 Range 证据缺失。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'manifest 必须提升指纹静态资源 Content-Range 证据。', + reason: + 'Pingora 切流证据包自测必须防止 manifest 静态响应头摘要退化。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveMissingStaticHeadersSummaryFails', + reason: + 'Pingora 切流证据包自测必须覆盖 direct live JSON 缺少静态响应头证据的负例。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveStaticHeaderDiagnosticsFails', + reason: + 'Pingora 切流证据包自测必须覆盖静态 Range / 304 摘要不完整时失败。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'manifest 必须提升普通静态资源 ETag 304 证据。', + reason: + 'Pingora 切流证据包自测必须防止 manifest 普通静态 304 摘要退化。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'summaryDirectLiveStaticHeaders', + reason: + 'Pingora 证据根目录审计必须读取阶段 manifest 的 direct live 静态响应头摘要。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-audit.mjs', + includes: 'directLiveStaticHeaders: latest.summaryDirectLiveStaticHeaders', + reason: + 'Pingora 证据根目录审计输出必须提升阶段 direct live 静态响应头摘要。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根审计输出必须提升 post-enable direct live 指纹静态 Cache-Control 摘要。', + reason: + 'Pingora 证据根目录审计自测必须覆盖 direct live 静态响应头摘要输出。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-audit.mjs', + includes: '证据根审计输出必须提升 post-enable direct live access log 摘要。', + reason: + 'Pingora 证据根目录审计自测必须覆盖 direct live access log 摘要输出。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '静态 GET / HEAD / 304 / Range 结果还必须在 `direct-live.json` 中保留白名单响应头', + reason: + 'Pingora 试点文档必须说明 direct live JSON 会归档静态响应头白名单证据。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '`cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`', + reason: + 'Pingora 试点文档必须固定 direct live 静态响应头证据白名单。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '静态摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 / Last-Modified 304 证据时，证据包都会直接记为 `CRITICAL`', + reason: + 'Pingora 试点文档必须说明静态响应头证据不完整会让证据包失败。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '总审计输出会把阶段 manifest 中的 `directLiveAccessLog` 与 `directLiveStaticHeaders` 一并带到对应 `phases[]` 项', + reason: + 'Pingora 试点文档必须说明最终证据根审计会输出 direct live 摘要。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'direct live JSON 只给静态 GET / HEAD / 304 / Range 写入白名单 `headers`', + reason: + 'Nginx README 必须说明 Pingora direct live 静态头部证据归档口径。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'manifest.summary.directLiveStaticHeaders', + reason: + 'Nginx README 必须说明切流 manifest 会提升 direct live 静态响应头摘要。', + }, + { + file: 'deploy/nginx/README.md', + includes: '证据包会记为 `CRITICAL`', + reason: + 'Nginx README 必须说明 direct live 静态响应头摘要缺关键证据时会阻断证据包。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: 'API / WSS 检查不落原始响应头', + reason: + '团队共享决策必须记录 direct live 只归档静态白名单响应头。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: 'directLiveStaticHeaders', + reason: + '团队共享决策必须记录切流 manifest 提升 direct live 静态响应头摘要。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: '静态头摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 / Last-Modified 304 证据时整包记为 `CRITICAL`', + reason: + '团队共享决策必须记录静态响应头摘要不完整会阻断证据包。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: '不要把 `direct-live.json` 当成只有状态码的摘要', + reason: + '团队共享踩坑必须提醒 direct live JSON 需要可复盘静态响应头证据。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: 'manifest.summary.directLiveStaticHeaders', + reason: + '团队共享踩坑必须提醒切流 manifest 需要可快速复盘静态响应头摘要。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: '摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 或 Last-Modified 304 证据，证据包会直接记为 `CRITICAL`', + reason: + '团队共享踩坑必须提醒静态响应头摘要不完整会阻断证据包。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'directLiveAccessLog', + reason: + 'Pingora 切流证据包 manifest 必须提升 direct live access log 摘要，便于快速复盘。', + }, + { + file: 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + includes: 'direct live JSON missing direct-access-log result', + reason: + 'Pingora 切流证据包必须在 direct live JSON 缺少 access log 结果时标记 CRITICAL。', + }, + { + file: 'scripts/check-pingora-cutover-evidence-bundle.mjs', + includes: 'assertDirectLiveMissingAccessLogSummaryFails', + reason: + 'Pingora 切流证据包自测必须覆盖 direct live JSON 缺少 access log 结果的负例。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--apply 必须同时提供 --direct-pingora-access-log', + reason: 'Pingora 直连启用 apply 必须强制 access log 落盘校验参数。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct_live_args+=(--json)', + reason: + 'Pingora 直连启用 apply 后必须用 JSON 模式运行 direct live，才能校验结构化接流证据。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct live JSON 缺少 direct-access-log 结构化结果', + reason: + 'Pingora 直连启用 apply 必须拒绝缺少 direct-access-log 结构化结果的 direct live 输出。', + }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: 'direct-access-log evidence checked=', + reason: + 'Pingora 直连启用 apply 必须在 direct access log 结构化证据通过后输出摘要。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: '--apply 缺少 --direct-pingora-access-log 必须被拒绝', + reason: 'Pingora 直连启用自测必须覆盖缺少 access log 参数的负例。', + }, + { + file: 'scripts/check-pingora-direct-enable.mjs', + includes: 'assertApplyFailsWhenDirectLiveAccessLogJsonMissing', + reason: + 'Pingora 直连启用自测必须覆盖 direct live 退出 0 但缺少 access log JSON 证据的负例。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: '--pingora-access-log', + reason: 'Pingora 本地网关 smoke 必须把临时 access log 传给 direct live smoke。', + }, + { + file: 'server-rs/crates/pingora-gateway/src/main.rs', + includes: 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS', + reason: + 'Pingora 网关必须暴露上游读取超时配置，避免正式切换时和 Nginx 超时口径漂移。', + }, { file: 'scripts/deploy/production-api-deploy.sh', includes: 'production-health-patrol.mjs', reason: 'API deploy 必须把健康巡检脚本复制到 current release。', }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少数据库备份脚本', + reason: 'API deploy 缺少数据库备份脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少生产健康巡检脚本', + reason: 'API deploy 缺少健康巡检脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: 'WORKSPACE_BACKUP_SCRIPT_SOURCE', + reason: 'API deploy 不应再从部署工作区兜底复制数据库备份脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: 'WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE', + reason: 'API deploy 不应再从部署工作区兜底复制健康巡检脚本。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + excludes: '回退使用部署工作区脚本', + reason: 'API deploy 不应再用部署工作区脚本掩盖发布产物缺失。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'pingora-gateway.sha256', + reason: 'API deploy 看到 Pingora 影子网关产物时必须校验 checksum 后复制。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'systemctl is-active --quiet "${service_name}"', + reason: + 'API deploy 看到 Pingora 影子网关产物时必须先确认 shadow service 已 active。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'systemctl try-restart "${service_name}"', + reason: + 'API deploy 看到 Pingora 影子网关产物且 shadow service 已 active 时必须随 current release 切换重启。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'Pingora 影子服务未处于 active，跳过自动重启', + reason: + 'API deploy 不能在 Pingora shadow service 未运行时主动拉起新公网入口候选服务。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRestartsActivePingoraWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 Pingora 产物发布后 active shadow service 自动重启。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeploySkipsInactivePingoraWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 时不会被主动拉起。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'assertApiReleaseCanIncludePingoraGatewayArtifact', + reason: 'API release 动态烟测必须覆盖显式 include Pingora 影子网关产物。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: '--skip-pingora-gateway-build', + reason: 'API release 动态烟测应能用假二进制快速覆盖 Pingora 产物布局。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'assertPingoraEnvProductionDefaults', + reason: 'API release 动态烟测必须校验 Pingora env 的生产安全默认值。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertPingoraEnvProductionDefaults', + reason: 'API deploy 动态烟测必须校验 current release 内 Pingora env 的生产安全默认值。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED', + reason: 'Pingora 网关配置示例必须保留接流保护总开关。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1', + reason: 'Pingora 网关配置示例必须显式保留默认单实例接流保护口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false', + reason: 'Pingora 网关配置示例必须默认不确认共享保护层。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED', + reason: 'Pingora 网关配置示例必须暴露 gzip 总开关。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN', + reason: 'Pingora 网关配置示例必须暴露可选 TLS 直连入口。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN', + reason: 'Pingora 网关配置示例必须暴露可选 HTTP 到 HTTPS 重定向入口。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + reason: 'Pingora 网关配置示例必须暴露压缩算法白名单。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'Pingora 正式化口径固定为 gzip-only', + reason: 'Pingora 网关配置示例必须明确当前正式化压缩口径。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Brotli', + reason: 'Pingora 试点文档必须说明 Brotli 尚未进入 Pingora 的边界。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Pingora 正式化口径固定为 gzip-only', + reason: 'Pingora 试点文档必须把 Brotli 取舍落成明确结论。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + excludes: '补齐 Brotli 取舍', + reason: 'Pingora 试点文档不能继续把 Brotli 取舍留成待办。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: 'Pingora Brotli 不能只看 Content-Encoding', + reason: '团队共享踩坑必须记录 Pingora Brotli 端到端解压风险。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL=5', + reason: 'Pingora 网关配置示例必须对齐 Nginx gzip 压缩等级。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES=1024', + reason: 'Pingora 网关配置示例必须对齐 Nginx gzip_min_length。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL=no-cache', + reason: 'Pingora 网关配置示例必须暴露 HTML / SPA fallback 缓存头。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable', + reason: 'Pingora 网关配置示例必须暴露指纹静态资源长期缓存头。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Vite 指纹静态资源', + reason: 'Pingora 试点文档必须说明直连静态缓存头口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: + 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS=3600', + reason: 'Pingora 网关配置示例必须保留上游 API 读取超时口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: + 'GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS=3600', + reason: 'Pingora 网关配置示例必须保留长连接读取超时口径。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT', + reason: 'Pingora 网关配置示例必须保留通用 API 并发保护参数。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_RATE_PER_SECOND', + reason: 'Pingora 网关配置示例必须保留公开列表 RPS 保护参数。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE', + reason: 'Pingora 网关配置示例必须暴露 access log 文件路径。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'API 上游未收到 X-Forwarded-Host', + reason: 'Pingora gateway smoke 必须断言 API 上游收到 Nginx parity 的 X-Forwarded-Host。', + }, + { + file: 'scripts/check-pingora-gateway-smoke.mjs', + includes: 'API 上游 X-Forwarded-For 未追加 TCP 对端 IP', + reason: + 'Pingora gateway smoke 必须断言 X-Forwarded-For 追加真实 TCP 对端 IP。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'API 代理头', + reason: 'Pingora 试点文档必须说明 API 代理头 parity 门禁。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'X-Real-IP', + reason: 'Pingora 试点文档必须说明 X-Real-IP 的 TCP 对端 IP 口径。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', + reason: 'Pingora direct live env 示例必须暴露 direct access log 复核路径。', + }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES=2000', + reason: 'Pingora direct live env 示例必须暴露 direct access log tail 行数。', + }, + { + file: 'deploy/nginx/README.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: 'Nginx README 的 Pingora 直连 runbook 必须包含 access log 落盘校验参数。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: 'Pingora 试点文档必须展示 direct access log 落盘校验参数。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: '生产运维文档必须展示 Pingora 直连 access log 落盘校验参数。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: '直连日志证据补充', + reason: '团队共享决策必须记录 Pingora 直连 access log 证据门禁。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: '--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log', + reason: '团队共享踩坑必须记录 Pingora 直连 access log 参数。', + }, + { + file: 'deploy/pingora/pingora-gateway.env.example', + includes: 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED', + reason: 'Pingora 网关配置示例必须暴露 X-Forwarded-For 信任确认开关。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'Brotli', + reason: 'Pingora 试点文档必须明确 gzip 以外的压缩边界。', + }, + { + file: 'deploy/nginx/README.md', + includes: '直连 Pingora 时，不把 Brotli parity 作为切换门禁', + reason: 'Nginx 压缩文档必须明确 Pingora Brotli 不作为直连切换门禁。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: + 'Genarrative-Api-Deploy` 只能从上游构建归档复制并执行 `build//scripts/deploy/production-api-deploy.sh`', + reason: 'Pingora 试点文档必须明确 API Deploy 执行入口来自 build 产物。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: 'Pingora 试点文档必须明确多实例接流保护确认边界。', + }, + { + file: 'docs/【开发运维】本地开发验证与生产运维-2026-05-15.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: '生产运维文档必须明确多实例 Pingora 与进程内保护的启动 / preflight 边界。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: 'GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1', + reason: '团队共享决策必须记录 Pingora 多实例接流保护边界。', + }, + { + file: '.hermes/shared-memory/decision-log.md', + includes: '禁止继续执行部署工作区根部脚本', + reason: '团队共享决策必须记录 API Deploy 不再执行 workspace 根部脚本。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: 'API deploy 脚本本身也不能继续用部署工作区根部', + reason: + '团队共享踩坑必须记录 workspace 根部 deploy 脚本会掩盖发布包布局问题。', + }, + { + file: '.hermes/shared-memory/pitfalls.md', + includes: '备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底', + reason: '团队共享踩坑必须记录备份/巡检脚本也不能使用 workspace fallback。', + }, + { + file: 'deploy/nginx/README.md', + includes: 'Pingora 影子网关已通过', + reason: 'Nginx 压缩文档必须说明 Pingora gzip parity 口径。', + }, { file: 'jenkins/Jenkinsfile.production-api-build', includes: 'scripts/ops/production-health-patrol.mjs', reason: 'API Build 归档必须包含健康巡检脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs', + reason: 'API Build 归档必须包含 Pingora current release 自审脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API Build 归档必须包含 Pingora 切换状态快照脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据包脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API Build 归档必须包含 Pingora 切换命令证据脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据验真脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs', + reason: 'API Build 归档必须包含 Pingora 切换证据根目录审计脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs', + reason: 'API Build 归档必须包含健康巡检 env 复核脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs', + reason: 'API Build 归档必须包含 Pingora release readiness 聚合门禁脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs', + reason: 'API Build 归档必须包含 Pingora canary live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs', + reason: 'API Build 归档必须包含 Pingora canary access log 对账脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh', + reason: 'API Build 归档必须包含 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh', + reason: 'API Build 归档必须包含 build 产物内的维护开启脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh', + reason: 'API Build 归档必须包含 build 产物内的维护关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API Build 归档必须包含 Pingora health patrol env 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**', + reason: 'API Build 归档必须包含 env 示例目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**', + reason: 'API Build 归档必须包含 Nginx/Pingora canary snippet。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + excludes: ',scripts/deploy/production-api-deploy.sh', + reason: 'API Build 不应再归档工作区根部 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'INCLUDE_PINGORA_GATEWAY', + reason: 'API Build 必须保留 Pingora 影子网关可选构建参数。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: 'command -v cmake', + reason: 'API Build 勾选 Pingora 时必须前置检查 cmake。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: '构建 Pingora 影子网关缺少 C++ 编译器', + reason: 'API Build 勾选 Pingora 时必须前置检查 C++ 编译器。', + }, { file: 'jenkins/Jenkinsfile.production-api-deploy', includes: 'scripts/ops/production-health-patrol.mjs', reason: 'API Deploy 复制上游产物时必须包含健康巡检脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora current release 自审脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换状态快照脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据包脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换命令证据脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据验真脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora 切换证据根目录审计脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs', + reason: 'API Deploy 复制上游产物时必须包含健康巡检 env 复核脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora release readiness 聚合门禁脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora canary live smoke 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs', + reason: 'API Deploy 复制上游产物时必须包含 Pingora canary access log 对账脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh', + reason: 'API Deploy 必须复制并执行 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + '"build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh"', + reason: 'API Deploy 执行入口必须是 build 产物内的 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh', + reason: 'API Deploy 必须复制 build 产物内的维护开启脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs', + reason: 'API Deploy 必须复制 build 产物内的 Pingora health patrol env 切换脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh', + reason: 'API Deploy 必须复制 build 产物内的维护关闭脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/deploy/env/**', + reason: 'API Deploy 必须从上游 build 产物复制 env 示例目录。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'build/${params.BUILD_VERSION}/deploy/nginx/**', + reason: 'API Deploy 必须从上游 build 产物复制 Nginx/Pingora canary snippet。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'NGINX_DEPLOY_DIR_SOURCE', + reason: 'API Deploy 必须把发布产物内 deploy/nginx 复制到 current release。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + excludes: ',scripts/deploy/production-api-deploy.sh', + reason: 'API Deploy 不应再复制工作区根部 deploy 脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: 'pingora-gateway.sha256', + reason: + 'API Deploy 复制上游产物时必须保留 Pingora 影子网关可选 checksum 产物。', + }, +]; + +const nodeEnvFileCommandFiles = [ + 'package.json', + 'deploy/systemd/genarrative-database-backup.service', + 'scripts/deploy/production-stdb-publish.sh', + 'scripts/deploy/pingora-direct-enable.sh', + 'scripts/deploy/pingora-direct-rollback.sh', ]; let failed = false; for (const check of checks) { const content = readFileSync(check.file, 'utf8'); - if (!content.includes(check.includes)) { + if (check.includes && !content.includes(check.includes)) { failed = true; console.error( `[check:production-ops] ${check.file} 缺少 ${check.includes}。${check.reason}`, ); } + if (check.excludes && content.includes(check.excludes)) { + failed = true; + console.error( + `[check:production-ops] ${check.file} 不应包含 ${check.excludes}。${check.reason}`, + ); + } +} + +for (const file of nodeEnvFileCommandFiles) { + const content = readFileSync(file, 'utf8'); + const commandText = content.replace(/\\\r?\n\s*/g, ' '); + const unsafeCommands = commandText.match( + /\bnode\b(?!\s+--)(?=[^;&|\n]*\.mjs)(?=[^;&|\n]*--env-file)[^;&|\n]*/g, + ); + if (!unsafeCommands) { + continue; + } + + failed = true; + for (const command of unsafeCommands) { + console.error( + `[check:production-ops] ${file} 中的 Node 命令必须用 node -- 分隔业务 --env-file 参数: ${command.trim()}`, + ); + } } if (failed) { diff --git a/scripts/database-backup-to-oss.mjs b/scripts/database-backup-to-oss.mjs index b01cf5571..4c5587b3d 100644 --- a/scripts/database-backup-to-oss.mjs +++ b/scripts/database-backup-to-oss.mjs @@ -20,8 +20,8 @@ const UNSIGNED_PAYLOAD = 'UNSIGNED-PAYLOAD'; function usage() { console.log(`用法: npm run database:backup:oss -- [--data-dir ] [--work-dir ] [--bucket ] [--object-prefix ] [--keep-local] - node scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] - node scripts/database-backup-to-oss.mjs --upload-archive + node -- scripts/database-backup-to-oss.mjs [--stop-service spacetimedb.service] [--restart-service-after genarrative-api.service] [--defer-upload] + node -- scripts/database-backup-to-oss.mjs --upload-archive 说明: 将 SpacetimeDB 数据目录打包成 .tar.gz，并上传到阿里云 OSS 指定 bucket。 diff --git a/scripts/deploy/pingora-direct-enable.sh b/scripts/deploy/pingora-direct-enable.sh new file mode 100644 index 000000000..99ca08846 --- /dev/null +++ b/scripts/deploy/pingora-direct-enable.sh @@ -0,0 +1,909 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" +SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}" +DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service" +SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_DIRECT_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}" +DEFAULT_TEMPLATE_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf" +TEMPLATE_PATH="${GENARRATIVE_PINGORA_DIRECT_TEMPLATE_PATH:-${DEFAULT_TEMPLATE_PATH}}" +DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}" +PREFLIGHT_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-direct-preflight.mjs}" +PREFLIGHT_ENV_FILE="${GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE:-}" +CURRENT_RELEASE_AUDIT_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_SCRIPT:-${REPO_ROOT}/scripts/ops/pingora-current-release-audit.mjs}" +CURRENT_RELEASE_ROOT="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_ROOT:-${REPO_ROOT}}" +CURRENT_RELEASE_AUDIT_TIMEOUT_MS="${GENARRATIVE_PINGORA_DIRECT_CURRENT_RELEASE_AUDIT_TIMEOUT_MS:-}" +PREFLIGHT_SYSTEMD="false" +PREFLIGHT_CHECK_CERT_READABLE="false" +PREFLIGHT_CHECK_SERVICE_ENV_FILE="false" +PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE="false" +PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE="false" +PREFLIGHT_CHECK_PORTS_FREE="false" +DIRECT_LIVE_SCRIPT="${GENARRATIVE_PINGORA_DIRECT_LIVE_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-direct-live.mjs}" +DIRECT_HTTPS_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL:-}" +DIRECT_HTTP_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL:-}" +DIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_HOST:-}" +DIRECT_REDIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST:-}" +DIRECT_PROBE_TOKEN="${GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN:-}" +DIRECT_PINGORA_ACCESS_LOG="${GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG:-}" +DIRECT_ACCESS_LOG_SINCE_LINES="${GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES:-}" +DIRECT_SPACETIME_DATABASE="${GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE:-}" +DIRECT_TIMEOUT_MS="${GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS:-}" +APPLY="false" +STATUS_AFTER="true" +VERIFY_SYSTEMD_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-direct-enable.sh [--apply] [--preflight-env-file ] [--preflight-systemd] [--preflight-check-cert-readable] [--preflight-check-service-env-file] [--preflight-check-service-user-cert-readable] [--preflight-check-service-binary-executable] [--preflight-check-ports-free] + [--direct-https-base-url ] [--direct-http-base-url ] [--direct-host ] [--direct-redirect-host ] + [--direct-spacetime-database ] [--direct-probe-token ] [--direct-pingora-access-log ] [--direct-access-log-since-lines ] [--direct-timeout-ms ] [--direct-live-script ] [--no-postcheck] + [--current-release-audit-script ] [--current-release-root ] [--current-release-audit-timeout-ms ] + [--service ] [--service-unit-path ] [--template-path ] [--dropin-path ] + +说明: + 启用 Pingora 直连低端口入口：把已准备好的 direct-entry systemd drop-in 模板安装到 + genarrative-pingora-gateway.service.d/direct-entry.conf，执行 daemon-reload， + 重启 genarrative-pingora-gateway.service，让服务具备绑定 80/443 的最小 capability。 + +默认是 dry-run，只打印将执行的命令；必须显式传 --apply 才会修改系统状态。 +该脚本不写入 TLS env、不复制证书、不停止 Nginx；--apply 会先执行 current release 自包含自审， +确认发布包已携带 Pingora 网关且 systemd ExecStart 指向随包二进制；--apply 时必须提供 --preflight-env-file、 +--preflight-check-cert-readable、--preflight-check-service-env-file、--preflight-check-service-user-cert-readable、--preflight-check-service-binary-executable、--preflight-check-ports-free 和 direct live smoke 参数， +先通过 direct preflight，安装并重启 Pingora，再通过 direct live smoke 后才视为启用成功。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + --no-postcheck) + VERIFY_SYSTEMD_AFTER="false" + shift + ;; + --service) + SERVICE_NAME="${2:-}" + if [[ -z "${SERVICE_NAME}" ]]; then + echo "[pingora-direct-enable] --service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --service-unit-path) + SERVICE_UNIT_PATH="${2:-}" + if [[ -z "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-enable] --service-unit-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --template-path) + TEMPLATE_PATH="${2:-}" + if [[ -z "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] --template-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --dropin-path) + DROPIN_PATH="${2:-}" + if [[ -z "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] --dropin-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-script) + PREFLIGHT_SCRIPT="${2:-}" + if [[ -z "${PREFLIGHT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --preflight-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-env-file) + PREFLIGHT_ENV_FILE="${2:-}" + if [[ -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] --preflight-env-file 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-audit-script) + CURRENT_RELEASE_AUDIT_SCRIPT="${2:-}" + if [[ -z "${CURRENT_RELEASE_AUDIT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --current-release-audit-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-root) + CURRENT_RELEASE_ROOT="${2:-}" + if [[ -z "${CURRENT_RELEASE_ROOT}" ]]; then + echo "[pingora-direct-enable] --current-release-root 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --current-release-audit-timeout-ms) + CURRENT_RELEASE_AUDIT_TIMEOUT_MS="${2:-}" + if [[ -z "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" ]]; then + echo "[pingora-direct-enable] --current-release-audit-timeout-ms 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --preflight-systemd) + PREFLIGHT_SYSTEMD="true" + shift + ;; + --preflight-check-cert-readable) + PREFLIGHT_CHECK_CERT_READABLE="true" + shift + ;; + --preflight-check-service-env-file) + PREFLIGHT_CHECK_SERVICE_ENV_FILE="true" + shift + ;; + --preflight-check-service-user-cert-readable) + PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE="true" + shift + ;; + --preflight-check-service-binary-executable) + PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE="true" + shift + ;; + --preflight-check-ports-free) + PREFLIGHT_CHECK_PORTS_FREE="true" + shift + ;; + --direct-live-script) + DIRECT_LIVE_SCRIPT="${2:-}" + if [[ -z "${DIRECT_LIVE_SCRIPT}" ]]; then + echo "[pingora-direct-enable] --direct-live-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-https-base-url) + DIRECT_HTTPS_BASE_URL="${2:-}" + if [[ -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-https-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-http-base-url) + DIRECT_HTTP_BASE_URL="${2:-}" + if [[ -z "${DIRECT_HTTP_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-http-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-host) + DIRECT_HOST="${2:-}" + if [[ -z "${DIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --direct-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-redirect-host) + DIRECT_REDIRECT_HOST="${2:-}" + if [[ -z "${DIRECT_REDIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --direct-redirect-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-probe-token) + DIRECT_PROBE_TOKEN="${2:-}" + if [[ -z "${DIRECT_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-enable] --direct-probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-pingora-access-log) + DIRECT_PINGORA_ACCESS_LOG="${2:-}" + if [[ -z "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + echo "[pingora-direct-enable] --direct-pingora-access-log 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-access-log-since-lines) + DIRECT_ACCESS_LOG_SINCE_LINES="${2:-}" + if [[ -z "${DIRECT_ACCESS_LOG_SINCE_LINES}" ]]; then + echo "[pingora-direct-enable] --direct-access-log-since-lines 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-spacetime-database) + DIRECT_SPACETIME_DATABASE="${2:-}" + if [[ -z "${DIRECT_SPACETIME_DATABASE}" ]]; then + echo "[pingora-direct-enable] --direct-spacetime-database 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --direct-timeout-ms) + DIRECT_TIMEOUT_MS="${2:-}" + if [[ -z "${DIRECT_TIMEOUT_MS}" ]]; then + echo "[pingora-direct-enable] --direct-timeout-ms 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + *) + echo "[pingora-direct-enable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-direct-enable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +reject_filesystem_root_path() { + local label="$1" + local value="$2" + if is_filesystem_root_path "${value}"; then + echo "[pingora-direct-enable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +reject_control_characters "--service" "${SERVICE_NAME}" +reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_control_characters "--template-path" "${TEMPLATE_PATH}" +reject_control_characters "--dropin-path" "${DROPIN_PATH}" +reject_control_characters "--preflight-script" "${PREFLIGHT_SCRIPT}" +reject_control_characters "--preflight-env-file" "${PREFLIGHT_ENV_FILE}" +reject_control_characters "--current-release-audit-script" "${CURRENT_RELEASE_AUDIT_SCRIPT}" +reject_control_characters "--current-release-root" "${CURRENT_RELEASE_ROOT}" +reject_control_characters "--current-release-audit-timeout-ms" "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" +reject_control_characters "--direct-live-script" "${DIRECT_LIVE_SCRIPT}" +reject_control_characters "--direct-https-base-url" "${DIRECT_HTTPS_BASE_URL}" +reject_control_characters "--direct-http-base-url" "${DIRECT_HTTP_BASE_URL}" +reject_control_characters "--direct-host" "${DIRECT_HOST}" +reject_control_characters "--direct-redirect-host" "${DIRECT_REDIRECT_HOST}" +reject_control_characters "--direct-probe-token" "${DIRECT_PROBE_TOKEN}" +reject_control_characters "--direct-pingora-access-log" "${DIRECT_PINGORA_ACCESS_LOG}" +reject_control_characters "--direct-access-log-since-lines" "${DIRECT_ACCESS_LOG_SINCE_LINES}" +reject_control_characters "--direct-spacetime-database" "${DIRECT_SPACETIME_DATABASE}" +reject_control_characters "--direct-timeout-ms" "${DIRECT_TIMEOUT_MS}" + +reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_filesystem_root_path "--template-path" "${TEMPLATE_PATH}" +reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}" +reject_filesystem_root_path "--preflight-env-file" "${PREFLIGHT_ENV_FILE}" +reject_filesystem_root_path "--preflight-script" "${PREFLIGHT_SCRIPT}" +reject_filesystem_root_path "--current-release-audit-script" "${CURRENT_RELEASE_AUDIT_SCRIPT}" +reject_filesystem_root_path "--current-release-root" "${CURRENT_RELEASE_ROOT}" +reject_filesystem_root_path "--direct-live-script" "${DIRECT_LIVE_SCRIPT}" +reject_filesystem_root_path "--direct-pingora-access-log" "${DIRECT_PINGORA_ACCESS_LOG}" + +if [[ "${TEMPLATE_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --template-path 必须是绝对路径: ${TEMPLATE_PATH}" >&2 + exit 1 +fi + +if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ "${DROPIN_PATH}" != /* ]]; then + echo "[pingora-direct-enable] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2 + exit 1 +fi + +if [[ -n "${PREFLIGHT_ENV_FILE}" && "${PREFLIGHT_ENV_FILE}" != /* ]]; then + echo "[pingora-direct-enable] --preflight-env-file 必须是绝对路径: ${PREFLIGHT_ENV_FILE}" >&2 + exit 1 +fi + +if [[ "${PREFLIGHT_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --preflight-script 必须是绝对路径: ${PREFLIGHT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${CURRENT_RELEASE_AUDIT_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --current-release-audit-script 必须是绝对路径: ${CURRENT_RELEASE_AUDIT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${CURRENT_RELEASE_ROOT}" != /* ]]; then + echo "[pingora-direct-enable] --current-release-root 必须是绝对路径: ${CURRENT_RELEASE_ROOT}" >&2 + exit 1 +fi + +if [[ "${DIRECT_LIVE_SCRIPT}" != /* ]]; then + echo "[pingora-direct-enable] --direct-live-script 必须是绝对路径: ${DIRECT_LIVE_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-env-file，避免跳过直连 env / 证书预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_CERT_READABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-cert-readable，避免跳过 TLS 证书和私钥可读预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-env-file，避免检查的 env 文件和 systemd service 实际读取的 EnvironmentFile 不一致。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-user-cert-readable，避免跳过 systemd 服务用户读取 TLS 证书和私钥预检。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-service-binary-executable，避免 current release 缺少可执行 pingora-gateway 时才在重启阶段失败。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && "${PREFLIGHT_CHECK_PORTS_FREE}" != "true" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --preflight-check-ports-free，避免 Nginx 或其它进程仍占用 80/443。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-https-base-url，启用后验证 Pingora HTTPS 直连入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HTTP_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-http-base-url，启用后验证 HTTP redirect / ACME 入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-host，启用后用正式域名 Host/SNI 验证直连入口。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_REDIRECT_HOST}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-redirect-host，启用后验证 HTTP redirect Location host。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-pingora-access-log，启用后验证直连请求已写入 Pingora access log。" >&2 + exit 1 +fi + +if [[ -n "${DIRECT_PINGORA_ACCESS_LOG}" && "${DIRECT_PINGORA_ACCESS_LOG}" != /* ]]; then + echo "[pingora-direct-enable] --direct-pingora-access-log 必须是绝对路径: ${DIRECT_PINGORA_ACCESS_LOG}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${DIRECT_SPACETIME_DATABASE}" ]]; then + echo "[pingora-direct-enable] --apply 必须同时提供 --direct-spacetime-database，启用后验证目标库 WSS subscribe。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] direct-entry 模板不存在: ${TEMPLATE_PATH}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-enable] 主 service 模板不存在: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${CURRENT_RELEASE_AUDIT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] current release 自审脚本不存在: ${CURRENT_RELEASE_AUDIT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${PREFLIGHT_SCRIPT}" ]]; then + echo "[pingora-direct-enable] direct preflight 脚本不存在: ${PREFLIGHT_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && ! -f "${DIRECT_LIVE_SCRIPT}" ]]; then + echo "[pingora-direct-enable] direct live smoke 脚本不存在: ${DIRECT_LIVE_SCRIPT}" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" ]]; then + DROPIN_DIR="$(dirname "${DROPIN_PATH}")" + if [[ -L "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-enable] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-enable] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -L "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-enable] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2 + exit 1 + fi +fi + +run_cmd() { + echo "+ $*" + if [[ "${APPLY}" == "true" ]]; then + "$@" + fi +} + +print_redacted_command() { + local previous="" output=() + for arg in "$@"; do + if [[ "${previous}" == "--probe-token" || "${previous}" == "--direct-probe-token" ]]; then + output+=("") + else + output+=("${arg}") + fi + previous="${arg}" + done + echo "+ ${output[*]}" +} + +run_systemctl() { + if [[ "${APPLY}" != "true" ]]; then + run_cmd systemctl "$@" + return + fi + if command -v systemctl >/dev/null 2>&1; then + run_cmd systemctl "$@" + else + echo "[pingora-direct-enable] 未找到 systemctl，无法执行: systemctl $*" >&2 + exit 1 + fi +} + +run_preflight() { + if [[ -z "${PREFLIGHT_ENV_FILE}" ]]; then + echo "[pingora-direct-enable] 未提供 --preflight-env-file，仅执行脚本 dry-run；--apply 会强制要求预检。" + return + fi + + preflight_args=( + "${PREFLIGHT_SCRIPT}" + --env-file "${PREFLIGHT_ENV_FILE}" + --require-live-env + ) + if [[ "${PREFLIGHT_SYSTEMD}" == "true" ]]; then + preflight_args+=(--systemd-cat) + fi + if [[ "${PREFLIGHT_CHECK_CERT_READABLE}" == "true" ]]; then + preflight_args+=(--check-cert-readable) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" == "true" ]]; then + preflight_args+=(--check-service-env-file) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE}" == "true" ]]; then + preflight_args+=(--check-service-user-cert-readable) + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE}" == "true" ]]; then + preflight_args+=(--check-service-binary-executable) + fi + if [[ "${PREFLIGHT_CHECK_PORTS_FREE}" == "true" ]]; then + preflight_args+=(--check-ports-free) + fi + + echo "+ node -- ${preflight_args[*]}" + node -- "${preflight_args[@]}" +} + +run_current_release_audit() { + audit_args=( + "${CURRENT_RELEASE_AUDIT_SCRIPT}" + --release-root "${CURRENT_RELEASE_ROOT}" + --require-pingora-gateway + --systemd-show + --systemd-service "${SERVICE_NAME}" + ) + if [[ -n "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}" ]]; then + audit_args+=(--timeout-ms "${CURRENT_RELEASE_AUDIT_TIMEOUT_MS}") + fi + + echo "+ node -- ${audit_args[*]}" + if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] dry-run：--apply 前会先执行 current release 自包含自审。" + return + fi + + node -- "${audit_args[@]}" +} + +systemd_env_file_matches() { + local expected="$1" + local line value word normalized + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == EnvironmentFile=* ]] || continue + value="${line#EnvironmentFile=}" + [[ -n "${value}" ]] || continue + for word in ${value}; do + normalized="${word%\"}" + normalized="${normalized#\"}" + normalized="${normalized%\'}" + normalized="${normalized#\'}" + normalized="${normalized#-}" + if [[ "${normalized}" == "${expected}" ]]; then + return 0 + fi + done + done <<<"${unit_content}" + return 1 +} + +read_expected_exec_start() { + local line value + if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then + return 1 + fi + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == ExecStart=* ]] || continue + value="${line#ExecStart=}" + [[ -n "${value}" ]] || continue + printf '%s\n' "${value}" + return 0 + done <"${SERVICE_UNIT_PATH}" + return 1 +} + +first_exec_word() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ "${value}" == \"* ]]; then + value="${value#\"}" + printf '%s\n' "${value%%\"*}" + return + fi + printf '%s\n' "${value%%[[:space:]]*}" +} + +systemd_show_exec_start_matches() { + local expected_exec="$1" + local expected_binary actual_show actual_binary + expected_binary="$(first_exec_word "${expected_exec}")" + [[ -n "${expected_binary}" ]] || return 1 + actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)" + actual_binary="${actual_show}" + if [[ "${actual_show}" == *"path="* ]]; then + actual_binary="${actual_show#*path=}" + actual_binary="${actual_binary%% ;*}" + elif [[ "${actual_show}" == ExecStart=* ]]; then + actual_binary="${actual_show#ExecStart=}" + actual_binary="$(first_exec_word "${actual_binary}")" + else + actual_binary="$(first_exec_word "${actual_binary}")" + fi + [[ "${actual_binary}" == "${expected_binary}" ]] +} + +verify_systemd_exec_start() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 Pingora ExecStart 指向核验。" + return + fi + + expected_exec_start="$(read_expected_exec_start || true)" + if [[ -z "${expected_exec_start}" ]]; then + echo "[pingora-direct-enable] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2 + exit 1 + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + echo "[pingora-direct-enable] dry-run：--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl，无法核验 Pingora ExecStart。" >&2 + exit 1 + fi + + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + if ! systemd_show_exec_start_matches "${expected_exec_start}"; then + echo "[pingora-direct-enable] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start}，请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2 + exit 1 + fi +} + +verify_systemd_dropin() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 systemd drop-in 生效核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl cat ${SERVICE_NAME}" + echo "[pingora-direct-enable] dry-run：--apply 后会校验 systemd 最终配置包含 CAP_NET_BIND_SERVICE。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl，无法核验 drop-in 是否生效。" >&2 + exit 1 + fi + + echo "+ systemctl cat ${SERVICE_NAME}" + unit_content="$(systemctl cat "${SERVICE_NAME}")" + if [[ "${unit_content}" != *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-enable] systemctl cat 未显示 AmbientCapabilities=CAP_NET_BIND_SERVICE，direct-entry drop-in 未生效。" >&2 + exit 1 + fi + if [[ "${unit_content}" != *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-enable] systemctl cat 未显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE，direct-entry drop-in 未生效。" >&2 + exit 1 + fi + if [[ "${PREFLIGHT_CHECK_SERVICE_ENV_FILE}" == "true" && -n "${PREFLIGHT_ENV_FILE}" ]]; then + if ! systemd_env_file_matches "${PREFLIGHT_ENV_FILE}"; then + echo "[pingora-direct-enable] systemctl cat 未显示 EnvironmentFile=${PREFLIGHT_ENV_FILE}，service 实际读取的 env 与 preflight env 不一致。" >&2 + exit 1 + fi + fi +} + +verify_service_active_after_restart() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-enable] 已跳过 Pingora 重启后状态核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl is-active ${SERVICE_NAME}" + echo "[pingora-direct-enable] dry-run：--apply 后会校验 Pingora service 为 active。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-enable] 未找到 systemctl，无法核验 Pingora 重启后状态。" >&2 + exit 1 + fi + + echo "+ systemctl is-active ${SERVICE_NAME}" + service_state="$(systemctl is-active "${SERVICE_NAME}")" + if [[ "${service_state}" != "active" ]]; then + echo "[pingora-direct-enable] Pingora 重启后状态不是 active: ${service_state}" >&2 + exit 1 + fi +} + +run_direct_live_smoke_after_enable() { + if [[ -z "${DIRECT_HTTPS_BASE_URL}" ]]; then + echo "+ node ${DIRECT_LIVE_SCRIPT} --https-base-url --http-base-url --host --redirect-host --spacetime-database --require-wss-upgrade" + echo "[pingora-direct-enable] dry-run：--apply 后会执行 direct live smoke；--apply 会强制要求 direct live 参数。" + return + fi + + direct_live_args=( + "${DIRECT_LIVE_SCRIPT}" + --https-base-url "${DIRECT_HTTPS_BASE_URL}" + ) + if [[ -n "${DIRECT_HTTP_BASE_URL}" ]]; then + direct_live_args+=(--http-base-url "${DIRECT_HTTP_BASE_URL}") + fi + if [[ -n "${DIRECT_HOST}" ]]; then + direct_live_args+=(--host "${DIRECT_HOST}") + fi + if [[ -n "${DIRECT_REDIRECT_HOST}" ]]; then + direct_live_args+=(--redirect-host "${DIRECT_REDIRECT_HOST}") + fi + if [[ -n "${DIRECT_PROBE_TOKEN}" ]]; then + direct_live_args+=(--probe-token "${DIRECT_PROBE_TOKEN}") + fi + if [[ -n "${DIRECT_PINGORA_ACCESS_LOG}" ]]; then + direct_live_args+=(--pingora-access-log "${DIRECT_PINGORA_ACCESS_LOG}") + fi + if [[ -n "${DIRECT_ACCESS_LOG_SINCE_LINES}" ]]; then + direct_live_args+=(--access-log-since-lines "${DIRECT_ACCESS_LOG_SINCE_LINES}") + fi + if [[ -n "${DIRECT_SPACETIME_DATABASE}" ]]; then + direct_live_args+=(--spacetime-database "${DIRECT_SPACETIME_DATABASE}") + fi + direct_live_args+=(--require-wss-upgrade) + if [[ -n "${DIRECT_TIMEOUT_MS}" ]]; then + direct_live_args+=(--timeout-ms "${DIRECT_TIMEOUT_MS}") + fi + direct_live_args+=(--json) + + print_redacted_command node "${direct_live_args[@]}" + if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] dry-run：--apply 后会验证 Pingora HTTPS / HTTP redirect / WSS 直连入口，并解析 direct-access-log 结构化证据。" + return + fi + + set +e + direct_live_output="$(node "${direct_live_args[@]}")" + direct_live_status=$? + set -e + printf '%s\n' "${direct_live_output}" + if [[ "${direct_live_status}" -ne 0 ]]; then + exit "${direct_live_status}" + fi + DIRECT_LIVE_OUTPUT="${direct_live_output}" node -- <<'NODE' +const stdout = process.env.DIRECT_LIVE_OUTPUT || ''; + +const jsonText = extractFirstJsonObject(stdout); +if (!jsonText) { + console.error( + '[pingora-direct-enable] direct live stdout 中未找到 JSON 结果对象，无法确认 direct-access-log 结构化证据。', + ); + process.exit(1); +} + +let parsed; +try { + parsed = JSON.parse(jsonText); +} catch (error) { + console.error( + `[pingora-direct-enable] direct live JSON 解析失败: ${error.message}`, + ); + process.exit(1); +} + +const accessLogCheck = parsed?.results?.find( + (item) => item?.name === 'direct-access-log', +); +if (!accessLogCheck) { + console.error( + '[pingora-direct-enable] direct live JSON 缺少 direct-access-log 结构化结果。', + ); + process.exit(1); +} + +const checked = toNonNegativeInteger(accessLogCheck.checked); +const matchedCount = toNonNegativeInteger(accessLogCheck.matchedCount); +const missingCount = toNonNegativeInteger(accessLogCheck.missingCount); +const mismatchCount = toNonNegativeInteger(accessLogCheck.mismatchCount); +const missingDetailsCount = Array.isArray(accessLogCheck.missing) + ? accessLogCheck.missing.length + : null; +const mismatchDetailsCount = Array.isArray(accessLogCheck.mismatches) + ? accessLogCheck.mismatches.length + : null; + +if ( + checked === null || + matchedCount === null || + missingCount !== 0 || + mismatchCount !== 0 || + matchedCount !== checked || + missingDetailsCount !== 0 || + mismatchDetailsCount !== 0 +) { + console.error( + `[pingora-direct-enable] direct-access-log 结构化证据未通过: checked=${checked ?? '-'} matched=${matchedCount ?? '-'} missing=${missingCount ?? '-'} mismatches=${mismatchCount ?? '-'}`, + ); + process.exit(1); +} + +console.log( + `[pingora-direct-enable] direct-access-log evidence checked=${checked} matched=${matchedCount} missing=0 mismatches=0`, +); + +function extractFirstJsonObject(text) { + const startIndex = text.indexOf('{'); + if (startIndex < 0) { + return ''; + } + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + return ''; +} + +function toNonNegativeInteger(value) { + const number = Number(value); + if (!Number.isInteger(number) || number < 0) { + return null; + } + return number; +} +NODE +} + +echo "[pingora-direct-enable] apply=${APPLY} service=${SERVICE_NAME} template=${TEMPLATE_PATH} dropin=${DROPIN_PATH}" + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-enable] 当前是 dry-run；确认 preflight 通过后追加 --apply。" + if [[ ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-direct-enable] dry-run 提示：当前模板不存在，--apply 会失败: ${TEMPLATE_PATH}" >&2 + fi +fi + +run_current_release_audit +run_preflight +run_cmd install -d -m 0755 "$(dirname "${DROPIN_PATH}")" +run_cmd install -m 0644 "${TEMPLATE_PATH}" "${DROPIN_PATH}" +run_systemctl daemon-reload +run_systemctl restart "${SERVICE_NAME}" +verify_systemd_dropin +verify_systemd_exec_start +verify_service_active_after_restart +run_direct_live_smoke_after_enable + +if [[ "${STATUS_AFTER}" == "true" ]]; then + run_systemctl status "${SERVICE_NAME}" --no-pager +fi + +cat <} --http-base-url ${DIRECT_HTTP_BASE_URL:-} --host ${DIRECT_HOST:-} --redirect-host ${DIRECT_REDIRECT_HOST:-} --pingora-access-log ${DIRECT_PINGORA_ACCESS_LOG:-} --spacetime-database ${DIRECT_SPACETIME_DATABASE:-} --require-wss-upgrade --json 已通过，且 direct-access-log 结构化证据 matched=checked、missing=0、mismatches=0 +EOF diff --git a/scripts/deploy/pingora-direct-rollback.sh b/scripts/deploy/pingora-direct-rollback.sh new file mode 100644 index 000000000..f5b56ba8a --- /dev/null +++ b/scripts/deploy/pingora-direct-rollback.sh @@ -0,0 +1,768 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SERVICE_NAME="${GENARRATIVE_PINGORA_GATEWAY_SERVICE:-genarrative-pingora-gateway.service}" +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)" +DEFAULT_SERVICE_UNIT_PATH="${REPO_ROOT}/deploy/systemd/genarrative-pingora-gateway.service" +SERVICE_UNIT_PATH="${GENARRATIVE_PINGORA_ROLLBACK_SERVICE_UNIT_PATH:-${DEFAULT_SERVICE_UNIT_PATH}}" +DROPIN_PATH="${GENARRATIVE_PINGORA_DIRECT_DROPIN_PATH:-/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SMOKE_URL="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_URL:-}" +NGINX_SMOKE_HOST="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_HOST:-}" +NGINX_SMOKE_EXPECT_BODY="${GENARRATIVE_NGINX_ROLLBACK_SMOKE_EXPECT_BODY:-}" +CURL_BINARY="${GENARRATIVE_CURL_BINARY:-curl}" +HEALTH_PATROL_ENV_FILE="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_FILE:-}" +HEALTH_PATROL_ENV_CHECK_SCRIPT="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_ENV_CHECK_SCRIPT:-}" +HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL:-}" +HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_EXPECTED_PUBLIC_HOST:-}" +HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="${GENARRATIVE_PINGORA_ROLLBACK_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST:-auto}" +PINGORA_SHADOW_PROBE_URL="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_URL:-}" +PINGORA_SHADOW_PROBE_TOKEN="${GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN:-}" +APPLY="false" +RELOAD_NGINX="false" +STATUS_AFTER="true" +VERIFY_SYSTEMD_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-direct-rollback.sh [--apply] [--reload-nginx] [--nginx-smoke-url ] [--nginx-smoke-host ] [--nginx-smoke-expect-body ] [--curl-binary ] [--health-patrol-env-file ] [--health-patrol-env-check-script ] [--health-patrol-expected-public-base-url ] [--health-patrol-expected-public-host |--health-patrol-require-empty-public-host] [--pingora-shadow-probe-url ] [--pingora-shadow-probe-token ] [--no-postcheck] [--service ] [--service-unit-path ] [--dropin-path ] [--nginx-service ] [--nginx-binary ] + +说明: + 回退 Pingora 直连低端口入口：移除 direct-entry systemd drop-in，执行 daemon-reload， + 重启 genarrative-pingora-gateway.service，让服务回到 shadow/high-port 口径。 + +默认是 dry-run，只打印将执行的命令；必须显式传 --apply 才会修改系统状态。 +--apply 必须同时传 --reload-nginx，让脚本执行 nginx reload， +脚本会先运行 nginx -t，通过后再 reload，并在 reload 后确认 Nginx 仍处于 active 状态。 +--apply 还必须提供 --nginx-smoke-url，reload 后用 curl 验证 Nginx 入口已经真实响应。 +若提供 --nginx-smoke-expect-body，脚本会要求 smoke 响应体包含该片段，避免 HTTP 200 命中错误入口。 +脚本会以 current release 随包主 service 模板的 ExecStart 为真相源， +在重启 Pingora 后核验 systemd 最终 ExecStart 仍指向同一个 pingora-gateway 二进制。 +如果提供 --health-patrol-env-file，脚本会在 Nginx smoke 之后复核 health patrol 已切回 nginx 模式， +并可校验回退后的 public base URL / Host 是否恢复为切换前记录值。 +如果同时提供 --pingora-shadow-probe-url 与 --pingora-shadow-probe-token，脚本会在回退后 +验证 Pingora shadow 高端口探针仍返回 gateway=pingora-shadow。 +EOF +} + +resolve_default_health_patrol_env_check_script() { + printf '%s\n' "${REPO_ROOT}/scripts/check-production-health-patrol-env.mjs" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --reload-nginx) + RELOAD_NGINX="true" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + --no-postcheck) + VERIFY_SYSTEMD_AFTER="false" + shift + ;; + --service) + SERVICE_NAME="${2:-}" + if [[ -z "${SERVICE_NAME}" ]]; then + echo "[pingora-direct-rollback] --service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --service-unit-path) + SERVICE_UNIT_PATH="${2:-}" + if [[ -z "${SERVICE_UNIT_PATH}" ]]; then + echo "[pingora-direct-rollback] --service-unit-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --dropin-path) + DROPIN_PATH="${2:-}" + if [[ -z "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] --dropin-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-direct-rollback] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-direct-rollback] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-url) + NGINX_SMOKE_URL="${2:-}" + if [[ -z "${NGINX_SMOKE_URL}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-host) + NGINX_SMOKE_HOST="${2:-}" + if [[ -z "${NGINX_SMOKE_HOST}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-smoke-expect-body) + NGINX_SMOKE_EXPECT_BODY="${2:-}" + if [[ -z "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + echo "[pingora-direct-rollback] --nginx-smoke-expect-body 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --curl-binary) + CURL_BINARY="${2:-}" + if [[ -z "${CURL_BINARY}" ]]; then + echo "[pingora-direct-rollback] --curl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-env-file) + HEALTH_PATROL_ENV_FILE="${2:-}" + if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-file 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-env-check-script) + HEALTH_PATROL_ENV_CHECK_SCRIPT="${2:-}" + if [[ -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-check-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-expected-public-base-url) + HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL="${2:-}" + if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-expected-public-host) + HEALTH_PATROL_EXPECTED_PUBLIC_HOST="${2:-}" + if [[ -z "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --health-patrol-require-empty-public-host) + HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST="true" + shift + ;; + --pingora-shadow-probe-url) + PINGORA_SHADOW_PROBE_URL="${2:-}" + if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --pingora-shadow-probe-token) + PINGORA_SHADOW_PROBE_TOKEN="${2:-}" + if [[ -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + *) + echo "[pingora-direct-rollback] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +if [[ -n "${HEALTH_PATROL_ENV_FILE}" && -z "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + HEALTH_PATROL_ENV_CHECK_SCRIPT="$(resolve_default_health_patrol_env_check_script)" +fi + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-direct-rollback] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +reject_filesystem_root_path() { + local label="$1" + local value="$2" + if is_filesystem_root_path "${value}"; then + echo "[pingora-direct-rollback] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +reject_control_characters "--service" "${SERVICE_NAME}" +reject_control_characters "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_control_characters "--dropin-path" "${DROPIN_PATH}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-smoke-url" "${NGINX_SMOKE_URL}" +reject_control_characters "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" +reject_control_characters "--nginx-smoke-expect-body" "${NGINX_SMOKE_EXPECT_BODY}" +reject_control_characters "--curl-binary" "${CURL_BINARY}" +reject_control_characters "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" +reject_control_characters "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" +reject_control_characters "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" +reject_control_characters "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" +reject_control_characters "--health-patrol-require-empty-public-host" "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" +reject_control_characters "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" +reject_control_characters "--pingora-shadow-probe-token" "${PINGORA_SHADOW_PROBE_TOKEN}" + +reject_filesystem_root_path "--service-unit-path" "${SERVICE_UNIT_PATH}" +reject_filesystem_root_path "--dropin-path" "${DROPIN_PATH}" +reject_filesystem_root_path "--nginx-binary" "${NGINX_BINARY}" +reject_filesystem_root_path "--curl-binary" "${CURL_BINARY}" +reject_filesystem_root_path "--health-patrol-env-file" "${HEALTH_PATROL_ENV_FILE}" +reject_filesystem_root_path "--health-patrol-env-check-script" "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" + +if [[ "${DROPIN_PATH}" != /* ]]; then + echo "[pingora-direct-rollback] --dropin-path 必须是绝对路径: ${DROPIN_PATH}" >&2 + exit 1 +fi + +if [[ "${SERVICE_UNIT_PATH}" != /* ]]; then + echo "[pingora-direct-rollback] --service-unit-path 必须是绝对路径: ${SERVICE_UNIT_PATH}" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_ENV_FILE}" && "${HEALTH_PATROL_ENV_FILE}" != /* ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-file 必须是绝对路径: ${HEALTH_PATROL_ENV_FILE}" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" && "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" != /* ]]; then + echo "[pingora-direct-rollback] --health-patrol-env-check-script 必须是绝对路径: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 + exit 1 +fi + +require_absolute_path_if_path_like() { + local label="$1" + local value="$2" + if [[ "${value}" == */* && "${value}" != /* ]]; then + echo "[pingora-direct-rollback] ${label} 包含路径分隔符时必须是绝对路径: ${value}" >&2 + exit 1 + fi +} + +require_absolute_path_if_path_like "--nginx-binary" "${NGINX_BINARY}" +require_absolute_path_if_path_like "--curl-binary" "${CURL_BINARY}" + +if [[ -n "${PINGORA_SHADOW_PROBE_URL}" && -z "${PINGORA_SHADOW_PROBE_TOKEN}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-url 必须同时提供 --pingora-shadow-probe-token。" >&2 + exit 1 +fi + +if [[ -n "${PINGORA_SHADOW_PROBE_TOKEN}" && -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + echo "[pingora-direct-rollback] --pingora-shadow-probe-token 必须同时提供 --pingora-shadow-probe-url。" >&2 + exit 1 +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" && "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" == "true" ]]; then + echo "[pingora-direct-rollback] --health-patrol-expected-public-host 和 --health-patrol-require-empty-public-host 不能同时使用。" >&2 + exit 1 +fi + +validate_nginx_smoke_host() { + validate_host_value "--nginx-smoke-host" "${NGINX_SMOKE_HOST}" +} + +validate_host_value() { + local label="$1" + local value="$2" + if [[ "${value}" == *"://"* || "${value}" == */* || "${value}" == *\?* || "${value}" == *"#"* || "${value}" =~ [[:space:]] ]]; then + echo "[pingora-direct-rollback] ${label} 只能是 host 或 host:port，不能包含 URL、路径、查询、片段或空白字符: ${value}" >&2 + exit 1 + fi +} + +is_loopback_smoke_url() { + local url="$1" + local authority host + case "${url}" in + http://*|https://*) ;; + *) return 1 ;; + esac + authority="${url#*://}" + authority="${authority%%/*}" + authority="${authority%%\?*}" + authority="${authority%%#*}" + authority="${authority##*@}" + if [[ "${authority}" == \[*\]* ]]; then + host="${authority#\[}" + host="${host%%\]*}" + else + host="${authority%%:*}" + fi + host="${host,,}" + [[ "${host}" == "localhost" || "${host}" == "127."* || "${host}" == "::1" ]] +} + +validate_http_url() { + local label="$1" + local url="$2" + case "${url}" in + http://*|https://*) ;; + *) + echo "[pingora-direct-rollback] ${label} 必须是 http(s) URL: ${url}" >&2 + exit 1 + ;; + esac +} + +if [[ -n "${NGINX_SMOKE_HOST}" ]]; then + validate_nginx_smoke_host +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + validate_host_value "--health-patrol-expected-public-host" "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" +fi + +if [[ -n "${PINGORA_SHADOW_PROBE_URL}" ]]; then + validate_http_url "--pingora-shadow-probe-url" "${PINGORA_SHADOW_PROBE_URL}" +fi + +if [[ -n "${NGINX_SMOKE_URL}" ]]; then + validate_http_url "--nginx-smoke-url" "${NGINX_SMOKE_URL}" +fi + +if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + validate_http_url "--health-patrol-expected-public-base-url" "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" +fi + +if [[ "${APPLY}" == "true" && "${RELOAD_NGINX}" != "true" ]]; then + echo "[pingora-direct-rollback] --apply 必须同时提供 --reload-nginx，确保回退后 Nginx 公网入口重新加载并保持 active。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_URL}" ]]; then + echo "[pingora-direct-rollback] --apply 必须同时提供 --nginx-smoke-url，确保回退后 Nginx 入口真实可访问。" >&2 + exit 1 +fi + +if [[ "${APPLY}" == "true" && -z "${NGINX_SMOKE_HOST}" ]] && is_loopback_smoke_url "${NGINX_SMOKE_URL}"; then + echo "[pingora-direct-rollback] --nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host，避免回退 smoke 命中默认 vhost。" >&2 + exit 1 +fi + +run_cmd() { + echo "+ $*" + if [[ "${APPLY}" == "true" ]]; then + "$@" + fi +} + +run_systemctl() { + if [[ "${APPLY}" != "true" ]]; then + run_cmd systemctl "$@" + return + fi + if command -v systemctl >/dev/null 2>&1; then + run_cmd systemctl "$@" + else + echo "[pingora-direct-rollback] 未找到 systemctl，无法执行: systemctl $*" >&2 + exit 1 + fi +} + +read_expected_exec_start() { + local line value + if [[ ! -f "${SERVICE_UNIT_PATH}" ]]; then + return 1 + fi + while IFS= read -r line; do + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ "${line}" == ExecStart=* ]] || continue + value="${line#ExecStart=}" + [[ -n "${value}" ]] || continue + printf '%s\n' "${value}" + return 0 + done <"${SERVICE_UNIT_PATH}" + return 1 +} + +first_exec_word() { + local value="$1" + value="${value#"${value%%[![:space:]]*}"}" + value="${value%"${value##*[![:space:]]}"}" + if [[ "${value}" == \"* ]]; then + value="${value#\"}" + printf '%s\n' "${value%%\"*}" + return + fi + printf '%s\n' "${value%%[[:space:]]*}" +} + +systemd_show_exec_start_matches() { + local expected_exec="$1" + local expected_binary actual_show actual_binary + expected_binary="$(first_exec_word "${expected_exec}")" + [[ -n "${expected_binary}" ]] || return 1 + actual_show="$(systemctl show "${SERVICE_NAME}" --property=ExecStart --value --no-pager)" + actual_binary="${actual_show}" + if [[ "${actual_show}" == *"path="* ]]; then + actual_binary="${actual_show#*path=}" + actual_binary="${actual_binary%% ;*}" + elif [[ "${actual_show}" == ExecStart=* ]]; then + actual_binary="${actual_show#ExecStart=}" + actual_binary="$(first_exec_word "${actual_binary}")" + else + actual_binary="$(first_exec_word "${actual_binary}")" + fi + [[ "${actual_binary}" == "${expected_binary}" ]] +} + +verify_systemd_exec_start_after_rollback() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Pingora ExecStart 指向核验。" + return + fi + + expected_exec_start="$(read_expected_exec_start || true)" + if [[ -z "${expected_exec_start}" ]]; then + echo "[pingora-direct-rollback] 无法从主 service 模板读取 ExecStart: ${SERVICE_UNIT_PATH}" >&2 + exit 1 + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + echo "[pingora-direct-rollback] dry-run：--apply 后会核验 systemd ExecStart 指向主 service 模板中的 ${expected_exec_start}。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 Pingora ExecStart。" >&2 + exit 1 + fi + + echo "+ systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager" + if ! systemd_show_exec_start_matches "${expected_exec_start}"; then + echo "[pingora-direct-rollback] systemctl show ExecStart 未指向主 service 模板中的 ${expected_exec_start}，请先修正 ${SERVICE_NAME} 指向 current release 网关二进制。" >&2 + exit 1 + fi +} + +verify_systemd_dropin_removed() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 systemd drop-in 移除核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl cat ${SERVICE_NAME}" + echo "[pingora-direct-rollback] dry-run：--apply 后会校验 systemd 最终配置不再包含 CAP_NET_BIND_SERVICE。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 drop-in 是否已移除。" >&2 + exit 1 + fi + + echo "+ systemctl cat ${SERVICE_NAME}" + unit_content="$(systemctl cat "${SERVICE_NAME}")" + if [[ "${unit_content}" == *"AmbientCapabilities=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-rollback] systemctl cat 仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE，direct-entry drop-in 未移除。" >&2 + exit 1 + fi + if [[ "${unit_content}" == *"CapabilityBoundingSet=CAP_NET_BIND_SERVICE"* ]]; then + echo "[pingora-direct-rollback] systemctl cat 仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE，direct-entry drop-in 未移除。" >&2 + exit 1 + fi +} + +verify_nginx_active_after_reload() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Nginx reload 后状态核验。" + return + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ systemctl is-active ${NGINX_SERVICE}" + echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会 reload Nginx 并校验 service 仍为 active。" + return + fi + + if ! command -v systemctl >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 systemctl，无法核验 Nginx reload 后状态。" >&2 + exit 1 + fi + + echo "+ systemctl is-active ${NGINX_SERVICE}" + nginx_state="$(systemctl is-active "${NGINX_SERVICE}")" + if [[ "${nginx_state}" != "active" ]]; then + echo "[pingora-direct-rollback] Nginx reload 后状态不是 active: ${nginx_state}" >&2 + exit 1 + fi +} + +verify_nginx_smoke_after_reload() { + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Nginx reload 后 smoke 核验。" + return + fi + + if [[ -z "${NGINX_SMOKE_URL}" ]]; then + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 " + echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会用 --nginx-smoke-url 验证 Nginx 入口。" + return + fi + + smoke_args=( + --fail + --silent + --show-error + --max-time + 5 + ) + if [[ -n "${NGINX_SMOKE_HOST}" ]]; then + smoke_args+=(-H "Host: ${NGINX_SMOKE_HOST}") + fi + smoke_args+=("${NGINX_SMOKE_URL}") + + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${CURL_BINARY} ${smoke_args[*]}" + if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会验证 Nginx smoke URL 可访问，且响应体包含预期片段。" + else + echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会验证 Nginx smoke URL 可访问。" + fi + return + fi + + if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 curl 可执行文件，无法验证 Nginx smoke URL: ${CURL_BINARY}" >&2 + exit 1 + fi + + echo "+ ${CURL_BINARY} ${smoke_args[*]}" + smoke_body="$("${CURL_BINARY}" "${smoke_args[@]}")" + if [[ -n "${NGINX_SMOKE_EXPECT_BODY}" ]]; then + if [[ "${smoke_body}" != *"${NGINX_SMOKE_EXPECT_BODY}"* ]]; then + echo "[pingora-direct-rollback] Nginx smoke 响应缺少预期片段: ${NGINX_SMOKE_EXPECT_BODY}" >&2 + echo "[pingora-direct-rollback] Nginx smoke 响应前 500 字符: ${smoke_body:0:500}" >&2 + exit 1 + fi + echo "[pingora-direct-rollback] nginx smoke evidence matched expected body fragment." + else + echo "[pingora-direct-rollback] nginx smoke evidence curl --fail passed without body fragment check." + fi +} + +test_nginx_config_before_reload() { + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${NGINX_BINARY} -t" + echo "[pingora-direct-rollback] dry-run：--apply --reload-nginx 后会先执行 nginx -t，通过后再 reload。" + return + fi + + if ! command -v "${NGINX_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 Nginx 可执行文件，无法执行: ${NGINX_BINARY} -t" >&2 + exit 1 + fi + + echo "+ ${NGINX_BINARY} -t" + "${NGINX_BINARY}" -t +} + +validate_dropin_path_for_apply() { + if [[ "${APPLY}" != "true" ]]; then + return + fi + + DROPIN_DIR="$(dirname "${DROPIN_PATH}")" + if [[ -L "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-rollback] drop-in 目录不能是符号链接: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_DIR}" && ! -d "${DROPIN_DIR}" ]]; then + echo "[pingora-direct-rollback] drop-in 父路径必须是目录: ${DROPIN_DIR}" >&2 + exit 1 + fi + if [[ -L "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] drop-in 目标不能是符号链接: ${DROPIN_PATH}" >&2 + exit 1 + fi + if [[ -e "${DROPIN_PATH}" && ! -f "${DROPIN_PATH}" ]]; then + echo "[pingora-direct-rollback] drop-in 目标已存在但不是普通文件: ${DROPIN_PATH}" >&2 + exit 1 + fi +} + +verify_health_patrol_env_after_rollback() { + if [[ -z "${HEALTH_PATROL_ENV_FILE}" ]]; then + return + fi + + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 health patrol env 回退后复核。" + return + fi + + health_patrol_args=( + "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" + --env-file + "${HEALTH_PATROL_ENV_FILE}" + --expected-gateway-mode + nginx + ) + if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" ]]; then + health_patrol_args+=( + --expected-public-base-url + "${HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL}" + ) + fi + if [[ -n "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" ]]; then + health_patrol_args+=( + --expected-public-host + "${HEALTH_PATROL_EXPECTED_PUBLIC_HOST}" + ) + elif [[ "${HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST}" != "false" ]]; then + health_patrol_args+=(--require-empty-public-host) + fi + + if [[ "${APPLY}" != "true" ]]; then + echo "+ node -- ${health_patrol_args[*]}" + echo "[pingora-direct-rollback] dry-run：--apply 后会复核 health patrol env 已切回 nginx，且 public base URL / Host 已恢复为预期值。" + return + fi + + if ! command -v node >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 node，无法复核 health patrol env。" >&2 + exit 1 + fi + if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT}" ]]; then + echo "[pingora-direct-rollback] health patrol env 复核脚本不存在: ${HEALTH_PATROL_ENV_CHECK_SCRIPT}" >&2 + exit 1 + fi + + echo "+ node -- ${health_patrol_args[*]}" + node -- "${health_patrol_args[@]}" +} + +verify_pingora_shadow_probe_after_rollback() { + if [[ -z "${PINGORA_SHADOW_PROBE_URL}" ]]; then + return + fi + + if [[ "${VERIFY_SYSTEMD_AFTER}" != "true" ]]; then + echo "[pingora-direct-rollback] 已跳过 Pingora shadow 探针回退后核验。" + return + fi + + shadow_args=( + --fail + --silent + --show-error + --max-time + 5 + -H + "X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_TOKEN}" + "${PINGORA_SHADOW_PROBE_URL}" + ) + + if [[ "${APPLY}" != "true" ]]; then + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" + echo "[pingora-direct-rollback] dry-run：--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。" + return + fi + + if ! command -v "${CURL_BINARY}" >/dev/null 2>&1; then + echo "[pingora-direct-rollback] 未找到 curl 可执行文件，无法验证 Pingora shadow 探针: ${CURL_BINARY}" >&2 + exit 1 + fi + + echo "+ ${CURL_BINARY} --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: ${PINGORA_SHADOW_PROBE_URL}" + shadow_body="$("${CURL_BINARY}" "${shadow_args[@]}")" + if [[ "${shadow_body}" != *'"ok":true'* || "${shadow_body}" != *'"gateway":"pingora-shadow"'* ]]; then + echo "[pingora-direct-rollback] Pingora shadow 探针响应不是预期 JSON: ${shadow_body}" >&2 + exit 1 + fi +} + +echo "[pingora-direct-rollback] apply=${APPLY} service=${SERVICE_NAME} dropin=${DROPIN_PATH}" + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-direct-rollback] 当前是 dry-run；确认无误后追加 --apply。" +fi + +validate_dropin_path_for_apply +test_nginx_config_before_reload + +if [[ -e "${DROPIN_PATH}" || -L "${DROPIN_PATH}" ]]; then + run_cmd rm -f "${DROPIN_PATH}" +else + echo "[pingora-direct-rollback] direct-entry drop-in 不存在，视为已移除: ${DROPIN_PATH}" +fi + +run_systemctl daemon-reload +run_systemctl restart "${SERVICE_NAME}" +verify_systemd_dropin_removed +verify_systemd_exec_start_after_rollback + +run_systemctl reload "${NGINX_SERVICE}" +verify_nginx_active_after_reload +verify_nginx_smoke_after_reload +verify_health_patrol_env_after_rollback +verify_pingora_shadow_probe_after_rollback + +if [[ "${STATUS_AFTER}" == "true" ]]; then + run_systemctl status "${SERVICE_NAME}" --no-pager +fi + +cat <} 已通过 +- 如传入 --nginx-smoke-expect-body，Nginx smoke 响应体已匹配预期片段 +- 如传入 --health-patrol-env-file，health patrol env 已复核为 nginx 模式且 public base URL / Host 符合预期 +- 如传入 --pingora-shadow-probe-url，Pingora shadow 高端口探针已复核为 gateway=pingora-shadow +- 公网入口已回到 Nginx 或其它正式前置入口 +EOF diff --git a/scripts/deploy/pingora-health-patrol-env-switch.mjs b/scripts/deploy/pingora-health-patrol-env-switch.mjs new file mode 100644 index 000000000..60f5df19a --- /dev/null +++ b/scripts/deploy/pingora-health-patrol-env-switch.mjs @@ -0,0 +1,391 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + chownSync, + chmodSync, + lstatSync, + mkdtempSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const TARGET_KEYS = [ + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST', +]; +const DRY_RUN_ENV_FILE_MODE = 0o600; + +const config = parseArgs(process.argv.slice(2)); +if (config.apply) { + assertEnvFileIsWritableRegularFile(config.envFile); +} +const originalText = readFileSync(config.envFile, 'utf8'); +const nextValues = { + GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE: config.gatewayMode, + GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL: normalizeBaseUrl( + config.publicBaseUrl, + '--public-base-url', + ), + GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST: config.clearPublicHost + ? '' + : config.publicHost, +}; +const nextText = rewriteEnvFile(originalText, nextValues); +const checkFile = writeDryRunEnv(nextText); + +try { + console.log( + `[pingora-health-patrol-env-switch] apply=${config.apply} env=${config.envFile} mode=${config.gatewayMode}`, + ); + for (const key of TARGET_KEYS) { + console.log(`~ ${key}=${nextValues[key]}`); + } + + runHealthPatrolEnvCheck(checkFile); + + if (!config.apply) { + console.log( + '[pingora-health-patrol-env-switch] 当前是 dry-run；追加 --apply 后才会写入 env 文件。', + ); + } else { + writeEnvFileAtomically(config.envFile, nextText); + } + + console.log('[pingora-health-patrol-env-switch] 完成'); +} finally { + rmSync(path.dirname(checkFile), { recursive: true, force: true }); +} + +function parseArgs(argv) { + const result = { + envFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + gatewayMode: process.env.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || '', + publicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '', + publicHost: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || null, + clearPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_CLEAR_PUBLIC_HOST', + ), + checkScript: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_CHECK_SCRIPT || + defaultEnvCheckScript(), + apply: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--env-file': + result.envFile = requireValue(argv, ++index, arg); + break; + case '--gateway-mode': + result.gatewayMode = requireValue(argv, ++index, arg); + break; + case '--public-base-url': + result.publicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--public-host': + result.publicHost = requireValue(argv, ++index, arg); + break; + case '--clear-public-host': + result.clearPublicHost = true; + break; + case '--check-script': + result.checkScript = requireValue(argv, ++index, arg); + break; + case '--apply': + result.apply = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.envFile)) { + throw new Error(`--env-file 必须是绝对路径: ${result.envFile}`); + } + validateNoControlCharacters(result.envFile, '--env-file'); + if (isFilesystemRootPath(result.envFile)) { + throw new Error('--env-file 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.checkScript)) { + throw new Error(`--check-script 必须是绝对路径: ${result.checkScript}`); + } + validateNoControlCharacters(result.checkScript, '--check-script'); + if (isFilesystemRootPath(result.checkScript)) { + throw new Error('--check-script 不能是文件系统根目录。'); + } + if (!GATEWAY_MODES.has(result.gatewayMode)) { + throw new Error( + `--gateway-mode 只支持 nginx 或 pingora-direct: ${result.gatewayMode}`, + ); + } + if (!result.publicBaseUrl) { + throw new Error('必须提供 --public-base-url。'); + } + normalizeBaseUrl(result.publicBaseUrl, '--public-base-url'); + if (result.publicHost !== null && result.clearPublicHost) { + throw new Error('--public-host 和 --clear-public-host 不能同时使用。'); + } + if (result.publicHost === null && !result.clearPublicHost) { + throw new Error( + '必须显式提供 --public-host 或 --clear-public-host，避免沿用旧 Host 覆盖。', + ); + } + if (result.publicHost !== null) { + validateHostOption(result.publicHost, '--public-host'); + } + if ( + result.gatewayMode === 'pingora-direct' && + isLoopbackUrl(result.publicBaseUrl) && + result.clearPublicHost + ) { + throw new Error( + 'Pingora direct 模式使用本机 public base URL 时必须提供 --public-host，避免巡检命中默认 vhost。', + ); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/deploy/pingora-health-patrol-env-switch.mjs --gateway-mode --public-base-url (--public-host |--clear-public-host) [options] + +Options: + --env-file health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。 + --gateway-mode nginx 或 pingora-direct。 + --public-base-url 写入 GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL。 + --public-host 写入 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST。 + --clear-public-host 清空 GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST。 + --check-script env 复核脚本，默认使用同一 release 内的 check-production-health-patrol-env.mjs。 + --apply 写入 env 文件；默认只 dry-run 并用临时 env 复核。 +`); +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function defaultEnvCheckScript() { + const scriptPath = fileURLToPath(import.meta.url); + const releaseRoot = path.resolve(path.dirname(scriptPath), '../..'); + return path.join(releaseRoot, 'scripts/check-production-health-patrol-env.mjs'); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function rewriteEnvFile(text, nextValuesByKey) { + const lines = text.replace(/\r\n/gu, '\n').split('\n'); + if (lines.at(-1) === '') { + lines.pop(); + } + + const seen = new Set(); + const output = []; + for (const line of lines) { + const assignment = parseAssignmentLine(line); + if (!assignment || !TARGET_KEYS.includes(assignment.key)) { + output.push(line); + continue; + } + if (seen.has(assignment.key)) { + throw new Error(`health patrol env 中存在重复配置: ${assignment.key}`); + } + seen.add(assignment.key); + output.push( + `${assignment.leading}${assignment.exportPrefix}${assignment.key}=${nextValuesByKey[assignment.key]}`, + ); + } + + for (const key of TARGET_KEYS) { + if (!seen.has(key)) { + output.push(`${key}=${nextValuesByKey[key]}`); + } + } + + return `${output.join('\n')}\n`; +} + +function parseAssignmentLine(line) { + const match = line.match(/^(\s*)(export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/u); + if (!match) { + return null; + } + return { + leading: match[1] || '', + exportPrefix: match[2] || '', + key: match[3], + }; +} + +function writeDryRunEnv(content) { + const dir = mkdtempSync( + path.join(tmpdir(), 'genarrative-health-patrol-env-switch-'), + ); + const filePath = path.join(dir, 'health-patrol.env'); + writeFileSync(filePath, content, { + encoding: 'utf8', + mode: DRY_RUN_ENV_FILE_MODE, + }); + chmodSync(filePath, DRY_RUN_ENV_FILE_MODE); + return filePath; +} + +function writeEnvFileAtomically(filePath, content) { + const currentStat = assertEnvFileIsWritableRegularFile(filePath); + const currentMode = currentStat.mode & 0o777; + const tempFile = path.join( + path.dirname(filePath), + `.${path.basename(filePath)}.${process.pid}.${Date.now()}.tmp`, + ); + try { + writeFileSync(tempFile, content, { encoding: 'utf8', mode: currentMode }); + chownSync(tempFile, currentStat.uid, currentStat.gid); + chmodSync(tempFile, currentMode); + assertEnvFileIsWritableRegularFile(filePath); + renameSync(tempFile, filePath); + } catch (error) { + rmSync(tempFile, { force: true }); + throw error; + } +} + +function assertEnvFileIsWritableRegularFile(filePath) { + const currentStat = lstatSync(filePath); + if (currentStat.isSymbolicLink()) { + throw new Error( + `--env-file 不能是符号链接；请直接传真实 health patrol env 文件路径: ${filePath}`, + ); + } + if (!currentStat.isFile()) { + throw new Error(`--env-file 必须指向普通文件: ${filePath}`); + } + return currentStat; +} + +function runHealthPatrolEnvCheck(envFile) { + const checkArgs = [ + config.checkScript, + '--env-file', + envFile, + '--expected-gateway-mode', + config.gatewayMode, + '--expected-public-base-url', + normalizeBaseUrl(config.publicBaseUrl, '--public-base-url'), + ]; + if (config.clearPublicHost) { + checkArgs.push('--require-empty-public-host'); + } else { + checkArgs.push('--expected-public-host', config.publicHost); + } + + console.log(`+ node ${checkArgs.join(' ')}`); + const result = spawnSync(process.execPath, ['--', ...checkArgs], { + encoding: 'utf8', + }); + if (result.stdout) { + process.stdout.write(result.stdout); + } + if (result.stderr) { + process.stderr.write(result.stderr); + } + if (result.status !== 0) { + process.exit(result.status || 1); + } +} + +function normalizeBaseUrl(value, label) { + validateNoControlCharacters(value, label); + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + return parsed.toString().replace(/\/+$/u, ''); + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + validateNoControlCharacters(raw, label); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/u.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符。`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value ?? ''))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function isLoopbackUrl(value) { + try { + const hostname = new URL(value).hostname.toLowerCase(); + return ( + hostname === 'localhost' || + hostname === '::1' || + hostname === '[::1]' || + hostname.startsWith('127.') + ); + } catch { + return false; + } +} diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 14029c520..64039e711 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -5,11 +5,12 @@ set -euo pipefail usage() { cat <<'EOF' 用法: - ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] + ./scripts/deploy/production-api-deploy.sh --source-dir build/ [--version ] [--release-root /opt/genarrative/releases] [--current-link /opt/genarrative/current] [--service genarrative-api.service] [--pingora-service genarrative-pingora-gateway.service] [--health-url http://127.0.0.1:8082/readyz] [--api-env-file /etc/genarrative/api-server.env] [--database genarrative-prod] [--spacetime-server-url http://127.0.0.1:3101] 说明: 进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。 若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。 + 若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后仅对已 active 的 Pingora 影子服务执行 try-restart 并复核 active；未运行时不会主动拉起。 失败时保留维护模式。 EOF } @@ -24,6 +25,16 @@ require_argument() { fi } +require_absolute_path() { + local value="$1" + local label="$2" + + if [[ ! "${value}" = /* ]]; then + echo "[production-api-deploy] ${label} 必须使用绝对路径: ${value}" >&2 + exit 1 + fi +} + validate_spacetime_database_name() { local database="$1" @@ -223,17 +234,76 @@ ensure_runtime_env_and_dirs() { fi } +restart_pingora_if_active() { + local service_name="$1" + + if ! systemctl is-active --quiet "${service_name}"; then + echo "[production-api-deploy] Pingora 影子服务未处于 active，跳过自动重启: ${service_name}" + return + fi + + echo "[production-api-deploy] 发布包包含 Pingora，重启已运行的影子服务: ${service_name}" + systemctl try-restart "${service_name}" + if ! systemctl is-active --quiet "${service_name}"; then + echo "[production-api-deploy] Pingora 影子服务重启后不是 active: ${service_name}" >&2 + exit 1 + fi +} + +validate_release_manifest() { + local manifest_path="$1" + local require_pingora="$2" + + if [[ ! -f "${manifest_path}" ]]; then + echo "[production-api-deploy] 发布产物缺少 release-manifest.json: ${manifest_path}" >&2 + exit 1 + fi + + node - "${manifest_path}" "${require_pingora}" <<'NODE' +const fs = require('fs'); + +const manifestPath = process.argv[2]; +const requirePingora = process.argv[3] === '1'; +let manifest; +try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); +} catch (error) { + console.error(`[production-api-deploy] release-manifest.json 不是合法 JSON: ${error.message}`); + process.exit(1); +} + +const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; +const hasApiServer = artifacts.some((artifact) => artifact?.path === 'api-server'); +if (!hasApiServer) { + console.error('[production-api-deploy] release-manifest.json 缺少 api-server artifact。'); + process.exit(1); +} + +if (requirePingora) { + const hasPingora = artifacts.some((artifact) => artifact?.path === 'pingora-gateway'); + if (!hasPingora) { + console.error('[production-api-deploy] release-manifest.json 缺少 pingora-gateway artifact。'); + process.exit(1); + } +} +NODE +} + SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" SOURCE_DIR="" VERSION="" RELEASE_ROOT="/opt/genarrative/releases" CURRENT_LINK="/opt/genarrative/current" SERVICE_NAME="genarrative-api.service" +PINGORA_SERVICE_NAME="genarrative-pingora-gateway.service" HEALTH_URL="http://127.0.0.1:8082/readyz" API_ENV_FILE="/etc/genarrative/api-server.env" DATABASE="" SPACETIME_SERVER_URL="" DEPLOY_COMPLETED=0 +PINGORA_INCLUDED=0 +RELEASE_DIR="" +STAGING_RELEASE_DIR="" while [[ $# -gt 0 ]]; do case "$1" in @@ -261,6 +331,10 @@ while [[ $# -gt 0 ]]; do SERVICE_NAME="${2:?缺少 --service 的值}" shift 2 ;; + --pingora-service) + PINGORA_SERVICE_NAME="${2:?缺少 --pingora-service 的值}" + shift 2 + ;; --health-url) HEALTH_URL="${2:?缺少 --health-url 的值}" shift 2 @@ -286,6 +360,9 @@ while [[ $# -gt 0 ]]; do done require_argument "${SOURCE_DIR}" "--source-dir" +require_absolute_path "${RELEASE_ROOT}" "--release-root" +require_absolute_path "${CURRENT_LINK}" "--current-link" +require_absolute_path "${API_ENV_FILE}" "--api-env-file" if [[ -n "${DATABASE}" ]]; then validate_spacetime_database_name "${DATABASE}" @@ -299,8 +376,13 @@ fi SOURCE_DIR="$(cd "${SOURCE_DIR}" && pwd)" VERSION="${VERSION:-$(basename "${SOURCE_DIR}")}" -if [[ ! "${VERSION}" =~ ^[0-9A-Za-z._-]+$ ]]; then - echo "[production-api-deploy] --version 只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2 +if [[ ! "${VERSION}" =~ ^[0-9A-Za-z][0-9A-Za-z._-]*$ ]]; then + echo "[production-api-deploy] --version 必须以数字或字母开头，且只能包含数字、字母、点、下划线和短横线: ${VERSION}" >&2 + exit 1 +fi + +if [[ "${VERSION}" == "." || "${VERSION}" == ".." ]]; then + echo "[production-api-deploy] --version 不能是点目录: ${VERSION}" >&2 exit 1 fi @@ -309,9 +391,34 @@ if [[ ! -f "${SOURCE_DIR}/api-server" || ! -f "${SOURCE_DIR}/api-server.sha256" exit 1 fi +RELEASE_DIR="${RELEASE_ROOT}/${VERSION}" +STAGING_RELEASE_DIR="${RELEASE_ROOT}/.${VERSION}.staging.$$" + +if [[ -e "${RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 目标 release 已存在，拒绝覆盖或合并旧文件: ${RELEASE_DIR}" >&2 + exit 1 +fi + +if [[ -e "${STAGING_RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 临时 staging release 已存在: ${STAGING_RELEASE_DIR}" >&2 + exit 1 +fi + +if [[ -e "${CURRENT_LINK}" && ! -L "${CURRENT_LINK}" ]]; then + echo "[production-api-deploy] current 链接路径已存在但不是符号链接，拒绝覆盖: ${CURRENT_LINK}" >&2 + exit 1 +fi + +cleanup_staging_release() { + if [[ -n "${STAGING_RELEASE_DIR:-}" && -d "${STAGING_RELEASE_DIR}" ]]; then + rm -rf "${STAGING_RELEASE_DIR}" + fi +} + on_exit() { local exit_code=$? if [[ "${exit_code}" -ne 0 && "${DEPLOY_COMPLETED}" -ne 1 ]]; then + cleanup_staging_release echo "[production-api-deploy] 部署失败，保持维护模式。" >&2 fi exit "${exit_code}" @@ -319,52 +426,186 @@ on_exit() { trap on_exit EXIT -"${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" +bash "${SCRIPT_DIR}/maintenance-on.sh" "api deploy ${VERSION}" echo "[production-api-deploy] 校验 api-server" ( cd "${SOURCE_DIR}" sha256sum -c api-server.sha256 + if [[ -f pingora-gateway || -f pingora-gateway.sha256 ]]; then + if [[ ! -f pingora-gateway || ! -f pingora-gateway.sha256 ]]; then + echo "[production-api-deploy] pingora-gateway 与 pingora-gateway.sha256 必须同时存在。" >&2 + exit 1 + fi + sha256sum -c pingora-gateway.sha256 + fi ) +if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then + PINGORA_INCLUDED=1 +fi +validate_release_manifest "${SOURCE_DIR}/release-manifest.json" "${PINGORA_INCLUDED}" -RELEASE_DIR="${RELEASE_ROOT}/${VERSION}" -mkdir -p "${RELEASE_DIR}" -cp "${SOURCE_DIR}/api-server" "${RELEASE_DIR}/api-server" -chmod +x "${RELEASE_DIR}/api-server" +mkdir -p "${RELEASE_ROOT}" +mkdir "${STAGING_RELEASE_DIR}" +RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}" +cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server" +chmod +x "${RELEASE_CONTENT_DIR}/api-server" +if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then + cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway" + chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway" + echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核并重启已运行的 ${PINGORA_SERVICE_NAME}" +fi BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" -WORKSPACE_BACKUP_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/database-backup-to-oss.mjs" HEALTH_PATROL_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" -WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE="$(cd "${SCRIPT_DIR}/../.." && pwd)/scripts/ops/production-health-patrol.mjs" -mkdir -p "${RELEASE_DIR}/scripts" "${RELEASE_DIR}/scripts/ops" +PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" +PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" +PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" +PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" +PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" +PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" +PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" +PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" +PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +PINGORA_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/pingora" +SYSTEMD_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/systemd" +NGINX_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/nginx" +ENV_DEPLOY_DIR_SOURCE="${SOURCE_DIR}/deploy/env" +mkdir -p "${RELEASE_CONTENT_DIR}/scripts" "${RELEASE_CONTENT_DIR}/scripts/deploy" "${RELEASE_CONTENT_DIR}/scripts/ops" "${RELEASE_CONTENT_DIR}/deploy" if [[ ! -f "${BACKUP_SCRIPT_SOURCE}" ]]; then - if [[ -f "${WORKSPACE_BACKUP_SCRIPT_SOURCE}" ]]; then - echo "[production-api-deploy] 发布产物缺少 scripts/database-backup-to-oss.mjs，回退使用部署工作区脚本；请重新触发包含该脚本的 API 构建。" >&2 - BACKUP_SCRIPT_SOURCE="${WORKSPACE_BACKUP_SCRIPT_SOURCE}" - else - echo "[production-api-deploy] 缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 - exit 1 - fi + echo "[production-api-deploy] 发布产物缺少数据库备份脚本: ${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" >&2 + exit 1 fi -cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs" -chmod 0644 "${RELEASE_DIR}/scripts/database-backup-to-oss.mjs" +cp "${BACKUP_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/database-backup-to-oss.mjs" if [[ ! -f "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - if [[ -f "${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - echo "[production-api-deploy] 发布产物缺少 scripts/ops/production-health-patrol.mjs，回退使用部署工作区脚本；请重新触发包含该脚本的 API 构建。" >&2 - HEALTH_PATROL_SCRIPT_SOURCE="${WORKSPACE_HEALTH_PATROL_SCRIPT_SOURCE}" - else - echo "[production-api-deploy] 未找到生产健康巡检脚本，跳过复制；genarrative-health-patrol.service 会因脚本缺失而跳过执行。" >&2 - HEALTH_PATROL_SCRIPT_SOURCE="" - fi + echo "[production-api-deploy] 发布产物缺少生产健康巡检脚本: ${SOURCE_DIR}/scripts/ops/production-health-patrol.mjs" >&2 + exit 1 fi -if [[ -n "${HEALTH_PATROL_SCRIPT_SOURCE}" ]]; then - cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs" - chmod 0644 "${RELEASE_DIR}/scripts/ops/production-health-patrol.mjs" +cp "${HEALTH_PATROL_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/production-health-patrol.mjs" +if [[ ! -f "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora current release 自审脚本: ${SOURCE_DIR}/scripts/ops/pingora-current-release-audit.mjs" >&2 + exit 1 fi +cp "${PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-current-release-audit.mjs" +if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据包脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_BUNDLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" +if [[ ! -f "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换命令证据脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_COMMAND_EVIDENCE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据验真脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" +if [[ ! -f "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换证据根目录审计脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CUTOVER_EVIDENCE_AUDIT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" +if [[ ! -f "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少生产健康巡检 env 复核脚本: ${SOURCE_DIR}/scripts/check-production-health-patrol-env.mjs" >&2 + exit 1 +fi +cp "${HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-production-health-patrol-env.mjs" +if [[ ! -f "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora release readiness 聚合门禁脚本: ${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" >&2 + exit 1 +fi +cp "${PINGORA_RELEASE_READINESS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-release-readiness.mjs" +if [[ ! -f "${PINGORA_ENABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" >&2 + exit 1 +fi +cp "${PINGORA_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-enable.sh" +if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连回退脚本: ${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" >&2 + exit 1 +fi +cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" +if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2 + exit 1 +fi +cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2 + exit 1 +fi +cp "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-preflight.mjs" +if [[ ! -f "${PINGORA_LIVE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 直连 live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" >&2 + exit 1 +fi +cp "${PINGORA_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-direct-live.mjs" +if [[ ! -f "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora canary live smoke 脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CANARY_LIVE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-live.mjs" +if [[ ! -f "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora canary access log 对账脚本: ${SOURCE_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" >&2 + exit 1 +fi +cp "${PINGORA_CANARY_ACCESS_LOG_PARITY_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" +if [[ ! -d "${PINGORA_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora 配置目录: ${SOURCE_DIR}/deploy/pingora" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/pingora" +cp -R "${PINGORA_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/pingora" +if [[ ! -d "${SYSTEMD_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 systemd 配置目录: ${SOURCE_DIR}/deploy/systemd" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/systemd" +cp -R "${SYSTEMD_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/systemd" +if [[ ! -d "${NGINX_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Nginx 配置目录: ${SOURCE_DIR}/deploy/nginx" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/nginx" +cp -R "${NGINX_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/nginx" +if [[ ! -d "${ENV_DEPLOY_DIR_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少环境变量示例目录: ${SOURCE_DIR}/deploy/env" >&2 + exit 1 +fi +rm -rf "${RELEASE_CONTENT_DIR}/deploy/env" +cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env" -if [[ -f "${SOURCE_DIR}/release-manifest.json" ]]; then - cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_DIR}/release-manifest.api-server.json" -fi +cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json" if [[ -n "${DATABASE}" ]]; then echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}" @@ -379,7 +620,17 @@ fi ensure_runtime_env_and_dirs "${API_ENV_FILE}" mkdir -p "$(dirname "${CURRENT_LINK}")" -ln -sfn "${RELEASE_DIR}" "${CURRENT_LINK}" +if [[ -e "${RELEASE_DIR}" ]]; then + echo "[production-api-deploy] 目标 release 在发布过程中出现，拒绝合并 staging: ${RELEASE_DIR}" >&2 + exit 1 +fi +mv -T "${STAGING_RELEASE_DIR}" "${RELEASE_DIR}" +STAGING_RELEASE_DIR="" +ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}" + +if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then + restart_pingora_if_active "${PINGORA_SERVICE_NAME}" +fi echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}" systemctl restart "${SERVICE_NAME}" @@ -387,7 +638,7 @@ systemctl restart "${SERVICE_NAME}" echo "[production-api-deploy] 等待 readiness: ${HEALTH_URL}" for _ in {1..30}; do if curl -fsS "${HEALTH_URL}" >/dev/null; then - "${SCRIPT_DIR}/maintenance-off.sh" + bash "${SCRIPT_DIR}/maintenance-off.sh" DEPLOY_COMPLETED=1 echo "[production-api-deploy] 完成: ${RELEASE_DIR}/api-server" exit 0 diff --git a/scripts/deploy/production-stdb-publish.sh b/scripts/deploy/production-stdb-publish.sh index cdd60d41b..0f894adc7 100644 --- a/scripts/deploy/production-stdb-publish.sh +++ b/scripts/deploy/production-stdb-publish.sh @@ -174,7 +174,7 @@ prepare_async_backup() { ASYNC_BACKUP_STATUS_FILE="$(mktemp /tmp/genarrative-stdb-backup-status.XXXXXX.json)" echo "[production-stdb-publish] publish 前生成本地冷备份，随后会异步上传 OSS" - node "${ASYNC_BACKUP_SCRIPT}" \ + node -- "${ASYNC_BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ @@ -200,7 +200,7 @@ start_async_backup_upload() { mkdir -p "$(dirname "${ASYNC_BACKUP_ARCHIVE}")" ASYNC_BACKUP_LOG="$(dirname "${ASYNC_BACKUP_ARCHIVE}")/${DATABASE}-upload.log" echo "[production-stdb-publish] 后台上传本地备份到 OSS: ${ASYNC_BACKUP_ARCHIVE}" - nohup node "${ASYNC_BACKUP_SCRIPT}" \ + nohup node -- "${ASYNC_BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --upload-archive "${ASYNC_BACKUP_ARCHIVE}" \ --manifest-file "${ASYNC_BACKUP_MANIFEST}" \ @@ -254,7 +254,7 @@ case "${BACKUP_MODE}" in fi echo "[production-stdb-publish] publish 前同步执行 OSS 冷备份，失败会阻断发布" - node "${BACKUP_SCRIPT}" \ + node -- "${BACKUP_SCRIPT}" \ --env-file /etc/genarrative/api-server.env \ --data-dir "${SPACETIME_ROOT_DIR}" \ --database "${DATABASE}" \ diff --git a/scripts/jenkins-server-provision.sh b/scripts/jenkins-server-provision.sh index 9f399e84e..a6a1d9ec2 100755 --- a/scripts/jenkins-server-provision.sh +++ b/scripts/jenkins-server-provision.sh @@ -600,10 +600,15 @@ restore_nginx_default_sites_enabled() { install_nginx_config_with_rollback() { local config_target="/etc/nginx/conf.d/genarrative.conf" local snippet_target="/etc/nginx/snippets/genarrative-maintenance.conf" + local pingora_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-canary.conf" + local pingora_realpath_canary_snippet_target="/etc/nginx/snippets/genarrative-pingora-realpath-canary.conf" local config_source - local rendered_config rendered_snippet config_backup snippet_backup disabled_sites + local rendered_config rendered_snippet rendered_pingora_canary_snippet rendered_pingora_realpath_canary_snippet + local config_backup snippet_backup pingora_canary_snippet_backup pingora_realpath_canary_snippet_backup disabled_sites local had_config="false" local had_snippet="false" + local had_pingora_canary_snippet="false" + local had_pingora_realpath_canary_snippet="false" run_cmd mkdir -p /etc/nginx/snippets /etc/nginx/conf.d if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then @@ -617,6 +622,8 @@ install_nginx_config_with_rollback() { echo "+ render ${config_source} -> ${config_target}" echo "+ install -m 0644 deploy/nginx/snippets/genarrative-maintenance.conf ${snippet_target}" + echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-canary.conf ${pingora_canary_snippet_target}" + echo "+ install -m 0644 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf ${pingora_realpath_canary_snippet_target}" if [[ "${DRY_RUN}" == "true" ]]; then echo "+ disable /etc/nginx/sites-enabled/default* if present" @@ -627,8 +634,12 @@ install_nginx_config_with_rollback() { rendered_config="$(mktemp)" rendered_snippet="$(mktemp)" + rendered_pingora_canary_snippet="$(mktemp)" + rendered_pingora_realpath_canary_snippet="$(mktemp)" config_backup="$(mktemp)" snippet_backup="$(mktemp)" + pingora_canary_snippet_backup="$(mktemp)" + pingora_realpath_canary_snippet_backup="$(mktemp)" disabled_sites="$(mktemp)" if [[ "${NGINX_CONFIG_MODE}" == "production-https" ]]; then validate_nginx_tls @@ -637,6 +648,8 @@ install_nginx_config_with_rollback() { render_nginx_development_http_config >"${rendered_config}" fi cp deploy/nginx/snippets/genarrative-maintenance.conf "${rendered_snippet}" + cp deploy/nginx/snippets/genarrative-pingora-canary.conf "${rendered_pingora_canary_snippet}" + cp deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf "${rendered_pingora_realpath_canary_snippet}" if [[ -f "${config_target}" ]]; then cp -p "${config_target}" "${config_backup}" @@ -646,9 +659,19 @@ install_nginx_config_with_rollback() { cp -p "${snippet_target}" "${snippet_backup}" had_snippet="true" fi + if [[ -f "${pingora_canary_snippet_target}" ]]; then + cp -p "${pingora_canary_snippet_target}" "${pingora_canary_snippet_backup}" + had_pingora_canary_snippet="true" + fi + if [[ -f "${pingora_realpath_canary_snippet_target}" ]]; then + cp -p "${pingora_realpath_canary_snippet_target}" "${pingora_realpath_canary_snippet_backup}" + had_pingora_realpath_canary_snippet="true" + fi install -m 0644 "${rendered_config}" "${config_target}" install -m 0644 "${rendered_snippet}" "${snippet_target}" + install -m 0644 "${rendered_pingora_canary_snippet}" "${pingora_canary_snippet_target}" + install -m 0644 "${rendered_pingora_realpath_canary_snippet}" "${pingora_realpath_canary_snippet_target}" disable_nginx_default_sites_enabled "${disabled_sites}" if ! nginx -t; then @@ -663,14 +686,24 @@ install_nginx_config_with_rollback() { else rm -f "${snippet_target}" fi + if [[ "${had_pingora_canary_snippet}" == "true" ]]; then + cp -p "${pingora_canary_snippet_backup}" "${pingora_canary_snippet_target}" + else + rm -f "${pingora_canary_snippet_target}" + fi + if [[ "${had_pingora_realpath_canary_snippet}" == "true" ]]; then + cp -p "${pingora_realpath_canary_snippet_backup}" "${pingora_realpath_canary_snippet_target}" + else + rm -f "${pingora_realpath_canary_snippet_target}" + fi restore_nginx_default_sites_enabled "${disabled_sites}" - rm -f "${rendered_config}" "${rendered_snippet}" "${config_backup}" "${snippet_backup}" "${disabled_sites}" + rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}" exit 1 fi echo "+ nginx -s reload" nginx -s reload - rm -f "${rendered_config}" "${rendered_snippet}" "${config_backup}" "${snippet_backup}" "${disabled_sites}" + rm -f "${rendered_config}" "${rendered_snippet}" "${rendered_pingora_canary_snippet}" "${rendered_pingora_realpath_canary_snippet}" "${config_backup}" "${snippet_backup}" "${pingora_canary_snippet_backup}" "${pingora_realpath_canary_snippet_backup}" "${disabled_sites}" } cleanup_placeholder_nginx_config() { @@ -746,11 +779,16 @@ require_path deploy/systemd/genarrative-database-backup.service require_path deploy/systemd/genarrative-database-backup.timer require_path deploy/systemd/genarrative-health-patrol.service require_path deploy/systemd/genarrative-health-patrol.timer +require_path deploy/systemd/genarrative-pingora-gateway.service +require_path deploy/systemd/genarrative-pingora-gateway-direct-entry.conf require_path deploy/systemd/otelcol-contrib.service require_path deploy/otelcol/genarrative-debug.yaml require_path deploy/nginx/genarrative.conf require_path deploy/nginx/genarrative-dev-http.conf require_path deploy/nginx/snippets/genarrative-maintenance.conf +require_path deploy/nginx/snippets/genarrative-pingora-canary.conf +require_path deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf +require_path deploy/logrotate/genarrative-pingora-gateway require_path deploy/env/api-server.env.example require_path scripts/deploy/maintenance-on.sh require_path scripts/deploy/maintenance-off.sh @@ -764,7 +802,7 @@ echo "[server-provision] target=${DEPLOY_TARGET}, dry_run=${DRY_RUN}, nginx_conf run_cmd id require_root_for_real_provision install_nginx_brotli_modules -run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol +run_cmd mkdir -p "${SPACETIME_ROOT}" "${RELEASE_ROOT}" "$(dirname "${CURRENT_LINK}")" "$(dirname "${WEB_LINK}")" /etc/genarrative /etc/genarrative/pingora /var/lib/genarrative/maintenance /var/lib/genarrative/auth /var/lib/genarrative/tracking-outbox /var/lib/genarrative/database-backups /var/lib/genarrative/health-patrol /var/log/genarrative if ! id spacetimedb >/dev/null 2>&1; then run_cmd useradd --system --home-dir "${SPACETIME_ROOT}" --shell /usr/sbin/nologin spacetimedb @@ -780,6 +818,7 @@ fi run_cmd chown -R spacetimedb:spacetimedb "${SPACETIME_ROOT}" run_cmd chown -R genarrative:genarrative /opt/genarrative /var/lib/genarrative /srv/genarrative +run_cmd chown -R genarrative:genarrative /var/log/genarrative install_genarrative_openssl_runtime if [[ ! -x "${SPACETIME_BIN_SOURCE}" ]]; then @@ -807,6 +846,9 @@ install_file "${database_backup_service}" /etc/systemd/system/genarrative-databa install_file deploy/systemd/genarrative-database-backup.timer /etc/systemd/system/genarrative-database-backup.timer 0644 install_file "${health_patrol_service}" /etc/systemd/system/genarrative-health-patrol.service 0644 install_file deploy/systemd/genarrative-health-patrol.timer /etc/systemd/system/genarrative-health-patrol.timer 0644 +install_file deploy/systemd/genarrative-pingora-gateway.service /etc/systemd/system/genarrative-pingora-gateway.service 0644 +install_file deploy/systemd/genarrative-pingora-gateway-direct-entry.conf /etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf 0644 +install_file deploy/logrotate/genarrative-pingora-gateway /etc/logrotate.d/genarrative-pingora-gateway 0644 rm -f "${spacetimedb_service}" "${api_service}" "${database_backup_service}" "${health_patrol_service}" if [[ ! -f "${API_ENV_FILE}" ]]; then diff --git a/scripts/ops/pingora-current-release-audit.mjs b/scripts/ops/pingora-current-release-audit.mjs new file mode 100644 index 000000000..1a2c7b968 --- /dev/null +++ b/scripts/ops/pingora-current-release-audit.mjs @@ -0,0 +1,666 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const REQUIRED_ARTIFACTS = [ + { path: 'api-server', executable: true }, + { path: 'scripts/database-backup-to-oss.mjs' }, + { path: 'scripts/ops/production-health-patrol.mjs' }, + { path: 'scripts/ops/pingora-current-release-audit.mjs' }, + { path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' }, + { path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' }, + { path: 'scripts/check-production-health-patrol-env.mjs' }, + { path: 'scripts/check-pingora-release-readiness.mjs' }, + { path: 'scripts/check-pingora-direct-preflight.mjs' }, + { path: 'scripts/check-pingora-direct-live.mjs' }, + { path: 'scripts/check-pingora-canary-live.mjs' }, + { path: 'scripts/check-pingora-canary-access-log-parity.mjs' }, + { path: 'scripts/deploy/pingora-direct-enable.sh', executable: true }, + { path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true }, + { path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', executable: true }, + { path: 'deploy/systemd/genarrative-pingora-gateway.service' }, + { path: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf' }, + { path: 'deploy/nginx/snippets/genarrative-pingora-canary.conf' }, + { path: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf' }, + { path: 'deploy/env/health-patrol.env.example' }, + { path: 'deploy/env/pingora-direct-live.env.example' }, + { path: 'deploy/env/pingora-canary-live.env.example' }, + { path: 'deploy/pingora/pingora-gateway.env.example' }, + { path: 'deploy/pingora/nginx-route-parity.matrix.json' }, +]; +const REQUIRED_DIRS = [ + 'deploy/systemd', + 'deploy/nginx', + 'deploy/env', + 'deploy/pingora', +]; + +const config = parseArgs(process.argv.slice(2)); +const audit = await buildAudit(config); + +console.log(`${JSON.stringify(audit, null, 2)}\n`); + +if (audit.summary.status === 'CRITICAL' && !config.warnOnly) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY', + ), + systemdShow: readBoolEnv( + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW', + ), + systemdService: + process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SERVICE || + 'genarrative-pingora-gateway.service', + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS', + ), + warnOnly: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--systemd-show': + result.systemdShow = true; + break; + case '--systemd-service': + result.systemdService = requireValue(argv, ++index, arg); + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--warn-only': + result.warnOnly = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.releaseRoot)) { + throw new Error('--release-root 必须是绝对路径。'); + } + validateNoControlCharacters(result.releaseRoot, '--release-root'); + if (isFilesystemRootPath(result.releaseRoot)) { + throw new Error('--release-root 不能是文件系统根目录。'); + } + validateNoControlCharacters(result.systemdService, '--systemd-service'); + if (!result.systemdService || /[\s/]/u.test(result.systemdService)) { + throw new Error('--systemd-service 必须是 systemd unit 名称。'); + } + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-current-release-audit.mjs [options] + +Options: + --release-root current release 根目录，默认 /opt/genarrative/current。 + --require-pingora-gateway 要求 current release 已包含 pingora-gateway 且可执行。 + --systemd-show 读取 systemctl show，确认 service ExecStart 指向 current release 的 pingora-gateway。 + --systemd-service systemd service 名，默认 genarrative-pingora-gateway.service。 + --timeout-ms systemctl 超时，默认 5000。 + --warn-only 即使出现 CRITICAL 也以退出码 0 结束，只用于人工盘点。 + +该脚本只读检查 current release 自包含能力，不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +CRITICAL 默认会让进程以退出码 1 结束，适合作为切换窗口前置门禁。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +async function buildAudit(input) { + const artifacts = await inspectArtifacts(input); + const directories = await inspectDirectories(input); + const pingoraGateway = await inspectPingoraGateway(input); + const checksums = await inspectChecksums(input, pingoraGateway); + const releaseManifest = await inspectReleaseManifest(input, pingoraGateway); + const systemd = input.systemdShow + ? await inspectSystemd(input) + : { + checked: false, + status: 'OK', + diagnostics: [], + }; + const summary = summarize([ + ...artifacts.map((artifact) => artifact.status), + ...directories.map((directory) => directory.status), + pingoraGateway.status, + ...checksums.map((checksum) => checksum.status), + releaseManifest.status, + systemd.status, + ]); + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + releaseRoot: input.releaseRoot, + summary, + artifacts, + directories, + pingoraGateway, + checksums, + releaseManifest, + systemd, + }; +} + +async function inspectArtifacts(input) { + const artifacts = []; + for (const artifact of REQUIRED_ARTIFACTS) { + artifacts.push(await inspectArtifact(input.releaseRoot, artifact)); + } + return artifacts; +} + +async function inspectDirectories(input) { + const directories = []; + for (const relativePath of REQUIRED_DIRS) { + const fullPath = path.join(input.releaseRoot, relativePath); + const diagnostics = []; + let exists = false; + let status = 'OK'; + try { + const fileStat = await stat(fullPath); + exists = fileStat.isDirectory(); + if (!exists) { + diagnostics.push('路径存在但不是目录。'); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`目录不存在或不可读: ${error.message}`); + status = 'CRITICAL'; + } + directories.push({ + path: relativePath, + exists, + status, + diagnostics, + }); + } + return directories; +} + +async function inspectArtifact(releaseRoot, artifact) { + const fullPath = path.join(releaseRoot, artifact.path); + const diagnostics = []; + let exists = false; + let type = 'missing'; + let executable = false; + let status = 'OK'; + + try { + const fileStat = await stat(fullPath); + exists = true; + type = fileStat.isFile() + ? 'file' + : fileStat.isDirectory() + ? 'directory' + : 'other'; + if (type !== 'file') { + diagnostics.push('路径存在但不是文件。'); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`文件不存在或不可读: ${error.message}`); + status = 'CRITICAL'; + } + + if (exists && artifact.executable) { + try { + await access(fullPath, fsConstants.X_OK); + executable = true; + } catch (error) { + diagnostics.push(`文件不可执行: ${error.message}`); + status = 'CRITICAL'; + } + } + + return { + path: artifact.path, + expectedExecutable: Boolean(artifact.executable), + exists, + type, + executable, + status, + diagnostics, + }; +} + +async function inspectPingoraGateway(input) { + const binaryPath = path.join(input.releaseRoot, 'pingora-gateway'); + const diagnostics = []; + let included = false; + let executable = false; + let status = 'OK'; + + try { + const fileStat = await stat(binaryPath); + included = fileStat.isFile(); + if (!included) { + diagnostics.push('pingora-gateway 路径存在但不是文件。'); + status = 'CRITICAL'; + } + } catch (error) { + if (input.requirePingoraGateway) { + diagnostics.push(`缺少 pingora-gateway: ${error.message}`); + status = 'CRITICAL'; + } + } + + if (included) { + try { + await access(binaryPath, fsConstants.X_OK); + executable = true; + } catch (error) { + diagnostics.push(`pingora-gateway 不可执行: ${error.message}`); + status = 'CRITICAL'; + } + } + + return { + required: input.requirePingoraGateway, + included, + path: 'pingora-gateway', + absolutePath: binaryPath, + executable, + status, + diagnostics, + }; +} + +async function inspectChecksums(input, pingoraGateway) { + const checksumTargets = [ + { + path: 'api-server', + checksumPath: 'api-server.sha256', + required: true, + }, + { + path: 'pingora-gateway', + checksumPath: 'pingora-gateway.sha256', + required: input.requirePingoraGateway || pingoraGateway.included, + }, + ]; + + const checksums = []; + for (const target of checksumTargets) { + checksums.push(await inspectChecksum(input.releaseRoot, target)); + } + return checksums; +} + +async function inspectChecksum(releaseRoot, target) { + const diagnostics = []; + const artifactPath = path.join(releaseRoot, target.path); + const checksumPath = path.join(releaseRoot, target.checksumPath); + let status = 'OK'; + let checked = false; + let exists = false; + let expectedSha256 = ''; + let computedSha256 = ''; + let checksumFileTarget = ''; + + if (!target.required) { + return { + path: target.path, + checksumPath: target.checksumPath, + required: false, + checked, + exists, + checksumFileTarget, + expectedSha256, + computedSha256, + matches: null, + status, + diagnostics, + }; + } + + try { + const checksumText = await readFile(checksumPath, 'utf8'); + exists = true; + const parsed = parseSha256File(checksumText); + expectedSha256 = parsed.sha256; + checksumFileTarget = parsed.fileName; + if (path.basename(checksumFileTarget) !== path.basename(target.path)) { + diagnostics.push( + `${target.checksumPath} 指向了非预期文件: ${checksumFileTarget}`, + ); + status = 'CRITICAL'; + } + } catch (error) { + diagnostics.push(`checksum 文件不存在、不可读或格式错误: ${error.message}`); + status = 'CRITICAL'; + } + + try { + computedSha256 = await sha256File(artifactPath); + checked = true; + } catch (error) { + diagnostics.push(`无法读取待校验文件: ${error.message}`); + status = 'CRITICAL'; + } + + const matches = + checked && Boolean(expectedSha256) && computedSha256 === expectedSha256; + if (checked && expectedSha256 && !matches) { + diagnostics.push( + `${target.path} sha256 不匹配: expected ${expectedSha256}, actual ${computedSha256}`, + ); + status = 'CRITICAL'; + } + + return { + path: target.path, + checksumPath: target.checksumPath, + required: target.required, + checked, + exists, + checksumFileTarget, + expectedSha256, + computedSha256, + matches, + status, + diagnostics, + }; +} + +function parseSha256File(content) { + const firstLine = content.split(/\r?\n/u).find((line) => line.trim()); + if (!firstLine) { + throw new Error('checksum 文件为空'); + } + const match = firstLine.match(/^([0-9a-fA-F]{64})\s+(.+)$/u); + if (!match) { + throw new Error('checksum 行必须是 sha256sum 输出格式'); + } + return { + sha256: match[1].toLowerCase(), + fileName: match[2].replace(/^\*/u, '').trim(), + }; +} + +async function sha256File(filePath) { + const content = await readFile(filePath); + return createHash('sha256').update(content).digest('hex'); +} + +async function inspectReleaseManifest(input, pingoraGateway) { + const candidates = [ + 'release-manifest.api-server.json', + 'release-manifest.json', + ]; + const diagnostics = []; + let status = 'OK'; + let pathUsed = ''; + let manifest = null; + + for (const candidate of candidates) { + try { + const content = await readFile(path.join(input.releaseRoot, candidate), 'utf8'); + manifest = JSON.parse(content); + pathUsed = candidate; + break; + } catch (error) { + diagnostics.push(`${candidate} 不可用: ${error.message}`); + } + } + + if (!manifest) { + return { + checked: true, + path: '', + candidates, + componentType: '', + artifacts: [], + status: 'CRITICAL', + diagnostics, + }; + } + + diagnostics.length = 0; + const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; + const apiArtifact = artifacts.find((artifact) => artifact?.path === 'api-server'); + const pingoraArtifact = artifacts.find( + (artifact) => artifact?.path === 'pingora-gateway', + ); + + if (manifest.component_type !== 'api-server') { + diagnostics.push( + `release manifest component_type 应为 api-server，实际 ${manifest.component_type || '(空)'}`, + ); + status = 'CRITICAL'; + } + if (!apiArtifact) { + diagnostics.push('release manifest 缺少 api-server artifact。'); + status = 'CRITICAL'; + } else if (apiArtifact.checksum_path !== 'api-server.sha256') { + diagnostics.push( + `api-server artifact checksum_path 应为 api-server.sha256，实际 ${apiArtifact.checksum_path || '(空)'}`, + ); + status = 'CRITICAL'; + } + + if (input.requirePingoraGateway || pingoraGateway.included) { + if (!pingoraArtifact) { + diagnostics.push('release manifest 缺少 pingora-gateway artifact。'); + status = 'CRITICAL'; + } else if (pingoraArtifact.checksum_path !== 'pingora-gateway.sha256') { + diagnostics.push( + `pingora-gateway artifact checksum_path 应为 pingora-gateway.sha256，实际 ${pingoraArtifact.checksum_path || '(空)'}`, + ); + status = 'CRITICAL'; + } + } else if (pingoraArtifact) { + diagnostics.push( + 'release manifest 登记了 pingora-gateway，但 current release 未包含该二进制。', + ); + status = 'CRITICAL'; + } + + return { + checked: true, + path: pathUsed, + candidates, + componentType: manifest.component_type || '', + artifacts: artifacts.map((artifact) => ({ + component: artifact?.component || '', + path: artifact?.path || '', + checksumPath: artifact?.checksum_path || '', + })), + status, + diagnostics, + }; +} + +async function inspectSystemd(input) { + const expectedBinary = path.join(input.releaseRoot, 'pingora-gateway'); + const result = await runCommand( + 'systemctl', + [ + 'show', + input.systemdService, + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + const show = parseSystemctlShow(result.stdout); + const diagnostics = []; + let status = 'OK'; + + if (result.code !== 0) { + diagnostics.push(`systemctl show 失败: ${result.stderr || result.error}`); + status = 'CRITICAL'; + } + + const execStart = show.ExecStart || ''; + if (result.code === 0 && !execStart.includes(expectedBinary)) { + diagnostics.push( + `ExecStart 未指向 current release 网关二进制: expected ${expectedBinary}, actual ${execStart || '(空)'}`, + ); + status = 'CRITICAL'; + } + + return { + checked: true, + service: input.systemdService, + status, + expectedBinary, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart, + command: result.command, + diagnostics, + }; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +function runCommand(command, args, input) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + + return new Promise((resolve) => { + const startedAt = new Date(); + const child = execFile( + command, + args, + { + encoding: 'utf8', + timeout: input.timeoutMs, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + const finishedAt = new Date(); + resolve({ + command: [command, ...args], + code: error?.code ?? 0, + signal: error?.signal ?? null, + error: error ? error.message : '', + stdout: stdout || '', + stderr: stderr || '', + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + }); + }, + ); + child.on('error', (error) => { + const finishedAt = new Date(); + resolve({ + command: [command, ...args], + code: 1, + signal: null, + error: error.message, + stdout: '', + stderr: '', + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + }); + }); + }); +} + +function summarize(statuses) { + const criticalCount = statuses.filter((status) => status === 'CRITICAL').length; + const warningCount = statuses.filter((status) => status === 'WARNING').length; + return { + status: criticalCount > 0 ? 'CRITICAL' : warningCount > 0 ? 'WARNING' : 'OK', + criticalCount, + warningCount, + }; +} diff --git a/scripts/ops/pingora-cutover-command-evidence.mjs b/scripts/ops/pingora-cutover-command-evidence.mjs new file mode 100644 index 000000000..9fdce149b --- /dev/null +++ b/scripts/ops/pingora-cutover-command-evidence.mjs @@ -0,0 +1,518 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { + access, + chmod, + lstat, + mkdir, + readFile, + stat, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; + +const BUNDLE_DIR_MODE = 0o750; +const EVIDENCE_FILE_MODE = 0o640; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); +const SECRET_ENV_KEYS = [ + 'GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN', + 'GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN', + 'GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN', + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN', +]; + +const config = parseArgs(process.argv.slice(2)); +const secretValues = collectSecretValues(config.commandArgs, process.env); +const startedAt = new Date(); +const bundleDir = await createBundleDir( + config.outputRoot, + config.phase, + config.commandName, + startedAt, +); +const run = await runCommand(config.command, config.commandArgs, secretValues); +const finishedAt = new Date(); + +const stdoutPath = path.join(bundleDir, 'command.stdout.txt'); +const stderrPath = path.join(bundleDir, 'command.stderr.txt'); +const recordPath = path.join(bundleDir, 'command-record.json'); +const manifestPath = path.join(bundleDir, 'manifest.json'); +const exitCode = + run.signal + ? run.exitCode ?? 1 + : run.exitCode === null || run.exitCode === undefined + ? run.error + ? 1 + : 0 + : run.exitCode; +const status = exitCode === 0 && !run.signal && !run.error ? 'OK' : 'FAILED'; + +await writeEvidenceFile(stdoutPath, redactSecrets(run.stdout, secretValues)); +await writeEvidenceFile(stderrPath, redactSecrets(run.stderr, secretValues)); + +const commandRecord = { + schemaVersion: 1, + name: config.commandName, + phase: config.phase, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + ...(config.expectedExecutable + ? { expectedExecutable: config.expectedExecutable } + : {}), + executable: redactSecrets(config.command, secretValues), + args: redactSecretArgs(config.commandArgs, secretValues), + command: formatCommand(config.command, config.commandArgs, secretValues), + cwd: process.cwd(), + exitCode, + signal: run.signal, + error: run.error || null, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + stdoutPath: path.basename(stdoutPath), + stderrPath: path.basename(stderrPath), +}; +await writeEvidenceFile( + recordPath, + `${JSON.stringify(commandRecord, null, 2)}\n`, +); +const artifactFiles = { + stdout: await buildEvidenceFileMetadata(stdoutPath), + stderr: await buildEvidenceFileMetadata(stderrPath), + commandRecord: await buildEvidenceFileMetadata(recordPath), +}; + +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: config.phase, + commandName: config.commandName, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + ...(config.expectedExecutable + ? { expectedExecutable: config.expectedExecutable } + : {}), + outputRoot: config.outputRoot, + bundleDir, + summary: { + status, + exitCode, + signal: run.signal, + }, + files: { + manifest: path.basename(manifestPath), + ...artifactFiles, + }, + command: commandRecord, +}; +await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + +console.log( + `${JSON.stringify( + { + ok: status === 'OK', + phase: config.phase, + commandName: config.commandName, + cutoverRunId: config.cutoverRunId || null, + expectedExecutable: config.expectedExecutable || null, + status, + exitCode, + bundleDir, + manifestPath, + stdoutPath, + stderrPath, + commandRecordPath: recordPath, + }, + null, + 2, + )}\n`, +); + +if (exitCode !== 0) { + process.exit(exitCode); +} + +function parseArgs(argv) { + const separatorIndex = argv.indexOf('--'); + const optionArgs = + separatorIndex < 0 ? argv : argv.slice(0, separatorIndex); + if (optionArgs.includes('-h') || optionArgs.includes('--help')) { + printUsage(); + process.exit(0); + } + if (separatorIndex < 0) { + throw new Error('必须使用 -- 分隔证据参数和真实命令。'); + } + + const command = argv[separatorIndex + 1] || ''; + const commandArgs = argv.slice(separatorIndex + 2); + const result = { + phase: + process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_PHASE || 'manual', + commandName: + process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_NAME || 'command', + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + expectedExecutable: + process.env.GENARRATIVE_PINGORA_CUTOVER_EXPECTED_EXECUTABLE || '', + requiredArgs: parseRequiredArgsEnv( + process.env.GENARRATIVE_PINGORA_CUTOVER_REQUIRED_ARGS || '', + ), + outputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + command, + commandArgs, + }; + + for (let index = 0; index < optionArgs.length; index += 1) { + const arg = optionArgs[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(optionArgs, ++index, arg); + break; + case '--command-name': + result.commandName = requireValue(optionArgs, ++index, arg); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(optionArgs, ++index, arg); + break; + case '--expected-executable': + result.expectedExecutable = requireValue(optionArgs, ++index, arg); + break; + case '--require-arg': + result.requiredArgs.push(requireAnyValue(optionArgs, ++index, arg)); + break; + case '--output-root': + result.outputRoot = requireValue(optionArgs, ++index, arg); + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-command-evidence.mjs [options] -- [args...] + +Options: + --phase 写入 manifest 的阶段标签，例如 enable-apply / rollback-apply。 + --command-name 写入 manifest 的命令名，例如 pingora-direct-rollback-apply。 + --cutover-run-id 可选，写入 manifest 的本次切换批次 ID；正式 runbook 会用同一 ID 串联所有阶段和命令证据。 + --expected-executable + 可选，要求 -- 后面的真实命令精确等于该绝对路径；正式 runbook 用于绑定 apply 证据和 current release 随包脚本。 + --require-arg 可选，要求 -- 后面的真实命令参数包含该值；正式 runbook 用于在执行前确认 apply 证据包含 --apply。 + --output-root 证据输出根目录，默认 /var/log/genarrative/pingora-cutover-evidence。 + +该脚本会执行 -- 后面的真实命令，并把 stdout、stderr、退出码、脱敏后的命令记录和 manifest 写入 --output-root 下的新证据目录；失败时仍保留证据并返回真实命令的退出码。 +`); +} + +function validateConfig(config) { + validateSafeName(config.phase, '--phase'); + validateSafeName(config.commandName, '--command-name'); + if (config.cutoverRunId) { + validateSafeName(config.cutoverRunId, '--cutover-run-id'); + } + if (config.expectedExecutable) { + validateSafeExecutablePath(config.expectedExecutable, '--expected-executable'); + } + if (!path.isAbsolute(config.outputRoot)) { + throw new Error('--output-root 必须是绝对路径。'); + } + if (path.resolve(config.outputRoot) === path.parse(path.resolve(config.outputRoot)).root) { + throw new Error('--output-root 不能是文件系统根目录。'); + } + validateNoControlCharacters(config.outputRoot, '--output-root'); + if (!config.command) { + throw new Error('-- 后必须提供真实命令。'); + } + validateCommandPath(config.command); + for (const commandArg of config.commandArgs) { + validateCommandArg(commandArg); + } + if ( + config.expectedExecutable && + config.command !== config.expectedExecutable + ) { + throw new Error( + `真实命令与 --expected-executable 不一致: expected ${config.expectedExecutable}, got ${config.command}`, + ); + } + for (const requiredArg of config.requiredArgs) { + validateRequiredArg(requiredArg, '--require-arg'); + if (!config.commandArgs.includes(requiredArg)) { + throw new Error(`真实命令参数缺少 --require-arg 要求的 ${requiredArg}`); + } + } +} + +function validateCommandPath(command) { + validateSafeExecutablePath(command, '真实命令'); +} + +function validateCommandArg(value) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error('真实命令参数不能包含换行或 NUL 字符。'); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateSafeExecutablePath(command, label) { + const prefix = label.startsWith('--') ? `${label} ` : label; + if (!path.isAbsolute(command)) { + throw new Error(`${prefix}必须是绝对路径。`); + } + if (path.resolve(command) === path.parse(path.resolve(command)).root) { + throw new Error(`${prefix}不能是文件系统根目录。`); + } + if (/[\0\r\n]/u.test(command)) { + throw new Error(`${prefix}不能包含换行或 NUL 字符。`); + } +} + +function parseRequiredArgsEnv(value) { + return String(value || '') + .split(',') + .map((item) => item.trim()) + .filter(Boolean); +} + +function validateRequiredArg(value, label) { + if ( + typeof value !== 'string' || + value.length === 0 || + /[\0\r\n]/u.test(value) + ) { + throw new Error(`${label} 必须是非空且不包含换行的字符串。`); + } +} + +function validateSafeName(value, label) { + const text = String(value || ''); + if (!/^[0-9A-Za-z._-]+$/u.test(text)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function requireAnyValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +async function createBundleDir(outputRoot, phase, commandName, date) { + await validateOutputRootForWriting(outputRoot); + await mkdir(outputRoot, { recursive: true }); + await validateOutputRootForWriting(outputRoot); + await access(outputRoot, fsConstants.W_OK); + const baseName = `${formatTimestampForPath(date)}-${phase}-${commandName}`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? '' : `-${attempt}`; + const bundleDir = path.join(outputRoot, `${baseName}${suffix}`); + try { + await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE }); + await chmod(bundleDir, BUNDLE_DIR_MODE); + return bundleDir; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + } + } + throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`); +} + +async function validateOutputRootForWriting(outputRoot) { + const target = path.resolve(outputRoot); + const root = path.parse(target).root; + let current = root; + const segments = path + .relative(root, target) + .split(path.sep) + .filter(Boolean); + + for (const segment of segments) { + current = path.join(current, segment); + let stats; + try { + stats = await lstat(current); + } catch (error) { + if (error?.code === 'ENOENT') { + return; + } + throw error; + } + + if (stats.isSymbolicLink()) { + if (current === target) { + throw new Error(`--output-root 不能是符号链接: ${current}`); + } + throw new Error(`--output-root 上级目录不能是符号链接: ${current}`); + } + + if (!stats.isDirectory()) { + if (current === target) { + throw new Error(`--output-root 已存在但不是目录: ${current}`); + } + throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`); + } + } +} + +async function writeEvidenceFile(filePath, content) { + await writeFile(filePath, content, { + encoding: 'utf8', + flag: 'wx', + mode: EVIDENCE_FILE_MODE, + }); + await chmod(filePath, EVIDENCE_FILE_MODE); +} + +async function buildEvidenceFileMetadata(filePath) { + const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + return { + path: path.basename(filePath), + sizeBytes: stats.size, + sha256: createHash('sha256').update(content).digest('hex'), + }; +} + +function runCommand(command, args, secrets) { + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: process.cwd(), + env: process.env, + shell: false, + windowsHide: true, + }); + let stdout = ''; + let stderr = ''; + let spawnError = ''; + + child.stdout?.on('data', (chunk) => { + const text = String(chunk); + stdout += text; + process.stdout.write(redactSecrets(text, secrets)); + }); + child.stderr?.on('data', (chunk) => { + const text = String(chunk); + stderr += text; + process.stderr.write(redactSecrets(text, secrets)); + }); + child.on('error', (error) => { + spawnError = error.message; + }); + child.on('close', (exitCode, signal) => { + resolve({ + exitCode, + signal: signal || null, + error: spawnError, + stdout, + stderr, + }); + }); + }); +} + +function collectSecretValues(args, env) { + const values = []; + for (let index = 0; index < args.length; index += 1) { + const inlineValue = getInlineSecretFlagValue(args[index]); + if (inlineValue) { + values.push(inlineValue); + continue; + } + if (index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) && args[index]) { + values.push(args[index]); + } + } + for (const key of SECRET_ENV_KEYS) { + const value = String(env[key] || ''); + if (value) { + values.push(value); + } + } + return [...new Set(values)].filter((value) => value.length >= 4); +} + +function redactSecrets(text, secrets) { + let redacted = String(text || ''); + for (const secret of secrets) { + redacted = redacted.split(secret).join(''); + } + return redacted; +} + +function formatCommand(command, args, secrets) { + return [redactSecrets(command, secrets), ...redactSecretArgs(args, secrets)].join(' '); +} + +function redactSecretArgs(args, secrets = []) { + return args.map((arg, index) => { + const redacted = + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) + ? '' + : redactInlineSecretArg(arg); + return redactSecrets(redacted, secrets); + }); +} + +function getInlineSecretFlagValue(arg) { + const text = String(arg || ''); + for (const flag of SECRET_VALUE_FLAGS) { + const prefix = `${flag}=`; + if (text.startsWith(prefix)) { + return text.slice(prefix.length); + } + } + return ''; +} + +function redactInlineSecretArg(arg) { + const text = String(arg || ''); + for (const flag of SECRET_VALUE_FLAGS) { + if (text.startsWith(`${flag}=`)) { + return `${flag}=`; + } + } + return arg; +} + +function formatTimestampForPath(date) { + return date + .toISOString() + .replace(/[-:]/gu, '') + .replace(/\.\d{3}Z$/u, 'Z'); +} diff --git a/scripts/ops/pingora-cutover-evidence-audit.mjs b/scripts/ops/pingora-cutover-evidence-audit.mjs new file mode 100644 index 000000000..6b9f5787c --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-audit.mjs @@ -0,0 +1,1919 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, lstat, readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const DEFAULT_EVIDENCE_ROOT = '/var/log/genarrative/pingora-cutover-evidence'; +const DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); +const CUTOVER_TIMELINE_ORDER = [ + { type: 'phase', phase: 'pre-cutover' }, + { + type: 'command', + phase: 'enable-apply', + commandName: 'pingora-direct-enable-apply', + }, + { type: 'phase', phase: 'post-enable' }, + { + type: 'command', + phase: 'rollback-apply', + commandName: 'pingora-direct-rollback-apply', + }, + { type: 'phase', phase: 'post-rollback' }, +]; + +const config = parseArgs(process.argv.slice(2)); +await validateReadOnlyDirectory(config.evidenceRoot, '--evidence-root'); +await validateVerifierScript(config.verifyScript); + +const discovery = await discoverEvidence(config.evidenceRoot, { + allowExtraRootEntries: config.allowExtraRootEntries, +}); +const audit = await buildAudit(config, discovery); + +console.log(`${JSON.stringify(audit, null, 2)}\n`); + +if (!audit.ok) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + evidenceRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + DEFAULT_EVIDENCE_ROOT, + verifyScript: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT || + path.join(SCRIPT_DIR, 'pingora-cutover-evidence-verify.mjs'), + timelineMaxSpanMs: parsePositiveInteger( + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS || + DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS, + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS', + ), + requiredCutoverRunId: + process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + allowExtraRootEntries: false, + requiredPhases: [], + requiredPhaseDirectLiveAccessLog: [], + requiredPhaseDirectLiveStaticHeaders: [], + requiredCommands: [], + requiredCommandExecutables: [], + requiredCommandArgs: [], + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--evidence-root': + result.evidenceRoot = requireValue(argv, ++index, arg); + break; + case '--verify-script': + result.verifyScript = requireValue(argv, ++index, arg); + break; + case '--phase': + case '--require-phase': + result.requiredPhases.push(requireValue(argv, ++index, arg)); + break; + case '--require-command': + result.requiredCommands.push(requireValue(argv, ++index, arg)); + break; + case '--require-phase-direct-live-access-log': + result.requiredPhaseDirectLiveAccessLog.push( + requireValue(argv, ++index, arg), + ); + break; + case '--require-phase-direct-live-static-headers': + result.requiredPhaseDirectLiveStaticHeaders.push( + requireValue(argv, ++index, arg), + ); + break; + case '--require-command-executable': + result.requiredCommandExecutables.push(requireValue(argv, ++index, arg)); + break; + case '--require-command-arg': + result.requiredCommandArgs.push(requireValue(argv, ++index, arg)); + break; + case '--timeline-max-span-ms': + result.timelineMaxSpanMs = parsePositiveInteger( + requireValue(argv, ++index, arg), + '--timeline-max-span-ms', + ); + break; + case '--require-cutover-run-id': + result.requiredCutoverRunId = requireValue(argv, ++index, arg); + break; + case '--allow-extra-root-entries': + result.allowExtraRootEntries = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + for (const [label, value] of [ + ['--evidence-root', result.evidenceRoot], + ['--verify-script', result.verifyScript], + ]) { + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + validateNoControlCharacters(value, label); + } + result.requiredPhases = normalizeRequiredPhases(result.requiredPhases); + result.requiredPhaseDirectLiveAccessLog = normalizeRequiredPhases( + result.requiredPhaseDirectLiveAccessLog, + ); + result.requiredPhaseDirectLiveStaticHeaders = normalizeRequiredPhases( + result.requiredPhaseDirectLiveStaticHeaders, + ); + const requiredCommands = normalizeRequiredCommands(result.requiredCommands); + result.requiredCommandExecutables = normalizeRequiredCommandExecutables( + result.requiredCommandExecutables, + ); + result.requiredCommandArgs = normalizeRequiredCommandArgs( + result.requiredCommandArgs, + ); + result.requiredCommands = mergeRequiredCommands( + requiredCommands, + result.requiredCommandExecutables, + result.requiredCommandArgs, + ); + if (result.requiredCutoverRunId) { + validateSafeCutoverRunId( + result.requiredCutoverRunId, + '--require-cutover-run-id', + ); + } + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-audit.mjs [options] + +Options: + --evidence-root 证据根目录，默认 ${DEFAULT_EVIDENCE_ROOT}。 + --verify-script 证据验真脚本，默认同目录 pingora-cutover-evidence-verify.mjs。 + --require-phase 要求并验真某个阶段的最新证据；可重复。--phase 是别名。 + --require-command : + 要求并验真某个切换命令证据；可重复。 + --require-phase-direct-live-access-log + 要求某个阶段 manifest.summary.directLiveAccessLog 已存在且 direct live request_id access log 对账成功；正式切换默认用于 post-enable。 + --require-phase-direct-live-static-headers + 要求某个阶段 manifest.summary.directLiveStaticHeaders 已存在且包含可判定的 direct live 静态响应头摘要；正式切换默认用于 post-enable。 + --require-command-executable :: + 要求某个命令证据的 manifest.expectedExecutable、manifest.command.executable 和 command-record.json executable 与该绝对路径一致；可重复。 + --require-command-arg :: + 要求某个命令证据的 manifest.command.args 和 command-record.json args 都包含该参数；可重复。 + --timeline-max-span-ms + 标准五段切换时间线允许的最大跨度，默认 ${DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS}ms，可用 GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS 覆盖。 + --require-cutover-run-id + 只接受 manifest.cutoverRunId 与该 ID 一致的阶段和命令证据，避免混入其它切换批次。 + --allow-extra-root-entries + 允许证据根目录中存在非证据目录条目；默认拒绝，正式切换归档不应使用。 + +该脚本只读扫描 Pingora 直连切换证据根目录，按 manifest.phase 找到每个阶段的最新证据目录，也可按 manifest.phase + manifest.commandName 找到真实切换命令证据，并调用随包证据验真脚本的 --require-summary-ok 严格模式校验 schemaVersion=1 manifest.files 的 sizeBytes / sha256 与 manifest.summary.status=OK；所有候选证据都必须带合法 manifest.generatedAt，最新证据选择和标准时间线证明只使用该字段，不用目录 mtime 兜底；若同一阶段或同一命令的最新 manifest.generatedAt 重复，则拒绝按目录名打平并要求重新归档或清理证据根目录；命令证据的 manifest.command 与 command-record.json 也必须是 schemaVersion=1；可用 --require-phase-direct-live-access-log 要求指定阶段必须带 direct live access log 对账摘要，防止旧 post-enable 证据包缺少 request_id 反查复盘入口；可用 --require-phase-direct-live-static-headers 要求指定阶段必须带 direct live 静态响应头摘要，防止旧 post-enable 证据包缺少 Cache-Control、ETag、Last-Modified、Range 和 304 复盘入口；证据根目录默认只能包含带 manifest.json 的证据目录，不允许夹带普通文件、无 manifest 目录或符号链接；不会修改证据目录、不会 reload systemd、不会访问 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function normalizeRequiredPhases(phases) { + const seen = new Set(); + const normalized = []; + for (const phase of phases) { + const text = String(phase || '').trim(); + validateSafePhase(text, '--require-phase'); + if (!seen.has(text)) { + seen.add(text); + normalized.push(text); + } + } + return normalized; +} + +function normalizeRequiredCommands(commands) { + const seen = new Set(); + const normalized = []; + for (const command of commands) { + const text = String(command || '').trim(); + const parts = text.split(':'); + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command 必须使用 : 格式。', + ); + } + const [phase, commandName] = parts; + validateSafePhase(phase, '--require-command phase'); + validateSafeCommandName(commandName, '--require-command commandName'); + const key = `${phase}:${commandName}`; + if (!seen.has(key)) { + seen.add(key); + normalized.push({ phase, commandName }); + } + } + return normalized; +} + +function normalizeRequiredCommandExecutables(commandExecutables) { + const seen = new Map(); + const normalized = []; + for (const item of commandExecutables) { + const text = String(item || '').trim(); + const parts = text.split(':'); + if (parts.length < 3 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command-executable 必须使用 :: 格式。', + ); + } + const [phase, commandName, ...pathParts] = parts; + const executable = pathParts.join(':'); + validateSafePhase(phase, '--require-command-executable phase'); + validateSafeCommandName( + commandName, + '--require-command-executable commandName', + ); + if (!path.isAbsolute(executable)) { + throw new Error('--require-command-executable executable 必须是绝对路径。'); + } + if (isFilesystemRootPath(executable)) { + throw new Error( + '--require-command-executable executable 不能是文件系统根目录。', + ); + } + if (/[\0\r\n]/u.test(executable)) { + throw new Error( + '--require-command-executable executable 不能包含换行或 NUL 字符。', + ); + } + const key = `${phase}:${commandName}`; + const previousExecutable = seen.get(key); + if (previousExecutable && previousExecutable !== executable) { + throw new Error( + '--require-command-executable 不能为同一个 : 指定多个不同路径。', + ); + } + if (!previousExecutable) { + seen.set(key, executable); + normalized.push({ phase, commandName, executable }); + } + } + return normalized; +} + +function normalizeRequiredCommandArgs(commandArgs) { + const seen = new Set(); + const normalized = []; + for (const item of commandArgs) { + const text = String(item || '').trim(); + const parts = text.split(':'); + if (parts.length < 3 || !parts[0] || !parts[1]) { + throw new Error( + '--require-command-arg 必须使用 :: 格式。', + ); + } + const [phase, commandName, ...argParts] = parts; + const requiredArg = argParts.join(':'); + validateSafePhase(phase, '--require-command-arg phase'); + validateSafeCommandName(commandName, '--require-command-arg commandName'); + validateRequiredCommandArg(requiredArg, '--require-command-arg arg'); + const key = `${phase}:${commandName}:${requiredArg}`; + if (!seen.has(key)) { + seen.add(key); + normalized.push({ phase, commandName, arg: requiredArg }); + } + } + return normalized; +} + +function mergeRequiredCommands( + requiredCommands, + requiredCommandExecutables, + requiredCommandArgs, +) { + const seen = new Set(); + const merged = []; + for (const command of [ + ...requiredCommands, + ...requiredCommandExecutables, + ...requiredCommandArgs, + ]) { + const key = `${command.phase}:${command.commandName}`; + if (!seen.has(key)) { + seen.add(key); + merged.push({ + phase: command.phase, + commandName: command.commandName, + }); + } + } + return merged; +} + +function findRequiredCommandExecutable(requiredCommandExecutables, command) { + return ( + requiredCommandExecutables.find( + (item) => + item.phase === command.phase && item.commandName === command.commandName, + )?.executable || null + ); +} + +function findRequiredCommandArgs(requiredCommandArgs, command) { + return requiredCommandArgs + .filter( + (item) => + item.phase === command.phase && item.commandName === command.commandName, + ) + .map((item) => item.arg); +} + +function validateSafePhase(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateSafeCommandName(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateSafeCutoverRunId(value, label) { + if (!isSafePhase(value)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateRequiredCommandArg(value, label) { + if ( + typeof value !== 'string' || + value.length === 0 || + /[\0\r\n]/u.test(value) + ) { + throw new Error(`${label} 必须是非空且不包含换行的字符串。`); + } +} + +function isSafePhase(value) { + return /^[0-9A-Za-z._-]+$/u.test(String(value || '')); +} + +function parsePositiveInteger(value, label) { + const text = String(value || '').trim(); + if (!/^[1-9]\d*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + const parsed = Number(text); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label} 超出 JavaScript 安全整数范围。`); + } + return parsed; +} + +async function validateReadOnlyDirectory(dirPath, label) { + const stats = await lstat(dirPath); + if (stats.isSymbolicLink()) { + throw new Error(`${label} 不能是符号链接: ${dirPath}`); + } + if (!stats.isDirectory()) { + throw new Error(`${label} 必须是目录: ${dirPath}`); + } + await access(dirPath, fsConstants.R_OK | fsConstants.X_OK); +} + +async function validateVerifierScript(scriptPath) { + const stats = await lstat(scriptPath); + if (stats.isSymbolicLink()) { + throw new Error(`--verify-script 不能是符号链接: ${scriptPath}`); + } + if (!stats.isFile()) { + throw new Error(`--verify-script 必须是文件: ${scriptPath}`); + } + await access(scriptPath, fsConstants.R_OK); +} + +async function discoverEvidence(evidenceRoot, options = {}) { + const allowExtraRootEntries = Boolean(options.allowExtraRootEntries); + const entries = await readdir(evidenceRoot, { withFileTypes: true }); + const candidates = []; + const diagnostics = []; + + for (const entry of entries) { + const entryPath = path.join(evidenceRoot, entry.name); + const linkStats = await lstat(entryPath); + if (linkStats.isSymbolicLink()) { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: '证据根目录下不能包含符号链接条目。', + }); + } + continue; + } + if (!linkStats.isDirectory()) { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: linkStats.isFile() + ? '证据根目录只能包含证据目录，发现普通文件。' + : '证据根目录只能包含证据目录，发现非目录条目。', + }); + } + continue; + } + + const manifestPath = path.join(entryPath, 'manifest.json'); + let manifestStats; + try { + manifestStats = await lstat(manifestPath); + } catch (error) { + if (error?.code === 'ENOENT') { + if (!allowExtraRootEntries) { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: '证据根目录只能包含带 manifest.json 的证据目录。', + }); + } + } else { + diagnostics.push({ + path: entry.name, + status: 'CRITICAL', + reason: `读取 manifest 状态失败: ${error.message}`, + }); + } + continue; + } + if (manifestStats.isSymbolicLink()) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 不能是符号链接。', + }); + continue; + } + if (!manifestStats.isFile()) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 必须是普通文件。', + }); + continue; + } + + const manifest = await readJsonManifest(manifestPath, entry.name, diagnostics); + if (!manifest) { + continue; + } + if (manifest.schemaVersion !== 1) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest.schemaVersion 必须是 1。', + }); + continue; + } + if (!manifest.phase || typeof manifest.phase !== 'string') { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 缺少 phase 字符串。', + }); + continue; + } + if (!isSafePhase(manifest.phase)) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.phase 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + const topLevelCommandName = + manifest.commandName === undefined ? null : manifest.commandName; + const embeddedCommandName = + manifest.command?.name === undefined ? null : manifest.command.name; + if ( + topLevelCommandName !== null && + (typeof topLevelCommandName !== 'string' || + !isSafePhase(topLevelCommandName)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.commandName 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + if ( + embeddedCommandName !== null && + (typeof embeddedCommandName !== 'string' || + !isSafePhase(embeddedCommandName)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.command.name 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + if ( + topLevelCommandName !== null && + embeddedCommandName !== null && + topLevelCommandName !== embeddedCommandName + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: `manifest.commandName 与 manifest.command.name 必须一致，manifest.commandName=${formatNullable(topLevelCommandName)}，manifest.command.name=${formatNullable(embeddedCommandName)}。`, + }); + continue; + } + const commandName = topLevelCommandName || embeddedCommandName; + const cutoverRunId = + manifest.cutoverRunId === undefined ? null : manifest.cutoverRunId; + if ( + cutoverRunId !== null && + (typeof cutoverRunId !== 'string' || !isSafePhase(cutoverRunId)) + ) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.cutoverRunId 只能包含 ASCII 字母、数字、点、下划线或短横线。', + }); + continue; + } + const topLevelExpectedExecutable = + manifest.expectedExecutable === undefined + ? null + : manifest.expectedExecutable; + const embeddedExpectedExecutable = + manifest.command?.expectedExecutable === undefined + ? null + : manifest.command.expectedExecutable; + const expectedExecutableDiagnostics = [ + ...validateOptionalAbsolutePath( + topLevelExpectedExecutable, + 'manifest.expectedExecutable', + ), + ...validateOptionalAbsolutePath( + embeddedExpectedExecutable, + 'manifest.command.expectedExecutable', + ), + ]; + if (expectedExecutableDiagnostics.length > 0) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: expectedExecutableDiagnostics.join(' '), + }); + continue; + } + const generatedAt = normalizeIsoTime(manifest.generatedAt); + if (!generatedAt) { + diagnostics.push({ + path: entry.name, + manifestPath, + status: 'CRITICAL', + reason: + 'manifest.generatedAt 必须是合法 ISO 时间；证据最新选择和时间线证明不能依赖目录 mtime。', + }); + continue; + } + candidates.push({ + phase: manifest.phase, + cutoverRunId, + bundleDir: entryPath, + manifestPath, + directoryName: entry.name, + generatedAt, + summaryStatus: manifest.summary?.status || null, + summaryExitCode: manifest.summary?.exitCode ?? null, + summarySignal: manifest.summary?.signal ?? null, + summaryDirectLiveAccessLog: manifest.summary?.directLiveAccessLog ?? null, + summaryDirectLiveStaticHeaders: + manifest.summary?.directLiveStaticHeaders ?? null, + commandName, + manifestCommand: manifest.command || null, + commandRecordFileName: manifest.files?.commandRecord?.path || null, + stdoutFileName: manifest.files?.stdout?.path || null, + stderrFileName: manifest.files?.stderr?.path || null, + commandStartedAt: normalizeIsoTime(manifest.command?.startedAt), + commandFinishedAt: normalizeIsoTime(manifest.command?.finishedAt), + commandDurationMs: manifest.command?.durationMs ?? null, + expectedExecutable: + topLevelExpectedExecutable ?? embeddedExpectedExecutable ?? null, + executable: manifest.command?.executable || null, + }); + } + + return { candidates, diagnostics }; +} + +function validateOptionalAbsolutePath(value, label) { + if (value === null) { + return []; + } + return validateRequiredAbsolutePath(value, label); +} + +function validateRequiredAbsolutePath(value, label) { + if (typeof value !== 'string' || value.length === 0) { + return [`${label} 必须是绝对路径字符串。`]; + } + if (!path.isAbsolute(value)) { + return [`${label} 必须是绝对路径。`]; + } + if (isFilesystemRootPath(value)) { + return [`${label} 不能是文件系统根目录。`]; + } + if (/[\0\r\n]/u.test(value)) { + return [`${label} 不能包含换行或 NUL 字符。`]; + } + return []; +} + +async function readJsonManifest(manifestPath, directoryName, diagnostics) { + let content; + try { + content = await readFile(manifestPath, 'utf8'); + } catch (error) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: `读取 manifest 失败: ${error.message}`, + }); + return null; + } + try { + const manifest = JSON.parse(content); + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: 'manifest 顶层必须是 JSON object。', + }); + return null; + } + return manifest; + } catch (error) { + diagnostics.push({ + path: directoryName, + manifestPath, + status: 'CRITICAL', + reason: `manifest 不是合法 JSON: ${error.message}`, + }); + return null; + } +} + +async function buildAudit(config, discovery) { + const phasesToCheck = + config.requiredPhases.length > 0 + ? config.requiredPhases + : config.requiredCommands.length > 0 + ? [] + : sortedUnique( + discovery.candidates + .filter((candidate) => candidate.commandName === null) + .map((candidate) => candidate.phase), + ); + const phases = []; + + for (const phase of phasesToCheck) { + const requiredAccessLog = config.requiredPhaseDirectLiveAccessLog.includes( + phase, + ); + const requiredStaticHeaders = config.requiredPhaseDirectLiveStaticHeaders.includes( + phase, + ); + const allPhaseCandidates = discovery.candidates.filter( + (candidate) => candidate.phase === phase && candidate.commandName === null, + ); + const candidates = filterCandidatesByCutoverRunId( + allPhaseCandidates, + config.requiredCutoverRunId, + ) + .sort(compareEvidenceCandidates); + if (candidates.length === 0) { + phases.push({ + phase, + status: 'MISSING', + candidateCount: 0, + allCandidateCount: allPhaseCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + directLiveAccessLog: null, + directLiveStaticHeaders: null, + diagnostics: [ + missingEvidenceDiagnostic( + '该阶段', + config.requiredCutoverRunId, + allPhaseCandidates.length, + ), + ], + }); + continue; + } + + const latestSelection = selectUniqueLatestEvidence(candidates); + if (!latestSelection.ok) { + phases.push({ + phase, + status: 'AMBIGUOUS_LATEST', + candidateCount: candidates.length, + allCandidateCount: allPhaseCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latestSelection.generatedAt, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + directLiveAccessLog: null, + directLiveStaticHeaders: null, + ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, + diagnostics: latestSelection.diagnostics, + }); + continue; + } + + const latest = latestSelection.latest; + const verify = await runVerifier(config.verifyScript, latest.bundleDir); + const manifestCheck = checkPhaseManifest(latest, { + requiredAccessLog, + requiredStaticHeaders, + }); + phases.push({ + phase, + status: phaseStatus(verify, manifestCheck), + candidateCount: candidates.length, + latestBundleDir: latest.bundleDir, + manifestPath: latest.manifestPath, + cutoverRunId: latest.cutoverRunId, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latest.generatedAt, + summaryStatus: latest.summaryStatus, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + directLiveAccessLog: latest.summaryDirectLiveAccessLog, + directLiveStaticHeaders: latest.summaryDirectLiveStaticHeaders, + commandName: latest.commandName, + verify, + manifestCheck, + diagnostics: [ + ...(verify.ok ? [] : ['最新证据目录 manifest 验真失败。']), + ...manifestCheck.diagnostics, + ], + }); + } + + const commands = []; + for (const requiredCommand of config.requiredCommands) { + const allCommandCandidates = discovery.candidates.filter( + (candidate) => + candidate.phase === requiredCommand.phase && + candidate.commandName === requiredCommand.commandName, + ); + const candidates = filterCandidatesByCutoverRunId( + allCommandCandidates, + config.requiredCutoverRunId, + ) + .sort(compareEvidenceCandidates); + if (candidates.length === 0) { + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: 'MISSING', + candidateCount: 0, + allCandidateCount: allCommandCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + diagnostics: [ + missingEvidenceDiagnostic( + '该切换命令', + config.requiredCutoverRunId, + allCommandCandidates.length, + ), + ], + }); + continue; + } + + const latestSelection = selectUniqueLatestEvidence(candidates); + if (!latestSelection.ok) { + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: 'AMBIGUOUS_LATEST', + candidateCount: candidates.length, + allCandidateCount: allCommandCandidates.length, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latestSelection.generatedAt, + ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, + diagnostics: latestSelection.diagnostics, + }); + continue; + } + + const latest = latestSelection.latest; + const verify = await runVerifier(config.verifyScript, latest.bundleDir); + const expectedExecutable = findRequiredCommandExecutable( + config.requiredCommandExecutables, + requiredCommand, + ); + const requiredArgs = findRequiredCommandArgs( + config.requiredCommandArgs, + requiredCommand, + ); + const manifestCheck = await checkCommandManifest( + latest, + expectedExecutable, + requiredArgs, + ); + commands.push({ + phase: requiredCommand.phase, + commandName: requiredCommand.commandName, + status: commandStatus(verify, manifestCheck), + candidateCount: candidates.length, + latestBundleDir: latest.bundleDir, + manifestPath: latest.manifestPath, + cutoverRunId: latest.cutoverRunId, + requiredCutoverRunId: config.requiredCutoverRunId || null, + generatedAt: latest.generatedAt, + summaryStatus: latest.summaryStatus, + summaryExitCode: latest.summaryExitCode, + summarySignal: latest.summarySignal, + expectedExecutable: latest.expectedExecutable, + executable: latest.executable, + commandRecordPath: manifestCheck.commandRecordPath, + stdoutPath: manifestCheck.stdoutPath, + stderrPath: manifestCheck.stderrPath, + commandStartedAt: manifestCheck.commandRecordStartedAt, + commandFinishedAt: manifestCheck.commandRecordFinishedAt, + commandDurationMs: manifestCheck.commandRecordDurationMs, + requiredExecutable: expectedExecutable, + requiredArgs, + verify, + manifestCheck, + diagnostics: [ + ...(verify.ok ? [] : ['最新命令证据目录 manifest 验真失败。']), + ...manifestCheck.diagnostics, + ], + }); + } + + const criticalDiagnostics = discovery.diagnostics.filter( + (item) => item.status === 'CRITICAL', + ); + const timeline = buildTimelineAudit( + phases, + commands, + config.timelineMaxSpanMs, + ); + const failedPhases = phases.filter((phase) => phase.status !== 'OK'); + const failedCommands = commands.filter((command) => command.status !== 'OK'); + const ok = + phases.length + commands.length > 0 && + failedPhases.length === 0 && + failedCommands.length === 0 && + criticalDiagnostics.length === 0 && + timeline.ok; + + return { + ok, + generatedAt: new Date().toISOString(), + evidenceRoot: config.evidenceRoot, + verifyScript: config.verifyScript, + requiredCutoverRunId: config.requiredCutoverRunId || null, + allowExtraRootEntries: config.allowExtraRootEntries, + timelineMaxSpanMs: config.timelineMaxSpanMs, + requiredPhases: config.requiredPhases, + requiredCommands: config.requiredCommands, + requiredCommandExecutables: config.requiredCommandExecutables, + requiredCommandArgs: config.requiredCommandArgs, + discoveredCount: discovery.candidates.length, + checkedCount: phases.length + commands.length, + failedCount: + failedPhases.length + + failedCommands.length + + criticalDiagnostics.length + + timeline.failedCount, + diagnostics: discovery.diagnostics, + summary: buildOperatorSummary({ + ok, + phases, + commands, + criticalDiagnostics, + timeline, + checkedCount: phases.length + commands.length, + failedCount: + failedPhases.length + + failedCommands.length + + criticalDiagnostics.length + + timeline.failedCount, + }), + timeline, + phases, + commands, + }; +} + +function buildOperatorSummary(input) { + const directLiveEvidence = input.phases + .filter( + (phase) => + phase.requiredDirectLiveAccessLog || + phase.requiredDirectLiveStaticHeaders, + ) + .map((phase) => { + const accessLogRequired = phase.requiredDirectLiveAccessLog === true; + const staticHeadersRequired = + phase.requiredDirectLiveStaticHeaders === true; + const accessLogSummary = phase.directLiveAccessLog ?? null; + const staticHeadersSummary = phase.directLiveStaticHeaders ?? null; + const accessLogChecked = accessLogRequired || accessLogSummary !== null; + const staticHeadersChecked = + staticHeadersRequired || staticHeadersSummary !== null; + const firstDiagnostic = phase.diagnostics?.[0] ?? null; + return { + phase: phase.phase, + status: phase.status, + latestBundleDir: phase.latestBundleDir ?? null, + manifestPath: phase.manifestPath ?? null, + cutoverRunId: phase.cutoverRunId ?? null, + generatedAt: phase.generatedAt ?? null, + accessLog: { + required: accessLogRequired, + ok: accessLogChecked + ? phase.manifestCheck?.directLiveAccessLogCheck?.ok === true + : null, + reason: accessLogChecked + ? phase.manifestCheck?.directLiveAccessLogCheck?.reason ?? + firstDiagnostic + : null, + summary: accessLogSummary, + }, + staticHeaders: { + required: staticHeadersRequired, + ok: staticHeadersChecked + ? phase.manifestCheck?.directLiveStaticHeadersCheck?.ok === true + : null, + reason: staticHeadersChecked + ? phase.manifestCheck?.directLiveStaticHeadersCheck?.reason ?? + firstDiagnostic + : null, + summary: staticHeadersSummary, + }, + }; + }); + + return { + status: input.ok ? 'OK' : 'CRITICAL', + checkedCount: input.checkedCount, + failedCount: input.failedCount, + failedItems: [ + ...input.criticalDiagnostics.map((diagnostic) => ({ + type: 'evidence-root', + status: diagnostic.status, + path: diagnostic.path ?? null, + manifestPath: diagnostic.manifestPath ?? null, + reason: diagnostic.reason ?? null, + })), + ...input.phases + .filter((phase) => phase.status !== 'OK') + .map((phase) => ({ + type: 'phase', + phase: phase.phase, + status: phase.status, + latestBundleDir: phase.latestBundleDir ?? null, + manifestPath: phase.manifestPath ?? null, + cutoverRunId: phase.cutoverRunId ?? null, + generatedAt: phase.generatedAt ?? null, + diagnostics: phase.diagnostics ?? [], + })), + ...input.commands + .filter((command) => command.status !== 'OK') + .map((command) => ({ + type: 'command', + phase: command.phase, + commandName: command.commandName, + status: command.status, + latestBundleDir: command.latestBundleDir ?? null, + manifestPath: command.manifestPath ?? null, + cutoverRunId: command.cutoverRunId ?? null, + generatedAt: command.generatedAt ?? null, + diagnostics: command.diagnostics ?? [], + })), + ...(input.timeline.ok + ? [] + : [ + { + type: 'timeline', + status: 'FAILED', + diagnostics: input.timeline.diagnostics ?? [], + failureBreakdown: input.timeline.failureBreakdown ?? null, + spanMs: input.timeline.spanMs ?? null, + maxSpanMs: input.timeline.maxSpanMs ?? null, + }, + ]), + ], + directLiveEvidence, + timeline: { + checked: input.timeline.checked, + ok: input.timeline.ok, + failedCount: input.timeline.failedCount, + spanMs: input.timeline.spanMs, + maxSpanMs: input.timeline.maxSpanMs, + firstGeneratedAt: input.timeline.firstGeneratedAt, + lastGeneratedAt: input.timeline.lastGeneratedAt, + }, + }; +} + +function filterCandidatesByCutoverRunId(candidates, requiredCutoverRunId) { + if (!requiredCutoverRunId) { + return candidates; + } + return candidates.filter( + (candidate) => candidate.cutoverRunId === requiredCutoverRunId, + ); +} + +function missingEvidenceDiagnostic(label, requiredCutoverRunId, allCandidateCount) { + if (!requiredCutoverRunId) { + return `没有找到${label}的证据 manifest。`; + } + if (allCandidateCount > 0) { + return `没有找到${label}且 cutoverRunId=${requiredCutoverRunId} 的证据 manifest；已有同名证据属于其它切换批次或缺少 cutoverRunId。`; + } + return `没有找到${label}的证据 manifest，无法满足 cutoverRunId=${requiredCutoverRunId}。`; +} + +function selectUniqueLatestEvidence(candidates) { + const latest = candidates.at(-1); + const latestGeneratedAt = latest?.generatedAt || null; + const ambiguousCandidates = candidates.filter( + (candidate) => candidate.generatedAt === latestGeneratedAt, + ); + if (ambiguousCandidates.length > 1) { + const ambiguousBundleDirs = ambiguousCandidates.map( + (candidate) => candidate.bundleDir, + ); + return { + ok: false, + latest: null, + generatedAt: latestGeneratedAt, + ambiguousBundleDirs, + diagnostics: [ + `最新证据 manifest.generatedAt=${latestGeneratedAt} 出现重复，不能按目录名打平选择；请重新归档或清理证据根目录。`, + `重复 generatedAt 的证据目录: ${ambiguousBundleDirs.join(', ')}`, + ], + }; + } + return { + ok: true, + latest, + generatedAt: latestGeneratedAt, + ambiguousBundleDirs: [], + diagnostics: [], + }; +} + +function checkPhaseManifest(candidate, options = {}) { + const { + requiredAccessLog = false, + requiredStaticHeaders = false, + } = options; + const diagnostics = []; + if (candidate.commandName !== null) { + diagnostics.push( + `最新阶段证据不能带 manifest.commandName，实际为 ${formatNullable(candidate.commandName)}。`, + ); + } + if (candidate.summaryStatus !== 'OK') { + diagnostics.push( + `最新阶段证据 manifest.summary.status 必须是 OK，实际为 ${formatNullable(candidate.summaryStatus)}。`, + ); + } + const accessLogCheck = checkDirectLiveAccessLogSummary( + candidate.summaryDirectLiveAccessLog, + ); + if (requiredAccessLog && !accessLogCheck.ok) { + diagnostics.push( + `最新阶段证据 manifest.summary.directLiveAccessLog 必须包含 direct live access log 对账摘要：${accessLogCheck.reason}`, + ); + } + const staticHeadersCheck = checkDirectLiveStaticHeadersSummary( + candidate.summaryDirectLiveStaticHeaders, + ); + if (requiredStaticHeaders && !staticHeadersCheck.ok) { + diagnostics.push( + `最新阶段证据 manifest.summary.directLiveStaticHeaders 必须包含 direct live 静态响应头摘要：${staticHeadersCheck.reason}`, + ); + } + return { + ok: diagnostics.length === 0, + expectedSummaryStatus: 'OK', + actualSummaryStatus: candidate.summaryStatus, + requiredDirectLiveAccessLog: requiredAccessLog, + requiredDirectLiveStaticHeaders: requiredStaticHeaders, + directLiveAccessLogCheck: accessLogCheck, + directLiveStaticHeadersCheck: staticHeadersCheck, + diagnostics, + }; +} + +function checkDirectLiveAccessLogSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { + return { ok: false, reason: '字段缺失或不是对象。' }; + } + if (summary.present !== true) { + return { ok: false, reason: 'present 必须为 true。' }; + } + const checked = Number(summary.checked); + const matchedCount = Number(summary.matchedCount); + const missingCount = Number(summary.missingCount); + const mismatchCount = Number(summary.mismatchCount); + const failures = []; + if (!Number.isInteger(checked) || checked <= 0) { + failures.push('checked 必须是正整数'); + } + if (!Number.isInteger(matchedCount) || matchedCount !== checked) { + failures.push('matchedCount 必须等于 checked'); + } + if (!Number.isInteger(missingCount) || missingCount !== 0) { + failures.push('missingCount 必须为 0'); + } + if (!Number.isInteger(mismatchCount) || mismatchCount !== 0) { + failures.push('mismatchCount 必须为 0'); + } + return failures.length === 0 + ? { ok: true, reason: null } + : { ok: false, reason: failures.join('；') }; +} + +function checkDirectLiveStaticHeadersSummary(summary) { + if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { + return { ok: false, reason: '字段缺失或不是对象。' }; + } + if (summary.present !== true) { + return { ok: false, reason: 'present 必须为 true。' }; + } + if (summary.skipped) { + return { + ok: false, + reason: `静态检查被跳过：${formatNullable(summary.skipReason)}`, + }; + } + const targets = [summary.normal, summary.fingerprinted].filter( + (target) => target && target.present, + ); + if (targets.length === 0) { + return { ok: false, reason: '缺少 normal 或 fingerprinted 静态资产摘要。' }; + } + const failures = []; + for (const [label, target] of [ + ['normal', summary.normal], + ['fingerprinted', summary.fingerprinted], + ]) { + if (!target || !target.present) { + continue; + } + if (target.ok !== true) { + failures.push(`${label}.ok 不是 true`); + } + for (const key of [ + 'cacheControl', + 'etag', + 'lastModified', + 'acceptRanges', + 'rangeContentRange', + ]) { + if (typeof target[key] !== 'string' || target[key].length === 0) { + failures.push(`${label}.${key} 缺失`); + } + } + if (target.rangeStatusCode !== 206) { + failures.push(`${label}.rangeStatusCode 不是 206`); + } + if (target.etag304StatusCode !== 304) { + failures.push(`${label}.etag304StatusCode 不是 304`); + } + if (target.lastModified304StatusCode !== 304) { + failures.push(`${label}.lastModified304StatusCode 不是 304`); + } + } + return failures.length === 0 + ? { ok: true, reason: null } + : { ok: false, reason: failures.join('；') }; +} + +async function checkCommandManifest( + candidate, + expectedExecutable = null, + requiredArgs = [], +) { + const diagnostics = []; + if (candidate.summaryStatus !== 'OK') { + diagnostics.push( + `最新命令证据 manifest.summary.status 必须是 OK，实际为 ${formatNullable(candidate.summaryStatus)}。`, + ); + } + if (candidate.summaryExitCode !== 0) { + diagnostics.push( + `最新命令证据 manifest.summary.exitCode 必须是 0，实际为 ${formatNullable(candidate.summaryExitCode)}。`, + ); + } + if (candidate.summarySignal !== null) { + diagnostics.push( + `最新命令证据 manifest.summary.signal 必须为空，实际为 ${formatNullable(candidate.summarySignal)}。`, + ); + } + if (expectedExecutable) { + if (candidate.expectedExecutable !== expectedExecutable) { + diagnostics.push( + `最新命令证据 manifest.expectedExecutable 必须是 ${expectedExecutable}，实际为 ${formatNullable(candidate.expectedExecutable)}。`, + ); + } + if (candidate.executable !== expectedExecutable) { + diagnostics.push( + `最新命令证据 command.executable 必须是 ${expectedExecutable}，实际为 ${formatNullable(candidate.executable)}。`, + ); + } + } + diagnostics.push(...checkCommandOutputEvidenceFileNames(candidate)); + const embeddedCommandCheck = checkCommandRecordObject( + candidate.manifestCommand, + 'manifest.command', + candidate, + expectedExecutable, + requiredArgs, + ); + diagnostics.push(...embeddedCommandCheck.diagnostics); + + const commandRecordCheck = await readCommandRecord(candidate); + diagnostics.push(...commandRecordCheck.diagnostics); + if (commandRecordCheck.record) { + const fileCommandCheck = checkCommandRecordObject( + commandRecordCheck.record, + 'command-record.json', + candidate, + expectedExecutable, + requiredArgs, + ); + diagnostics.push(...fileCommandCheck.diagnostics); + if (embeddedCommandCheck.record) { + diagnostics.push( + ...compareCommandRecords( + embeddedCommandCheck.record, + commandRecordCheck.record, + ), + ); + } + } + return { + ok: diagnostics.length === 0, + expectedSummaryStatus: 'OK', + actualSummaryStatus: candidate.summaryStatus, + expectedExitCode: 0, + actualExitCode: candidate.summaryExitCode, + expectedSignal: null, + actualSignal: candidate.summarySignal, + expectedExecutable, + actualExpectedExecutable: candidate.expectedExecutable, + actualExecutable: candidate.executable, + stdoutPath: candidate.stdoutFileName, + stderrPath: candidate.stderrFileName, + commandRecordStdoutPath: commandRecordCheck.record?.stdoutPath ?? null, + commandRecordStderrPath: commandRecordCheck.record?.stderrPath ?? null, + commandRecordPath: commandRecordCheck.path, + commandRecordReadOk: commandRecordCheck.ok, + commandRecordExpectedExecutable: + commandRecordCheck.record?.expectedExecutable ?? null, + commandRecordExecutable: commandRecordCheck.record?.executable ?? null, + commandRecordStartedAt: normalizeIsoTime(commandRecordCheck.record?.startedAt), + commandRecordFinishedAt: normalizeIsoTime(commandRecordCheck.record?.finishedAt), + commandRecordDurationMs: commandRecordCheck.record?.durationMs ?? null, + requiredArgs, + diagnostics, + }; +} + +function checkCommandOutputEvidenceFileNames(candidate) { + const diagnostics = []; + for (const [field, fileName] of [ + ['stdout', candidate.stdoutFileName], + ['stderr', candidate.stderrFileName], + ]) { + if (!fileName) { + diagnostics.push( + `最新命令证据 manifest.files.${field}.path 必须存在。`, + ); + continue; + } + if (!isSafeEvidenceFileName(fileName)) { + diagnostics.push( + `最新命令证据 manifest.files.${field}.path 只能是证据目录内的普通文件名，实际为 ${formatNullable(fileName)}。`, + ); + } + } + return diagnostics; +} + +async function readCommandRecord(candidate) { + const fileName = candidate.commandRecordFileName; + if (!fileName) { + return { + ok: false, + path: null, + record: null, + diagnostics: ['最新命令证据 manifest.files.commandRecord.path 必须存在。'], + }; + } + if (!isSafeEvidenceFileName(fileName)) { + return { + ok: false, + path: null, + record: null, + diagnostics: [ + `最新命令证据 manifest.files.commandRecord.path 只能是证据目录内的普通文件名，实际为 ${formatNullable(fileName)}。`, + ], + }; + } + + const recordPath = path.join(candidate.bundleDir, fileName); + try { + const stats = await lstat(recordPath); + if (stats.isSymbolicLink()) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 不能是符号链接。'], + }; + } + if (!stats.isFile()) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 必须是普通文件。'], + }; + } + const content = await readFile(recordPath, 'utf8'); + const record = JSON.parse(content); + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: ['最新命令证据 command-record.json 顶层必须是 JSON object。'], + }; + } + return { ok: true, path: recordPath, record, diagnostics: [] }; + } catch (error) { + return { + ok: false, + path: recordPath, + record: null, + diagnostics: [ + `读取最新命令证据 command-record.json 失败: ${error.message}`, + ], + }; + } +} + +function checkCommandRecordObject( + record, + label, + candidate, + expectedExecutable, + requiredArgs = [], +) { + const diagnostics = []; + if (!record || typeof record !== 'object' || Array.isArray(record)) { + return { + ok: false, + record: null, + diagnostics: [`最新命令证据 ${label} 必须是 JSON object。`], + }; + } + const checks = [ + ['schemaVersion', 1], + ['phase', candidate.phase], + ['name', candidate.commandName], + ['exitCode', candidate.summaryExitCode], + ['signal', candidate.summarySignal], + ]; + if (candidate.cutoverRunId !== null || record.cutoverRunId !== undefined) { + checks.push(['cutoverRunId', candidate.cutoverRunId]); + } + if ( + candidate.expectedExecutable !== null || + record.expectedExecutable !== undefined + ) { + checks.push(['expectedExecutable', candidate.expectedExecutable]); + } + for (const [field, expected] of checks) { + const actual = record[field] ?? null; + if (actual !== expected) { + diagnostics.push( + `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)}，实际为 ${formatNullable(actual)}。`, + ); + } + } + for (const [field, expected] of [ + ['stdoutPath', candidate.stdoutFileName], + ['stderrPath', candidate.stderrFileName], + ]) { + const actual = record[field] ?? null; + if (actual !== expected) { + diagnostics.push( + `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)}，实际为 ${formatNullable(actual)}。`, + ); + } + } + diagnostics.push(...checkCommandRecordInvocation(record, label)); + diagnostics.push( + ...validateRequiredAbsolutePath(record.executable, `${label}.executable`), + ); + const effectiveExpectedExecutable = + expectedExecutable || candidate.expectedExecutable; + if (effectiveExpectedExecutable) { + if (record.expectedExecutable !== effectiveExpectedExecutable) { + diagnostics.push( + `最新命令证据 ${label}.expectedExecutable 必须是 ${effectiveExpectedExecutable}，实际为 ${formatNullable(record.expectedExecutable)}。`, + ); + } + if (record.executable !== effectiveExpectedExecutable) { + diagnostics.push( + `最新命令证据 ${label}.executable 必须是 ${effectiveExpectedExecutable}，实际为 ${formatNullable(record.executable)}。`, + ); + } + } + diagnostics.push(...checkRequiredCommandArgs(record, label, requiredArgs)); + diagnostics.push(...checkCommandRecordTimes(record, label, candidate)); + return { + ok: diagnostics.length === 0, + record, + diagnostics, + }; +} + +function checkRequiredCommandArgs(record, label, requiredArgs) { + const diagnostics = []; + if (requiredArgs.length === 0) { + return diagnostics; + } + if (!Array.isArray(record.args)) { + diagnostics.push( + `最新命令证据 ${label}.args 必须是字符串数组，才能校验必需参数 ${formatNullable(requiredArgs)}。`, + ); + return diagnostics; + } + for (const requiredArg of requiredArgs) { + if (!record.args.includes(requiredArg)) { + diagnostics.push( + `最新命令证据 ${label}.args 必须包含 ${formatNullable(requiredArg)}。`, + ); + } + } + return diagnostics; +} + +function checkCommandRecordInvocation(record, label) { + const diagnostics = []; + if (!Array.isArray(record.args)) { + diagnostics.push(`最新命令证据 ${label}.args 必须是字符串数组。`); + } else if (record.args.some((item) => typeof item !== 'string')) { + diagnostics.push(`最新命令证据 ${label}.args 必须只包含字符串。`); + } else if (record.args.some((item) => /[\0\r\n]/u.test(item))) { + diagnostics.push( + `最新命令证据 ${label}.args 不能包含换行或 NUL 字符。`, + ); + } + if (typeof record.command !== 'string' || record.command.length === 0) { + diagnostics.push(`最新命令证据 ${label}.command 必须是非空字符串。`); + } + if (typeof record.cwd !== 'string' || !path.isAbsolute(record.cwd)) { + diagnostics.push(`最新命令证据 ${label}.cwd 必须是绝对路径字符串。`); + } + if (record.error !== null && typeof record.error !== 'string') { + diagnostics.push(`最新命令证据 ${label}.error 必须是字符串或 null。`); + } + return diagnostics; +} + +function checkCommandRecordTimes(record, label, candidate) { + const diagnostics = []; + const startedAt = normalizeIsoTime(record.startedAt); + const finishedAt = normalizeIsoTime(record.finishedAt); + if (!startedAt) { + diagnostics.push(`最新命令证据 ${label}.startedAt 必须是合法 ISO 时间。`); + } + if (!finishedAt) { + diagnostics.push(`最新命令证据 ${label}.finishedAt 必须是合法 ISO 时间。`); + } + if (startedAt && finishedAt) { + const startedMs = Date.parse(startedAt); + const finishedMs = Date.parse(finishedAt); + if (finishedMs < startedMs) { + diagnostics.push( + `最新命令证据 ${label}.finishedAt 不能早于 startedAt。`, + ); + } + if (candidate.generatedAt) { + const generatedMs = Date.parse(candidate.generatedAt); + if (generatedMs < finishedMs) { + diagnostics.push( + `最新命令证据 manifest.generatedAt 不能早于 ${label}.finishedAt。`, + ); + } + } + const durationMs = record.durationMs; + if (!Number.isSafeInteger(durationMs) || durationMs < 0) { + diagnostics.push( + `最新命令证据 ${label}.durationMs 必须是非负安全整数。`, + ); + } else { + const actualDurationMs = finishedMs - startedMs; + if (durationMs !== actualDurationMs) { + diagnostics.push( + `最新命令证据 ${label}.durationMs 必须等于 finishedAt - startedAt，实际记录 ${durationMs}ms，计算值 ${actualDurationMs}ms。`, + ); + } + } + } + return diagnostics; +} + +function compareCommandRecords(embeddedRecord, fileRecord) { + const diagnostics = []; + const scalarFields = [ + 'schemaVersion', + 'name', + 'phase', + 'cutoverRunId', + 'expectedExecutable', + 'executable', + 'command', + 'cwd', + 'error', + 'stdoutPath', + 'stderrPath', + 'exitCode', + 'signal', + ]; + for (const field of scalarFields) { + const embedded = embeddedRecord[field] ?? null; + const file = fileRecord[field] ?? null; + if (embedded !== file) { + diagnostics.push( + `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致，manifest=${formatNullable(embedded)}，command-record=${formatNullable(file)}。`, + ); + } + } + for (const field of ['args']) { + const embedded = embeddedRecord[field] ?? null; + const file = fileRecord[field] ?? null; + if (!jsonValuesEqual(embedded, file)) { + diagnostics.push( + `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致，manifest=${formatNullable(embedded)}，command-record=${formatNullable(file)}。`, + ); + } + } + return diagnostics; +} + +function jsonValuesEqual(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function phaseStatus(verify, manifestCheck) { + if (!manifestCheck.ok) { + return 'MANIFEST_FAILED'; + } + if (!verify.ok) { + return 'VERIFY_FAILED'; + } + return 'OK'; +} + +function commandStatus(verify, manifestCheck) { + if (!manifestCheck.ok) { + return 'MANIFEST_FAILED'; + } + if (!verify.ok) { + return 'VERIFY_FAILED'; + } + return 'OK'; +} + +function formatNullable(value) { + return value === null || value === undefined ? '' : JSON.stringify(value); +} + +function isSafeEvidenceFileName(value) { + return ( + typeof value === 'string' && + value.length > 0 && + !/[\0\r\n]/u.test(value) && + !path.isAbsolute(value) && + !value.includes('/') && + !value.includes('\\') && + value !== '.' && + value !== '..' + ); +} + +function buildTimelineAudit(phases, commands, timelineMaxSpanMs) { + const selected = []; + const missing = []; + for (const expected of CUTOVER_TIMELINE_ORDER) { + const item = + expected.type === 'phase' + ? phases.find((phase) => phase.phase === expected.phase) + : commands.find( + (command) => + command.phase === expected.phase && + command.commandName === expected.commandName, + ); + if (!item) { + missing.push(expected); + continue; + } + selected.push({ + ...expected, + generatedAt: item.generatedAt, + latestBundleDir: item.latestBundleDir, + manifestPath: item.manifestPath, + cutoverRunId: item.cutoverRunId, + status: item.status, + }); + } + + if (selected.length === 0 || missing.length > 0) { + return { + ok: true, + checked: false, + failedCount: 0, + reason: + selected.length === 0 + ? '未要求标准直连切换时间线证据。' + : '仅在标准直连切换时间线的阶段和命令证据都被要求时检查顺序。', + maxSpanMs: timelineMaxSpanMs, + spanMs: null, + firstGeneratedAt: null, + lastGeneratedAt: null, + expectedOrder: CUTOVER_TIMELINE_ORDER, + missing, + items: selected, + diagnostics: [], + }; + } + + const diagnostics = []; + const failureBreakdown = { + nonOkItems: 0, + missingGeneratedAt: 0, + cutoverRunIdMismatch: 0, + outOfOrder: 0, + spanExceeded: 0, + }; + const nonOkItems = selected.filter((item) => item.status !== 'OK'); + failureBreakdown.nonOkItems = nonOkItems.length; + if (nonOkItems.length > 0) { + diagnostics.push( + `标准切换时间线包含非 OK 证据: ${nonOkItems + .map((item) => `${formatTimelineItem(item)}=${item.status}`) + .join(', ')}。`, + ); + } + const timelineCutoverRunIds = selected.map((item) => ({ + item: formatTimelineItem(item), + cutoverRunId: item.cutoverRunId, + })); + const hasDeclaredCutoverRunId = timelineCutoverRunIds.some( + (item) => item.cutoverRunId !== null, + ); + const distinctCutoverRunIds = new Set( + timelineCutoverRunIds.map((item) => item.cutoverRunId ?? ''), + ); + if (hasDeclaredCutoverRunId && distinctCutoverRunIds.size > 1) { + failureBreakdown.cutoverRunIdMismatch = 1; + diagnostics.push( + `标准切换时间线包含不同 cutoverRunId 或缺少 cutoverRunId: ${timelineCutoverRunIds + .map((item) => `${item.item}=${formatNullable(item.cutoverRunId)}`) + .join(', ')}，疑似混入不同切换批次证据。`, + ); + } + const parsedTimes = []; + for (const item of selected) { + if (!item.generatedAt) { + failureBreakdown.missingGeneratedAt += 1; + diagnostics.push( + `${formatTimelineItem(item)} 缺少合法 manifest.generatedAt，无法证明切换时间线。`, + ); + parsedTimes.push(null); + continue; + } + parsedTimes.push(Date.parse(item.generatedAt)); + } + for (let index = 1; index < selected.length; index += 1) { + const previous = selected[index - 1]; + const current = selected[index]; + const previousTime = parsedTimes[index - 1]; + const currentTime = parsedTimes[index]; + if ( + previousTime !== null && + currentTime !== null && + currentTime < previousTime + ) { + failureBreakdown.outOfOrder += 1; + diagnostics.push( + `${formatTimelineItem(current)} 的 manifest.generatedAt 早于 ${formatTimelineItem(previous)}，疑似混入不同切换窗口证据。`, + ); + } + } + + let firstGeneratedAt = null; + let lastGeneratedAt = null; + let spanMs = null; + if (parsedTimes.every((time) => time !== null && !Number.isNaN(time))) { + const firstTime = parsedTimes[0]; + const lastTime = parsedTimes.at(-1); + firstGeneratedAt = selected[0].generatedAt; + lastGeneratedAt = selected.at(-1).generatedAt; + spanMs = lastTime - firstTime; + if (spanMs > timelineMaxSpanMs) { + failureBreakdown.spanExceeded = 1; + diagnostics.push( + `标准切换时间线从 ${firstGeneratedAt} 到 ${lastGeneratedAt} 的时间跨度 ${spanMs}ms 超过 ${timelineMaxSpanMs}ms，疑似混入不同切换窗口证据。`, + ); + } + } + const timelineFailedCount = Object.values(failureBreakdown).reduce( + (sum, count) => sum + count, + 0, + ); + + return { + ok: diagnostics.length === 0, + checked: true, + failedCount: timelineFailedCount, + failureBreakdown, + maxSpanMs: timelineMaxSpanMs, + spanMs, + firstGeneratedAt, + lastGeneratedAt, + cutoverRunIds: timelineCutoverRunIds, + expectedOrder: CUTOVER_TIMELINE_ORDER, + items: selected, + diagnostics, + }; +} + +function formatTimelineItem(item) { + if (item.type === 'command') { + return `${item.phase}:${item.commandName}`; + } + return item.phase; +} + +function compareEvidenceCandidates(left, right) { + const leftTime = Date.parse(left.generatedAt); + const rightTime = Date.parse(right.generatedAt); + if (leftTime !== rightTime) { + return leftTime - rightTime; + } + return left.directoryName.localeCompare(right.directoryName); +} + +function sortedUnique(values) { + return [...new Set(values)].sort((left, right) => left.localeCompare(right)); +} + +function normalizeIsoTime(value) { + if ( + !value || + typeof value !== 'string' || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(value) + ) { + return null; + } + const time = Date.parse(value); + if (Number.isNaN(time)) { + return null; + } + const normalized = new Date(time).toISOString(); + return normalized === value ? normalized : null; +} + +async function runVerifier(verifyScript, bundleDir) { + const result = await execFileJson(process.execPath, [ + '--', + verifyScript, + '--bundle-dir', + bundleDir, + '--require-summary-ok', + ]); + const parsed = parseJsonOutput(result.stdout); + return { + ok: + result.exitCode === 0 && + parsed.ok && + parsed.value?.ok === true && + parsed.value?.requireSummaryOk === true, + exitCode: result.exitCode, + stdoutJson: parsed.ok, + checkedCount: parsed.value?.checkedCount ?? null, + failedCount: parsed.value?.failedCount ?? null, + requireSummaryOk: parsed.value?.requireSummaryOk ?? null, + summary: parsed.value?.summary ?? null, + stderr: result.stderr, + error: result.error, + }; +} + +function execFileJson(command, args) { + return new Promise((resolve) => { + execFile( + command, + args, + { + encoding: 'utf8', + maxBuffer: 10 * 1024 * 1024, + shell: false, + }, + (error, stdout, stderr) => { + resolve({ + exitCode: + typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: stdout || '', + stderr: stderr || '', + error: error?.message || null, + }); + }, + ); + }); +} + +function parseJsonOutput(stdout) { + try { + return { ok: true, value: JSON.parse(stdout) }; + } catch { + return { ok: false, value: null }; + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} diff --git a/scripts/ops/pingora-cutover-evidence-bundle.mjs b/scripts/ops/pingora-cutover-evidence-bundle.mjs new file mode 100644 index 000000000..46765da95 --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-bundle.mjs @@ -0,0 +1,1146 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { + access, + chmod, + lstat, + mkdir, + readFile, + stat, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const BUNDLE_DIR_MODE = 0o750; +const EVIDENCE_FILE_MODE = 0o640; +const SECRET_VALUE_FLAGS = new Set([ + '--direct-probe-token', + '--probe-token', + '--pingora-shadow-probe-token', + '--rollback-pingora-shadow-probe-token', +]); +const DIRECT_LIVE_JSON_PATTERN = /\{\s*"ok"\s*:/u; + +const config = parseArgs(process.argv.slice(2)); +const startedAt = new Date(); +const bundleDir = await createBundleDir( + config.outputRoot, + config.phase, + startedAt, +); +const snapshotArgs = buildSnapshotArgs(config); +const snapshotRun = await runEvidenceCommand({ + name: 'pingora-cutover-status-snapshot', + executable: 'node', + args: [config.snapshotScript, ...snapshotArgs], + stdoutFile: 'snapshot.stdout.txt', + stderrFile: 'snapshot.stderr.txt', + jsonFile: 'snapshot.json', + parseErrorFile: 'snapshot-parse-error.txt', + commandFile: 'snapshot-command.json', + parseJson: parseSnapshotJson, +}); +const manifestPath = path.join(bundleDir, 'manifest.json'); +const snapshot = snapshotRun.parsed; +const snapshotStatus = snapshot.ok + ? snapshot.value?.summary?.status || + (snapshotRun.result.code === 0 ? 'OK' : 'CRITICAL') + : 'CRITICAL'; + +let directLiveRun = null; +if ( + config.runDirectLive && + snapshotRun.result.code === 0 && + snapshot.ok && + (!config.failOnCritical || snapshotStatus !== 'CRITICAL') +) { + directLiveRun = await runEvidenceCommand({ + name: 'pingora-direct-live', + executable: 'node', + args: [config.directLiveScript, ...buildDirectLiveArgs(config)], + stdoutFile: 'direct-live.stdout.txt', + stderrFile: 'direct-live.stderr.txt', + jsonFile: 'direct-live.json', + parseErrorFile: 'direct-live-parse-error.txt', + commandFile: 'direct-live-command.json', + parseJson: parseDirectLiveJson, + }); +} + +const directLiveAccessLog = summarizeDirectLiveAccessLog(directLiveRun); +const directLiveStaticHeaders = summarizeDirectLiveStaticHeaders(directLiveRun); +const directLiveStatus = directLiveRun + ? isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, + ) + ? 'OK' + : 'CRITICAL' + : 'SKIPPED'; +const snapshotCriticalCount = snapshot.ok + ? Number(snapshot.value?.summary?.criticalCount || 0) + : 1; +const directLiveCriticalCount = directLiveStatus === 'CRITICAL' ? 1 : 0; +const bundleStatus = + snapshotStatus === 'CRITICAL' || directLiveStatus === 'CRITICAL' + ? 'CRITICAL' + : snapshotStatus; +const manifest = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: config.phase, + ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}), + releaseRoot: config.releaseRoot, + outputRoot: config.outputRoot, + bundleDir, + summary: { + status: bundleStatus, + snapshotStatus, + snapshotExitCode: snapshotRun.result.code, + directLiveStatus, + directLiveExitCode: directLiveRun?.result.code ?? null, + directLiveAccessLog: directLiveAccessLog?.summary ?? null, + directLiveStaticHeaders: directLiveStaticHeaders?.summary ?? null, + criticalCount: snapshotCriticalCount + directLiveCriticalCount, + warningCount: snapshot.ok + ? Number(snapshot.value?.summary?.warningCount || 0) + : 0, + }, + files: { + manifest: path.basename(manifestPath), + snapshot: snapshot.ok ? snapshotRun.files.json : null, + snapshotParseError: snapshot.ok ? null : snapshotRun.files.parseError, + snapshotStdout: snapshotRun.files.stdout, + snapshotStderr: snapshotRun.files.stderr, + snapshotCommand: snapshotRun.files.command, + directLive: directLiveRun?.parsed.ok ? directLiveRun.files.json : null, + directLiveParseError: + directLiveRun && !directLiveRun.parsed.ok + ? directLiveRun.files.parseError + : null, + directLiveStdout: directLiveRun ? directLiveRun.files.stdout : null, + directLiveStderr: directLiveRun ? directLiveRun.files.stderr : null, + directLiveCommand: directLiveRun ? directLiveRun.files.command : null, + }, + commands: [ + snapshotRun.commandRecord, + ...(directLiveRun ? [directLiveRun.commandRecord] : []), + ], +}; +await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + +const directLiveOk = + !config.runDirectLive || + isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, + ); +const result = { + ok: + snapshotRun.result.code === 0 && + snapshot.ok && + (!config.failOnCritical || snapshotStatus !== 'CRITICAL') && + directLiveOk, + phase: config.phase, + cutoverRunId: config.cutoverRunId || null, + status: bundleStatus, + bundleDir, + manifestPath, + snapshotPath: snapshot.ok ? snapshotRun.jsonPath : null, + snapshotParseErrorPath: snapshot.ok ? null : snapshotRun.parseErrorPath, + directLivePath: directLiveRun?.parsed.ok ? directLiveRun.jsonPath : null, + directLiveParseErrorPath: + directLiveRun && !directLiveRun.parsed.ok + ? directLiveRun.parseErrorPath + : null, +}; +console.log(`${JSON.stringify(result, null, 2)}\n`); + +if (!result.ok) { + process.exit(1); +} + +function summarizeDirectLiveAccessLog(directLiveRun) { + if (!directLiveRun) { + return null; + } + if (!directLiveRun.parsed.ok) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON parse failed', + }, + }; + } + const accessLogCheck = directLiveRun.parsed.value?.results?.find( + (item) => item?.name === 'direct-access-log', + ); + if (!accessLogCheck) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON missing direct-access-log result', + }, + }; + } + + const summary = { + present: true, + logFile: accessLogCheck.logFile || '', + sinceLines: coerceNonNegativeInteger(accessLogCheck.sinceLines), + scannedLineCount: coerceNonNegativeInteger( + accessLogCheck.scannedLineCount, + ), + checked: coerceNonNegativeInteger(accessLogCheck.checked), + matchedCount: coerceNonNegativeInteger(accessLogCheck.matchedCount), + missingCount: coerceNonNegativeInteger(accessLogCheck.missingCount), + mismatchCount: coerceNonNegativeInteger(accessLogCheck.mismatchCount), + missingDetailsCount: Array.isArray(accessLogCheck.missing) + ? accessLogCheck.missing.length + : null, + mismatchDetailsCount: Array.isArray(accessLogCheck.mismatches) + ? accessLogCheck.mismatches.length + : null, + }; + const ok = + summary.checked !== null && + summary.matchedCount === summary.checked && + summary.missingCount === 0 && + summary.mismatchCount === 0 && + summary.missingDetailsCount === 0 && + summary.mismatchDetailsCount === 0; + + return { ok, summary }; +} + +function summarizeDirectLiveStaticHeaders(directLiveRun) { + if (!directLiveRun) { + return null; + } + if (!directLiveRun.parsed.ok) { + return { + ok: false, + summary: { + present: false, + reason: 'direct live JSON parse failed', + }, + }; + } + + const staticAssetCheck = directLiveRun.parsed.value?.results?.find( + (item) => item?.name === 'https-static-asset', + ); + if (!staticAssetCheck) { + return { + ok: false, + summary: { + present: false, + skipped: false, + reason: 'direct live JSON missing https-static-asset result', + }, + }; + } + if (staticAssetCheck.skipped) { + return { + ok: true, + summary: { + present: false, + skipped: true, + reason: staticAssetCheck.reason || '', + }, + }; + } + + const normal = summarizeStaticHeaderEntry(staticAssetCheck); + const fingerprinted = summarizeStaticHeaderEntry( + staticAssetCheck.fingerprinted, + { + expectedCacheControl: 'public, max-age=31536000, immutable', + }, + ); + const diagnostics = [ + ...normal.diagnostics.map((item) => `normal: ${item}`), + ...(fingerprinted.present + ? fingerprinted.diagnostics.map((item) => `fingerprinted: ${item}`) + : []), + ]; + const summary = { + present: true, + normal, + fingerprinted, + diagnostics, + }; + return { + ok: normal.ok && (!fingerprinted.present || fingerprinted.ok), + summary, + }; +} + +function summarizeStaticHeaderEntry(entry, options = {}) { + if (!entry || entry.skipped) { + return { + ok: true, + present: false, + skipped: Boolean(entry?.skipped), + reason: entry?.reason || '', + diagnostics: [], + }; + } + + const summary = { + present: true, + path: safePathFromUrl(entry.url), + statusCode: coerceNonNegativeInteger(entry.statusCode), + cacheControl: entry.headers?.['cache-control'] || '', + etag: entry.headers?.etag || '', + lastModified: entry.headers?.['last-modified'] || '', + acceptRanges: entry.headers?.['accept-ranges'] || '', + contentLength: entry.headers?.['content-length'] || '', + rangeStatusCode: coerceNonNegativeInteger(entry.range?.statusCode), + rangeContentRange: entry.range?.headers?.['content-range'] || '', + rangeContentLength: entry.range?.headers?.['content-length'] || '', + etag304StatusCode: coerceNonNegativeInteger( + entry.notModified?.etag?.statusCode, + ), + lastModified304StatusCode: coerceNonNegativeInteger( + entry.notModified?.lastModified?.statusCode, + ), + }; + const diagnostics = []; + if (summary.statusCode !== 200) { + diagnostics.push(`GET statusCode 应为 200，实际 ${formatValue(summary.statusCode)}`); + } + if (!summary.cacheControl) { + diagnostics.push('缺少 Cache-Control'); + } + if ( + options.expectedCacheControl && + summary.cacheControl !== options.expectedCacheControl + ) { + diagnostics.push( + `Cache-Control 应为 ${options.expectedCacheControl}，实际 ${summary.cacheControl || '-'}`, + ); + } + if (!summary.etag) { + diagnostics.push('缺少 ETag'); + } + if (!summary.lastModified) { + diagnostics.push('缺少 Last-Modified'); + } + if (summary.acceptRanges !== 'bytes') { + diagnostics.push( + `Accept-Ranges 应为 bytes，实际 ${summary.acceptRanges || '-'}`, + ); + } + if (!summary.contentLength) { + diagnostics.push('缺少 Content-Length'); + } + if (summary.rangeStatusCode !== 206) { + diagnostics.push( + `Range statusCode 应为 206，实际 ${formatValue(summary.rangeStatusCode)}`, + ); + } + if (!/^bytes 0-0\/\d+$/u.test(summary.rangeContentRange)) { + diagnostics.push( + `Range Content-Range 应匹配 bytes 0-0/，实际 ${summary.rangeContentRange || '-'}`, + ); + } + if (summary.rangeContentLength !== '1') { + diagnostics.push( + `Range Content-Length 应为 1，实际 ${summary.rangeContentLength || '-'}`, + ); + } + if (summary.etag304StatusCode !== 304) { + diagnostics.push( + `ETag 304 statusCode 应为 304，实际 ${formatValue(summary.etag304StatusCode)}`, + ); + } + if (summary.lastModified304StatusCode !== 304) { + diagnostics.push( + `Last-Modified 304 statusCode 应为 304，实际 ${formatValue(summary.lastModified304StatusCode)}`, + ); + } + + return { + ok: diagnostics.length === 0, + ...summary, + diagnostics, + }; +} + +function safePathFromUrl(value) { + if (!value) { + return ''; + } + try { + return new URL(value).pathname; + } catch { + return ''; + } +} + +function coerceNonNegativeInteger(value) { + const number = Number(value); + if (!Number.isInteger(number) || number < 0) { + return null; + } + return number; +} + +function isDirectLiveEvidenceOk( + directLiveRun, + directLiveAccessLog, + directLiveStaticHeaders, +) { + return Boolean( + directLiveRun && + directLiveRun.result.code === 0 && + directLiveRun.parsed.ok && + directLiveRun.parsed.value?.ok !== false && + directLiveAccessLog?.ok && + directLiveStaticHeaders?.ok, + ); +} + +function formatValue(value) { + return value === null || value === undefined ? '-' : String(value); +} + +function parseArgs(argv) { + const result = { + phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual', + cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + outputRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || + '/var/log/genarrative/pingora-cutover-evidence', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + snapshotScript: '', + directLiveScript: '', + runDirectLive: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_RUN_DIRECT_LIVE', + ), + directHttpsBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL || '', + directHttpBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '', + directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', + directRedirectHost: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', + directSpacetimeDatabase: + process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '', + directPingoraAccessLog: + process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '', + directAccessLogSinceLines: + process.env.GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES || '2000', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST', + ), + runHealthPatrol: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL', + ), + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(argv, ++index, arg); + break; + case '--cutover-run-id': + result.cutoverRunId = requireValue(argv, ++index, arg); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--output-root': + result.outputRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--snapshot-script': + result.snapshotScript = requireValue(argv, ++index, arg); + break; + case '--direct-live-script': + result.directLiveScript = requireValue(argv, ++index, arg); + break; + case '--run-direct-live': + result.runDirectLive = true; + break; + case '--direct-https-base-url': + result.directHttpsBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-http-base-url': + result.directHttpBaseUrl = requireValue(argv, ++index, arg); + break; + case '--direct-host': + result.directHost = requireValue(argv, ++index, arg); + break; + case '--direct-redirect-host': + result.directRedirectHost = requireValue(argv, ++index, arg); + break; + case '--direct-probe-token': + result.directProbeToken = requireValue(argv, ++index, arg); + break; + case '--direct-spacetime-database': + result.directSpacetimeDatabase = requireValue(argv, ++index, arg); + break; + case '--direct-pingora-access-log': + result.directPingoraAccessLog = requireValue(argv, ++index, arg); + break; + case '--direct-access-log-since-lines': + result.directAccessLogSinceLines = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + case '--run-health-patrol': + result.runHealthPatrol = true; + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + result.snapshotScript = + result.snapshotScript || + path.join( + result.releaseRoot, + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + ); + result.directLiveScript = + result.directLiveScript || + path.join(result.releaseRoot, 'scripts/check-pingora-direct-live.mjs'); + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-bundle.mjs [options] + +Options: + --phase 写入 manifest 的阶段标签，例如 pre-cutover / post-enable / post-rollback。 + --cutover-run-id 可选，写入 manifest 的本次切换批次 ID；正式 runbook 会用同一 ID 串联所有阶段和命令证据。 + --release-root current release 根目录，默认 /opt/genarrative/current。 + --output-root 证据包输出根目录，默认 /var/log/genarrative/pingora-cutover-evidence。 + --health-patrol-env-file health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。 + --snapshot-script 状态快照脚本，默认 current release 随包脚本。 + --direct-live-script direct live smoke 脚本，默认 current release 随包脚本。 + --run-direct-live 额外执行 direct live smoke，并把 request_id / access log 结果归档。 + --direct-https-base-url 透传给 direct live smoke 的 HTTPS 入口。 + --direct-http-base-url 透传给 direct live smoke 的 HTTP redirect / ACME 入口。 + --direct-host 透传给 direct live smoke 的正式 Host/SNI。 + --direct-redirect-host 透传给 direct live smoke 的 redirect Location host。 + --direct-probe-token 可选，透传给 direct live smoke 的内部探针 token；证据中会脱敏。 + --direct-spacetime-database 透传给 direct live smoke 的 SpacetimeDB 数据库名。 + --direct-pingora-access-log + 透传给 direct live smoke 的 Pingora access log 路径。 + --direct-access-log-since-lines + 透传给 direct live smoke 的 access log tail 行数，默认 2000。 + --expected-gateway-mode 可选，nginx 或 pingora-direct；会透传给状态快照。 + --expected-public-base-url 可选，要求 health-patrol public base URL 与该值一致。 + --expected-public-host 可选，要求 health-patrol public Host 与该值一致。 + --require-empty-public-host 可选，要求 health-patrol public Host 为空。 + --run-health-patrol 让状态快照执行 current release 随包生产巡检脚本并收录结果。 + --require-pingora-gateway 让状态快照自审 current release 的 Pingora 二进制、checksum 和 manifest。 + --timeout-ms 透传给状态快照脚本的 systemctl / 子检查超时，默认 5000。 + --fail-on-critical 如果快照中出现 CRITICAL，则证据包脚本也以退出码 1 结束。 + +该脚本只写 --output-root 下的新证据目录，且 --output-root 不能是符号链接或非目录路径；证据目录权限固定 0750，证据文件权限固定 0640；不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function validateConfig(config) { + validatePhase(config.phase); + if (config.cutoverRunId) { + validateSafeName(config.cutoverRunId, '--cutover-run-id'); + } + for (const [label, value] of [ + ['--release-root', config.releaseRoot], + ['--output-root', config.outputRoot], + ['--health-patrol-env-file', config.healthPatrolEnvFile], + ['--pingora-env-file', config.pingoraEnvFile], + ['--snapshot-script', config.snapshotScript], + ['--direct-live-script', config.directLiveScript], + ]) { + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (path.resolve(value) === path.parse(path.resolve(value)).root) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + } + if ( + config.expectedGatewayMode && + !GATEWAY_MODES.has(config.expectedGatewayMode) + ) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${config.expectedGatewayMode}`, + ); + } + if (config.expectedPublicHost !== null) { + validateHostOption(config.expectedPublicHost, '--expected-public-host'); + } + if (config.expectedPublicHost !== null && config.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + if (config.expectedPublicBaseUrl) { + validateHttpUrl(config.expectedPublicBaseUrl, '--expected-public-base-url'); + } + if (config.runDirectLive) { + validateDirectLiveConfig(config); + } + validateEvidenceCommandArgs('状态快照命令参数', buildSnapshotArgs(config)); + if (config.runDirectLive) { + validateEvidenceCommandArgs('direct live 命令参数', buildDirectLiveArgs(config)); + } +} + +function validateDirectLiveConfig(config) { + if (!config.directHttpsBaseUrl) { + throw new Error('--run-direct-live 必须提供 --direct-https-base-url。'); + } + validateHttpsUrl(config.directHttpsBaseUrl, '--direct-https-base-url'); + if (!config.directHttpBaseUrl) { + throw new Error('--run-direct-live 必须提供 --direct-http-base-url。'); + } + validateHttpOnlyUrl(config.directHttpBaseUrl, '--direct-http-base-url'); + if (!config.directHost) { + throw new Error('--run-direct-live 必须提供 --direct-host。'); + } + validateHostOption(config.directHost, '--direct-host'); + if (!config.directRedirectHost) { + throw new Error('--run-direct-live 必须提供 --direct-redirect-host。'); + } + validateHostOption(config.directRedirectHost, '--direct-redirect-host'); + if (!config.directSpacetimeDatabase) { + throw new Error('--run-direct-live 必须提供 --direct-spacetime-database。'); + } + validateNoControlCharacters( + config.directSpacetimeDatabase, + '--direct-spacetime-database', + ); + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(config.directSpacetimeDatabase)) { + throw new Error( + '--direct-spacetime-database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$。', + ); + } + if (config.directProbeToken) { + validateNoControlCharacters(config.directProbeToken, '--direct-probe-token'); + } + if (!config.directPingoraAccessLog) { + throw new Error('--run-direct-live 必须提供 --direct-pingora-access-log。'); + } + validateSafeAbsoluteFilePath( + config.directPingoraAccessLog, + '--direct-pingora-access-log', + ); + validateNoControlCharacters( + config.directAccessLogSinceLines, + '--direct-access-log-since-lines', + ); + if (!/^[1-9][0-9]*$/u.test(String(config.directAccessLogSinceLines))) { + throw new Error('--direct-access-log-since-lines 必须是正整数。'); + } +} + +function validatePhase(value) { + validateSafeName(value, '--phase'); +} + +function validateSafeName(value, label) { + const text = String(value || ''); + if (!/^[0-9A-Za-z._-]+$/u.test(text)) { + throw new Error( + `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, + ); + } +} + +function validateEvidenceCommandArgs(label, args) { + for (const arg of args) { + if (/[\0\r\n]/u.test(String(arg))) { + throw new Error(`${label}不能包含换行或 NUL 字符。`); + } + } +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (path.resolve(value) === path.parse(path.resolve(value)).root) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + validateNoControlCharacters(raw, label); + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function validateHttpUrl(value, label) { + validateNoControlCharacters(value, label); + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHttpsUrl(value, label) { + validateHttpUrl(value, label); + if (new URL(value).protocol !== 'https:') { + throw new Error(`${label} 必须使用 https://。`); + } +} + +function validateHttpOnlyUrl(value, label) { + validateHttpUrl(value, label); + if (new URL(value).protocol !== 'http:') { + throw new Error(`${label} 必须使用 http://。`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + validateNoControlCharacters(raw, label); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port`); + } +} + +async function createBundleDir(outputRoot, phase, date) { + await validateOutputRootForWriting(outputRoot); + await mkdir(outputRoot, { recursive: true }); + await validateOutputRootForWriting(outputRoot); + await access(outputRoot, fsConstants.W_OK); + const baseName = `${formatTimestampForPath(date)}-${phase}`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? '' : `-${attempt}`; + const bundleDir = path.join(outputRoot, `${baseName}${suffix}`); + try { + await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE }); + await chmod(bundleDir, BUNDLE_DIR_MODE); + return bundleDir; + } catch (error) { + if (error?.code !== 'EEXIST') { + throw error; + } + } + } + throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`); +} + +async function validateOutputRootForWriting(outputRoot) { + const target = path.resolve(outputRoot); + const root = path.parse(target).root; + let current = root; + const segments = path + .relative(root, target) + .split(path.sep) + .filter(Boolean); + + for (const segment of segments) { + current = path.join(current, segment); + let stats; + try { + stats = await lstat(current); + } catch (error) { + if (error?.code === 'ENOENT') { + return; + } + throw error; + } + + if (stats.isSymbolicLink()) { + if (current === target) { + throw new Error(`--output-root 不能是符号链接: ${current}`); + } + throw new Error(`--output-root 上级目录不能是符号链接: ${current}`); + } + + if (!stats.isDirectory()) { + if (current === target) { + throw new Error(`--output-root 已存在但不是目录: ${current}`); + } + throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`); + } + } +} + +async function writeEvidenceFile(filePath, content) { + await writeFile(filePath, content, { + encoding: 'utf8', + flag: 'wx', + mode: EVIDENCE_FILE_MODE, + }); + await chmod(filePath, EVIDENCE_FILE_MODE); +} + +async function buildEvidenceFileMetadata(filePath) { + const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + return { + path: path.basename(filePath), + sizeBytes: stats.size, + sha256: createHash('sha256').update(content).digest('hex'), + }; +} + +function formatTimestampForPath(date) { + return date + .toISOString() + .replace(/[-:]/gu, '') + .replace(/\.\d{3}Z$/u, 'Z'); +} + +function buildSnapshotArgs(config) { + return [ + '--phase', + config.phase, + '--release-root', + config.releaseRoot, + '--health-patrol-env-file', + config.healthPatrolEnvFile, + '--pingora-env-file', + config.pingoraEnvFile, + ...(config.expectedGatewayMode + ? ['--expected-gateway-mode', config.expectedGatewayMode] + : []), + ...(config.expectedPublicBaseUrl + ? ['--expected-public-base-url', config.expectedPublicBaseUrl] + : []), + ...(config.expectedPublicHost !== null + ? ['--expected-public-host', config.expectedPublicHost] + : []), + ...(config.requireEmptyPublicHost ? ['--require-empty-public-host'] : []), + ...(config.runHealthPatrol ? ['--run-health-patrol'] : []), + ...(config.requirePingoraGateway ? ['--require-pingora-gateway'] : []), + '--timeout-ms', + String(config.timeoutMs), + ...(config.failOnCritical ? ['--fail-on-critical'] : []), + ]; +} + +function buildDirectLiveArgs(config) { + return [ + '--https-base-url', + config.directHttpsBaseUrl, + '--http-base-url', + config.directHttpBaseUrl, + '--host', + config.directHost, + '--redirect-host', + config.directRedirectHost, + ...(config.directProbeToken + ? ['--probe-token', config.directProbeToken] + : []), + '--spacetime-database', + config.directSpacetimeDatabase, + '--require-wss-upgrade', + '--pingora-access-log', + config.directPingoraAccessLog, + '--access-log-since-lines', + config.directAccessLogSinceLines, + '--json', + ]; +} + +async function runEvidenceCommand({ + name, + executable, + args, + stdoutFile, + stderrFile, + jsonFile, + parseErrorFile, + commandFile, + parseJson, +}) { + const startedAt = new Date(); + const result = await runCommand(executable, args); + const finishedAt = new Date(); + const stdoutPath = path.join(bundleDir, stdoutFile); + const stderrPath = path.join(bundleDir, stderrFile); + const jsonPath = path.join(bundleDir, jsonFile); + const parseErrorPath = path.join(bundleDir, parseErrorFile); + const commandPath = path.join(bundleDir, commandFile); + + await writeEvidenceFile(stdoutPath, result.stdout); + await writeEvidenceFile(stderrPath, result.stderr || result.error); + + const parsed = parseJson(result.stdout); + let jsonMetadata = null; + let parseErrorMetadata = null; + if (parsed.ok) { + await writeEvidenceFile(jsonPath, `${JSON.stringify(parsed.value, null, 2)}\n`); + jsonMetadata = await buildEvidenceFileMetadata(jsonPath); + } else { + await writeEvidenceFile(parseErrorPath, `${parsed.error}\n`); + parseErrorMetadata = await buildEvidenceFileMetadata(parseErrorPath); + } + + const commandRecord = { + name, + executable, + args: redactSecretArgs(args), + command: formatCommand(executable, args), + exitCode: result.code, + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + durationMs: finishedAt.getTime() - startedAt.getTime(), + stdoutPath: path.basename(stdoutPath), + stderrPath: path.basename(stderrPath), + }; + await writeEvidenceFile( + commandPath, + `${JSON.stringify(commandRecord, null, 2)}\n`, + ); + const files = { + stdout: await buildEvidenceFileMetadata(stdoutPath), + stderr: await buildEvidenceFileMetadata(stderrPath), + json: jsonMetadata, + parseError: parseErrorMetadata, + command: await buildEvidenceFileMetadata(commandPath), + }; + + return { + result, + parsed, + stdoutPath, + stderrPath, + jsonPath, + parseErrorPath, + commandPath, + commandRecord, + files, + }; +} + +function runCommand(command, args) { + return new Promise((resolve) => { + execFile( + command, + args, + { + env: process.env, + windowsHide: true, + maxBuffer: 2 * 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function parseSnapshotJson(stdout) { + try { + return { ok: true, value: JSON.parse(stdout) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function parseDirectLiveJson(stdout) { + const match = DIRECT_LIVE_JSON_PATTERN.exec(stdout); + if (!match) { + return { + ok: false, + error: 'direct live stdout 中未找到 JSON 结果对象。', + }; + } + const jsonText = extractJsonObject(stdout, match.index); + if (!jsonText) { + return { + ok: false, + error: 'direct live stdout 中未找到完整 JSON 结果对象。', + }; + } + try { + return { ok: true, value: JSON.parse(jsonText) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function extractJsonObject(text, startIndex) { + let depth = 0; + let inString = false; + let escaped = false; + + for (let index = startIndex; index < text.length; index += 1) { + const char = text[index]; + if (inString) { + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === '"') { + inString = false; + } + continue; + } + + if (char === '"') { + inString = true; + continue; + } + if (char === '{') { + depth += 1; + continue; + } + if (char === '}') { + depth -= 1; + if (depth === 0) { + return text.slice(startIndex, index + 1); + } + } + } + + return ''; +} + +function formatCommand(command, args) { + return [command, ...redactSecretArgs(args)].join(' '); +} + +function redactSecretArgs(args) { + return args.map((arg, index) => + index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) ? '' : arg, + ); +} diff --git a/scripts/ops/pingora-cutover-evidence-verify.mjs b/scripts/ops/pingora-cutover-evidence-verify.mjs new file mode 100644 index 000000000..8c920a414 --- /dev/null +++ b/scripts/ops/pingora-cutover-evidence-verify.mjs @@ -0,0 +1,341 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { constants as fsConstants } from 'node:fs'; +import { access, lstat, readdir, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const config = parseArgs(process.argv.slice(2)); +const manifestPath = await resolveManifestPath(config); +const bundleDir = path.dirname(manifestPath); +await validateBundleDir(bundleDir); + +const manifest = await readManifest(manifestPath); +validateManifestSchemaVersion(manifest); +const { verifications, registeredFileNames } = + await verifyManifestFiles(bundleDir, manifest); +const extraFiles = config.allowExtraFiles + ? [] + : await findExtraEvidenceEntries(bundleDir, registeredFileNames); +const summaryVerification = config.requireSummaryOk + ? verifySummaryOk(manifest) + : null; +const failed = verifications.filter((item) => item.status !== 'OK'); +const failedCount = + failed.length + + extraFiles.length + + (summaryVerification && summaryVerification.status !== 'OK' ? 1 : 0); + +console.log( + `${JSON.stringify( + { + ok: failedCount === 0, + manifestPath, + bundleDir, + checkedCount: verifications.length, + failedCount, + allowExtraFiles: config.allowExtraFiles, + requireSummaryOk: config.requireSummaryOk, + summary: summaryVerification, + files: verifications, + extraFiles, + }, + null, + 2, + )}\n`, +); + +if (failedCount > 0) { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + manifestPath: '', + bundleDir: '', + allowExtraFiles: false, + requireSummaryOk: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--manifest': + result.manifestPath = requireValue(argv, ++index, arg); + break; + case '--bundle-dir': + result.bundleDir = requireValue(argv, ++index, arg); + break; + case '--allow-extra-files': + result.allowExtraFiles = true; + break; + case '--require-summary-ok': + result.requireSummaryOk = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (result.manifestPath && result.bundleDir) { + throw new Error('--manifest 和 --bundle-dir 只能二选一。'); + } + if (!result.manifestPath && !result.bundleDir) { + throw new Error('必须提供 --manifest 或 --bundle-dir。'); + } + for (const [label, value] of [ + ['--manifest', result.manifestPath], + ['--bundle-dir', result.bundleDir], + ]) { + if (!value) { + continue; + } + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } + validateNoControlCharacters(value, label); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-evidence-verify.mjs --manifest + node scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir + +Options: + --allow-extra-files 允许证据目录中存在 manifest.files 未登记的额外条目；默认拒绝，正式切换归档不应使用。 + --require-summary-ok 要求 manifest.summary.status 必须是 OK；正式切换 runbook 的即时验真步骤必须使用。 + +该脚本只读校验 Pingora 直连切换证据目录中的 schemaVersion=1 manifest.files 元数据，确认每个已登记文件的 path / sizeBytes / sha256 与实际文件一致，并默认拒绝证据目录中混入未登记文件、目录或符号链接；null 文件条目会跳过。正式切换 runbook 还会用 --require-summary-ok 把 manifest.summary.status 非 OK 的证据作为失败处理。脚本不会修改证据目录、不会 reload systemd、不会访问 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +async function resolveManifestPath(config) { + if (config.manifestPath) { + return path.resolve(config.manifestPath); + } + return path.join(path.resolve(config.bundleDir), 'manifest.json'); +} + +async function validateBundleDir(bundleDir) { + const resolved = path.resolve(bundleDir); + if (isFilesystemRootPath(resolved)) { + throw new Error('证据目录不能是文件系统根目录。'); + } + const stats = await lstat(resolved); + if (stats.isSymbolicLink()) { + throw new Error(`证据目录不能是符号链接: ${resolved}`); + } + if (!stats.isDirectory()) { + throw new Error(`证据目录必须是目录: ${resolved}`); + } + await access(resolved, fsConstants.R_OK | fsConstants.X_OK); +} + +async function readManifest(manifestPath) { + const stats = await lstat(manifestPath); + if (stats.isSymbolicLink()) { + throw new Error(`manifest 不能是符号链接: ${manifestPath}`); + } + if (!stats.isFile()) { + throw new Error(`manifest 必须是文件: ${manifestPath}`); + } + let content; + try { + content = await readFile(manifestPath, 'utf8'); + } catch (error) { + throw new Error(`读取 manifest 失败: ${error.message}`); + } + try { + return JSON.parse(content); + } catch (error) { + throw new Error(`manifest 不是合法 JSON: ${error.message}`); + } +} + +function validateManifestSchemaVersion(manifest) { + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + throw new Error('manifest 顶层必须是 JSON object。'); + } + if (manifest.schemaVersion !== 1) { + throw new Error('manifest.schemaVersion 必须是 1。'); + } +} + +async function verifyManifestFiles(bundleDir, manifest) { + if (!manifest.files || typeof manifest.files !== 'object') { + throw new Error('manifest 缺少 files object。'); + } + + const verifications = []; + const registeredFileNames = new Set(['manifest.json']); + for (const [key, value] of Object.entries(manifest.files)) { + if (value === null) { + continue; + } + if (typeof value === 'string') { + if (key === 'manifest' && value === 'manifest.json') { + continue; + } + throw new Error( + `manifest.files.${key} 必须使用 { path, sizeBytes, sha256 } 元数据对象。`, + ); + } + validateMetadata(key, value); + registeredFileNames.add(value.path); + verifications.push(await verifyMetadata(bundleDir, key, value)); + } + return { verifications, registeredFileNames }; +} + +function verifySummaryOk(manifest) { + const status = manifest?.summary?.status; + if (status === 'OK') { + return { + status: 'OK', + expectedStatus: 'OK', + actualStatus: status, + reason: null, + }; + } + return { + status: 'MISMATCH', + expectedStatus: 'OK', + actualStatus: status ?? null, + reason: 'manifest.summary.status is not OK', + }; +} + +async function findExtraEvidenceEntries(bundleDir, registeredFileNames) { + const entries = await readdir(bundleDir, { withFileTypes: true }); + const extras = []; + for (const entry of entries) { + if (registeredFileNames.has(entry.name)) { + continue; + } + extras.push({ + path: entry.name, + status: 'MISMATCH', + reason: entry.isSymbolicLink() + ? 'extra symlink not registered in manifest.files' + : entry.isDirectory() + ? 'extra directory not registered in manifest.files' + : 'extra file not registered in manifest.files', + }); + } + return extras.sort((left, right) => left.path.localeCompare(right.path)); +} + +function validateMetadata(key, value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`manifest.files.${key} 必须是元数据对象或 null。`); + } + if (typeof value.path !== 'string' || value.path.length === 0) { + throw new Error(`manifest.files.${key}.path 必须是非空字符串。`); + } + if ( + /[\0\r\n]/u.test(value.path) || + path.isAbsolute(value.path) || + value.path.includes('/') || + value.path.includes('\\') || + value.path === '.' || + value.path === '..' || + value.path.includes('..') + ) { + throw new Error( + `manifest.files.${key}.path 只能是证据目录内的安全文件名: ${value.path}`, + ); + } + if (!Number.isSafeInteger(value.sizeBytes) || value.sizeBytes < 0) { + throw new Error(`manifest.files.${key}.sizeBytes 必须是非负安全整数。`); + } + if (typeof value.sha256 !== 'string' || !/^[0-9a-f]{64}$/u.test(value.sha256)) { + throw new Error(`manifest.files.${key}.sha256 必须是 64 位小写十六进制。`); + } +} + +async function verifyMetadata(bundleDir, key, metadata) { + const filePath = path.join(bundleDir, metadata.path); + const expectedPath = path.resolve(bundleDir, metadata.path); + if (path.dirname(expectedPath) !== path.resolve(bundleDir)) { + throw new Error(`manifest.files.${key}.path 超出证据目录: ${metadata.path}`); + } + let stats; + let content; + try { + const linkStats = await lstat(filePath); + if (linkStats.isSymbolicLink()) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: 'file is symlink', + }; + } + if (!linkStats.isFile()) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: 'not a regular file', + }; + } + [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]); + } catch (error) { + return { + key, + path: metadata.path, + status: 'MISMATCH', + reason: error?.code === 'ENOENT' ? 'missing file' : error.message, + }; + } + + const actualSha256 = createHash('sha256').update(content).digest('hex'); + const sizeMatches = stats.size === metadata.sizeBytes; + const sha256Matches = actualSha256 === metadata.sha256; + return { + key, + path: metadata.path, + status: sizeMatches && sha256Matches ? 'OK' : 'MISMATCH', + expectedSizeBytes: metadata.sizeBytes, + actualSizeBytes: stats.size, + expectedSha256: metadata.sha256, + actualSha256, + reason: + sizeMatches && sha256Matches + ? null + : [ + ...(sizeMatches ? [] : ['size mismatch']), + ...(sha256Matches ? [] : ['sha256 mismatch']), + ].join(', '), + }; +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} diff --git a/scripts/ops/pingora-cutover-status-snapshot.mjs b/scripts/ops/pingora-cutover-status-snapshot.mjs new file mode 100644 index 000000000..c37ee8d69 --- /dev/null +++ b/scripts/ops/pingora-cutover-status-snapshot.mjs @@ -0,0 +1,939 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const STATUS_RANK = { + OK: 0, + WARNING: 1, + CRITICAL: 2, +}; + +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const SERVICES = [ + 'genarrative-api.service', + 'spacetimedb.service', + 'nginx.service', + 'genarrative-pingora-gateway.service', +]; +const RELEASE_ARTIFACTS = [ + 'scripts/ops/pingora-cutover-status-snapshot.mjs', + 'scripts/ops/pingora-current-release-audit.mjs', + 'scripts/ops/pingora-cutover-evidence-audit.mjs', + 'scripts/ops/pingora-cutover-evidence-bundle.mjs', + 'scripts/ops/pingora-cutover-command-evidence.mjs', + 'scripts/ops/pingora-cutover-evidence-verify.mjs', + 'scripts/ops/production-health-patrol.mjs', + 'scripts/check-production-health-patrol-env.mjs', + 'scripts/check-pingora-release-readiness.mjs', + 'scripts/check-pingora-direct-preflight.mjs', + 'scripts/check-pingora-direct-live.mjs', + 'scripts/check-pingora-canary-live.mjs', + 'scripts/check-pingora-canary-access-log-parity.mjs', + 'scripts/deploy/pingora-direct-enable.sh', + 'scripts/deploy/pingora-direct-rollback.sh', + 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'deploy/systemd/genarrative-pingora-gateway.service', + 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', + 'deploy/nginx/snippets/genarrative-pingora-canary.conf', + 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', + 'deploy/pingora/pingora-gateway.env.example', +]; +const SECRET_ENV_KEY_PATTERN = + /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu; + +const config = parseArgs(process.argv.slice(2)); +const snapshot = await buildSnapshot(config); + +console.log(`${JSON.stringify(snapshot, null, 2)}\n`); + +if (config.failOnCritical && snapshot.summary.status === 'CRITICAL') { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual', + releaseRoot: + process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || + '/opt/genarrative/current', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + expectedGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + expectedPublicBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', + expectedPublicHost: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, + requireEmptyPublicHost: readBoolEnv( + 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST', + ), + runHealthPatrol: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL', + ), + requirePingoraGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--phase': + result.phase = requireValue(argv, ++index, arg); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--expected-gateway-mode': + result.expectedGatewayMode = requireValue(argv, ++index, arg); + break; + case '--expected-public-base-url': + result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); + break; + case '--expected-public-host': + result.expectedPublicHost = requireValue(argv, ++index, arg); + break; + case '--require-empty-public-host': + result.requireEmptyPublicHost = true; + break; + case '--run-health-patrol': + result.runHealthPatrol = true; + break; + case '--require-pingora-gateway': + result.requirePingoraGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + if (!path.isAbsolute(result.releaseRoot)) { + throw new Error('--release-root 必须是绝对路径。'); + } + validateNoControlCharacters(result.releaseRoot, '--release-root'); + if (isFilesystemRootPath(result.releaseRoot)) { + throw new Error('--release-root 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.healthPatrolEnvFile)) { + throw new Error('--health-patrol-env-file 必须是绝对路径。'); + } + validateNoControlCharacters( + result.healthPatrolEnvFile, + '--health-patrol-env-file', + ); + if (isFilesystemRootPath(result.healthPatrolEnvFile)) { + throw new Error('--health-patrol-env-file 不能是文件系统根目录。'); + } + if (!path.isAbsolute(result.pingoraEnvFile)) { + throw new Error('--pingora-env-file 必须是绝对路径。'); + } + validateNoControlCharacters(result.pingoraEnvFile, '--pingora-env-file'); + if (isFilesystemRootPath(result.pingoraEnvFile)) { + throw new Error('--pingora-env-file 不能是文件系统根目录。'); + } + if ( + result.expectedGatewayMode && + !GATEWAY_MODES.has(result.expectedGatewayMode) + ) { + throw new Error( + `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`, + ); + } + if (result.expectedPublicHost !== null) { + validateHostOption(result.expectedPublicHost, '--expected-public-host'); + } + if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) { + throw new Error( + '--expected-public-host 和 --require-empty-public-host 不能同时使用。', + ); + } + if (result.expectedPublicBaseUrl) { + validateHttpUrl(result.expectedPublicBaseUrl, '--expected-public-base-url'); + } + + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-cutover-status-snapshot.mjs [options] + +Options: + --phase 写入 JSON 的阶段标签，例如 pre-cutover / post-enable / post-rollback。 + --release-root current release 根目录，默认 /opt/genarrative/current。 + --health-patrol-env-file health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。 + --expected-gateway-mode 可选，nginx 或 pingora-direct；提供后会检查 env 和 systemd capability 方向。 + --expected-public-base-url 可选，要求 health-patrol public base URL 与该值一致。 + --expected-public-host 可选，要求 health-patrol public Host 与该值一致。 + --require-empty-public-host 可选，要求 health-patrol public Host 为空。 + --run-health-patrol 读取 env 后执行 current release 随包生产巡检脚本并收录结果。 + --require-pingora-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。 + --timeout-ms systemctl / 子检查超时，默认 5000。 + --fail-on-critical 如果快照中出现 CRITICAL，则以退出码 1 结束。 + +该脚本只读采集状态，不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateHttpUrl(value, label) { + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error('unsupported protocol'); + } + } catch { + throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); + } +} + +function validateHostOption(value, label) { + const raw = String(value); + if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { + throw new Error( + `${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符`, + ); + } + try { + const parsed = new URL(`https://${raw}`); + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error('invalid host'); + } + } catch { + throw new Error(`${label} 不是合法的 host 或 host:port`); + } +} + +async function buildSnapshot(input) { + const context = { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + phase: input.phase, + releaseRoot: input.releaseRoot, + }; + const healthPatrolEnv = await inspectHealthPatrolEnv(input); + const pingoraEnv = await inspectPingoraEnv(input); + const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv); + delete healthPatrolEnv.secretValues; + delete pingoraEnv.secretValues; + const releaseArtifacts = await inspectReleaseArtifacts(input); + const systemd = await inspectSystemd(input); + const checks = await runChecks(input, healthPatrolEnv, secrets); + const summary = summarize([ + healthPatrolEnv.status, + pingoraEnv.status, + releaseArtifacts.status, + systemd.status, + ...checks.map((check) => check.status), + ]); + + return { + ...context, + summary, + healthPatrolEnv, + pingoraEnv, + releaseArtifacts, + systemd, + checks, + }; +} + +async function inspectHealthPatrolEnv(input) { + const parsed = await readEnvFile(input.healthPatrolEnvFile); + if (parsed.status === 'CRITICAL') { + return parsed; + } + + const values = parsed.values; + const publicHost = values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || ''; + const publicBaseUrl = + values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || ''; + const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || ''; + const diagnostics = []; + let status = 'OK'; + + if (input.expectedGatewayMode && gatewayMode !== input.expectedGatewayMode) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${input.expectedGatewayMode}，实际 ${gatewayMode || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if ( + input.expectedPublicBaseUrl && + normalizeBaseUrl(publicBaseUrl) !== normalizeBaseUrl(input.expectedPublicBaseUrl) + ) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${normalizeBaseUrl(input.expectedPublicBaseUrl)}，实际 ${publicBaseUrl || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if (input.expectedPublicHost !== null && publicHost !== input.expectedPublicHost) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${input.expectedPublicHost}，实际 ${publicHost || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + if (input.requireEmptyPublicHost && publicHost) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空，实际 ${publicHost}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + + return { + path: input.healthPatrolEnvFile, + status, + secretValues: collectSecretValuesFromEnv(values), + values: { + gatewayMode, + publicBaseUrl, + publicHost, + apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '', + spacetimeBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL || '', + pingoraBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + hasPingoraProbeToken: Boolean( + values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN, + ), + }, + diagnostics, + }; +} + +async function inspectPingoraEnv(input) { + const parsed = await readEnvFile(input.pingoraEnvFile); + if (parsed.status === 'CRITICAL') { + return { + ...parsed, + values: {}, + }; + } + const values = parsed.values; + return { + path: input.pingoraEnvFile, + status: 'OK', + secretValues: collectSecretValuesFromEnv(values), + values: { + listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '', + tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '', + httpRedirectListen: + values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '', + tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '', + tlsKeyFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE || '', + hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN), + accessLogFile: + values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '', + }, + diagnostics: [], + }; +} + +async function readEnvFile(filePath) { + let text; + try { + text = await readFile(filePath, 'utf8'); + } catch (error) { + return { + path: filePath, + status: 'CRITICAL', + values: {}, + diagnostics: [`无法读取 env 文件: ${error.message}`], + }; + } + + const values = {}; + const diagnostics = []; + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + values[key] = stripQuotes(line.slice(equalsIndex + 1).trim()); + } + + return { + path: filePath, + status: diagnostics.length > 0 ? 'WARNING' : 'OK', + values, + diagnostics, + }; +} + +function collectSecretValuesFromEnv(values) { + const secrets = []; + for (const [key, value] of Object.entries(values || {})) { + if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') { + continue; + } + const trimmed = value.trim(); + if (trimmed.length > 0) { + secrets.push(trimmed); + } + } + return secrets; +} + +function collectSecretValues(...envSnapshots) { + const secrets = new Set(); + for (const snapshot of envSnapshots) { + for (const value of snapshot?.secretValues || []) { + if (typeof value === 'string' && value.trim()) { + secrets.add(value.trim()); + } + } + } + return [...secrets].sort((left, right) => right.length - left.length); +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function normalizeBaseUrl(value) { + if (!value) { + return ''; + } + try { + return new URL(value).toString().replace(/\/+$/u, ''); + } catch { + return value; + } +} + +async function inspectReleaseArtifacts(input) { + const artifacts = []; + for (const relativePath of RELEASE_ARTIFACTS) { + const fullPath = path.join(input.releaseRoot, relativePath); + const artifact = { + path: relativePath, + exists: false, + executable: false, + status: 'CRITICAL', + }; + try { + const fileStat = await stat(fullPath); + artifact.exists = fileStat.isFile() || fileStat.isDirectory(); + if (fileStat.isFile()) { + try { + await access(fullPath, fsConstants.X_OK); + artifact.executable = true; + } catch { + artifact.executable = false; + } + } + artifact.status = artifact.exists ? 'OK' : 'CRITICAL'; + } catch { + artifact.status = 'CRITICAL'; + } + artifacts.push(artifact); + } + + const status = artifacts.some((artifact) => artifact.status === 'CRITICAL') + ? 'CRITICAL' + : 'OK'; + return { + status, + artifacts, + }; +} + +async function inspectSystemd(input) { + const services = []; + for (const service of SERVICES) { + services.push(await inspectService(service, input)); + } + const pingoraUnit = await inspectPingoraUnit(input); + return { + status: summarize([ + ...services.map((service) => service.status), + pingoraUnit.status, + ]).status, + services, + pingoraUnit, + }; +} + +async function inspectService(service, input) { + const expected = isServiceExpected(service, input.expectedGatewayMode); + const result = await runCommand('systemctl', ['is-active', service], input); + const state = result.stdout.trim() || result.stderr.trim() || result.error; + let status = 'OK'; + let summary = state || 'unknown'; + if (result.code === 0 && state === 'active') { + summary = 'active'; + } else if (expected) { + status = 'CRITICAL'; + } else { + summary = state ? `${state} (not required)` : 'not required'; + } + return { + name: service, + expectedActive: expected, + activeState: state, + status, + summary, + command: result.command, + }; +} + +function isServiceExpected(service, gatewayMode) { + if ( + service === 'genarrative-api.service' || + service === 'spacetimedb.service' + ) { + return Boolean(gatewayMode); + } + if (service === 'nginx.service') { + return gatewayMode === 'nginx'; + } + if (service === 'genarrative-pingora-gateway.service') { + return gatewayMode === 'pingora-direct'; + } + return false; +} + +async function inspectPingoraUnit(input) { + const catResult = await runCommand( + 'systemctl', + ['cat', 'genarrative-pingora-gateway.service'], + input, + ); + const showResult = await runCommand( + 'systemctl', + [ + 'show', + 'genarrative-pingora-gateway.service', + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + + const unitText = catResult.stdout; + const hasAmbientCapability = unitText.includes( + 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + ); + const hasCapabilityBoundingSet = unitText.includes( + 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + ); + const environmentFiles = [ + ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu), + ].map((match) => match[1].trim()); + const environmentFileMatchesPingoraEnvFile = environmentFilesInclude( + environmentFiles, + input.pingoraEnvFile, + ); + const show = parseSystemctlShow(showResult.stdout); + const diagnostics = []; + let status = 'OK'; + + if (catResult.code !== 0) { + diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`); + status = 'CRITICAL'; + } + if (showResult.code !== 0) { + diagnostics.push( + `systemctl show 失败: ${showResult.stderr || showResult.error}`, + ); + status = 'CRITICAL'; + } + if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) { + diagnostics.push( + `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`, + ); + status = 'CRITICAL'; + } + if (input.expectedGatewayMode === 'pingora-direct') { + if (!hasAmbientCapability || !hasCapabilityBoundingSet) { + diagnostics.push( + 'pingora-direct 模式必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + } + if (input.expectedGatewayMode === 'nginx') { + if (hasAmbientCapability || hasCapabilityBoundingSet) { + diagnostics.push( + 'nginx 模式下 systemd 最终配置不应残留 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + } + + return { + status, + hasAmbientCapability, + hasCapabilityBoundingSet, + environmentFiles, + environmentFileMatchesPingoraEnvFile, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart: show.ExecStart || '', + diagnostics, + commands: [catResult.command, showResult.command], + }; +} + +function environmentFilesInclude(environmentFiles, expectedPath) { + return environmentFiles.some((entry) => + splitSystemdEnvironmentFileEntry(entry).includes(expectedPath), + ); +} + +function splitSystemdEnvironmentFileEntry(entry) { + const files = []; + for (const word of String(entry || '').split(/\s+/u)) { + if (!word) { + continue; + } + let normalized = word.trim(); + if (normalized.startsWith('-')) { + normalized = normalized.slice(1); + } + if ( + (normalized.startsWith('"') && normalized.endsWith('"')) || + (normalized.startsWith("'") && normalized.endsWith("'")) + ) { + normalized = normalized.slice(1, -1); + } + if (normalized) { + files.push(normalized); + } + } + return files; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +async function runChecks(input, healthPatrolEnv, secrets) { + const checks = []; + checks.push(await runCurrentReleaseAudit(input, secrets)); + if (input.expectedGatewayMode) { + checks.push(await runHealthPatrolEnvCheck(input, secrets)); + } + if (input.runHealthPatrol) { + checks.push(await runProductionHealthPatrol(input, healthPatrolEnv, secrets)); + } + return checks; +} + +async function runCurrentReleaseAudit(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + const result = await runCommand( + 'node', + [ + '--', + script, + '--release-root', + input.releaseRoot, + ...(input.requirePingoraGateway ? ['--require-pingora-gateway'] : []), + '--systemd-show', + '--timeout-ms', + String(input.timeoutMs), + ], + input, + ); + const parsed = parseJsonObject(result.stdout); + const details = parsed.ok + ? { + summary: parsed.value.summary, + pingoraGateway: parsed.value.pingoraGateway, + checksums: parsed.value.checksums, + releaseManifest: parsed.value.releaseManifest, + systemd: parsed.value.systemd, + } + : { + parseError: parsed.error, + }; + const check = commandCheck('current-release-audit', result, secrets, details); + if (!parsed.ok) { + check.status = 'CRITICAL'; + check.stderr = trimForJson( + [check.stderr, `无法解析 current release 自审 JSON: ${parsed.error}`] + .filter(Boolean) + .join('\n'), + ); + } + return check; +} + +async function runHealthPatrolEnvCheck(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/check-production-health-patrol-env.mjs', + ); + const args = [ + '--', + script, + '--env-file', + input.healthPatrolEnvFile, + '--expected-gateway-mode', + input.expectedGatewayMode, + ]; + if (input.expectedPublicBaseUrl) { + args.push('--expected-public-base-url', input.expectedPublicBaseUrl); + } + if (input.expectedPublicHost !== null) { + args.push('--expected-public-host', input.expectedPublicHost); + } + if (input.requireEmptyPublicHost) { + args.push('--require-empty-public-host'); + } + const result = await runCommand('node', args, input); + return commandCheck('health-patrol-env-check', result, secrets); +} + +async function runProductionHealthPatrol(input, healthPatrolEnv, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/production-health-patrol.mjs', + ); + const rawEnv = await readEnvFile(input.healthPatrolEnvFile); + const env = { + ...process.env, + ...readEnvValuesForChild(rawEnv.status === 'CRITICAL' ? healthPatrolEnv : rawEnv), + }; + const result = await runCommand( + 'node', + ['--', script, '--json'], + input, + env, + ); + return commandCheck('production-health-patrol', result, secrets); +} + +function readEnvValuesForChild(healthPatrolEnv) { + if (!healthPatrolEnv.values) { + return {}; + } + const result = {}; + for (const [key, value] of Object.entries(healthPatrolEnv.values)) { + if (typeof value === 'string') { + result[key] = value; + } + } + return result; +} + +function commandCheck(name, result, secrets = [], details = undefined) { + return { + name, + status: result.code === 0 ? 'OK' : 'CRITICAL', + code: result.code, + command: result.command, + stdout: trimForJson(redactSecrets(result.stdout, secrets)), + stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)), + ...(details === undefined ? {} : { details }), + }; +} + +function redactSecrets(value, secrets) { + let text = String(value || ''); + for (const secret of secrets) { + if (!secret) { + continue; + } + text = text.split(secret).join(''); + } + return text; +} + +function runCommand(command, args, input, env = process.env) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + return new Promise((resolve) => { + execFile( + command, + args, + { + env, + timeout: input.timeoutMs, + windowsHide: true, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + command: formatCommand(command, args), + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + timedOut: Boolean(error?.killed), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function formatCommand(command, args) { + const secretValueFlags = new Set([ + '--pingora-probe-token', + '--probe-token', + '--direct-probe-token', + ]); + const redactedArgs = args.map((arg, index) => + index > 0 && secretValueFlags.has(args[index - 1]) ? '' : arg, + ); + return [command, ...redactedArgs].join(' '); +} + +function trimForJson(value) { + const text = String(value || '').trim(); + if (text.length <= 1000) { + return text; + } + return `${text.slice(0, 1000)}...`; +} + +function parseJsonObject(value) { + try { + return { ok: true, value: JSON.parse(value) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function summarize(statuses) { + const normalized = statuses.filter(Boolean); + const status = normalized.reduce( + (current, item) => maxStatus(current, item), + 'OK', + ); + return { + status, + criticalCount: normalized.filter((item) => item === 'CRITICAL').length, + warningCount: normalized.filter((item) => item === 'WARNING').length, + }; +} + +function maxStatus(left, right) { + return STATUS_RANK[right] > STATUS_RANK[left] ? right : left; +} diff --git a/scripts/ops/production-health-patrol.mjs b/scripts/ops/production-health-patrol.mjs index 219d8e295..50df72a25 100644 --- a/scripts/ops/production-health-patrol.mjs +++ b/scripts/ops/production-health-patrol.mjs @@ -1,10 +1,10 @@ #!/usr/bin/env node -import {execFile} from 'node:child_process'; +import { execFile } from 'node:child_process'; import http from 'node:http'; import https from 'node:https'; -import {mkdir, writeFile} from 'node:fs/promises'; -import {dirname} from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname } from 'node:path'; const STATUS_RANK = { OK: 0, @@ -23,6 +23,12 @@ const DEFAULT_SERVICES = [ 'spacetimedb.service', 'nginx.service', ]; +const PINGORA_DIRECT_SERVICES = [ + 'genarrative-api.service', + 'spacetimedb.service', + 'genarrative-pingora-gateway.service', +]; +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); function usage() { console.log(`Usage: @@ -32,7 +38,11 @@ Options: --api-base-url API direct base URL, default http://127.0.0.1:8082 --spacetime-base-url SpacetimeDB base URL, default http://127.0.0.1:3101 --public-base-url Nginx/public base URL, default http://127.0.0.1 + --public-host Optional public Host header, useful when probing 127.0.0.1 --public-path Public API path to probe; repeatable + --gateway-mode nginx or pingora-direct, default nginx + --pingora-base-url Optional Pingora shadow gateway base URL + --pingora-probe-token Optional Pingora shadow probe token --status-file Write the last patrol result as JSON --timeout-ms HTTP/command timeout, default 5000 --slow-ms Mark successful probes slower than this as WARNING, default 3000 @@ -44,15 +54,32 @@ Options: function readBoolEnv(name, fallback = false) { const value = process.env[name]; - if (!value) { + if (value === undefined || value === null || String(value).trim() === '') { return fallback; } - return ['1', 'true', 'yes', 'on'].includes(value.trim().toLowerCase()); + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); } -function parsePositiveInt(raw, fallback) { - const value = Number.parseInt(String(raw ?? ''), 10); - return Number.isFinite(value) && value > 0 ? value : fallback; +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const valueText = String(raw ?? '').trim(); + if (!/^[1-9]\d*$/.test(valueText)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(valueText, 10); } function parseArgs(argv) { @@ -65,17 +92,26 @@ function parseArgs(argv) { 'http://127.0.0.1:3101', publicBaseUrl: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || - process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || - 'http://127.0.0.1:8082', + 'http://127.0.0.1', + publicHost: process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '', publicPaths: [], + gatewayMode: process.env.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || 'nginx', + pingoraBaseUrl: + process.env.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + pingoraProbeToken: + process.env.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + process.env.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN || + '', statusFile: process.env.GENARRATIVE_HEALTH_PATROL_STATUS_FILE || '', - timeoutMs: parsePositiveInt( + timeoutMs: parseOptionalPositiveInt( process.env.GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS, 5000, + 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS', ), - slowMs: parsePositiveInt( + slowMs: parseOptionalPositiveInt( process.env.GENARRATIVE_HEALTH_PATROL_SLOW_MS, 3000, + 'GENARRATIVE_HEALTH_PATROL_SLOW_MS', ), failOnWarning: readBoolEnv('GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING'), skipJournal: readBoolEnv('GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL'), @@ -100,17 +136,35 @@ function parseArgs(argv) { case '--public-base-url': config.publicBaseUrl = requireValue(argv, ++index, arg); break; + case '--public-host': + config.publicHost = requireValue(argv, ++index, arg); + break; case '--public-path': config.publicPaths.push(requireValue(argv, ++index, arg)); break; + case '--gateway-mode': + config.gatewayMode = requireValue(argv, ++index, arg); + break; + case '--pingora-base-url': + config.pingoraBaseUrl = requireValue(argv, ++index, arg); + break; + case '--pingora-probe-token': + config.pingoraProbeToken = requireValue(argv, ++index, arg); + break; case '--status-file': config.statusFile = requireValue(argv, ++index, arg); break; case '--timeout-ms': - config.timeoutMs = parsePositiveInt(requireValue(argv, ++index, arg), 5000); + config.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + '--timeout-ms', + ); break; case '--slow-ms': - config.slowMs = parsePositiveInt(requireValue(argv, ++index, arg), 3000); + config.slowMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + '--slow-ms', + ); break; case '--fail-on-warning': config.failOnWarning = true; @@ -129,6 +183,15 @@ function parseArgs(argv) { if (config.publicPaths.length === 0) { config.publicPaths = DEFAULT_PUBLIC_PATHS; } + config.gatewayMode = config.gatewayMode.trim(); + if (!GATEWAY_MODES.has(config.gatewayMode)) { + throw new Error( + `--gateway-mode 只支持 nginx 或 pingora-direct: ${config.gatewayMode}`, + ); + } + if (config.publicHost) { + validateHostOption(config.publicHost, '--public-host'); + } return config; } @@ -147,9 +210,22 @@ function joinUrl(baseUrl, path) { return `${base}${suffix}`; } +function validateHostOption(value, flag) { + if ( + value.includes('://') || + value.includes('/') || + value.includes('?') || + value.includes('#') + ) { + throw new Error(`${flag} 只接受域名或 host:port，不要填 URL、路径或查询。`); + } +} + function maxStatus(checks) { return checks.reduce((current, check) => { - return STATUS_RANK[check.status] > STATUS_RANK[current] ? check.status : current; + return STATUS_RANK[check.status] > STATUS_RANK[current] + ? check.status + : current; }, 'OK'); } @@ -175,12 +251,7 @@ function runCommand(command, args, timeoutMs) { (error, stdout, stderr) => { resolve({ command: [command, ...args].join(' '), - code: - typeof error?.code === 'number' - ? error.code - : error - ? 1 - : 0, + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, signal: error?.signal || '', stdout: String(stdout || ''), stderr: String(stderr || ''), @@ -216,7 +287,7 @@ async function checkService(serviceName, timeoutMs) { ); } -function requestUrl(url, timeoutMs) { +function requestUrl(url, timeoutMs, headers = {}) { return new Promise((resolve) => { const startedAt = Date.now(); const parsed = new URL(url); @@ -229,6 +300,7 @@ function requestUrl(url, timeoutMs) { headers: { 'User-Agent': 'genarrative-health-patrol/1.0', Accept: 'application/json,text/plain,*/*', + ...headers, }, }, (response) => { @@ -262,9 +334,17 @@ function requestUrl(url, timeoutMs) { }); } -async function checkHttp(name, url, config) { - const result = await requestUrl(url, config.timeoutMs); - const curlCommand = `curl -fsS --max-time ${Math.ceil(config.timeoutMs / 1000)} ${url}`; +async function checkHttp(name, url, config, headers = {}) { + const result = await requestUrl(url, config.timeoutMs, headers); + const headerArgs = Object.entries(headers) + .map( + ([key, value]) => + ` -H '${key}: ${key.toLowerCase() === 'host' ? value : '***'}'`, + ) + .join(''); + const curlCommand = `curl -fsS --max-time ${Math.ceil( + config.timeoutMs / 1000, + )}${headerArgs} ${url}`; if (result.error) { return checkResult(name, 'CRITICAL', `请求失败: ${result.error}`, { @@ -299,20 +379,123 @@ async function checkHttp(name, url, config) { ); } - return checkResult(name, 'OK', `HTTP ${result.statusCode} ${result.elapsedMs}ms`, { - command: curlCommand, - elapsedMs: result.elapsedMs, + return checkResult( + name, + 'OK', + `HTTP ${result.statusCode} ${result.elapsedMs}ms`, + { + command: curlCommand, + elapsedMs: result.elapsedMs, + }, + ); +} + +async function checkPingoraShadow(config) { + if (!config.pingoraBaseUrl || !config.pingoraProbeToken) { + return checkResult( + 'pingora:shadow-probe', + 'WARNING', + 'Pingora shadow 巡检配置不完整，需同时配置 base URL 与 probe token', + ); + } + + const path = '/__genarrative_pingora/healthz'; + const url = joinUrl(config.pingoraBaseUrl, path); + const result = await requestUrl(url, config.timeoutMs, { + 'X-Genarrative-Pingora-Probe': config.pingoraProbeToken, }); + const command = `curl -fsS --max-time ${Math.ceil( + config.timeoutMs / 1000, + )} -H 'X-Genarrative-Pingora-Probe: ***' ${url}`; + + if (result.error) { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `请求失败: ${result.error}`, + { + command, + elapsedMs: result.elapsedMs, + }, + ); + } + + if (result.statusCode < 200 || result.statusCode >= 300) { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `HTTP ${result.statusCode}，耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + let payload; + try { + payload = JSON.parse(result.body); + } catch { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `响应不是合法 JSON，耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + if (payload?.ok !== true || payload?.gateway !== 'pingora-shadow') { + return checkResult( + 'pingora:shadow-probe', + 'CRITICAL', + `响应不是 Pingora shadow 探针，耗时 ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + body: result.body.trim(), + }, + ); + } + + if (result.elapsedMs > config.slowMs) { + return checkResult( + 'pingora:shadow-probe', + 'WARNING', + `HTTP ${result.statusCode} 但耗时偏高: ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + }, + ); + } + + return checkResult( + 'pingora:shadow-probe', + 'OK', + `HTTP ${result.statusCode} ${result.elapsedMs}ms`, + { + command, + elapsedMs: result.elapsedMs, + maintenance: Boolean(payload.maintenance), + }, + ); +} + +function gatewayServiceNames(config) { + return config.gatewayMode === 'pingora-direct' + ? PINGORA_DIRECT_SERVICES + : DEFAULT_SERVICES; } async function checkRecentJournal(config) { + const serviceNames = gatewayServiceNames(config); const args = [ - '-u', - 'genarrative-api.service', - '-u', - 'spacetimedb.service', - '-u', - 'nginx.service', + ...serviceNames.flatMap((serviceName) => ['-u', serviceName]), '--since', '15 minutes ago', '-p', @@ -326,10 +509,15 @@ async function checkRecentJournal(config) { const result = await runCommand('journalctl', args, config.timeoutMs); if (result.code !== 0) { - return checkResult('journal:recent-errors', 'WARNING', '无法读取最近错误日志', { - command: result.command, - stderr: result.stderr.trim() || result.error, - }); + return checkResult( + 'journal:recent-errors', + 'WARNING', + '无法读取最近错误日志', + { + command: result.command, + stderr: result.stderr.trim() || result.error, + }, + ); } const lines = result.stdout @@ -338,9 +526,14 @@ async function checkRecentJournal(config) { .filter((line) => line && line !== '-- No entries --'); if (lines.length === 0) { - return checkResult('journal:recent-errors', 'OK', '最近 15 分钟无 err..alert 日志', { - command: result.command, - }); + return checkResult( + 'journal:recent-errors', + 'OK', + '最近 15 分钟无 err..alert 日志', + { + command: result.command, + }, + ); } return checkResult( @@ -358,7 +551,7 @@ async function writeStatusFile(statusFile, payload) { if (!statusFile) { return; } - await mkdir(dirname(statusFile), {recursive: true}); + await mkdir(dirname(statusFile), { recursive: true }); await writeFile(statusFile, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); } @@ -423,12 +616,24 @@ async function main() { const config = parseArgs(process.argv.slice(2)); const checks = []; - for (const serviceName of DEFAULT_SERVICES) { + for (const serviceName of gatewayServiceNames(config)) { checks.push(await checkService(serviceName, config.timeoutMs)); } - checks.push(await checkHttp('api:/healthz', joinUrl(config.apiBaseUrl, '/healthz'), config)); - checks.push(await checkHttp('api:/readyz', joinUrl(config.apiBaseUrl, '/readyz'), config)); + checks.push( + await checkHttp( + 'api:/healthz', + joinUrl(config.apiBaseUrl, '/healthz'), + config, + ), + ); + checks.push( + await checkHttp( + 'api:/readyz', + joinUrl(config.apiBaseUrl, '/readyz'), + config, + ), + ); checks.push( await checkHttp( 'spacetimedb:/v1/ping', @@ -438,11 +643,21 @@ async function main() { ); for (const path of config.publicPaths) { + const headers = config.publicHost ? { Host: config.publicHost } : {}; checks.push( - await checkHttp(`public:${path}`, joinUrl(config.publicBaseUrl, path), config), + await checkHttp( + `public:${path}`, + joinUrl(config.publicBaseUrl, path), + config, + headers, + ), ); } + if (config.pingoraBaseUrl || config.pingoraProbeToken) { + checks.push(await checkPingoraShadow(config)); + } + if (!config.skipJournal) { checks.push(await checkRecentJournal(config)); } @@ -451,6 +666,7 @@ async function main() { status: maxStatus(checks), checkedAt: new Date().toISOString(), host: process.env.HOSTNAME || '', + gatewayMode: config.gatewayMode, checks, }; @@ -472,6 +688,8 @@ async function main() { } main().catch((error) => { - console.error(`[health-patrol] CRITICAL ${error instanceof Error ? error.message : String(error)}`); + console.error( + `[health-patrol] CRITICAL ${error instanceof Error ? error.message : String(error)}`, + ); process.exit(2); }); diff --git a/server-rs/Cargo.lock b/server-rs/Cargo.lock index aa52b22d9..b71702e19 100644 --- a/server-rs/Cargo.lock +++ b/server-rs/Cargo.lock @@ -59,6 +59,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -239,6 +245,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -263,6 +278,45 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "asn1-rs" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -308,6 +362,28 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.9" @@ -952,6 +1028,20 @@ dependencies = [ "num-traits", ] +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1056,6 +1146,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1235,6 +1331,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -2521,6 +2623,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "no_debug" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f23a60c850e1144fc1dd9435152e0cfdc7dd18725350b4243584118013a52a4" + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2589,6 +2697,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2729,6 +2846,30 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.117", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -2852,8 +2993,10 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54a75f2ff8e122aa80ab202dc865294fe59cd856c2a5dab2d3df6e122c93b941" dependencies = [ + "pingora-cache", "pingora-core", "pingora-http", + "pingora-load-balancing", "pingora-proxy", "pingora-timeout", ] @@ -2922,12 +3065,14 @@ dependencies = [ "nix", "once_cell", "openssl-probe", + "ouroboros", "parking_lot", "percent-encoding", "pingora-error", "pingora-http", "pingora-pool", "pingora-runtime", + "pingora-rustls", "pingora-timeout", "prometheus 0.13.4", "rand 0.8.6", @@ -2943,6 +3088,7 @@ dependencies = [ "tokio-test", "unicase", "windows-sys 0.59.0", + "x509-parser", "zstd", ] @@ -2958,10 +3104,13 @@ version = "0.1.0" dependencies = [ "async-trait", "bytes", + "httpdate", "mime_guess", "pingora", "pingora-http", "pingora-proxy", + "serde", + "serde_json", "shared-logging", "tokio", "tracing", @@ -2995,6 +3144,37 @@ dependencies = [ "pingora-error", ] +[[package]] +name = "pingora-ketama" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2a2e43a14f1d291fba7905542c7c1d1f89528f470b3cd48b6806e702ea772f" +dependencies = [ + "crc32fast", +] + +[[package]] +name = "pingora-load-balancing" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361b69af0234d2e4d10234e2efd106bb3b8147c575d52f45604a46aaf26def7a" +dependencies = [ + "arc-swap", + "async-trait", + "derivative", + "fnv", + "futures", + "http 1.4.0", + "log", + "pingora-core", + "pingora-error", + "pingora-http", + "pingora-ketama", + "pingora-runtime", + "rand 0.8.6", + "tokio", +] + [[package]] name = "pingora-lru" version = "0.8.1" @@ -3057,6 +3237,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "pingora-rustls" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2c12f20ac2eb8d708763aba7ce3c6a21bfed113b07cee79906225da5a500d28" +dependencies = [ + "log", + "no_debug", + "pingora-error", + "ring", + "rustls", + "rustls-native-certs", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "tokio-rustls", +] + [[package]] name = "pingora-timeout" version = "0.8.1" @@ -3299,6 +3496,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "version_check", + "yansi", +] + [[package]] name = "prometheus" version = "0.13.4" @@ -3634,7 +3844,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -3759,6 +3969,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.4" @@ -3778,6 +3997,8 @@ version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ + "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -3786,6 +4007,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.2.0", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3795,6 +4029,15 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -3811,6 +4054,7 @@ version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -4557,6 +4801,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.11.1" @@ -5853,6 +6103,29 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.2" diff --git a/server-rs/Cargo.toml b/server-rs/Cargo.toml index 953865a93..206231ce2 100644 --- a/server-rs/Cargo.toml +++ b/server-rs/Cargo.toml @@ -108,13 +108,14 @@ flate2 = "1" futures-util = "0.3" hmac = "0.12" http-body-util = "0.1" +httpdate = "1" hex = "0.4" image = { version = "0.25", default-features = false } jsonwebtoken = "9" langchainrust = "0.2.18" log = "0.4" mime_guess = "2.0.5" -pingora = { version = "0.8.1", default-features = false, features = ["proxy"] } +pingora = { version = "0.8.1", default-features = false, features = ["proxy", "rustls"] } pingora-http = { version = "0.8.1", default-features = false } pingora-proxy = { version = "0.8.1", default-features = false } rand_core = "0.6" diff --git a/server-rs/crates/pingora-gateway/Cargo.toml b/server-rs/crates/pingora-gateway/Cargo.toml index c4602aaf2..1c159c717 100644 --- a/server-rs/crates/pingora-gateway/Cargo.toml +++ b/server-rs/crates/pingora-gateway/Cargo.toml @@ -7,11 +7,16 @@ license.workspace = true [dependencies] async-trait = { workspace = true } bytes = { workspace = true } +httpdate = { workspace = true } mime_guess = { workspace = true } pingora = { workspace = true } pingora-http = { workspace = true } pingora-proxy = { workspace = true } shared-logging = { workspace = true } -tokio = { workspace = true, features = ["fs"] } +tokio = { workspace = true, features = ["fs", "io-util"] } tracing = { workspace = true } uuid = { workspace = true, features = ["v4"] } + +[dev-dependencies] +serde = { workspace = true } +serde_json = { workspace = true } diff --git a/server-rs/crates/pingora-gateway/src/main.rs b/server-rs/crates/pingora-gateway/src/main.rs index 45757fcdc..b9567f0ff 100644 --- a/server-rs/crates/pingora-gateway/src/main.rs +++ b/server-rs/crates/pingora-gateway/src/main.rs @@ -1,21 +1,32 @@ use std::{ - env, io, + collections::HashMap, + env, + fs::OpenOptions, + io, + io::SeekFrom, + io::Write, net::SocketAddr, path::{Component, Path, PathBuf}, - sync::Arc, - time::Instant, + sync::{Arc, Mutex}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, }; use async_trait::async_trait; use bytes::Bytes; +use httpdate::{fmt_http_date, parse_http_date}; use pingora::{ - Result as PingoraResult, + ConnectTimedout, ConnectionClosed, Error, ErrorSource, HTTPStatus, ReadError, ReadTimedout, + Result as PingoraResult, WriteError, WriteTimedout, + listeners::tls::TlsSettings, + modules::http::{HttpModule, HttpModuleBuilder, HttpModules, Module}, prelude::HttpPeer, + protocols::http::compression::ResponseCompressionCtx, server::{Server, configuration::Opt}, }; -use pingora_http::{Method, ResponseHeader}; -use pingora_proxy::{ProxyHttp, Session, http_proxy_service}; +use pingora_http::{Method, RequestHeader, ResponseHeader}; +use pingora_proxy::{FailToProxy, ProxyHttp, Session, http_proxy_service}; use shared_logging::{OtelConfig, init_tracing}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tracing::{error, info, warn}; use uuid::Uuid; @@ -26,11 +37,32 @@ const DEFAULT_WEB_ROOT: &str = "/srv/genarrative/web"; const DEFAULT_ACME_ROOT: &str = "/var/www/html"; const DEFAULT_MAINTENANCE_FILE: &str = "/var/lib/genarrative/maintenance/enabled"; const DEFAULT_MAX_API_BODY_BYTES: u64 = 64 * 1024 * 1024; +const DEFAULT_GZIP_LEVEL: u32 = 5; +const DEFAULT_GZIP_MIN_LENGTH_BYTES: u64 = 1024; +const DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS: u64 = 3_000; +const DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS: u64 = 60; +const DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS: u64 = 3_600; +const DEFAULT_HTML_CACHE_CONTROL: &str = "no-cache"; +const DEFAULT_ASSET_CACHE_CONTROL: &str = "public, max-age=31536000, immutable"; +const DEFAULT_STATIC_CACHE_CONTROL: &str = "no-cache"; +const DEFAULT_INSTANCE_COUNT: u32 = 1; const DEFAULT_LOG_FILTER: &str = "info,pingora=info,pingora_gateway=info"; +const SHADOW_PROBE_PATH: &str = "/__genarrative_pingora/healthz"; +const SHADOW_PROBE_HEADER: &str = "x-genarrative-pingora-probe"; +const PAYLOAD_TOO_LARGE_CONTEXT: &str = "genarrative_payload_too_large"; +const PROTECTION_STATE_TTL: Duration = Duration::from_secs(600); +const PROTECTION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60); #[derive(Clone, Debug)] struct GatewayConfig { listen_addr: String, + tls_listen_addr: Option, + tls_cert_file: Option, + tls_key_file: Option, + http_redirect_listen_addr: Option, + http_redirect_target_scheme: String, api_upstream: SocketAddr, spacetime_upstream: SocketAddr, web_root: PathBuf, @@ -38,17 +70,41 @@ struct GatewayConfig { maintenance_file: PathBuf, forwarded_proto: String, max_api_body_bytes: u64, + gzip_enabled: bool, + gzip_level: u32, + gzip_min_length_bytes: u64, + compression: CompressionConfig, + static_cache: StaticCacheConfig, + upstream_timeouts: UpstreamTimeoutConfig, + shadow_probe_token: Option, + trust_x_forwarded_for: bool, + instance_count: u32, + shared_protection_confirmed: bool, + protection: ProtectionConfig, log_filter: String, + access_log_file: Option, otel_enabled: bool, } impl GatewayConfig { fn from_env() -> io::Result { - Ok(Self { + let config = Self { listen_addr: read_env_or_default( "GENARRATIVE_PINGORA_GATEWAY_LISTEN", DEFAULT_LISTEN_ADDR, ), + tls_listen_addr: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN"), + tls_cert_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE") + .map(PathBuf::from), + tls_key_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE") + .map(PathBuf::from), + http_redirect_listen_addr: read_optional_env( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + ), + http_redirect_target_scheme: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME", + "https", + ), api_upstream: read_socket_addr_env( "GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM", DEFAULT_API_UPSTREAM, @@ -77,30 +133,621 @@ impl GatewayConfig { "GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES", DEFAULT_MAX_API_BODY_BYTES, )?, + gzip_enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED", true), + gzip_level: read_u32_env("GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL", DEFAULT_GZIP_LEVEL)?, + gzip_min_length_bytes: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES", + DEFAULT_GZIP_MIN_LENGTH_BYTES, + )?, + compression: CompressionConfig::from_env()?, + static_cache: StaticCacheConfig::from_env(), + upstream_timeouts: UpstreamTimeoutConfig::from_env()?, + shadow_probe_token: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN"), + trust_x_forwarded_for: read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR", + false, + ), + instance_count: read_u32_env( + "GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT", + DEFAULT_INSTANCE_COUNT, + )?, + shared_protection_confirmed: read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED", + false, + ), + protection: ProtectionConfig::from_env()?, log_filter: read_env_or_default("GENARRATIVE_PINGORA_GATEWAY_LOG", DEFAULT_LOG_FILTER), + access_log_file: read_optional_env("GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE") + .map(PathBuf::from), otel_enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED", false), + }; + config.validate()?; + Ok(config) + } + + fn validate(&self) -> io::Result<()> { + let listen_addr = + parse_socket_addr_config("GENARRATIVE_PINGORA_GATEWAY_LISTEN", &self.listen_addr)?; + + if listen_addr == self.api_upstream || listen_addr == self.spacetime_upstream { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_LISTEN 不能与上游地址相同", + )); + } + + let tls_listen_addr = if let Some(tls_listen_addr) = &self.tls_listen_addr { + let tls_listen_addr = parse_socket_addr_config( + "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + tls_listen_addr, + )?; + validate_distinct_listen_addr( + "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + tls_listen_addr, + listen_addr, + self.api_upstream, + self.spacetime_upstream, + )?; + validate_tls_file_config(&self.tls_cert_file, &self.tls_key_file)?; + Some(tls_listen_addr) + } else if self.tls_cert_file.is_some() || self.tls_key_file.is_some() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 TLS_CERT_FILE / TLS_KEY_FILE 时必须同时设置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN", + )); + } else { + None + }; + + if let Some(http_redirect_listen_addr) = &self.http_redirect_listen_addr { + let Some(tls_listen_addr) = tls_listen_addr else { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN 时必须同时设置 TLS_LISTEN / TLS_CERT_FILE / TLS_KEY_FILE", + )); + }; + let http_redirect_listen_addr = parse_socket_addr_config( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + http_redirect_listen_addr, + )?; + validate_distinct_listen_addr( + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN", + http_redirect_listen_addr, + listen_addr, + self.api_upstream, + self.spacetime_upstream, + )?; + if http_redirect_listen_addr == tls_listen_addr { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN 不能与 TLS_LISTEN 相同", + )); + } + validate_redirect_target_scheme(&self.http_redirect_target_scheme)?; + } + + if self.max_api_body_bytes == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES 必须大于 0", + )); + } + + if self.gzip_level > 9 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL 必须在 0..=9 之间", + )); + } + if self.gzip_min_length_bytes == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES 必须大于 0", + )); + } + self.compression.validate()?; + self.static_cache.validate()?; + self.upstream_timeouts.validate()?; + + if let Some(token) = self.shadow_probe_token.as_deref() { + validate_shadow_probe_token(token)?; + } + + if self.trust_x_forwarded_for + && !read_bool_env( + "GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED", + false, + ) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "启用 GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=true 前必须同时设置 GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=true，确认前置代理会清洗 X-Forwarded-For", + )); + } + + if let Some(access_log_file) = &self.access_log_file { + validate_access_log_file(access_log_file)?; + } + + self.protection.validate()?; + validate_instance_protection_boundary( + self.instance_count, + self.protection.enabled, + self.shared_protection_confirmed, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct CompressionConfig { + gzip: bool, + raw_algorithms: String, +} + +impl CompressionConfig { + fn from_env() -> io::Result { + let raw_algorithms = + read_env_or_default("GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS", "gzip"); + Self::parse(raw_algorithms) + } + + fn parse(raw_algorithms: String) -> io::Result { + let mut gzip = false; + + for algorithm in raw_algorithms + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + match algorithm.to_ascii_lowercase().as_str() { + "gzip" => gzip = true, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS 当前只支持 gzip，收到 {algorithm:?}" + ), + )); + } + } + } + + Ok(Self { + gzip, + raw_algorithms, }) } + + fn validate(&self) -> io::Result<()> { + if !self.gzip && !self.raw_algorithms.trim().is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS 未启用任何受支持算法", + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct UpstreamTimeoutConfig { + connect_timeout_ms: u64, + default_read_timeout_seconds: u64, + api_read_timeout_seconds: u64, + long_read_timeout_seconds: u64, + write_timeout_seconds: u64, +} + +impl UpstreamTimeoutConfig { + fn from_env() -> io::Result { + Ok(Self { + connect_timeout_ms: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS", + DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + )?, + default_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + )?, + api_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + )?, + long_read_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + )?, + write_timeout_seconds: read_u64_env( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS", + DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + )?, + }) + } + + fn validate(&self) -> io::Result<()> { + for (name, value) in [ + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS", + self.connect_timeout_ms, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS", + self.default_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS", + self.api_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS", + self.long_read_timeout_seconds, + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS", + self.write_timeout_seconds, + ), + ] { + if value == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} 必须大于 0"), + )); + } + } + + Ok(()) + } + + fn read_timeout_for_route(&self, _route: &RouteDecision, path: &str) -> Duration { + let seconds = if is_long_upstream_route(path) { + self.long_read_timeout_seconds + } else if is_generic_api_proxy_path(path) { + self.api_read_timeout_seconds + } else { + self.default_read_timeout_seconds + }; + Duration::from_secs(seconds) + } + + fn connect_timeout(&self) -> Duration { + Duration::from_millis(self.connect_timeout_ms) + } + + fn write_timeout(&self) -> Duration { + Duration::from_secs(self.write_timeout_seconds) + } +} + +#[derive(Clone, Debug)] +struct StaticCacheConfig { + html_cache_control: String, + asset_cache_control: String, + static_cache_control: String, +} + +impl StaticCacheConfig { + fn from_env() -> Self { + Self { + html_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL", + DEFAULT_HTML_CACHE_CONTROL, + ), + asset_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL", + DEFAULT_ASSET_CACHE_CONTROL, + ), + static_cache_control: read_env_or_default( + "GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL", + DEFAULT_STATIC_CACHE_CONTROL, + ), + } + } + + fn validate(&self) -> io::Result<()> { + for (name, value) in [ + ( + "GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL", + self.html_cache_control.as_str(), + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL", + self.asset_cache_control.as_str(), + ), + ( + "GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL", + self.static_cache_control.as_str(), + ), + ] { + if value.contains('\n') || value.contains('\r') || value.contains('\0') { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{name} 不能包含换行或 NUL 字符"), + )); + } + } + + Ok(()) + } } #[derive(Clone)] struct GenarrativeGateway { config: Arc, + protection: Arc, + mode: GatewayMode, } #[derive(Debug)] struct RequestContext { request_id: String, route: RouteDecision, + request_body_bytes_seen: u64, + protection_class: Option, + protection_client: String, + protection_key: Option, started_at: Instant, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GatewayMode { + Proxy, + HttpRedirect, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CompressionAlgorithm { + Gzip, +} + +impl CompressionAlgorithm { + fn header_value(self) -> &'static str { + match self { + Self::Gzip => "gzip", + } + } +} + +#[derive(Clone, Copy)] +struct CompressionRequestConfig { + enabled: bool, + gzip: bool, + min_length_bytes: u64, + gzip_level: u32, +} + +struct GatewayResponseCompressionBuilder { + config: CompressionRequestConfig, +} + +struct GatewayResponseCompression { + config: CompressionRequestConfig, + inner: ResponseCompressionCtx, +} + +impl HttpModuleBuilder for GatewayResponseCompressionBuilder { + fn init(&self) -> Module { + Box::new(GatewayResponseCompression { + config: self.config, + inner: ResponseCompressionCtx::new(self.config.gzip_level, false, false), + }) + } + + fn order(&self) -> i16 { + i16::MIN / 2 + } +} + +#[async_trait] +impl HttpModule for GatewayResponseCompression { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + async fn request_header_filter(&mut self, req: &mut RequestHeader) -> PingoraResult<()> { + normalize_accept_encoding_for_gateway_compression(req, self.config)?; + self.inner.request_filter(req); + Ok(()) + } + + async fn response_header_filter( + &mut self, + resp: &mut ResponseHeader, + _end_of_stream: bool, + ) -> PingoraResult<()> { + if !should_allow_gateway_compression_for_response(resp, self.config.min_length_bytes) { + self.inner.adjust_level(0); + } + self.inner.response_header_filter(resp, _end_of_stream); + Ok(()) + } + + fn response_body_filter( + &mut self, + body: &mut Option, + end_of_stream: bool, + ) -> PingoraResult<()> { + if !self.inner.is_enabled() { + return Ok(()); + } + if let Some(compressed) = self + .inner + .response_body_filter(body.as_ref(), end_of_stream) + { + *body = Some(compressed); + } + Ok(()) + } + + fn response_done_filter(&mut self) -> PingoraResult> { + if !self.inner.is_enabled() { + return Ok(None); + } + Ok(self.inner.response_body_filter(None, true)) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ProxyTarget { Api, Spacetime, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum ProtectionClass { + AdminApi, + GalleryList, + GalleryDetail, + Api, + Spacetime, +} + +impl ProtectionClass { + fn as_str(self) -> &'static str { + match self { + ProtectionClass::AdminApi => "admin_api", + ProtectionClass::GalleryList => "gallery_list", + ProtectionClass::GalleryDetail => "gallery_detail", + ProtectionClass::Api => "api", + ProtectionClass::Spacetime => "spacetime", + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RejectReason { + Concurrent, + Rate, +} + +impl RejectReason { + fn code(self) -> &'static str { + match self { + RejectReason::Concurrent => "GATEWAY_CONCURRENCY_LIMITED", + RejectReason::Rate => "GATEWAY_RATE_LIMITED", + } + } + + fn message(self) -> &'static str { + match self { + RejectReason::Concurrent => "服务繁忙，请稍后重试", + RejectReason::Rate => "请求过于频繁，请稍后重试", + } + } +} + +#[derive(Clone, Copy, Debug)] +struct ProtectionClassConfig { + max_concurrent: u32, + rate_per_second: u32, + burst: u32, +} + +#[derive(Clone, Debug)] +struct ProtectionConfig { + enabled: bool, + admin_api: ProtectionClassConfig, + gallery_list: ProtectionClassConfig, + gallery_detail: ProtectionClassConfig, + api: ProtectionClassConfig, + spacetime: ProtectionClassConfig, +} + +impl ProtectionConfig { + fn from_env() -> io::Result { + Ok(Self { + enabled: read_bool_env("GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED", true), + admin_api: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_ADMIN_API", + 64, + 30, + 16, + )?, + gallery_list: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST", + 320, + 5000, + 4096, + )?, + gallery_detail: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL", + 32, + 300, + 32, + )?, + api: ProtectionClassConfig::from_env("GENARRATIVE_PINGORA_GATEWAY_API", 64, 300, 64)?, + spacetime: ProtectionClassConfig::from_env( + "GENARRATIVE_PINGORA_GATEWAY_SPACETIME", + 256, + 1000, + 256, + )?, + }) + } + + fn class_config(&self, class: ProtectionClass) -> ProtectionClassConfig { + match class { + ProtectionClass::AdminApi => self.admin_api, + ProtectionClass::GalleryList => self.gallery_list, + ProtectionClass::GalleryDetail => self.gallery_detail, + ProtectionClass::Api => self.api, + ProtectionClass::Spacetime => self.spacetime, + } + } + + fn validate(&self) -> io::Result<()> { + for (name, config) in [ + ("admin_api", self.admin_api), + ("gallery_list", self.gallery_list), + ("gallery_detail", self.gallery_detail), + ("api", self.api), + ("spacetime", self.spacetime), + ] { + config.validate(name)?; + } + Ok(()) + } +} + +impl ProtectionClassConfig { + fn from_env( + prefix: &str, + max_concurrent: u32, + rate_per_second: u32, + burst: u32, + ) -> io::Result { + Ok(Self { + max_concurrent: read_u32_env(&format!("{prefix}_MAX_CONCURRENT"), max_concurrent)?, + rate_per_second: read_u32_env(&format!("{prefix}_RATE_PER_SECOND"), rate_per_second)?, + burst: read_u32_env(&format!("{prefix}_BURST"), burst)?, + }) + } + + fn is_disabled(&self) -> bool { + self.max_concurrent == 0 && self.rate_per_second == 0 + } + + fn bucket_capacity(&self) -> u32 { + self.rate_per_second.saturating_add(self.burst).max(1) + } + + fn validate(&self, name: &str) -> io::Result<()> { + if self.burst > 0 && self.rate_per_second == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "{name} 配置了 burst 但 RATE_PER_SECOND=0；若要关闭 RPS，请同时把 BURST 设为 0" + ), + )); + } + Ok(()) + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum StaticRoot { Web, @@ -116,6 +763,8 @@ enum StaticMode { #[derive(Clone, Debug, PartialEq, Eq)] enum LocalResponse { RedirectPermanent { location: &'static str }, + HttpToHttpsRedirect, + ShadowProbe, Static { root: StaticRoot, mode: StaticMode }, NotFound, } @@ -157,6 +806,141 @@ impl RouteDecision { RouteDecision::Local(_) => None, } } + + fn body_limit(&self) -> Option { + match self { + RouteDecision::Proxy { body_limit, .. } => *body_limit, + RouteDecision::Local(_) => None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +struct ProtectionKey { + class: ProtectionClass, + client: String, +} + +#[derive(Debug)] +struct ClientProtectionState { + in_flight: u32, + tokens: f64, + last_refill: Instant, + last_seen: Instant, +} + +#[derive(Debug)] +struct ProtectionState { + clients: HashMap, + last_cleanup: Instant, +} + +struct GatewayProtection { + config: ProtectionConfig, + state: Mutex, +} + +impl GatewayProtection { + fn new(config: ProtectionConfig) -> Self { + Self { + config, + state: Mutex::new(ProtectionState { + clients: HashMap::new(), + last_cleanup: Instant::now(), + }), + } + } + + fn try_acquire( + self: &Arc, + class: ProtectionClass, + client: String, + ) -> Result, RejectReason> { + if !self.config.enabled { + return Ok(None); + } + + let class_config = self.config.class_config(class); + if class_config.is_disabled() { + return Ok(None); + } + + let now = Instant::now(); + let key = ProtectionKey { class, client }; + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + cleanup_protection_state(&mut state, now); + + let entry = state + .clients + .entry(key.clone()) + .or_insert_with(|| ClientProtectionState { + in_flight: 0, + tokens: class_config.bucket_capacity() as f64, + last_refill: now, + last_seen: now, + }); + refill_tokens(entry, class_config, now); + entry.last_seen = now; + + if class_config.max_concurrent > 0 && entry.in_flight >= class_config.max_concurrent { + return Err(RejectReason::Concurrent); + } + + if class_config.rate_per_second > 0 && entry.tokens < 1.0 { + return Err(RejectReason::Rate); + } + + if class_config.rate_per_second > 0 { + entry.tokens -= 1.0; + } + entry.in_flight = entry.in_flight.saturating_add(1); + + Ok(Some(key)) + } + + fn release(&self, key: &ProtectionKey) { + let mut state = self + .state + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if let Some(entry) = state.clients.get_mut(key) { + entry.in_flight = entry.in_flight.saturating_sub(1); + entry.last_seen = Instant::now(); + } + } +} + +fn refill_tokens( + entry: &mut ClientProtectionState, + class_config: ProtectionClassConfig, + now: Instant, +) { + if class_config.rate_per_second == 0 { + entry.last_refill = now; + return; + } + + let elapsed = now.saturating_duration_since(entry.last_refill); + let refill = elapsed.as_secs_f64() * class_config.rate_per_second as f64; + if refill > 0.0 { + entry.tokens = (entry.tokens + refill).min(class_config.bucket_capacity() as f64); + entry.last_refill = now; + } +} + +fn cleanup_protection_state(state: &mut ProtectionState, now: Instant) { + if now.saturating_duration_since(state.last_cleanup) < PROTECTION_CLEANUP_INTERVAL { + return; + } + + state.clients.retain(|_, entry| { + entry.in_flight > 0 + || now.saturating_duration_since(entry.last_seen) <= PROTECTION_STATE_TTL + }); + state.last_cleanup = now; } fn main() -> std::result::Result<(), Box> { @@ -170,6 +954,8 @@ fn main() -> std::result::Result<(), Box> { info!( listen = %config.listen_addr, + tls_listen = ?config.tls_listen_addr, + http_redirect_listen = ?config.http_redirect_listen_addr, api_upstream = %config.api_upstream, spacetime_upstream = %config.spacetime_upstream, web_root = %config.web_root.display(), @@ -180,12 +966,45 @@ fn main() -> std::result::Result<(), Box> { let mut server = Server::new(Some(opt))?; server.bootstrap(); - let gateway = GenarrativeGateway { config }; + let protection = Arc::new(GatewayProtection::new(config.protection.clone())); + let gateway = GenarrativeGateway { + config: config.clone(), + protection: protection.clone(), + mode: GatewayMode::Proxy, + }; let listen_addr = gateway.config.listen_addr.clone(); let mut proxy = http_proxy_service(&server.configuration, gateway); proxy.add_tcp(&listen_addr); + if let Some(tls_listen_addr) = config.tls_listen_addr.as_deref() { + let cert_path = config + .tls_cert_file + .as_deref() + .expect("TLS cert file validated") + .to_string_lossy() + .into_owned(); + let key_path = config + .tls_key_file + .as_deref() + .expect("TLS key file validated") + .to_string_lossy() + .into_owned(); + let mut tls_settings = TlsSettings::intermediate(&cert_path, &key_path)?; + tls_settings.enable_h2(); + proxy.add_tls_with_settings(tls_listen_addr, None, tls_settings); + } server.add_service(proxy); + if let Some(http_redirect_listen_addr) = config.http_redirect_listen_addr.clone() { + let redirect_gateway = GenarrativeGateway { + config, + protection, + mode: GatewayMode::HttpRedirect, + }; + let mut redirect_service = http_proxy_service(&server.configuration, redirect_gateway); + redirect_service.add_tcp(&http_redirect_listen_addr); + server.add_service(redirect_service); + } + server.run_forever(); } @@ -193,10 +1012,30 @@ fn main() -> std::result::Result<(), Box> { impl ProxyHttp for GenarrativeGateway { type CTX = RequestContext; + fn init_downstream_modules(&self, modules: &mut HttpModules) { + let gzip_level = if self.config.gzip_enabled { + self.config.gzip_level + } else { + 0 + }; + modules.add_module(Box::new(GatewayResponseCompressionBuilder { + config: CompressionRequestConfig { + enabled: gzip_level > 0, + gzip: self.config.compression.gzip, + min_length_bytes: self.config.gzip_min_length_bytes, + gzip_level, + }, + })); + } + fn new_ctx(&self) -> Self::CTX { RequestContext { request_id: Uuid::new_v4().to_string(), route: RouteDecision::Local(LocalResponse::NotFound), + request_body_bytes_seen: 0, + protection_class: None, + protection_client: String::new(), + protection_key: None, started_at: Instant::now(), } } @@ -210,7 +1049,7 @@ impl ProxyHttp for GenarrativeGateway { Self::CTX: Send + Sync, { let path = session.req_header().uri.path(); - ctx.route = classify_path(path); + ctx.route = self.classify_request_path(path); apply_configured_body_limit(&mut ctx.route, self.config.max_api_body_bytes); ctx.request_id = resolve_request_id(session); @@ -224,36 +1063,97 @@ impl ProxyHttp for GenarrativeGateway { && let Some(content_length) = content_length(session) && content_length > limit { - respond_json( - session, - 413, - r#"{"ok":false,"error":{"code":"PAYLOAD_TOO_LARGE","message":"请求体过大"}}"#, - ) - .await?; + respond_json(session, 413, payload_too_large_body()).await?; return Ok(true); } + if let Some(protection_class) = protection_class_for_route(&ctx.route, path) { + let protection_client = + protection_client_id(session, self.config.trust_x_forwarded_for); + match self + .protection + .try_acquire(protection_class, protection_client.clone()) + { + Ok(key) => { + ctx.protection_class = Some(protection_class); + ctx.protection_client = protection_client; + ctx.protection_key = key; + } + Err(reason) => { + ctx.protection_class = Some(protection_class); + ctx.protection_client = protection_client; + respond_too_many_requests(session, reason).await?; + return Ok(true); + } + } + } + match &ctx.route { RouteDecision::Proxy { .. } => Ok(false), RouteDecision::Local(LocalResponse::RedirectPermanent { location }) => { respond_redirect(session, location).await?; Ok(true) } + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) => { + respond_http_to_https_redirect( + session, + self.config.http_redirect_target_scheme.as_str(), + ) + .await?; + Ok(true) + } + RouteDecision::Local(LocalResponse::ShadowProbe) => { + if !self.is_shadow_probe_authorized(session) { + respond_not_found(session).await?; + return Ok(true); + } + + respond_shadow_probe(session, self.is_maintenance_enabled()).await?; + Ok(true) + } RouteDecision::Local(LocalResponse::Static { root, mode }) => { let root_path = match root { StaticRoot::Web => &self.config.web_root, StaticRoot::Acme => &self.config.acme_root, }; - serve_static(session, root_path, *mode).await?; + serve_static(session, root_path, *root, *mode, &self.config.static_cache).await?; Ok(true) } RouteDecision::Local(LocalResponse::NotFound) => { - session.respond_error(404).await?; + respond_not_found(session).await?; Ok(true) } } } + async fn request_body_filter( + &self, + session: &mut Session, + body: &mut Option, + _end_of_stream: bool, + ctx: &mut Self::CTX, + ) -> PingoraResult<()> + where + Self::CTX: Send + Sync, + { + let Some(limit) = ctx.route.body_limit() else { + return Ok(()); + }; + + if let Some(body) = body.as_ref() { + ctx.request_body_bytes_seen = ctx + .request_body_bytes_seen + .saturating_add(body.len() as u64); + } + + if ctx.request_body_bytes_seen > limit { + respond_json(session, 413, payload_too_large_body()).await?; + return Error::e_explain(HTTPStatus(413), PAYLOAD_TOO_LARGE_CONTEXT); + } + + Ok(()) + } + async fn upstream_peer( &self, _session: &mut Session, @@ -264,7 +1164,16 @@ impl ProxyHttp for GenarrativeGateway { ProxyTarget::Spacetime => self.config.spacetime_upstream, }; - Ok(Box::new(HttpPeer::new(upstream, false, String::new()))) + let mut peer = HttpPeer::new(upstream, false, String::new()); + peer.options.connection_timeout = Some(self.config.upstream_timeouts.connect_timeout()); + peer.options.read_timeout = Some( + self.config + .upstream_timeouts + .read_timeout_for_route(&ctx.route, _session.req_header().uri.path()), + ); + peer.options.write_timeout = Some(self.config.upstream_timeouts.write_timeout()); + + Ok(Box::new(peer)) } async fn upstream_request_filter( @@ -280,7 +1189,7 @@ impl ProxyHttp for GenarrativeGateway { upstream_request .insert_header("X-Forwarded-Proto", self.config.forwarded_proto.as_str())?; - if let Some(host) = session.req_header().headers.get("host").cloned() { + if let Some(host) = host_or_authority(session) { upstream_request.insert_header("Host", host.clone())?; upstream_request.insert_header("X-Forwarded-Host", host)?; } @@ -305,12 +1214,15 @@ impl ProxyHttp for GenarrativeGateway { &self, _session: &mut Session, upstream_response: &mut ResponseHeader, - _ctx: &mut Self::CTX, + ctx: &mut Self::CTX, ) -> PingoraResult<()> where Self::CTX: Send + Sync, { upstream_response.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + if should_disable_accel_buffering(&ctx.route) { + upstream_response.insert_header("X-Accel-Buffering", "no")?; + } Ok(()) } @@ -328,40 +1240,483 @@ impl ProxyHttp for GenarrativeGateway { let elapsed_ms = ctx.started_at.elapsed().as_millis(); let method = session.req_header().method.as_str(); let path = session.req_header().uri.path(); + let uri = session.req_header().uri.to_string(); + let host = header_value(session, "host").unwrap_or_default(); + let client_ip = client_ip(session).unwrap_or_default(); + let content_length = content_length(session); + let protection_class = ctx.protection_class.map(|class| class.as_str()); + let protection_client = ctx.protection_client.as_str(); + let proxy_target = ctx + .route + .proxy_target() + .map(|target| format!("{target:?}")) + .unwrap_or_else(|| "Local".to_string()); + let upstream = match ctx.route.proxy_target() { + Some(ProxyTarget::Api) => self.config.api_upstream.to_string(), + Some(ProxyTarget::Spacetime) => self.config.spacetime_upstream.to_string(), + None => String::new(), + }; let route = format!("{:?}", ctx.route); + if let Some(key) = ctx.protection_key.take() { + self.protection.release(&key); + } - if let Some(error) = error { - error!( - request_id = %ctx.request_id, + write_access_log( + self.config.access_log_file.as_deref(), + AccessLogRecord { + request_id: &ctx.request_id, method, path, + uri: &uri, + host: &host, + client_ip: &client_ip, status, - route, + route: &route, + proxy_target: &proxy_target, + upstream: &upstream, + content_length, + body_bytes_seen: ctx.request_body_bytes_seen, + protection_class, + protection_client, elapsed_ms, - %error, - "Pingora gateway request failed" - ); + error: error.map(ToString::to_string), + }, + ); + + if let Some(error) = error { + if is_payload_too_large_error(error) { + warn!( + request_id = %ctx.request_id, + method, + path, + uri = %uri, + host = %host, + client_ip = %client_ip, + status, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, + elapsed_ms, + %error, + "Pingora gateway request rejected" + ); + } else { + error!( + request_id = %ctx.request_id, + method, + path, + uri = %uri, + host = %host, + client_ip = %client_ip, + status, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, + elapsed_ms, + %error, + "Pingora gateway request failed" + ); + } } else { info!( request_id = %ctx.request_id, method, path, + uri = %uri, + host = %host, + client_ip = %client_ip, status, - route, + route = %route, + proxy_target = %proxy_target, + upstream = %upstream, + content_length = ?content_length, + body_bytes_seen = ctx.request_body_bytes_seen, + protection_class = ?protection_class, + protection_client, elapsed_ms, "Pingora gateway request completed" ); } } + + fn suppress_error_log( + &self, + _session: &Session, + _ctx: &Self::CTX, + error: &pingora::Error, + ) -> bool { + is_payload_too_large_error(error) + } + + async fn fail_to_proxy( + &self, + session: &mut Session, + error: &pingora::Error, + ctx: &mut Self::CTX, + ) -> FailToProxy + where + Self::CTX: Send + Sync, + { + if is_payload_too_large_error(error) { + if session.response_written().is_none() { + respond_json(session, 413, payload_too_large_body()) + .await + .unwrap_or_else(|error| { + warn!(%error, "请求体超限响应写入失败"); + }); + } + + return FailToProxy { + error_code: 413, + can_reuse_downstream: false, + }; + } + + let status = match error.etype() { + ConnectTimedout | ReadTimedout | WriteTimedout => 504, + HTTPStatus(code) => *code, + _ => match error.esource() { + ErrorSource::Upstream => 502, + ErrorSource::Downstream => match error.etype() { + WriteError | ReadError | ConnectionClosed => 0, + _ => 400, + }, + ErrorSource::Internal | ErrorSource::Unset => 500, + }, + }; + if status > 0 { + let response_result = if ctx.route.proxy_target().is_some() { + respond_gateway_proxy_error(session, status).await + } else { + session.respond_error(status).await + }; + response_result.unwrap_or_else(|error| { + warn!(%error, "Pingora 错误响应写入失败"); + }); + } + + FailToProxy { + error_code: status, + can_reuse_downstream: false, + } + } +} + +struct AccessLogRecord<'a> { + request_id: &'a str, + method: &'a str, + path: &'a str, + uri: &'a str, + host: &'a str, + client_ip: &'a str, + status: u16, + route: &'a str, + proxy_target: &'a str, + upstream: &'a str, + content_length: Option, + body_bytes_seen: u64, + protection_class: Option<&'static str>, + protection_client: &'a str, + elapsed_ms: u128, + error: Option, +} + +fn write_access_log(path: Option<&Path>, record: AccessLogRecord<'_>) { + let Some(path) = path else { + return; + }; + + if let Some(parent) = path.parent() + && let Err(error) = std::fs::create_dir_all(parent) + { + warn!( + file = %path.display(), + %error, + "Pingora access log 目录创建失败" + ); + return; + } + + let line = format_access_log_line(&record); + match OpenOptions::new().create(true).append(true).open(path) { + Ok(mut file) => { + if let Err(error) = file.write_all(line.as_bytes()) { + warn!( + file = %path.display(), + %error, + "Pingora access log 写入失败" + ); + } + } + Err(error) => { + warn!( + file = %path.display(), + %error, + "Pingora access log 打开失败" + ); + } + } +} + +fn format_access_log_line(record: &AccessLogRecord<'_>) -> String { + format!( + concat!( + "request_id={}\tmethod={}\tpath={}\turi={}\thost={}\tclient_ip={}\tstatus={}\t", + "route={}\tproxy_target={}\tupstream={}\tcontent_length={}\tbody_bytes_seen={}\t", + "protection_class={}\tprotection_client={}\telapsed_ms={}\terror={}\n" + ), + escape_access_log_value(record.request_id), + escape_access_log_value(record.method), + escape_access_log_value(record.path), + escape_access_log_value(record.uri), + escape_access_log_value(record.host), + escape_access_log_value(record.client_ip), + record.status, + escape_access_log_value(record.route), + escape_access_log_value(record.proxy_target), + escape_access_log_value(record.upstream), + record + .content_length + .map(|value| value.to_string()) + .unwrap_or_else(|| "-".to_string()), + record.body_bytes_seen, + record.protection_class.unwrap_or("-"), + escape_access_log_value(record.protection_client), + record.elapsed_ms, + record + .error + .as_deref() + .map(escape_access_log_value) + .unwrap_or_else(|| "-".to_string()) + ) +} + +fn escape_access_log_value(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('\t', "\\t") + .replace('\n', "\\n") + .replace('\r', "\\r") +} + +fn should_disable_accel_buffering(route: &RouteDecision) -> bool { + route.proxy_target() == Some(ProxyTarget::Api) +} + +fn is_long_upstream_route(path: &str) -> bool { + is_spacetime_subscribe_path(path) || is_gallery_list_path(path) || is_gallery_detail_path(path) +} + +fn is_generic_api_proxy_path(path: &str) -> bool { + path == "/api" || path.starts_with("/api/") +} + +fn protection_class_for_route(route: &RouteDecision, path: &str) -> Option { + match route { + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if path.starts_with("/admin/api/") => Some(ProtectionClass::AdminApi), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if is_gallery_list_path(path) => Some(ProtectionClass::GalleryList), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } if is_gallery_detail_path(path) => Some(ProtectionClass::GalleryDetail), + RouteDecision::Proxy { + target: ProxyTarget::Api, + .. + } => Some(ProtectionClass::Api), + RouteDecision::Proxy { + target: ProxyTarget::Spacetime, + .. + } => Some(ProtectionClass::Spacetime), + RouteDecision::Local(_) => None, + } +} + +fn validate_shadow_probe_token(token: &str) -> io::Result<()> { + if token == "__GENARRATIVE_PINGORA_PROBE_TOKEN__" + || token.eq_ignore_ascii_case("changeme") + || token.len() < 16 + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN 必须使用 16 字符以上的非占位 token", + )); + } + Ok(()) +} + +fn validate_access_log_file(path: &Path) -> io::Result<()> { + if path.as_os_str().is_empty() { + return Ok(()); + } + + if path.exists() && path.is_dir() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!( + "GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE 不能指向目录：{}", + path.display() + ), + )); + } + + Ok(()) +} + +fn validate_instance_protection_boundary( + instance_count: u32, + protection_enabled: bool, + shared_protection_confirmed: bool, +) -> io::Result<()> { + if instance_count == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT 必须大于 0", + )); + } + + if protection_enabled && instance_count > 1 && !shared_protection_confirmed { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Pingora 接流保护当前默认是进程内状态；GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1 时必须设置 GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true，或关闭网关保护并由前置层承担", + )); + } + + Ok(()) +} + +fn parse_socket_addr_config(key: &str, value: &str) -> io::Result { + value.parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效 socket 地址 {value:?}：{error}"), + ) + }) +} + +fn validate_distinct_listen_addr( + key: &str, + listen_addr: SocketAddr, + primary_listen_addr: SocketAddr, + api_upstream: SocketAddr, + spacetime_upstream: SocketAddr, +) -> io::Result<()> { + if listen_addr == primary_listen_addr { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不能与 GENARRATIVE_PINGORA_GATEWAY_LISTEN 相同"), + )); + } + + if listen_addr == api_upstream || listen_addr == spacetime_upstream { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不能与上游地址相同"), + )); + } + + Ok(()) +} + +fn validate_tls_file_config( + cert_file: &Option, + key_file: &Option, +) -> io::Result<()> { + let cert_file = cert_file.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN 时必须设置 GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE", + ) + })?; + let key_file = key_file.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "配置 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN 时必须设置 GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE", + ) + })?; + + validate_existing_file( + "GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE", + cert_file.as_path(), + )?; + validate_existing_file( + "GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE", + key_file.as_path(), + )?; + Ok(()) +} + +fn validate_existing_file(key: &str, path: &Path) -> io::Result<()> { + let metadata = std::fs::metadata(path).map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 无法读取 {}：{error}", path.display()), + ) + })?; + if !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 必须指向文件：{}", path.display()), + )); + } + Ok(()) +} + +fn validate_redirect_target_scheme(scheme: &str) -> io::Result<()> { + if scheme == "https" { + return Ok(()); + } + + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME 当前只允许 https", + )) } impl GenarrativeGateway { fn is_maintenance_enabled(&self) -> bool { self.config.maintenance_file.exists() } + + fn is_shadow_probe_authorized(&self, session: &Session) -> bool { + let Some(expected) = self.config.shadow_probe_token.as_deref() else { + return false; + }; + + session + .req_header() + .headers + .get(SHADOW_PROBE_HEADER) + .and_then(|value| value.to_str().ok()) + .is_some_and(|actual| actual == expected) + } + + fn classify_request_path(&self, path: &str) -> RouteDecision { + match self.mode { + GatewayMode::Proxy => classify_path(path), + GatewayMode::HttpRedirect => classify_http_redirect_path(path), + } + } } fn classify_path(path: &str) -> RouteDecision { + if path == SHADOW_PROBE_PATH { + return RouteDecision::Local(LocalResponse::ShadowProbe); + } + if path == "/admin" { return RouteDecision::Local(LocalResponse::RedirectPermanent { location: "/admin/", @@ -439,6 +1794,17 @@ fn classify_path(path: &str) -> RouteDecision { }) } +fn classify_http_redirect_path(path: &str) -> RouteDecision { + if path.starts_with("/.well-known/acme-challenge/") { + return RouteDecision::Local(LocalResponse::Static { + root: StaticRoot::Acme, + mode: StaticMode::Exact, + }); + } + + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) +} + fn apply_configured_body_limit(route: &mut RouteDecision, max_api_body_bytes: u64) { if let RouteDecision::Proxy { target: ProxyTarget::Api, @@ -449,6 +1815,30 @@ fn apply_configured_body_limit(route: &mut RouteDecision, max_api_body_bytes: u6 } } +fn payload_too_large_body() -> &'static str { + r#"{"ok":false,"error":{"code":"PAYLOAD_TOO_LARGE","message":"请求体过大"}}"# +} + +fn gateway_proxy_error_body(status: u16) -> &'static str { + match status { + 502 => { + r#"{"ok":false,"error":{"code":"GATEWAY_UPSTREAM_ERROR","message":"上游服务不可用"}}"# + } + 504 => { + r#"{"ok":false,"error":{"code":"GATEWAY_UPSTREAM_TIMEOUT","message":"上游服务请求超时"}}"# + } + _ => r#"{"ok":false,"error":{"code":"GATEWAY_PROXY_ERROR","message":"网关代理失败"}}"#, + } +} + +fn is_payload_too_large_error(error: &pingora::Error) -> bool { + matches!(error.etype(), HTTPStatus(413)) + && error + .context + .as_ref() + .is_some_and(|context| context.as_str() == PAYLOAD_TOO_LARGE_CONTEXT) +} + fn is_gallery_list_path(path: &str) -> bool { matches!( path, @@ -470,44 +1860,353 @@ fn is_spacetime_subscribe_path(path: &str) -> bool { matches!(parts.as_slice(), ["v1", "database", _, "subscribe"]) } -async fn serve_static(session: &mut Session, root: &Path, mode: StaticMode) -> PingoraResult<()> { +struct StaticCandidate { + path: PathBuf, + cache_kind: StaticCacheKind, +} + +struct StaticResponseMetadata { + len: u64, + etag: String, + last_modified: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticRangeDecision { + Full, + Partial { start: u64, end: u64 }, + Unsatisfiable, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum StaticCacheKind { + Html, + FingerprintedAsset, + Other, +} + +async fn serve_static( + session: &mut Session, + root: &Path, + root_kind: StaticRoot, + mode: StaticMode, + cache_config: &StaticCacheConfig, +) -> PingoraResult<()> { let path = session.req_header().uri.path(); - let Some(candidate) = resolve_static_candidate(root, path, mode).await else { - session.respond_error(404).await?; + let Some(candidate) = resolve_static_candidate(root, root_kind, path, mode).await else { + respond_not_found(session).await?; return Ok(()); }; + if !is_static_read_method(&session.req_header().method) { + respond_static_method_not_allowed(session).await?; + return Ok(()); + } - let body = match tokio::fs::read(&candidate).await { + let metadata = match static_response_metadata(&candidate.path).await { + Ok(metadata) => metadata, + Err(error) => { + warn!( + path = %path, + file = %candidate.path.display(), + %error, + "静态文件 metadata 读取失败" + ); + respond_not_found(session).await?; + return Ok(()); + } + }; + let content_type = mime_guess::from_path(&candidate.path) + .first_or_octet_stream() + .essence_str() + .to_string(); + let cache_control = cache_control_for_static_candidate(&candidate, cache_config); + let last_modified = metadata.last_modified.map(fmt_http_date); + let mut headers = vec![ + ("Cache-Control".to_string(), cache_control.to_string()), + ("ETag".to_string(), metadata.etag.clone()), + ("Accept-Ranges".to_string(), "bytes".to_string()), + ]; + if let Some(last_modified) = last_modified.as_deref() { + headers.push(("Last-Modified".to_string(), last_modified.to_string())); + } + + if static_not_modified(session, &metadata) { + return respond_empty_with_headers(session, 304, Some(&content_type), &headers).await; + } + + match static_range_decision(session, &metadata) { + StaticRangeDecision::Full => {} + StaticRangeDecision::Partial { start, end } => { + let range_len = end - start + 1; + let mut range_headers = headers.clone(); + range_headers.push(( + "Content-Range".to_string(), + format!("bytes {start}-{end}/{}", metadata.len), + )); + if session.req_header().method == Method::HEAD { + return respond_head_with_headers( + session, + 206, + Some(&content_type), + &range_headers, + range_len, + ) + .await; + } + + let body = match read_static_range(&candidate.path, start, range_len).await { + Ok(bytes) => bytes, + Err(error) => { + warn!( + path = %path, + file = %candidate.path.display(), + %error, + "静态文件 Range 读取失败" + ); + respond_not_found(session).await?; + return Ok(()); + } + }; + + return respond_bytes_with_headers( + session, + 206, + Some(&content_type), + &range_headers, + body, + ) + .await; + } + StaticRangeDecision::Unsatisfiable => { + let mut range_headers = headers.clone(); + range_headers.push(( + "Content-Range".to_string(), + format!("bytes */{}", metadata.len), + )); + return respond_head_with_headers(session, 416, Some(&content_type), &range_headers, 0) + .await; + } + } + + if session.req_header().method == Method::HEAD { + return respond_head_with_headers( + session, + 200, + Some(&content_type), + &headers, + metadata.len, + ) + .await; + } + + let body = match tokio::fs::read(&candidate.path).await { Ok(bytes) => Bytes::from(bytes), Err(error) => { warn!( path = %path, - file = %candidate.display(), + file = %candidate.path.display(), %error, "静态文件读取失败" ); - session.respond_error(404).await?; + respond_not_found(session).await?; return Ok(()); } }; - let content_type = mime_guess::from_path(&candidate) - .first_or_octet_stream() - .essence_str() - .to_string(); - respond_bytes(session, 200, Some(&content_type), body).await + respond_bytes_with_headers(session, 200, Some(&content_type), &headers, body).await } -async fn resolve_static_candidate(root: &Path, path: &str, mode: StaticMode) -> Option { +fn is_static_read_method(method: &Method) -> bool { + *method == Method::GET || *method == Method::HEAD +} + +async fn read_static_range(path: &Path, start: u64, len: u64) -> io::Result { + let mut file = tokio::fs::File::open(path).await?; + file.seek(SeekFrom::Start(start)).await?; + let mut reader = file.take(len); + let capacity = usize::try_from(len) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Range 长度超过平台上限"))?; + let mut body = Vec::with_capacity(capacity); + reader.read_to_end(&mut body).await?; + Ok(Bytes::from(body)) +} + +async fn static_response_metadata(path: &Path) -> io::Result { + let metadata = tokio::fs::metadata(path).await?; + let len = metadata.len(); + let last_modified = metadata.modified().ok(); + Ok(StaticResponseMetadata { + len, + etag: build_static_etag(len, last_modified), + last_modified, + }) +} + +fn build_static_etag(len: u64, last_modified: Option) -> String { + let modified_secs = last_modified + .and_then(|modified| modified.duration_since(UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_secs()); + format!("W/\"{len:x}-{modified_secs:x}\"") +} + +fn static_not_modified(session: &Session, metadata: &StaticResponseMetadata) -> bool { + if session.req_header().method != Method::GET && session.req_header().method != Method::HEAD { + return false; + } + + if let Some(if_none_match) = header_value(session, "if-none-match") { + return if_none_match_matches(&if_none_match, &metadata.etag); + } + + let Some(last_modified) = metadata.last_modified else { + return false; + }; + header_value(session, "if-modified-since") + .and_then(|value| parse_http_date(&value).ok()) + .is_some_and(|since| truncate_system_time_to_seconds(last_modified) <= since) +} + +fn static_range_decision( + session: &Session, + metadata: &StaticResponseMetadata, +) -> StaticRangeDecision { + if session.req_header().method != Method::GET && session.req_header().method != Method::HEAD { + return StaticRangeDecision::Full; + } + + let Some(range) = header_value(session, "range") else { + return StaticRangeDecision::Full; + }; + if !static_if_range_matches(session, metadata) { + return StaticRangeDecision::Full; + } + parse_static_range(&range, metadata.len) +} + +fn static_if_range_matches(session: &Session, metadata: &StaticResponseMetadata) -> bool { + let Some(if_range) = header_value(session, "if-range") else { + return true; + }; + static_if_range_value_matches(&if_range, metadata) +} + +fn static_if_range_value_matches(if_range: &str, metadata: &StaticResponseMetadata) -> bool { + if let Ok(if_range_date) = parse_http_date(&if_range) { + return metadata.last_modified.is_some_and(|last_modified| { + truncate_system_time_to_seconds(last_modified) <= if_range_date + }); + } + + strong_etag_opaque_value(&if_range) + .is_some_and(|candidate| strong_etag_opaque_value(&metadata.etag) == Some(candidate)) +} + +fn parse_static_range(range: &str, file_len: u64) -> StaticRangeDecision { + let range = range.trim(); + let Some(unit) = range.get(..6) else { + return StaticRangeDecision::Full; + }; + if !unit.eq_ignore_ascii_case("bytes=") { + return StaticRangeDecision::Full; + } + let spec = &range[6..]; + if spec.contains(',') { + return StaticRangeDecision::Full; + } + if file_len == 0 { + return StaticRangeDecision::Unsatisfiable; + } + + let Some((start, end)) = spec.split_once('-') else { + return StaticRangeDecision::Full; + }; + let start = start.trim(); + let end = end.trim(); + + if start.is_empty() { + let Ok(suffix_len) = end.parse::() else { + return StaticRangeDecision::Full; + }; + if suffix_len == 0 { + return StaticRangeDecision::Unsatisfiable; + } + let start = file_len.saturating_sub(suffix_len); + return StaticRangeDecision::Partial { + start, + end: file_len - 1, + }; + } + + let Ok(start) = start.parse::() else { + return StaticRangeDecision::Full; + }; + if start >= file_len { + return StaticRangeDecision::Unsatisfiable; + } + if end.is_empty() { + return StaticRangeDecision::Partial { + start, + end: file_len - 1, + }; + } + + let Ok(mut end) = end.parse::() else { + return StaticRangeDecision::Full; + }; + if start > end { + return StaticRangeDecision::Unsatisfiable; + } + end = end.min(file_len - 1); + StaticRangeDecision::Partial { start, end } +} + +fn if_none_match_matches(value: &str, etag: &str) -> bool { + let expected = weak_etag_opaque_value(etag); + value + .split(',') + .map(str::trim) + .any(|candidate| candidate == "*" || weak_etag_opaque_value(candidate) == expected) +} + +fn weak_etag_opaque_value(value: &str) -> &str { + value.strip_prefix("W/").unwrap_or(value) +} + +fn strong_etag_opaque_value(value: &str) -> Option<&str> { + if value.trim().starts_with("W/") { + return None; + } + Some(value.trim()) +} + +fn truncate_system_time_to_seconds(value: SystemTime) -> SystemTime { + value + .duration_since(UNIX_EPOCH) + .map(|duration| UNIX_EPOCH + Duration::from_secs(duration.as_secs())) + .unwrap_or(UNIX_EPOCH) +} + +async fn resolve_static_candidate( + root: &Path, + root_kind: StaticRoot, + path: &str, + mode: StaticMode, +) -> Option { let candidate = sanitize_static_path(root, path)?; if is_regular_file(&candidate).await { - return Some(candidate); + return Some(StaticCandidate { + cache_kind: classify_static_cache_kind(root_kind, path, &candidate), + path: candidate, + }); } if is_directory(&candidate).await { let index = candidate.join("index.html"); if is_regular_file(&index).await { - return Some(index); + return Some(StaticCandidate { + path: index, + cache_kind: StaticCacheKind::Html, + }); } } @@ -515,15 +2214,70 @@ async fn resolve_static_candidate(root: &Path, path: &str, mode: StaticMode) -> StaticMode::Exact => None, StaticMode::SpaFallback if path.starts_with("/admin/") => { let fallback = root.join("admin").join("index.html"); - is_regular_file(&fallback).await.then_some(fallback) + is_regular_file(&fallback).await.then_some(StaticCandidate { + path: fallback, + cache_kind: StaticCacheKind::Html, + }) } StaticMode::SpaFallback => { let fallback = root.join("index.html"); - is_regular_file(&fallback).await.then_some(fallback) + is_regular_file(&fallback).await.then_some(StaticCandidate { + path: fallback, + cache_kind: StaticCacheKind::Html, + }) } } } +fn classify_static_cache_kind( + root_kind: StaticRoot, + request_path: &str, + file_path: &Path, +) -> StaticCacheKind { + if is_html_path(file_path) { + return StaticCacheKind::Html; + } + + if root_kind == StaticRoot::Web + && (request_path.starts_with("/assets/") || request_path.starts_with("/admin/assets/")) + && has_fingerprinted_file_name(file_path) + { + return StaticCacheKind::FingerprintedAsset; + } + + StaticCacheKind::Other +} + +fn cache_control_for_static_candidate<'a>( + candidate: &StaticCandidate, + config: &'a StaticCacheConfig, +) -> &'a str { + match candidate.cache_kind { + StaticCacheKind::Html => config.html_cache_control.as_str(), + StaticCacheKind::FingerprintedAsset => config.asset_cache_control.as_str(), + StaticCacheKind::Other => config.static_cache_control.as_str(), + } +} + +fn is_html_path(path: &Path) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("html")) +} + +fn has_fingerprinted_file_name(path: &Path) -> bool { + let Some(file_stem) = path.file_stem().and_then(|value| value.to_str()) else { + return false; + }; + file_stem + .rsplit_once('-') + .is_some_and(|(_, suffix)| is_asset_fingerprint(suffix)) +} + +fn is_asset_fingerprint(value: &str) -> bool { + value.len() >= 8 && value.bytes().all(|byte| byte.is_ascii_alphanumeric()) +} + fn sanitize_static_path(root: &Path, path: &str) -> Option { let mut result = root.to_path_buf(); for component in Path::new(path.trim_start_matches('/')).components() { @@ -582,6 +2336,36 @@ async fn respond_maintenance( async fn respond_redirect(session: &mut Session, location: &str) -> PingoraResult<()> { let mut header = ResponseHeader::build(301, Some(0))?; header.insert_header("Location", location)?; + header.insert_header("Content-Length", "0")?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_http_to_https_redirect( + session: &mut Session, + target_scheme: &str, +) -> PingoraResult<()> { + let Some(host) = host_or_authority(session).filter(|value| is_valid_redirect_host(value)) + else { + respond_json( + session, + 400, + r#"{"ok":false,"error":{"code":"BAD_REQUEST","message":"无效请求"}}"#, + ) + .await?; + return Ok(()); + }; + let location = format!( + "{target_scheme}://{host}{}", + session + .req_header() + .uri + .path_and_query() + .map_or("/", |value| value.as_str()) + ); + let mut header = ResponseHeader::build(301, Some(0))?; + header.insert_header("Location", location.as_str())?; + header.insert_header("Content-Length", "0")?; + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; session.write_response_header(Box::new(header), true).await } @@ -595,17 +2379,93 @@ async fn respond_json(session: &mut Session, status: u16, body: &str) -> Pingora .await } +async fn respond_not_found(session: &mut Session) -> PingoraResult<()> { + respond_bytes(session, 404, None, Bytes::new()).await +} + +async fn respond_static_method_not_allowed(session: &mut Session) -> PingoraResult<()> { + respond_head_with_headers( + session, + 405, + None, + &[("Allow".to_string(), "GET, HEAD".to_string())], + 0, + ) + .await +} + +async fn respond_gateway_proxy_error(session: &mut Session, status: u16) -> PingoraResult<()> { + respond_json(session, status, gateway_proxy_error_body(status)).await +} + +async fn respond_too_many_requests( + session: &mut Session, + reason: RejectReason, +) -> PingoraResult<()> { + let body = format!( + r#"{{"ok":false,"error":{{"code":"{}","message":"{}"}}}}"#, + reason.code(), + reason.message() + ); + let is_head = session.req_header().method == Method::HEAD; + let mut header = ResponseHeader::build(429, Some(body.len()))?; + header.insert_header("Content-Length", body.len().to_string())?; + header.insert_header("Content-Type", "application/json; charset=utf-8")?; + header.insert_header("Retry-After", "1")?; + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session + .write_response_header(Box::new(header), is_head) + .await?; + if !is_head { + session + .write_response_body(Some(Bytes::from(body)), true) + .await?; + } + Ok(()) +} + +async fn respond_shadow_probe( + session: &mut Session, + maintenance_enabled: bool, +) -> PingoraResult<()> { + let maintenance = if maintenance_enabled { "true" } else { "false" }; + let body = format!(r#"{{"ok":true,"gateway":"pingora-shadow","maintenance":{maintenance}}}"#); + respond_bytes( + session, + 200, + Some("application/json; charset=utf-8"), + Bytes::from(body), + ) + .await +} + async fn respond_bytes( session: &mut Session, status: u16, content_type: Option<&str>, body: Bytes, +) -> PingoraResult<()> { + respond_bytes_with_headers(session, status, content_type, &[], body).await +} + +async fn respond_bytes_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], + body: Bytes, ) -> PingoraResult<()> { let is_head = session.req_header().method == Method::HEAD; let mut header = ResponseHeader::build(status, Some(body.len()))?; + header.insert_header("Content-Length", body.len().to_string())?; if let Some(content_type) = content_type { header.insert_header("Content-Type", content_type)?; } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; session .write_response_header(Box::new(header), is_head || body.is_empty()) @@ -616,6 +2476,46 @@ async fn respond_bytes( Ok(()) } +async fn respond_head_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], + content_length: u64, +) -> PingoraResult<()> { + let mut header = ResponseHeader::build(status, Some(headers.len() + 3))?; + header.insert_header("Content-Length", content_length.to_string())?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session.write_response_header(Box::new(header), true).await +} + +async fn respond_empty_with_headers( + session: &mut Session, + status: u16, + content_type: Option<&str>, + headers: &[(String, String)], +) -> PingoraResult<()> { + let mut header = ResponseHeader::build(status, Some(headers.len() + 2))?; + if let Some(content_type) = content_type { + header.insert_header("Content-Type", content_type)?; + } + for (name, value) in headers { + if !value.is_empty() { + header.insert_header(name.clone(), value.as_str())?; + } + } + header.insert_header("X-Genarrative-Gateway", "pingora-shadow")?; + session.write_response_header(Box::new(header), true).await +} + fn resolve_request_id(session: &Session) -> String { session .req_header() @@ -628,20 +2528,42 @@ fn resolve_request_id(session: &Session) -> String { } fn content_length(session: &Session) -> Option { + header_value(session, "content-length")?.parse().ok() +} + +fn header_value(session: &Session, name: &str) -> Option { session .req_header() .headers - .get("content-length")? - .to_str() - .ok()? - .parse() - .ok() + .get(name) + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) +} + +fn host_or_authority(session: &Session) -> Option { + header_value(session, "host").or_else(|| { + session + .req_header() + .uri + .authority() + .map(|authority| authority.as_str().to_string()) + }) +} + +fn is_valid_redirect_host(host: &str) -> bool { + !host.is_empty() + && host.len() <= 255 + && host.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-' | b':' | b'[' | b']') + }) } fn client_ip(session: &Session) -> Option { session .as_downstream() .client_addr() + .and_then(|addr| addr.as_inet()) .map(|addr| addr.ip().to_string()) } @@ -656,6 +2578,26 @@ fn append_forwarded_for(session: &Session, client_ip: &str) -> String { .unwrap_or_else(|| client_ip.to_string()) } +fn protection_client_id(session: &Session, trust_x_forwarded_for: bool) -> String { + if trust_x_forwarded_for && let Some(forwarded_for) = first_forwarded_for(session) { + return forwarded_for; + } + + client_ip(session).unwrap_or_else(|| "unknown".to_string()) +} + +fn first_forwarded_for(session: &Session) -> Option { + session + .req_header() + .headers + .get("x-forwarded-for") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(',').next()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToOwned::to_owned) +} + fn is_upgrade_request(session: &Session) -> bool { session .req_header() @@ -665,6 +2607,85 @@ fn is_upgrade_request(session: &Session) -> bool { .is_some_and(|value| value.eq_ignore_ascii_case("websocket")) } +fn normalize_accept_encoding_for_gateway_compression( + req: &mut RequestHeader, + config: CompressionRequestConfig, +) -> PingoraResult<()> { + let algorithm = if config.enabled { + req.headers + .get("accept-encoding") + .and_then(|value| value.to_str().ok()) + .and_then(|value| select_compression_algorithm(value, config)) + } else { + None + }; + + if let Some(algorithm) = algorithm { + req.insert_header("Accept-Encoding", algorithm.header_value())?; + } else { + req.remove_header("accept-encoding"); + } + + Ok(()) +} + +fn should_allow_gateway_compression_for_response( + resp: &ResponseHeader, + min_length_bytes: u64, +) -> bool { + if resp.status.as_u16() != 200 + || resp.headers.get("content-range").is_some() + || resp.headers.get("content-encoding").is_some() + { + return false; + } + + resp.headers + .get("content-length") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .is_none_or(|content_length| content_length >= min_length_bytes) +} + +fn select_compression_algorithm( + value: &str, + config: CompressionRequestConfig, +) -> Option { + parse_accepted_encodings(value) + .into_iter() + .find_map(|coding| { + if coding.eq_ignore_ascii_case("gzip") && config.gzip { + Some(CompressionAlgorithm::Gzip) + } else { + None + } + }) +} + +fn parse_accepted_encodings(value: &str) -> Vec { + value + .split(',') + .filter_map(|item| { + let mut parts = item.split(';'); + let coding = parts.next()?.trim(); + if coding.is_empty() { + return None; + } + + let enabled = parts + .map(str::trim) + .filter_map(|param| param.split_once('=')) + .filter(|(name, _)| name.trim().eq_ignore_ascii_case("q")) + .all(|(_, value)| value.trim() != "0" && value.trim() != "0.0"); + if enabled { + Some(coding.to_string()) + } else { + None + } + }) + .collect() +} + fn read_env_or_default(key: &str, default_value: &str) -> String { env::var(key) .ok() @@ -672,6 +2693,13 @@ fn read_env_or_default(key: &str, default_value: &str) -> String { .unwrap_or_else(|| default_value.to_string()) } +fn read_optional_env(key: &str) -> Option { + env::var(key) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + fn read_bool_env(key: &str, default_value: bool) -> bool { env::var(key) .ok() @@ -683,6 +2711,18 @@ fn read_bool_env(key: &str, default_value: bool) -> bool { .unwrap_or(default_value) } +fn read_u32_env(key: &str, default_value: u32) -> io::Result { + match env::var(key) { + Ok(value) if !value.trim().is_empty() => value.trim().parse().map_err(|error| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("{key} 不是有效整数：{error}"), + ) + }), + _ => Ok(default_value), + } +} + fn read_u64_env(key: &str, default_value: u64) -> io::Result { match env::var(key) { Ok(value) if !value.trim().is_empty() => value.trim().parse().map_err(|error| { @@ -708,105 +2748,827 @@ fn read_socket_addr_env(key: &str, default_value: &str) -> io::Result, } - #[test] - fn applies_configured_body_limit_to_generic_api_routes() { - let mut route = classify_path("/api/creation-entry/config"); - apply_configured_body_limit(&mut route, 1024); + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RouteParityCase { + id: String, + sample_path: String, + expect: RouteExpectation, + } - assert_eq!( - route, - RouteDecision::Proxy { - target: ProxyTarget::Api, - body_limit: Some(1024), + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct RouteExpectation { + kind: String, + target: Option, + body_limit: Option, + root: Option, + mode: Option, + location: Option, + protection_class: Option, + } + + #[derive(Deserialize)] + #[serde(untagged)] + enum BodyLimitExpectation { + Named(String), + Bytes(u64), + } + + fn assert_route_matches_expectation(case: &RouteParityCase, route: &RouteDecision) { + match (case.expect.kind.as_str(), route) { + ("proxy", RouteDecision::Proxy { target, body_limit }) => { + assert_eq!( + Some(proxy_target_name(*target)), + case.expect.target.as_deref(), + "route parity proxy target mismatch: {} {}", + case.id, + case.sample_path + ); + assert_eq!( + *body_limit, + expected_body_limit(case.expect.body_limit.as_ref()), + "route parity body limit mismatch: {} {}", + case.id, + case.sample_path + ); } - ); + ("static", RouteDecision::Local(LocalResponse::Static { root, mode })) => { + assert_eq!( + Some(static_root_name(*root)), + case.expect.root.as_deref(), + "route parity static root mismatch: {} {}", + case.id, + case.sample_path + ); + assert_eq!( + Some(static_mode_name(*mode)), + case.expect.mode.as_deref(), + "route parity static mode mismatch: {} {}", + case.id, + case.sample_path + ); + } + ( + "redirect_permanent", + RouteDecision::Local(LocalResponse::RedirectPermanent { location }), + ) => { + assert_eq!( + Some(*location), + case.expect.location.as_deref(), + "route parity redirect location mismatch: {} {}", + case.id, + case.sample_path + ); + } + ("shadow_probe", RouteDecision::Local(LocalResponse::ShadowProbe)) => {} + ("not_found", RouteDecision::Local(LocalResponse::NotFound)) => {} + _ => panic!( + "route parity kind mismatch: {} {} expected {} got {:?}", + case.id, case.sample_path, case.expect.kind, route + ), + } + } + + fn proxy_target_name(target: ProxyTarget) -> &'static str { + match target { + ProxyTarget::Api => "api", + ProxyTarget::Spacetime => "spacetime", + } + } + + fn static_root_name(root: StaticRoot) -> &'static str { + match root { + StaticRoot::Web => "web", + StaticRoot::Acme => "acme", + } + } + + fn static_mode_name(mode: StaticMode) -> &'static str { + match mode { + StaticMode::Exact => "exact", + StaticMode::SpaFallback => "spa_fallback", + } + } + + fn expected_body_limit(expectation: Option<&BodyLimitExpectation>) -> Option { + match expectation { + None => None, + Some(BodyLimitExpectation::Bytes(bytes)) => Some(*bytes), + Some(BodyLimitExpectation::Named(name)) if name == "default" => { + Some(DEFAULT_MAX_API_BODY_BYTES) + } + Some(BodyLimitExpectation::Named(name)) => { + panic!("unknown route parity body limit expectation: {name}") + } + } + } + + fn test_gateway_config() -> GatewayConfig { + GatewayConfig { + listen_addr: "127.0.0.1:18081".to_string(), + tls_listen_addr: None, + tls_cert_file: None, + tls_key_file: None, + http_redirect_listen_addr: None, + http_redirect_target_scheme: "https".to_string(), + api_upstream: "127.0.0.1:8082".parse().unwrap(), + spacetime_upstream: "127.0.0.1:3101".parse().unwrap(), + web_root: PathBuf::from("/srv/genarrative/web"), + acme_root: PathBuf::from("/var/www/html"), + maintenance_file: PathBuf::from("/var/lib/genarrative/maintenance/enabled"), + forwarded_proto: "http".to_string(), + max_api_body_bytes: DEFAULT_MAX_API_BODY_BYTES, + gzip_enabled: true, + gzip_level: DEFAULT_GZIP_LEVEL, + gzip_min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + compression: CompressionConfig { + gzip: true, + raw_algorithms: "gzip".to_string(), + }, + static_cache: StaticCacheConfig { + html_cache_control: DEFAULT_HTML_CACHE_CONTROL.to_string(), + asset_cache_control: DEFAULT_ASSET_CACHE_CONTROL.to_string(), + static_cache_control: DEFAULT_STATIC_CACHE_CONTROL.to_string(), + }, + upstream_timeouts: UpstreamTimeoutConfig { + connect_timeout_ms: DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + }, + shadow_probe_token: None, + trust_x_forwarded_for: false, + instance_count: DEFAULT_INSTANCE_COUNT, + shared_protection_confirmed: false, + protection: ProtectionConfig { + enabled: true, + admin_api: ProtectionClassConfig { + max_concurrent: 64, + rate_per_second: 30, + burst: 16, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 320, + rate_per_second: 5000, + burst: 4096, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 32, + rate_per_second: 300, + burst: 32, + }, + api: ProtectionClassConfig { + max_concurrent: 64, + rate_per_second: 300, + burst: 64, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 256, + rate_per_second: 1000, + burst: 256, + }, + }, + log_filter: DEFAULT_LOG_FILTER.to_string(), + access_log_file: None, + otel_enabled: false, + } } #[test] - fn keeps_public_health_and_legacy_generated_paths_closed() { - assert_eq!( - classify_path("/healthz"), - RouteDecision::Local(LocalResponse::NotFound) - ); - assert_eq!( - classify_path("/readyz"), - RouteDecision::Local(LocalResponse::NotFound) - ); - assert_eq!( - classify_path("/healthz/internal"), - RouteDecision::Local(LocalResponse::NotFound) - ); - assert_eq!( - classify_path("/generated-puzzle-assets/foo.webp"), - RouteDecision::Local(LocalResponse::NotFound) - ); + fn matches_nginx_route_parity_matrix() { + let matrix: RouteParityMatrix = serde_json::from_str(ROUTE_PARITY_MATRIX_JSON) + .expect("route parity matrix should be valid JSON"); + assert_eq!(matrix.version, 1); + + for case in matrix.routes { + let route = classify_path(&case.sample_path); + assert_route_matches_expectation(&case, &route); + + let expected_protection_class = case.expect.protection_class.as_deref(); + let actual_protection_class = + protection_class_for_route(&route, &case.sample_path).map(ProtectionClass::as_str); + assert_eq!( + actual_protection_class, expected_protection_class, + "route parity protection class mismatch: {} {}", + case.id, case.sample_path + ); + } } #[test] - fn only_forwards_minimal_spacetime_public_routes() { - assert_eq!( - classify_path("/v1/database/genarrative/subscribe").proxy_target(), - Some(ProxyTarget::Spacetime) - ); - assert_eq!( - classify_path("/v1/identity").proxy_target(), - Some(ProxyTarget::Spacetime) - ); - assert_eq!( - classify_path("/v1/ping"), - RouteDecision::Local(LocalResponse::NotFound) - ); + fn applies_configured_body_limit_to_generic_api_routes_only() { + let mut generic_api = classify_path("/api/creation-entry/config"); + apply_configured_body_limit(&mut generic_api, 1024); + assert_eq!(generic_api.body_limit(), Some(1024)); + + let mut gallery = classify_path("/api/runtime/puzzle/gallery"); + apply_configured_body_limit(&mut gallery, 1024); + assert_eq!(gallery.body_limit(), None); } #[test] - fn classifies_static_shell_routes() { + fn http_redirect_mode_redirects_non_acme_routes_only() { assert_eq!( - classify_path("/admin"), - RouteDecision::Local(LocalResponse::RedirectPermanent { - location: "/admin/" - }) + classify_http_redirect_path("/api/creation-entry/config"), + RouteDecision::Local(LocalResponse::HttpToHttpsRedirect) ); assert_eq!( - classify_path("/admin/assets/index.js"), + classify_http_redirect_path("/.well-known/acme-challenge/token"), RouteDecision::Local(LocalResponse::Static { - root: StaticRoot::Web, + root: StaticRoot::Acme, mode: StaticMode::Exact, }) ); + } + + #[test] + fn identifies_streaming_payload_limit_errors() { + let error = Error::explain(HTTPStatus(413), PAYLOAD_TOO_LARGE_CONTEXT); + assert!(is_payload_too_large_error(&error)); + + let unrelated = Error::explain(HTTPStatus(413), "other"); + assert!(!is_payload_too_large_error(&unrelated)); + } + + #[test] + fn maps_gateway_proxy_errors_to_stable_json_codes() { + assert!(gateway_proxy_error_body(502).contains("GATEWAY_UPSTREAM_ERROR")); + assert!(gateway_proxy_error_body(504).contains("GATEWAY_UPSTREAM_TIMEOUT")); + assert!(gateway_proxy_error_body(500).contains("GATEWAY_PROXY_ERROR")); + } + + #[test] + fn formats_access_log_as_tab_separated_key_values() { + let line = format_access_log_line(&AccessLogRecord { + request_id: "req-1", + method: "GET", + path: "/api/test", + uri: "/api/test?a=1\t2", + host: "example.test", + client_ip: "127.0.0.1", + status: 200, + route: "Proxy", + proxy_target: "Api", + upstream: "127.0.0.1:8082", + content_length: None, + body_bytes_seen: 0, + protection_class: Some("api"), + protection_client: "127.0.0.1", + elapsed_ms: 12, + error: None, + }); + + assert!(line.contains("request_id=req-1\tmethod=GET\tpath=/api/test")); + assert!(line.contains("uri=/api/test?a=1\\t2")); + assert!(line.contains("content_length=-")); + assert!(line.ends_with("error=-\n")); + } + + #[test] + fn rejects_placeholder_or_short_shadow_probe_tokens() { + assert!(validate_shadow_probe_token("__GENARRATIVE_PINGORA_PROBE_TOKEN__").is_err()); + assert!(validate_shadow_probe_token("changeme").is_err()); + assert!(validate_shadow_probe_token("too-short").is_err()); + assert!(validate_shadow_probe_token("0123456789abcdef").is_ok()); + } + + #[test] + fn rejects_burst_without_rate_limit() { + let config = ProtectionClassConfig { + max_concurrent: 1, + rate_per_second: 0, + burst: 1, + }; + assert!(config.validate("api").is_err()); + } + + #[test] + fn rejects_access_log_directory_paths() { + let temp_dir = std::env::temp_dir(); + assert!(validate_access_log_file(&temp_dir).is_err()); + assert!(validate_access_log_file(&temp_dir.join("pingora-access.log")).is_ok()); + } + + #[test] + fn rejects_incomplete_tls_and_redirect_config() { + let config = GatewayConfig { + tls_listen_addr: Some("127.0.0.1:18443".to_string()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + http_redirect_listen_addr: Some("127.0.0.1:18080".to_string()), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + tls_listen_addr: Some("127.0.0.1:18081".to_string()), + tls_cert_file: Some(PathBuf::from("/missing/cert.pem")), + tls_key_file: Some(PathBuf::from("/missing/key.pem")), + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + } + + #[test] + fn validates_redirect_target_host_and_scheme() { + assert!(validate_redirect_target_scheme("https").is_ok()); + assert!(validate_redirect_target_scheme("http").is_err()); + assert!(is_valid_redirect_host("genarrative.world")); + assert!(is_valid_redirect_host("localhost:8443")); + assert!(is_valid_redirect_host("[::1]:8443")); + assert!(!is_valid_redirect_host("evil.test/path")); + assert!(!is_valid_redirect_host("evil.test\r\nx: y")); + } + + #[test] + fn builds_stable_static_validators() { + let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_123); + assert_eq!( - classify_path("/admin/settings"), - RouteDecision::Local(LocalResponse::Static { - root: StaticRoot::Web, - mode: StaticMode::SpaFallback, - }) + build_static_etag(4096, Some(modified)), + "W/\"1000-6553f17b\"" + ); + assert_eq!(build_static_etag(0, None), "W/\"0-0\""); + assert_eq!( + truncate_system_time_to_seconds(modified + Duration::from_millis(900)), + modified + ); + } + + #[test] + fn matches_static_if_none_match_values() { + assert!(if_none_match_matches("*", "W/\"1000-6553f17b\"")); + assert!(if_none_match_matches( + "\"other\", W/\"1000-6553f17b\"", + "W/\"1000-6553f17b\"" + )); + assert!(if_none_match_matches( + "\"1000-6553f17b\"", + "W/\"1000-6553f17b\"" + )); + assert!(!if_none_match_matches( + "\"other\", W/\"bad\"", + "W/\"1000-6553f17b\"" + )); + } + + #[test] + fn parses_static_byte_ranges() { + assert_eq!( + parse_static_range("bytes=0-3", 10), + StaticRangeDecision::Partial { start: 0, end: 3 } ); assert_eq!( - classify_path("/some/spa/path"), - RouteDecision::Local(LocalResponse::Static { - root: StaticRoot::Web, - mode: StaticMode::SpaFallback, - }) + parse_static_range("Bytes=5-", 10), + StaticRangeDecision::Partial { start: 5, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=-4", 10), + StaticRangeDecision::Partial { start: 6, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=-20", 10), + StaticRangeDecision::Partial { start: 0, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=8-20", 10), + StaticRangeDecision::Partial { start: 8, end: 9 } + ); + assert_eq!( + parse_static_range("bytes=10-11", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=4-3", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=-0", 10), + StaticRangeDecision::Unsatisfiable + ); + assert_eq!( + parse_static_range("bytes=0-1,3-4", 10), + StaticRangeDecision::Full + ); + assert_eq!( + parse_static_range("items=0-1", 10), + StaticRangeDecision::Full + ); + assert_eq!( + parse_static_range("bytes=0-1", 0), + StaticRangeDecision::Unsatisfiable + ); + } + + #[test] + fn matches_static_if_range_values() { + let modified = UNIX_EPOCH + Duration::from_secs(1_700_000_123); + let metadata = StaticResponseMetadata { + len: 4096, + etag: build_static_etag(4096, Some(modified)), + last_modified: Some(modified), + }; + + assert!(static_if_range_value_matches( + &fmt_http_date(modified + Duration::from_secs(30)), + &metadata + )); + assert!(!static_if_range_value_matches( + &fmt_http_date(modified - Duration::from_secs(30)), + &metadata + )); + assert!(!static_if_range_value_matches(&metadata.etag, &metadata)); + assert!(!static_if_range_value_matches( + "\"1000-6553f17b\"", + &metadata + )); + } + + #[test] + fn only_allows_static_read_methods() { + assert!(is_static_read_method(&Method::GET)); + assert!(is_static_read_method(&Method::HEAD)); + assert!(!is_static_read_method(&Method::POST)); + assert!(!is_static_read_method(&Method::PUT)); + } + + #[test] + fn rejects_invalid_gzip_level() { + let config = GatewayConfig { + gzip_level: 10, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_invalid_gzip_min_length() { + let config = GatewayConfig { + gzip_min_length_bytes: 0, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_static_cache_control_header_injection() { + let config = GatewayConfig { + static_cache: StaticCacheConfig { + html_cache_control: "no-cache\nX-Bad: yes".to_string(), + asset_cache_control: DEFAULT_ASSET_CACHE_CONTROL.to_string(), + static_cache_control: DEFAULT_STATIC_CACHE_CONTROL.to_string(), + }, + ..test_gateway_config() + }; + + assert!(config.validate().is_err()); + } + + #[test] + fn rejects_unconfirmed_multi_instance_protection() { + let config = GatewayConfig { + instance_count: 0, + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + instance_count: 2, + ..test_gateway_config() + }; + assert!(config.validate().is_err()); + + let config = GatewayConfig { + instance_count: 2, + shared_protection_confirmed: true, + ..test_gateway_config() + }; + assert!(config.validate().is_ok()); + + let config = GatewayConfig { + instance_count: 2, + protection: ProtectionConfig { + enabled: false, + ..test_gateway_config().protection + }, + ..test_gateway_config() + }; + assert!(config.validate().is_ok()); + } + + #[test] + fn rejects_zero_upstream_timeouts() { + assert!( + UpstreamTimeoutConfig { + connect_timeout_ms: 0, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: DEFAULT_UPSTREAM_API_READ_TIMEOUT_SECONDS, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + } + .validate() + .is_err() + ); + assert!( + UpstreamTimeoutConfig { + connect_timeout_ms: DEFAULT_UPSTREAM_CONNECT_TIMEOUT_MS, + default_read_timeout_seconds: DEFAULT_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS, + api_read_timeout_seconds: 0, + long_read_timeout_seconds: DEFAULT_UPSTREAM_LONG_READ_TIMEOUT_SECONDS, + write_timeout_seconds: DEFAULT_UPSTREAM_WRITE_TIMEOUT_SECONDS, + } + .validate() + .is_err() + ); + } + + #[test] + fn chooses_long_upstream_timeout_for_gallery_and_spacetime_routes() { + let timeouts = UpstreamTimeoutConfig { + connect_timeout_ms: 100, + default_read_timeout_seconds: 60, + api_read_timeout_seconds: 10, + long_read_timeout_seconds: 3600, + write_timeout_seconds: 20, + }; + let generic_api = classify_path("/api/creation-entry/config"); + let gallery = classify_path("/api/runtime/puzzle/gallery"); + let subscribe = classify_path("/v1/database/genarrative/subscribe"); + + assert_eq!( + timeouts.read_timeout_for_route(&generic_api, "/api/creation-entry/config"), + Duration::from_secs(10) + ); + assert_eq!( + timeouts.read_timeout_for_route(&gallery, "/api/runtime/puzzle/gallery"), + Duration::from_secs(3600) + ); + assert_eq!( + timeouts.read_timeout_for_route(&subscribe, "/v1/database/genarrative/subscribe"), + Duration::from_secs(3600) + ); + } + + #[test] + fn rejects_unsupported_compression_algorithms() { + assert!(CompressionConfig::parse("zstd".to_string()).is_err()); + assert!(CompressionConfig::parse("gzip,br".to_string()).is_err()); + } + + #[test] + fn selects_configured_gateway_compression_algorithm() { + let gzip_only = CompressionRequestConfig { + enabled: true, + gzip: true, + min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + gzip_level: DEFAULT_GZIP_LEVEL, + }; + + let mut req = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + req.insert_header("Accept-Encoding", "br, gzip").unwrap(); + normalize_accept_encoding_for_gateway_compression(&mut req, gzip_only).unwrap(); + assert_eq!( + req.headers + .get("accept-encoding") + .and_then(|value| value.to_str().ok()), + Some("gzip") + ); + + let mut br_only = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + br_only.insert_header("Accept-Encoding", "br").unwrap(); + normalize_accept_encoding_for_gateway_compression(&mut br_only, gzip_only).unwrap(); + assert!(br_only.headers.get("accept-encoding").is_none()); + + let mut gzip_disabled = RequestHeader::build(Method::GET, b"/assets/app.js", None).unwrap(); + gzip_disabled + .insert_header("Accept-Encoding", "gzip") + .unwrap(); + normalize_accept_encoding_for_gateway_compression( + &mut gzip_disabled, + CompressionRequestConfig { + enabled: false, + gzip: true, + min_length_bytes: DEFAULT_GZIP_MIN_LENGTH_BYTES, + gzip_level: DEFAULT_GZIP_LEVEL, + }, + ) + .unwrap(); + assert!(gzip_disabled.headers.get("accept-encoding").is_none()); + + assert_eq!(parse_accepted_encodings("br, gzip;q=1"), ["br", "gzip"]); + assert_eq!(parse_accepted_encodings("br, gzip;q=0"), ["br"]); + } + + #[test] + fn honors_configured_gzip_min_length() { + let mut small = ResponseHeader::build(200, Some(1023)).unwrap(); + small.insert_header("Content-Length", "1023").unwrap(); + small + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(!should_allow_gateway_compression_for_response( + &small, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut exact = ResponseHeader::build(200, Some(1024)).unwrap(); + exact.insert_header("Content-Length", "1024").unwrap(); + exact + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(should_allow_gateway_compression_for_response( + &exact, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut unknown_length = ResponseHeader::build(200, None).unwrap(); + unknown_length + .insert_header("Content-Type", "application/javascript") + .unwrap(); + assert!(should_allow_gateway_compression_for_response( + &unknown_length, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut partial = ResponseHeader::build(206, Some(1024)).unwrap(); + partial.insert_header("Content-Length", "1024").unwrap(); + partial + .insert_header("Content-Range", "bytes 0-1023/4096") + .unwrap(); + assert!(!should_allow_gateway_compression_for_response( + &partial, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + + let mut not_modified = ResponseHeader::build(304, Some(0)).unwrap(); + not_modified.insert_header("Content-Length", "0").unwrap(); + assert!(!should_allow_gateway_compression_for_response( + ¬_modified, + DEFAULT_GZIP_MIN_LENGTH_BYTES + )); + } + + #[test] + fn disables_accel_buffering_for_api_proxy_only() { + assert!(should_disable_accel_buffering(&classify_path( + "/api/creation-entry/config" + ))); + assert!(should_disable_accel_buffering(&classify_path( + "/api/runtime/puzzle/gallery" + ))); + assert!(!should_disable_accel_buffering(&classify_path( + "/v1/database/genarrative/subscribe" + ))); + assert!(!should_disable_accel_buffering(&classify_path( + "/assets/app.js" + ))); + } + + #[test] + fn maps_proxy_routes_to_protection_classes() { + let cases = [ + ("/admin/api/users", Some(ProtectionClass::AdminApi)), + ( + "/api/runtime/puzzle/gallery", + Some(ProtectionClass::GalleryList), + ), + ( + "/api/runtime/custom-world-gallery/profile-a/owner-b", + Some(ProtectionClass::GalleryDetail), + ), + ("/api/creation-entry/config", Some(ProtectionClass::Api)), + ( + "/v1/database/genarrative/subscribe", + Some(ProtectionClass::Spacetime), + ), + ("/assets/app.js", None), + ]; + + for (path, expected) in cases { + assert_eq!( + protection_class_for_route(&classify_path(path), path), + expected, + "path: {path}" + ); + } + } + + #[test] + fn protection_rejects_when_concurrency_is_exhausted() { + let config = ProtectionConfig { + enabled: true, + api: ProtectionClassConfig { + max_concurrent: 1, + rate_per_second: 0, + burst: 0, + }, + admin_api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + }; + let protection = Arc::new(GatewayProtection::new(config)); + let first = protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("first request should be accepted"); + + assert!(matches!( + protection.try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()), + Err(RejectReason::Concurrent) + )); + + if let Some(key) = first { + protection.release(&key); + } + assert!( + protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("request should be accepted after release") + .is_some() + ); + } + + #[test] + fn protection_rejects_when_rate_bucket_is_empty() { + let config = ProtectionConfig { + enabled: true, + api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 1, + burst: 0, + }, + admin_api: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_list: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + gallery_detail: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + spacetime: ProtectionClassConfig { + max_concurrent: 0, + rate_per_second: 0, + burst: 0, + }, + }; + let protection = Arc::new(GatewayProtection::new(config)); + let first = protection + .try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()) + .expect("first request should pass"); + if let Some(key) = first { + protection.release(&key); + } + + assert!(matches!( + protection.try_acquire(ProtectionClass::Api, "127.0.0.1".to_string()), + Err(RejectReason::Rate) + )); + assert!( + protection + .try_acquire(ProtectionClass::Api, "127.0.0.2".to_string()) + .expect("different client should have an independent bucket") + .is_some() ); } @@ -815,4 +3577,41 @@ mod tests { assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/app.js").is_some()); assert!(sanitize_static_path(Path::new("/srv/web"), "/assets/../secret").is_none()); } + + #[test] + fn classifies_static_cache_policy() { + let root = StaticRoot::Web; + assert_eq!( + classify_static_cache_kind(root, "/", Path::new("/srv/web/index.html")), + StaticCacheKind::Html + ); + assert_eq!( + classify_static_cache_kind( + root, + "/assets/index-B4dmVw0r.js", + Path::new("/srv/web/assets/index-B4dmVw0r.js"), + ), + StaticCacheKind::FingerprintedAsset + ); + assert_eq!( + classify_static_cache_kind( + root, + "/admin/assets/admin-Ca8f3012.css", + Path::new("/srv/web/admin/assets/admin-Ca8f3012.css"), + ), + StaticCacheKind::FingerprintedAsset + ); + assert_eq!( + classify_static_cache_kind(root, "/assets/app.js", Path::new("/srv/web/assets/app.js")), + StaticCacheKind::Other + ); + assert_eq!( + classify_static_cache_kind( + StaticRoot::Acme, + "/.well-known/acme-challenge/token-12345678", + Path::new("/var/www/html/.well-known/acme-challenge/token-12345678"), + ), + StaticCacheKind::Other + ); + } } From 9eb4937afb58a86ccb8ee5ae09db02c29a452fb9 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 20:44:44 +0800 Subject: [PATCH 03/22] =?UTF-8?q?=E8=A1=A5=E5=85=85Pingora=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E4=B8=8A=E6=B8=B8Docker=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 Docker canary 增加真实 api-server 与 SpacetimeDB 上游参数 允许 canary live 接受真实 SpacetimeDB identity GET 的 405 语义 同步运维文档记录真实上游 Docker 验收命令 --- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-pingora-canary-docker.mjs | 203 +++++++++++++++--- scripts/check-pingora-canary-live.mjs | 2 +- 3 files changed, 171 insertions(+), 36 deletions(-) diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index c4ea225fc..8db4badd0 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -311,7 +311,7 @@ current release 随包执行的 release readiness 必须追加 `--release-runtim 涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时，必须同步更新 `deploy/pingora/nginx-route-parity.matrix.json`，并运行 `npm run check:pingora-route-parity`；Rust 侧 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix` 会读取同一份矩阵验证 `classify_path` 的路由结果、body limit 和接流保护分组。 -Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 +Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。需要证明 Docker Nginx -> Pingora -> 真实本地服务时，先启动真实 api-server 与 SpacetimeDB，再运行 `node scripts/check-pingora-canary-docker.mjs --require-docker --real-upstreams --api-upstream 127.0.0.1: --spacetime-upstream 127.0.0.1: --web-root dist`；该模式不会启动内置 mock 上游，会先检查真实 `/healthz` 和 `/v1/ping`，随后复用同一组 live smoke 与 access log 对账。真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，canary live 只把它作为路径路由代表，不要求该 GET 创建 identity。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 Pingora 正式切换前必须额外运行聚合门禁：`node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。该门禁串起 Rust 单测、mock smoke、路由 parity、Nginx snippet 校验、Docker Nginx handoff、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、health patrol env 切换脚本烟测、current release 自审烟测、release readiness 计划自检、生产运维护栏、API release build 烟测、Pingora production release 真实构建烟测、API deploy release 烟测、目标机 live canary 和目标机真实 access log 对账；其中 `--require-live` 只能在目标 Nginx 已人工 include canary snippet 并 reload 后执行，并会强制要求 `--live-host`，避免只访问 `127.0.0.1` 命中 Nginx 默认 vhost。live smoke 成功后会立即读取 Nginx 与 Pingora access log 尾部记录，按 `request_id` 验证 `/__genarrative_pingora_canary/healthz` 和 `/__genarrative_pingora_canary/api/creation-entry/config` 已在两边落盘且 method/status/path 没有漂移。启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，它会用独立 Nginx access log 对账真实 `/api`、`/v1` 与 `/assets` 路径。live canary、direct live 和 access log 对账的显式毫秒超时或尾部行数参数都必须是正整数，非法值应先修参数再重跑，不能把默认值兜底后的结果当成切换证据；direct live 与 release readiness 读取的直连布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，拼写错误会直接失败，避免 `REQUIRE_WSS_UPGRADE`、preflight 开关或 `SKIP_WSS` 被悄悄当成 false。若验证 Pingora 直连公网入口，还必须追加 direct preflight 参数检查目标机 env、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和端口释放：`--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free`；该模式缺少 `--direct-http-base-url`、`--direct-host`、`--direct-redirect-host`、`--direct-pingora-access-log`、`--direct-preflight-systemd`、`--direct-preflight-check-cert-readable`、`--direct-preflight-check-service-env-file`、`--direct-preflight-check-service-user-cert-readable`、`--direct-preflight-check-service-binary-executable`、`--direct-preflight-check-ports-free` 、缺少 `--direct-health-patrol-env-file` 或缺少 `--direct-spacetime-database` 会直接失败，且会拒绝 `--direct-skip-wss`，并默认要求 WSS subscribe 返回 101，证明 HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id + path/status 对账、health patrol direct 模式、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性、80/443 端口释放与目标 SpacetimeDB 长连接都能从直连入口透传。`check-pingora-release-readiness.mjs --help` 中的正式直连和只生成 runbook 示例也必须显式带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，避免值班人员复制示例后被 `--require-direct` 自身拦住。普通本机提交前可先跑 `npm run check:pingora-release-readiness`，但该默认模式不能替代目标机切换窗口的强制门禁。直连切换前还应先运行 `npm run plan:pingora-direct-cutover -- --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --rollback-nginx-smoke-url http://127.0.0.1/healthz --rollback-health-patrol-public-base-url <切换前Nginx巡检入口>` 生成只读 JSON runbook；若切换参数提供 `--direct-probe-token`，启用后复核会继续透传 direct probe token 检查内部探针，runbook JSON 只显示 ``。runbook 会列出 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply、用 `/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply` 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、回退前用同一脚本预置回 `nginx` 并恢复切换前 public base URL / Host、rollback apply、回退后 health patrol env Nginx 模式复核，供当班人员逐条审阅。direct enable / rollback `--apply` 都会拒绝符号链接形式的 systemd drop-in 目录或目标文件，并拒绝已存在但不是普通文件的目标，避免把低端口 capability 写入非预期位置，或在回退窗口误删 / 误判非预期 systemd 位置；health patrol env 切换脚本 `--apply` 会保留原文件权限和 owner/group，此时 `--env-file` 必须直接指向真实普通文件，不能是符号链接。如果现场 env 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 diff --git a/scripts/check-pingora-canary-docker.mjs b/scripts/check-pingora-canary-docker.mjs index c50efc86d..e7bcae174 100644 --- a/scripts/check-pingora-canary-docker.mjs +++ b/scripts/check-pingora-canary-docker.mjs @@ -72,10 +72,35 @@ async function main() { const accessLogFile = path.join(logsRoot, 'pingora.access.log'); await mkdir(logsRoot, { recursive: true }); await chmod(logsRoot, 0o777); - await prepareStaticRoots(webRoot, acmeRoot); - const api = await startApiMock(); - const spacetime = await startSpacetimeMock(); + const gatewayWebRoot = config.webRoot || webRoot; + const gatewayAcmeRoot = config.acmeRoot || acmeRoot; + if (!config.webRoot) { + await prepareStaticWebRoot(webRoot); + } + if (!config.acmeRoot) { + await prepareAcmeRoot(acmeRoot); + } + + const realUpstreams = config.realUpstreams || config.apiUpstream || config.spacetimeUpstream; + if (realUpstreams && (!config.apiUpstream || !config.spacetimeUpstream)) { + throw new Error( + '真实上游模式必须同时提供 --api-upstream 与 --spacetime-upstream。', + ); + } + + const api = realUpstreams + ? { upstream: config.apiUpstream, state: null } + : await startApiMock(); + const spacetime = realUpstreams + ? { upstream: config.spacetimeUpstream, state: null } + : await startSpacetimeMock(); + if (realUpstreams) { + console.log( + `[pingora-canary-docker] 使用真实上游 api=${api.upstream} spacetime=${spacetime.upstream} webRoot=${gatewayWebRoot}`, + ); + await assertRealUpstreamsReady(api.upstream, spacetime.upstream); + } const pingoraPort = await getFreePort(); const nginxPort = await getFreePort(); const realpathNginxPort = await getFreePort(); @@ -97,10 +122,10 @@ async function main() { env: { ...smokeEnv(), GENARRATIVE_PINGORA_GATEWAY_LISTEN: `0.0.0.0:${pingoraPort}`, - GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: `127.0.0.1:${api.port}`, - GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: `127.0.0.1:${spacetime.port}`, - GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: webRoot, - GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: acmeRoot, + GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM: api.upstream, + GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM: spacetime.upstream, + GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT: gatewayWebRoot, + GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT: gatewayAcmeRoot, GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE: path.join( tempRoot, 'maintenance', @@ -280,27 +305,29 @@ async function main() { '/assets/app.js', ]); - ensure( - api.state.requests.some( - (request) => request.url === '/api/creation-entry/config', - ), - 'Docker Nginx canary 未把 API 代表路径交给 mock api-server', - ); - ensure( - spacetime.state.requests.some((request) => request.url === '/v1/identity'), - 'Docker Nginx canary 未把 SpacetimeDB identity 代表路径交给 mock SpacetimeDB', - ); - ensure( - api.state.requests.filter( - (request) => request.url === '/api/creation-entry/config', - ).length >= 2, - 'Docker Nginx realpath canary 未把真实 API 代表路径交给 mock api-server', - ); - ensure( - spacetime.state.requests.filter((request) => request.url === '/v1/identity') - .length >= 2, - 'Docker Nginx realpath canary 未把真实 SpacetimeDB identity 路径交给 mock SpacetimeDB', - ); + if (!realUpstreams) { + ensure( + api.state.requests.some( + (request) => request.url === '/api/creation-entry/config', + ), + 'Docker Nginx canary 未把 API 代表路径交给 mock api-server', + ); + ensure( + spacetime.state.requests.some((request) => request.url === '/v1/identity'), + 'Docker Nginx canary 未把 SpacetimeDB identity 代表路径交给 mock SpacetimeDB', + ); + ensure( + api.state.requests.filter( + (request) => request.url === '/api/creation-entry/config', + ).length >= 2, + 'Docker Nginx realpath canary 未把真实 API 代表路径交给 mock api-server', + ); + ensure( + spacetime.state.requests.filter((request) => request.url === '/v1/identity') + .length >= 2, + 'Docker Nginx realpath canary 未把真实 SpacetimeDB identity 路径交给 mock SpacetimeDB', + ); + } } function parseArgs(argv) { @@ -310,6 +337,23 @@ function parseArgs(argv) { requireDocker: false, skipBuild: false, verbose: false, + realUpstreams: false, + apiUpstream: normalizeOptionalHostPort( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM, + 'GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM', + ), + spacetimeUpstream: normalizeOptionalHostPort( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM, + 'GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM', + ), + webRoot: normalizeOptionalDirectory( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT, + 'GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT', + ), + acmeRoot: normalizeOptionalDirectory( + process.env.GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT, + 'GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT', + ), }; for (let index = 0; index < argv.length; index += 1) { @@ -325,10 +369,21 @@ Options: --pull Pull the image when it is not present locally. --require-docker Treat missing Docker/image/host-network as failure instead of skip. --skip-build Reuse an existing pingora-gateway debug binary. + --real-upstreams Require real api-server and SpacetimeDB upstream arguments. + --api-upstream + Use a real api-server upstream instead of the built-in mock. + --spacetime-upstream + Use a real SpacetimeDB upstream instead of the built-in mock. + --web-root Serve an existing web root, for example dist/. + --acme-root Serve an existing ACME root; temp root is used by default. --verbose Print Pingora and Docker Nginx process logs. Environment aliases: GENARRATIVE_PINGORA_CANARY_DOCKER_IMAGE + GENARRATIVE_PINGORA_CANARY_REAL_API_UPSTREAM + GENARRATIVE_PINGORA_CANARY_REAL_SPACETIME_UPSTREAM + GENARRATIVE_PINGORA_CANARY_REAL_WEB_ROOT + GENARRATIVE_PINGORA_CANARY_REAL_ACME_ROOT GENARRATIVE_PINGORA_GATEWAY_BINARY 默认不会拉取镜像；本机或 CI 要做强验收时建议： @@ -348,6 +403,24 @@ Environment aliases: case '--skip-build': result.skipBuild = true; break; + case '--real-upstreams': + result.realUpstreams = true; + break; + case '--api-upstream': + result.apiUpstream = normalizeHostPort(requireValue(argv, ++index, arg), arg); + break; + case '--spacetime-upstream': + result.spacetimeUpstream = normalizeHostPort( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--web-root': + result.webRoot = normalizeDirectory(requireValue(argv, ++index, arg), arg); + break; + case '--acme-root': + result.acmeRoot = normalizeDirectory(requireValue(argv, ++index, arg), arg); + break; case '--verbose': result.verbose = true; break; @@ -367,6 +440,56 @@ function requireValue(argv, index, flag) { return value; } +function normalizeOptionalHostPort(value, label) { + if (!value) { + return ''; + } + return normalizeHostPort(value, label); +} + +function normalizeHostPort(value, label) { + validateNoControlCharacters(value, label); + const raw = String(value || '').trim(); + if (!raw) { + throw new Error(`${label} 不能为空。`); + } + if (raw.includes('://') || /[\s/?#@]/u.test(raw)) { + throw new Error(`${label} 必须是 host:port，不能包含 scheme、路径、查询、片段或空白字符。`); + } + try { + const parsed = new URL(`http://${raw}`); + const port = Number.parseInt(parsed.port, 10); + if (!parsed.hostname || !Number.isInteger(port) || port <= 0 || port > 65535) { + throw new Error('invalid host:port'); + } + } catch { + throw new Error(`${label} 必须是合法 host:port。`); + } + return raw; +} + +function normalizeOptionalDirectory(value, label) { + if (!value) { + return ''; + } + return normalizeDirectory(value, label); +} + +function normalizeDirectory(value, label) { + validateNoControlCharacters(value, label); + const resolved = path.resolve(repoRoot, String(value || '').trim()); + if (!existsSync(resolved)) { + throw new Error(`${label} 不存在：${resolved}`); + } + return resolved; +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value ?? ''))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + function ensureDockerReady() { const result = spawnSync( 'docker', @@ -425,12 +548,9 @@ function skipOrFail(message) { return false; } -async function prepareStaticRoots(webRoot, acmeRoot) { +async function prepareStaticWebRoot(webRoot) { await mkdir(path.join(webRoot, 'admin', 'assets'), { recursive: true }); await mkdir(path.join(webRoot, 'assets'), { recursive: true }); - await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), { - recursive: true, - }); await writeFile(path.join(webRoot, 'index.html'), '
site-shell
'); await writeFile( @@ -445,6 +565,12 @@ async function prepareStaticRoots(webRoot, acmeRoot) { path.join(webRoot, 'admin', 'assets', 'admin.js'), 'console.log("admin asset");', ); +} + +async function prepareAcmeRoot(acmeRoot) { + await mkdir(path.join(acmeRoot, '.well-known', 'acme-challenge'), { + recursive: true, + }); await writeFile( path.join(acmeRoot, '.well-known', 'acme-challenge', 'token'), 'acme-token', @@ -501,7 +627,7 @@ async function startApiMock() { }); const port = await listen(server); - return { port, state }; + return { upstream: `127.0.0.1:${port}`, state }; } async function startSpacetimeMock() { @@ -542,7 +668,16 @@ async function startSpacetimeMock() { }); const port = await listen(server); - return { port, state }; + return { upstream: `127.0.0.1:${port}`, state }; +} + +async function assertRealUpstreamsReady(apiUpstream, spacetimeUpstream) { + await waitForHttp(`http://${apiUpstream}/healthz`, 200, { + label: '等待真实 api-server 就绪', + }); + await waitForHttp(`http://${spacetimeUpstream}/v1/ping`, 200, { + label: '等待真实 SpacetimeDB 就绪', + }); } async function renderNginxConfig({ diff --git a/scripts/check-pingora-canary-live.mjs b/scripts/check-pingora-canary-live.mjs index 759c2810c..0527a32d0 100644 --- a/scripts/check-pingora-canary-live.mjs +++ b/scripts/check-pingora-canary-live.mjs @@ -242,7 +242,7 @@ async function main() { { name: 'spacetime-identity', path: '/v1/identity', - expectedStatuses: [200, 401, 403, 404, 503], + expectedStatuses: [200, 401, 403, 404, 405, 503], }, { name: 'web-assets', From c3f7d1f2223c29454b9cdeffe587842356c0ce7c Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 21:25:36 +0800 Subject: [PATCH 04/22] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20Pingora=20=E7=9B=B4?= =?UTF-8?q?=E8=BF=9E=E9=AA=8C=E6=94=B6=E5=8F=A3=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 允许 direct live 接受 SpacetimeDB identity 的 405 真实响应 同步 Nginx README 与 Pingora 试点文档的直连验收说明 在 dev 服务器用真实 API、SpacetimeDB 和静态目录完成 shadow 验收 --- deploy/nginx/README.md | 2 +- docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md | 2 +- scripts/check-pingora-direct-live.mjs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 767615d63..d33842cf5 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -49,7 +49,7 @@ - API release 必须把 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 -- 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。 +- 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。 - direct live 在 HTTPS 根路径返回 `200` 且 HTML 中发现 `/assets/` 或 `/admin/assets/` 引用时，会自动请求该静态资源，确认 `Cache-Control`、`ETag`、`Last-Modified`、`Accept-Ranges: bytes`、`HEAD` 头响应、`If-None-Match` / `If-Modified-Since` 的 `304` 响应和 `Range: bytes=0-0` 的 `206 + Content-Range` 响应，并纳入 Pingora access log method/path/status 对账；若首页引用 Vite 指纹资源，还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 以及指纹资源 GET / HEAD / 304 / Range access log 证据。维护模式、非 HTML 或发布包首页没有资产引用时该项标记为 skipped，不阻断维护窗口。 - 直连启用前先 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`；验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`，脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` 已从 systemd 最终配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 smoke URL 证明 Nginx 入口真实可访问；传入 `--nginx-smoke-expect-body` 时还会要求响应体包含该片段。仓库工作区可用 `npm run deploy:pingora-direct-enable -- --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log` 和 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`。两个脚本默认 dry-run；启用脚本 `--apply` 必须先通过 current release 自审，并带 env、证书可读、service EnvironmentFile 一致性、服务用户证书可读、service 二进制可执行、端口释放预检、direct live 参数和 Pingora access log 参数，回退脚本 `--apply` 必须带 `--reload-nginx` 和 `--nginx-smoke-url`；如果 health patrol env 已预先切回 Nginx，也可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让回退脚本在 Nginx smoke 后复核 `gateway mode=nginx` 且 public base URL / Host 已恢复，若切换前 Nginx 巡检本来需要 Host 覆盖，则把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`；如果还要同时证明 shadow 高端口仍健康，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。本机 `npm run check:pingora-direct-enable` / `npm run check:pingora-direct-rollback` 会验证 dry-run 不会安装或删除临时 drop-in、current release 自审失败时启用脚本不会安装 drop-in，并展示启用 / 回退后的 systemd 最终配置核验、ExecStart 指向核验、Pingora active 核验、direct live smoke、Nginx 语法检查、reload 状态核验、Nginx smoke 及响应体片段核验、可选 health patrol env 复核和可选 shadow probe 复核。 - `npm run check:pingora-current-release-audit` 会烟测 current release 自审脚本；正式直连 runbook 会先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，只读确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、`release-manifest.api-server.json` 已登记 `pingora-gateway`、`pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 8cafabdec..46375cb9d 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -213,7 +213,7 @@ GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST=<域名或host:port> \ npm run check:pingora-direct-live ``` -该脚本只读检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS `/v1/database//subscribe` 握手、generated 禁止入口、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 入口到 HTTPS 的 301 和 ACME challenge 静态读取；配置 `--pingora-access-log` 后还会为每个请求生成 `X-Request-Id` 并反查 Pingora access log，逐条比对同一 `request_id` 的 method、path 与 status，证明直连流量真实进入 Pingora，且未发生日志侧方法 / 路径 / 状态漂移。HTTP 响应必须带 `X-Genarrative-Gateway: pingora-shadow`，TLS ALPN 必须协商到 `h2`，WSS 成功握手时必须保留 `v2.bsatn.spacetimedb` 子协议。release readiness `--require-direct` 会自动要求 WSS 返回 101，并要求提供 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、显式 SpacetimeDB 数据库名和 Pingora access log 路径，让 HTTP/2、HTTP redirect / ACME 入口、正式证书域名、跳转目标域名、目标库 subscribe 和 request_id + method/path/status 对账证据都纳入硬门禁；单独运行 direct live smoke 时也可加 `--require-wss-upgrade` 强制同一口径。direct live 在首页发现 `/assets/` 或 `/admin/assets/` 引用时会额外执行静态资源 GET、HEAD、`If-None-Match` / `If-Modified-Since` 条件请求和 `Range: bytes=0-0` 探针，确认静态头、HEAD 头响应、`304` 协商缓存、`206 + Content-Range` 以及不压缩 Range / 304 响应均进入同一 access log 证据链；这些静态 GET / HEAD / 304 / Range 的 JSON 结果会写入白名单 `headers`，只保留 `cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`，方便切换后复盘缓存和 Range 响应头。若发现 Vite 指纹资源，还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 及其 GET / HEAD / 304 / Range method/path/status 证据，避免旧 tab chunk 缓存口径在直连后退化。direct live 的 HTTPS / HTTP base URL、Host、redirect Host、probe token、额外 path、SpacetimeDB 数据库名、access log 路径、timeout 和布尔 env 都不能包含换行或 NUL；脚本会在发起 HTTPS / HTTP / WSS 请求前失败，避免污染参数进入请求头、URL、日志对账或 JSON 证据。环境变量示例见 `deploy/env/pingora-direct-live.env.example`。生产证书必须可被系统信任；`--insecure-tls` 只允许本机自签证书 smoke 使用。`--skip-wss` 只允许单独 direct live 临时排障，release readiness `--require-direct` 会直接拒绝。 +该脚本只读检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS `/v1/database//subscribe` 握手、generated 禁止入口、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 入口到 HTTPS 的 301 和 ACME challenge 静态读取；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；配置 `--pingora-access-log` 后还会为每个请求生成 `X-Request-Id` 并反查 Pingora access log，逐条比对同一 `request_id` 的 method、path 与 status，证明直连流量真实进入 Pingora，且未发生日志侧方法 / 路径 / 状态漂移。HTTP 响应必须带 `X-Genarrative-Gateway: pingora-shadow`，TLS ALPN 必须协商到 `h2`，WSS 成功握手时必须保留 `v2.bsatn.spacetimedb` 子协议。release readiness `--require-direct` 会自动要求 WSS 返回 101，并要求提供 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、显式 SpacetimeDB 数据库名和 Pingora access log 路径，让 HTTP/2、HTTP redirect / ACME 入口、正式证书域名、跳转目标域名、目标库 subscribe 和 request_id + method/path/status 对账证据都纳入硬门禁；单独运行 direct live smoke 时也可加 `--require-wss-upgrade` 强制同一口径。direct live 在首页发现 `/assets/` 或 `/admin/assets/` 引用时会额外执行静态资源 GET、HEAD、`If-None-Match` / `If-Modified-Since` 条件请求和 `Range: bytes=0-0` 探针，确认静态头、HEAD 头响应、`304` 协商缓存、`206 + Content-Range` 以及不压缩 Range / 304 响应均进入同一 access log 证据链；这些静态 GET / HEAD / 304 / Range 的 JSON 结果会写入白名单 `headers`，只保留 `cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`，方便切换后复盘缓存和 Range 响应头。若发现 Vite 指纹资源，还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 及其 GET / HEAD / 304 / Range method/path/status 证据，避免旧 tab chunk 缓存口径在直连后退化。direct live 的 HTTPS / HTTP base URL、Host、redirect Host、probe token、额外 path、SpacetimeDB 数据库名、access log 路径、timeout 和布尔 env 都不能包含换行或 NUL；脚本会在发起 HTTPS / HTTP / WSS 请求前失败，避免污染参数进入请求头、URL、日志对账或 JSON 证据。环境变量示例见 `deploy/env/pingora-direct-live.env.example`。生产证书必须可被系统信任；`--insecure-tls` 只允许本机自签证书 smoke 使用。`--skip-wss` 只允许单独 direct live 临时排障，release readiness `--require-direct` 会直接拒绝。 生产发布包默认不携带 Pingora，避免现有 API 流水线被影子网关构建依赖影响。需要部署影子服务时，构建机先确保有 `cmake`、C/C++ 编译器和 Rust target，再显式执行；`build-production-release.sh` 会在真实构建 Pingora 前 fail-fast 检查 `cmake`、C 编译器和 C++ 编译器，Jenkins `INCLUDE_PINGORA_GATEWAY=true` 时也会先检查这些工具： diff --git a/scripts/check-pingora-direct-live.mjs b/scripts/check-pingora-direct-live.mjs index a52a70937..b9a8f7659 100644 --- a/scripts/check-pingora-direct-live.mjs +++ b/scripts/check-pingora-direct-live.mjs @@ -331,7 +331,7 @@ async function main() { { name: 'https-spacetime-identity', url: joinUrl(config.httpsBaseUrl, '/v1/identity'), - expectedStatuses: [200, 401, 403, 404, 503], + expectedStatuses: [200, 401, 403, 404, 405, 503], assertHeader: assertPingoraGatewayHeader, }, { From 55df19da9fc87f4da7ad04268fdcbea0493f76a7 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 21:48:26 +0800 Subject: [PATCH 05/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20dev=20?= =?UTF-8?q?=E9=98=B6=E6=AE=B5=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 记录 dev 持久 shadow service 真实链路验收结果 记录 dev Nginx 前缀 canary 验收和恢复状态 补充 canary 前必须先完成持久 shadow 验收的口径 --- deploy/nginx/README.md | 1 + ...开发运维】Pingora独立网关试点-2026-06-11.md | 29 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index d33842cf5..555bda2e5 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -63,6 +63,7 @@ - `deploy/nginx/snippets/genarrative-pingora-canary.conf` 是默认不启用的人工前缀 canary 入口，只在需要验证 Nginx -> Pingora handoff 时临时 include。它使用 `/__genarrative_pingora_canary/` 前缀改写后转发到 `127.0.0.1:18081`，并返回 `X-Genarrative-Nginx-Handoff: pingora-canary`。 - `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary。它只能在 Nginx `http` 上下文人工 include，默认监听 `127.0.0.1:18083`，覆盖 `/api/creation-entry/config`、`/v1/identity`、`/v1/database//subscribe`、`/assets/app.js` 和拒绝入口，并写入独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`；不要把它 include 到生产 `443` server 内作为 location 覆盖。 +- 启用 Nginx canary 前，目标机必须已经有持久 `genarrative-pingora-gateway.service` 运行在 `127.0.0.1:18081`，且带真实 `api-server`、SpacetimeDB、静态目录和 `/var/log/genarrative/pingora-gateway.access.log` 完成本机 shadow 验收；不要从一次性 `/tmp` 网关进程直接切到 Nginx handoff。 - 启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，同时做两个 snippet 的静态护栏和真实 `nginx -t`。本机或 CI 可运行 `npm run check:pingora-canary-docker`，用 Docker Nginx、真实 Pingora 和 mock 上游复现前缀 canary 与真实路径 canary 两条 handoff 链路。 - 前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`，再用 `scripts/check-pingora-canary-access-log-parity.mjs` 对账 `/var/log/nginx/genarrative.access.log` 与 `/var/log/genarrative/pingora-gateway.access.log`。 - 真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 46375cb9d..1f593521d 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -312,7 +312,7 @@ node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch 当前 canary 只提供人工验证入口，不做随机流量抽样。启用步骤： 1. 在本地或 CI 先执行 `npm run check:pingora-canary-docker`；需要强制真实 Docker Nginx 链路时执行 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`。该脚本会同时验证 Docker Nginx access log 与 Pingora access log 的 `request_id` 对账，不只看 handoff 响应头。 -2. 确认 `genarrative-pingora-gateway.service` 已运行，且 `GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN` 非空。 +2. 确认 `genarrative-pingora-gateway.service` 已作为持久 shadow service 运行在 `127.0.0.1:18081`，`GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN` 非空，并已用真实 `api-server`、SpacetimeDB、静态 Web 目录和 access log 做过本机回环验收；不要直接从一次性 `/tmp` 进程进入 Nginx canary。 3. 在目标机复制 `/etc/nginx/snippets/genarrative-pingora-canary.conf` 为临时启用版本，替换 `__GENARRATIVE_PINGORA_PROBE_TOKEN__`。 4. 在 `server {}` 内人工 include 该 snippet，并保持 `allow 127.0.0.1; allow ::1; deny all;` 或改成当次可信来源。 5. 执行 `npm run check:nginx-pingora-canary`；目标机或 CI 有 Nginx 时执行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，再执行 `nginx -t && nginx -s reload`。 @@ -321,6 +321,33 @@ node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch 8. 执行 current release 随包 `scripts/check-pingora-canary-access-log-parity.mjs --nginx-log-file /var/log/nginx/genarrative.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_canary/healthz --path /__genarrative_pingora_canary/api/creation-entry/config`，确认同一 `request_id`、`path`、`status` 和 `proxy_target` 与 Nginx access log 可对齐。对账脚本的日志路径、canary prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 都不能包含换行或 NUL；若 access log 行里解析出的 URI / path 含控制字符，脚本也会把对应行记为失败，避免污染值进入 JSON 对账输出。 9. 验证结束后移除 include 并 reload Nginx；不要把该前缀入口当作正式公网 URL。 +## dev shadow service 验收记录 + +2026-06-17 已在 dev 机安装 `genarrative-pingora-gateway.service` 作为持久 shadow service。服务使用 `/opt/genarrative/current/pingora-gateway`，读取 `/etc/genarrative/pingora-gateway.env`，只监听 `127.0.0.1:18081`，上游保持 `api-server=127.0.0.1:8082`、`SpacetimeDB=127.0.0.1:3101`、静态目录 `/srv/genarrative/web`，access log 写入 `/var/log/genarrative/pingora-gateway.access.log`。probe token 只保存在目标机 env 文件，不进入仓库、终端日志或证据包。 + +本阶段验收只证明 shadow service 和真实上游链路可用，不启用 Nginx include，不 reload Nginx，不绑定公网 `80/443`。验收结果： + +- `systemctl is-active genarrative-pingora-gateway.service` 返回 `active`，`NRestarts=0`。 +- `GET /__genarrative_pingora/healthz` 带 `X-Genarrative-Pingora-Probe` 返回 `200` 和 `gateway=pingora-shadow`，不带 token 返回 `404`。 +- `GET /api/creation-entry/config` 经 Pingora 转发到真实 `api-server`，返回 `200 application/json`。 +- `GET /` 从真实 Web 目录返回 `200 text/html`。 +- `GET /v1/identity` 经 Pingora 转发到真实 SpacetimeDB，返回 `405 Method Not Allowed`；这是当前 SpacetimeDB 对 GET identity 的真实语义，只作为路径转发代表。 +- `GET /v1/database/genarrative-prod/subscribe?compression=Brotli` WebSocket Upgrade 经 Pingora 转发到真实 SpacetimeDB，返回 `101 Switching Protocols`，并带 `X-Genarrative-Gateway: pingora-shadow`。 +- `/var/log/genarrative/pingora-gateway.access.log` 已记录 healthz、API、静态、SpacetimeDB identity 和 WSS subscribe 的 `request_id`、`path`、`status`、`proxy_target` 与 `upstream`。 + +dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清理旧 `/tmp/genarrative-*` 临时部署目录、apt cache，并将 journald 收敛到约 `512M`，`df -h /` 从 `100%` 降到约 `92%`。后续 canary / 证据归档前仍应复核 `df -h /`，避免 access log、Nginx reload 或证据归档阶段被磁盘空间干扰。 + +## dev Nginx prefix canary 验收记录 + +2026-06-17 已在 dev 机做过一次临时 Nginx prefix canary。步骤是将 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 渲染到 `/etc/nginx/snippets/genarrative-pingora-canary.conf`，只在 `dev.genarrative.world` 的本机 HTTP `server {}` 内临时 include，执行 `nginx -t && nginx -s reload` 后用 `http://127.0.0.1` 与 `Host: dev.genarrative.world` 做 loopback 验收。验证结束后已恢复 `/etc/nginx/conf.d/genarrative.conf` 备份并 reload Nginx；当前正常公网入口仍由 Nginx 原配置承接，Pingora 继续只作为 `127.0.0.1:18081` shadow service 运行。 + +本阶段验收只证明 Nginx -> Pingora 的前缀 handoff 可用，不做真实路径 canary，不切 `80/443` 到 Pingora。验收结果： + +- `node /tmp/check-pingora-canary-live.mjs --base-url http://127.0.0.1 --host dev.genarrative.world --json` 通过：healthz `200`、API config `200`、SpacetimeDB identity `405`、静态代表路径 `404`、generated 拒绝路径 `404`，全部带 `X-Genarrative-Nginx-Handoff: pingora-canary`。 +- 通过 Nginx 前缀 canary 访问 `/__genarrative_pingora_canary/v1/database/genarrative-prod/subscribe?compression=Brotli`，WebSocket Upgrade 返回 `101 Switching Protocols`，并带 `sec-websocket-protocol: v2.bsatn.spacetimedb`、`X-Genarrative-Nginx-Handoff: pingora-canary` 和 `X-Genarrative-Gateway: pingora-shadow`。 +- `node /tmp/check-pingora-canary-access-log-parity.mjs --nginx-log-file /var/log/nginx/genarrative.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ... --json` 对账 `6/6 matched`，`missingCount=0`，`mismatchCount=0`；覆盖 healthz、API config、SpacetimeDB identity、WSS subscribe、静态代表路径和 generated 拒绝路径。 +- 恢复后 `grep genarrative-pingora-canary /etc/nginx/conf.d/genarrative.conf` 无匹配，`GET http://127.0.0.1/__genarrative_pingora_canary/healthz` 在正常 Nginx HTTP 入口回到 `301`，证明临时 canary include 已移除；`genarrative-pingora-gateway.service` 仍为 `active` 且 `NRestarts=0`。 + ## 环境变量 | 变量 | 默认值 | 说明 | From e5713944f390b2843eb7d919c739cf8df78c3f1d Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 21:52:49 +0800 Subject: [PATCH 06/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20=E7=9C=9F?= =?UTF-8?q?=E5=AE=9E=E8=B7=AF=E5=BE=84=20canary=20=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 记录 dev 真实路径 canary 的 live smoke 和 WSS 验收 记录 Nginx 与 Pingora access log 真实路径对账结果 记录恢复 Nginx 配置并释放 18083 端口 --- .../【开发运维】Pingora独立网关试点-2026-06-11.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 1f593521d..3d8296c48 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -348,6 +348,17 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - `node /tmp/check-pingora-canary-access-log-parity.mjs --nginx-log-file /var/log/nginx/genarrative.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ... --json` 对账 `6/6 matched`，`missingCount=0`，`mismatchCount=0`；覆盖 healthz、API config、SpacetimeDB identity、WSS subscribe、静态代表路径和 generated 拒绝路径。 - 恢复后 `grep genarrative-pingora-canary /etc/nginx/conf.d/genarrative.conf` 无匹配，`GET http://127.0.0.1/__genarrative_pingora_canary/healthz` 在正常 Nginx HTTP 入口回到 `301`，证明临时 canary include 已移除；`genarrative-pingora-gateway.service` 仍为 `active` 且 `NRestarts=0`。 +## dev Nginx realpath canary 验收记录 + +2026-06-17 已在 dev 机做过一次临时 Nginx realpath canary。步骤是将 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 渲染到 `/etc/nginx/snippets/genarrative-pingora-realpath-canary.conf`，并在 Nginx `http` 上下文临时 include 一个只监听 `127.0.0.1:18083` 的独立 `server {}`；执行 `nginx -t && nginx -s reload` 后，用 `http://127.0.0.1:18083` 与 `Host: genarrative-pingora-realpath-canary.local` 做 loopback 验收。验证结束后已恢复 `/etc/nginx/conf.d/genarrative.conf` 备份并 reload Nginx；`127.0.0.1:18083` 已释放，正常公网入口仍由 Nginx 原配置承接，Pingora 继续只作为 `127.0.0.1:18081` shadow service 运行。 + +本阶段验收只证明真实路径 Nginx -> Pingora handoff 可用，不切 `80/443` 到 Pingora。验收结果： + +- `node /tmp/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host genarrative-pingora-realpath-canary.local --json` 通过：healthz `200`、API config `200`、SpacetimeDB identity `405`、静态代表路径 `404`、generated 拒绝路径 `404`，全部带 `X-Genarrative-Nginx-Handoff: pingora-realpath-canary`。 +- 通过真实路径 canary 访问 `/v1/database/genarrative-prod/subscribe?compression=Brotli`，WebSocket Upgrade 返回 `101 Switching Protocols`，并带 `sec-websocket-protocol: v2.bsatn.spacetimedb`、`X-Genarrative-Nginx-Handoff: pingora-realpath-canary` 和 `X-Genarrative-Gateway: pingora-shadow`。 +- `node /tmp/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ... --json` 对账 `6/6 matched`，`missingCount=0`，`mismatchCount=0`；除 healthz 探针映射到 `__genarrative_pingora/healthz` 外，API config、SpacetimeDB identity、WSS subscribe、静态代表路径和 generated 拒绝路径的 Nginx path 与 Pingora path 完全一致。 +- 恢复后 `grep genarrative-pingora-realpath-canary /etc/nginx/conf.d/genarrative.conf` 无匹配，`ss -ltnp | grep :18083` 无监听，证明临时 realpath canary server 已移除；`genarrative-pingora-gateway.service` 仍为 `active` 且 `NRestarts=0`。 + ## 环境变量 | 变量 | 默认值 | 说明 | From 2bf569562e8450d88d5abd18645091de8f2f773b Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 22:03:58 +0800 Subject: [PATCH 07/22] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20API=20=E9=83=A8?= =?UTF-8?q?=E7=BD=B2=E8=87=AA=E6=B5=8B=20worker=20=E6=A8=A1=E6=8B=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐 fake systemctl 对默认 worker 自动启用后的 list-units 模拟 避免 production-api-deploy 自测在 worker 自动启用路径误失败 --- scripts/check-production-api-deploy.mjs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 9e7e76044..36ebece13 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -907,6 +907,7 @@ function prepareFixture(name) { const maintenanceFile = path.join(root, 'maintenance', 'enabled'); const fakeBin = path.join(root, 'bin'); const commandsLog = path.join(root, 'commands.log'); + const workerStateFile = path.join(root, 'worker-service-enabled'); const version = `20260614-${name}`; mkdirSync(sourceDir, { recursive: true }); @@ -1086,8 +1087,21 @@ function prepareFixture(name) { path.join(fakeBin, 'systemctl'), [ '#!/usr/bin/env bash', + 'set -euo pipefail', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, - 'if [[ "$1 $2 $3" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then', + 'worker_state_file="${FAKE_WORKER_STATE_FILE}"', + 'if [[ "$1" == "list-units" ]]; then', + ' pattern="${@: -1}"', + ' if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" && -f "${worker_state_file}" ]]; then', + ' printf "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker\\n"', + ' fi', + ' exit 0', + 'fi', + 'if [[ "$1 $2" == "enable --now" && "${3:-}" == "genarrative-external-generation-worker@1.service" ]]; then', + ' printf "enabled\\n" > "${worker_state_file}"', + ' exit 0', + 'fi', + 'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then', ' exit 3', 'fi', 'exit 0', @@ -1136,6 +1150,7 @@ function prepareFixture(name) { maintenanceFile, fakeBin, commandsLog, + workerStateFile, version, }; } @@ -1204,6 +1219,7 @@ function runDeploy(fixture, options = {}) { options.createReleaseDuringCopy === true ? 'true' : 'false', FAKE_RELEASE_ROOT: fixture.releaseRoot, FAKE_RELEASE_VERSION: fixture.version, + FAKE_WORKER_STATE_FILE: fixture.workerStateFile, }, }, ); From 68bf3b698a885ee2cd248dcba81e4b57460000f8 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 22:09:03 +0800 Subject: [PATCH 08/22] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20API=20=E5=8F=91?= =?UTF-8?q?=E5=B8=83=20checksum=20=E5=85=A5=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 部署脚本将 api-server 与 Pingora checksum 一起复制到 current release 部署自测覆盖 current release checksum 自审依赖 同步 Pingora 正式发布链路文档和共享排障记忆 --- deploy/nginx/README.md | 2 +- docs/project-memory/shared-memory/pitfalls.md | 2 +- .../【开发运维】Pingora独立网关试点-2026-06-11.md | 2 +- scripts/check-production-api-deploy.mjs | 8 ++++++++ scripts/deploy/production-api-deploy.sh | 2 ++ 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 555bda2e5..4000e7481 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -46,7 +46,7 @@ - Pingora 网关默认仍只监听本机 shadow 端口，不绑定公网 `80/443`。 - 需要评估直连入口时，显式配置 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` 和 `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`；网关只读取现有证书文件，不负责 Certbot / ACME 自动续期。 - 需要 HTTP 到 HTTPS 跳转时，再显式配置 `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`；该入口除 `/.well-known/acme-challenge/*` 外统一 301 到 HTTPS，ACME challenge 仍从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取。 -- API release 必须把 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 - 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 681bb753c..a65fc8c9e 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -18,7 +18,7 @@ ## Pingora 直连 80/443 不能只改 env - 现象：`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`，但 `genarrative-pingora-gateway.service` 启动失败，日志出现低端口绑定权限错误。 -- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 +- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 - 处理：确认真实 TLS 证书和 redirect env 已写入 `/etc/genarrative/pingora-gateway.env`、service 模板和 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含这份 env、当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都能读取证书链 / 私钥、current release 的 `pingora-gateway` 已存在且可执行、Nginx 或其它进程已释放 `80/443` 后，先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成只读 JSON runbook，并逐条审阅 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础 readiness、direct enable dry-run、direct enable apply、启用后 `--require-direct` 复核、rollback dry-run、rollback apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；runbook 只用于审阅，不修改系统。正式 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 或回退 smoke 各自验证到不同入口；同时必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检需要 Host 覆盖，再追加 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 把现场巡检入口覆盖成仓库默认值；如需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文。随后先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，再 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`，最后执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，由脚本先跑 direct preflight，再安装 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`、用 `systemctl is-active` 确认服务 active，再以 JSON 模式执行 direct live smoke，验证 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘，并要求 `direct-access-log` 结构化结果 `matchedCount == checked`、`missingCount=0`、`mismatchCount=0`；如果 direct live 退出 0 但缺少该结构化证据，也必须视为启用失败。直连启用后同步调整 `/etc/genarrative/health-patrol.env`：设置 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，本机打 `127.0.0.1` 时设置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`，否则巡检会继续按 Nginx 模式误报。验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 或 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>`；回退脚本先跑 `nginx -t`，通过后才移除 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 已移除、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`，随后 reload Nginx、确认 Nginx service 仍为 active，并用 curl smoke URL 证明公网入口已回到 Nginx；回退脚本 `--apply` 不允许省略 `--reload-nginx` 或 `--nginx-smoke-url`，当 smoke URL 指向本机地址时必须同时提供 `--nginx-smoke-host <域名>`，且 host 值不能包含 URL、路径或查询；回退后把 health patrol gateway mode 改回 `nginx`，恢复切换前 public base URL / Host，并用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url <切换前Nginx巡检入口> --require-empty-public-host` 复核；若切换前 Nginx 巡检需要 Host 覆盖，则把 `--require-empty-public-host` 换成 `--expected-public-host <切换前Host>`。若 env 已在回退命令前切回 Nginx，也可给 rollback 脚本追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让它在 Nginx smoke 后自动复核；切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若要同时证明 Pingora shadow 高端口仍活着，追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应为 `gateway=pingora-shadow`。 - 处理补充：不要在切换窗口手工编辑 `/etc/genarrative/health-patrol.env` 的三项网关变量；使用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切到直连，回退前用同一脚本传 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>` 并按切换前记录选择 `--clear-public-host` 或 `--public-host <切换前Host>`。脚本只改 gateway mode / public base URL / public Host，并立即复用随包 env 复核脚本，减少空 Host 和旧值残留；生产巡检、env 复核和 env 切换脚本读取的布尔 env 都必须是明确布尔值，非法值直接失败，不能把拼写错误当成 false；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，直接用 `node script.mjs --env-file ...` 或 shebang 执行 `.mjs --env-file ...` 都可能让 Node 抢走业务参数；所有这类命令都必须写成 `node -- script.mjs --env-file ...`，或通过已内置 `node --` 的 npm script 执行。 - 踩坑补充：health patrol env 切换脚本必须先复核权限固定为 `0600` 的临时目标 env 再写真实文件，真实 env 原子替换时保持原文件权限和 owner/group；如果随包 env 复核脚本失败，`--apply` 应失败且真实 env 保持原样，避免“切换脚本失败但巡检配置已半改”的状态。`--apply` 的 `--env-file` 必须直接指向真实普通文件，不能传符号链接；如果 `/etc/genarrative/health-patrol.env` 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 3d8296c48..27af7dc4e 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -225,7 +225,7 @@ Jenkins `Genarrative-Api-Build` 对应参数是 `INCLUDE_PINGORA_GATEWAY`，默 API release 还必须携带 `scripts/check-pingora-release-readiness.mjs` 与 `scripts/check-pingora-canary-live.mjs`。前者支撑 current release 的 `--release-runtime-only` 聚合复核，后者支撑目标 Nginx canary live smoke；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败。 -发布包会额外包含 `pingora-gateway` 与 `pingora-gateway.sha256`。`production-api-deploy.sh` 看到这两个文件时会校验并复制到 `/opt/genarrative/current/pingora-gateway`，供 shadow systemd 模板使用；没有这两个文件时现有 API 发布行为不变。API 发布仍只重启 `genarrative-api.service`；如果 `genarrative-pingora-gateway.service` 已经处于 `active`，部署脚本会在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制。若 Pingora 服务未运行，部署脚本只打印跳过信息，不主动拉起影子服务，也不改变默认接流边界。 +发布包会额外包含 `pingora-gateway` 与 `pingora-gateway.sha256`。`production-api-deploy.sh` 看到这两个文件时会校验并把 `pingora-gateway` / `pingora-gateway.sha256` 一起复制到 `/opt/genarrative/current`，供 shadow systemd 模板和 current release 自审使用；`api-server.sha256` 也必须随 `api-server` 一起进入 current release。没有这两个 Pingora 文件时现有 API 发布行为不变。API 发布仍只重启 `genarrative-api.service`；如果 `genarrative-pingora-gateway.service` 已经处于 `active`，部署脚本会在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制。若 Pingora 服务未运行，部署脚本只打印跳过信息，不主动拉起影子服务，也不改变默认接流边界。 也可以复制 `deploy/pingora/pingora-gateway.env.example` 到部署环境的非 Git 配置文件，由 systemd 或容器注入。仓库提供 `deploy/systemd/genarrative-pingora-gateway.service` 作为影子服务模板，默认读取 `/etc/genarrative/pingora-gateway.env`，仍只应监听本机高端口，再由 Nginx 或本机 smoke 主动访问。Server-Provision 会把主 service 安装到 `/etc/systemd/system/genarrative-pingora-gateway.service`，并把直连低端口 drop-in 模板安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为参考和手动覆盖来源；该模板不会默认生效，`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 默认使用随 current release 发布的 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是可选的 Nginx -> Pingora 前缀 canary 模板，Server-Provision 会安装到 `/etc/nginx/snippets/`，但主站配置默认不 include；启用前必须把 `__GENARRATIVE_PINGORA_PROBE_TOKEN__` 替换为真实 token，并确认 `allow/deny` 来源边界符合当次验证窗口。 diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 36ebece13..21843139c 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -157,6 +157,10 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() { path.join(releaseDir, 'release-manifest.api-server.json'), 'current release 必须包含 API release manifest 副本。', ); + assertFileExists( + path.join(releaseDir, 'api-server.sha256'), + 'current release 必须包含 api-server checksum，供随包 current release 自审校验。', + ); assertFileExists( path.join(releaseDir, 'deploy/systemd/genarrative-pingora-gateway.service'), 'current release 必须包含 systemd 主 service 模板。', @@ -254,6 +258,10 @@ function assertDeployRestartsActivePingoraWhenArtifactIncluded() { path.join(releaseDir, 'pingora-gateway'), '包含 Pingora 的发布包部署后 current release 必须包含网关二进制。', ); + assertFileExists( + path.join(releaseDir, 'pingora-gateway.sha256'), + '包含 Pingora 的发布包部署后 current release 必须包含网关 checksum。', + ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index ec3714f8e..7a017686b 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -598,9 +598,11 @@ mkdir -p "${RELEASE_ROOT}" mkdir "${STAGING_RELEASE_DIR}" RELEASE_CONTENT_DIR="${STAGING_RELEASE_DIR}" cp "${SOURCE_DIR}/api-server" "${RELEASE_CONTENT_DIR}/api-server" +cp "${SOURCE_DIR}/api-server.sha256" "${RELEASE_CONTENT_DIR}/api-server.sha256" chmod +x "${RELEASE_CONTENT_DIR}/api-server" if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway" + cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256" chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway" echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核并重启已运行的 ${PINGORA_SERVICE_NAME}" fi From 206ea48878523b38b258e0335ffa27430c9cbc7a Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 22:15:15 +0800 Subject: [PATCH 09/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20dev=20?= =?UTF-8?q?=E6=AD=A3=E5=BC=8F=E5=8F=91=E5=B8=83=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 dev API release 正式路径部署证据 记录 current release 自审、runtime-only 门禁和 shadow WSS 验收结果 明确公网入口仍由 Nginx 承接且临时发布包已清理 --- ...开发运维】Pingora独立网关试点-2026-06-11.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 27af7dc4e..0832bf487 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -359,6 +359,24 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - `node /tmp/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ... --json` 对账 `6/6 matched`，`missingCount=0`，`mismatchCount=0`；除 healthz 探针映射到 `__genarrative_pingora/healthz` 外，API config、SpacetimeDB identity、WSS subscribe、静态代表路径和 generated 拒绝路径的 Nginx path 与 Pingora path 完全一致。 - 恢复后 `grep genarrative-pingora-realpath-canary /etc/nginx/conf.d/genarrative.conf` 无匹配，`ss -ltnp | grep :18083` 无监听，证明临时 realpath canary server 已移除；`genarrative-pingora-gateway.service` 仍为 `active` 且 `NRestarts=0`。 +## dev API release 正式路径验收记录 + +2026-06-17 已在 dev 机用正式 API release 路径部署包含 Pingora 的发布包 `dev-pingora-api-20260617140915`。发布包由分支 `codex/pingoranginx` 的 `68bf3b698a885ee2cd248dcba81e4b57460000f8` 构建，包含 `api-server`、`api-server.sha256`、`pingora-gateway`、`pingora-gateway.sha256`、`release-manifest.json` 和随包部署 / 自审脚本。部署命令从上传到 `/tmp/dev-pingora-api-20260617140915` 的发布包内执行 `scripts/deploy/production-api-deploy.sh`，目标 release root 为 `/opt/genarrative/releases`，current link 为 `/opt/genarrative/current`。 + +本阶段验收只证明 API release 的正式打包、复制、current 切换、服务重启和 shadow Pingora 随 current 重启可用，不切 `80/443` 到 Pingora。验收结果： + +- `production-api-deploy.sh` 进入维护模式后完成 `api-server.sha256` 和 `pingora-gateway.sha256` 校验，复制二进制、checksum、manifest 和 Pingora 直连依赖到新 release，再把 `/opt/genarrative/current` 切到 `/opt/genarrative/releases/dev-pingora-api-20260617140915`，最后退出维护模式。 +- 部署时 `genarrative-pingora-gateway.service` 已为 `active`，脚本在 current 切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核 active；`genarrative-api.service`、外部生成 worker 和 worker controller 也完成重启和 active 等待。 +- `/opt/genarrative/current/release-manifest.api-server.json` 记录 `api-server` 和 `pingora-gateway` 两个 artifact，`cd /opt/genarrative/current && sha256sum -c api-server.sha256 && sha256sum -c pingora-gateway.sha256` 均为 `OK`。 +- `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show` 返回 `summary.status=OK`，确认 current release 自包含、checksum 匹配、`pingora-gateway` 可执行，且 systemd `ExecStart` 指向 `/opt/genarrative/current/pingora-gateway`。 +- `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only` 通过，证明目标机可以只依赖 current release 运行 runtime-only 发布自审，不需要源码 checkout 或 Jenkins 工作区。 +- 部署后 `nginx`、`genarrative-api`、`genarrative-pingora-gateway`、`spacetimedb`、`genarrative-external-generation-controller.service` 和 `genarrative-external-generation-worker@1.service` 均为 `active`，API / Pingora / worker 相关服务 `NRestarts=0`。 +- `GET http://127.0.0.1:8082/readyz` 返回 `{"ok":true,"ready":true}`；`GET http://127.0.0.1:8082/api/creation-entry/config` 返回 `200`。 +- `GET http://127.0.0.1:18081/api/creation-entry/config` 经 Pingora shadow 返回 `200`；`GET http://127.0.0.1:18081/v1/identity` 返回真实 SpacetimeDB 语义 `405`。 +- 使用完整 WebSocket 握手头 `Sec-WebSocket-Protocol: v2.bsatn.spacetimedb` 访问 `http://127.0.0.1:18081/v1/database/genarrative-prod/subscribe?compression=Brotli`，HTTP code 为 `101`；curl 的超时退出只发生在 WebSocket 升级后连接保持阶段。少传该子协议时 SpacetimeDB 会返回 `400 no valid protocol selected`，不能作为网关失败证据。 +- Nginx 仍监听公网 `80/443`，Pingora 仍只监听 `127.0.0.1:18081`；`127.0.0.1:18083` 无监听。`https://dev.genarrative.world/` 与 `https://dev.genarrative.world/api/creation-entry/config` 通过本机 `--resolve` 验收均为 `200`，证明正常公网入口仍由 Nginx 承接。 +- 上传到 `/tmp/dev-pingora-api-20260617140915` 的临时发布包已删除；新 release 目录约 `57M`，清理后 dev 根盘约 `92%` 使用率。后续正式证据归档前仍需复核磁盘余量。 + ## 环境变量 | 变量 | 默认值 | 说明 | From 057522fc2e83ff2108984842afab1df94761025f Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 22:40:05 +0800 Subject: [PATCH 10/22] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20Pingora=20TLS=20?= =?UTF-8?q?=E8=AF=81=E4=B9=A6=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 Pingora TLS 证书同步脚本，默认 dry-run，apply 时原子复制证书到私有目录并复核服务用户可读。 将证书同步脚本纳入 API release 构建、部署复制和 current release 自审清单。 补充发布包、deploy、自审和脚本自身门禁，覆盖缺失脚本、symlink 目标与权限边界。 更新 Pingora env 示例、运维文档和项目踩坑记录，明确不要放宽 Let’s Encrypt 原路径权限。 --- deploy/nginx/README.md | 3 +- deploy/pingora/pingora-gateway.env.example | 8 +- docs/project-memory/shared-memory/pitfalls.md | 3 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 20 +- package.json | 1 + scripts/build-production-release.sh | 4 +- .../check-pingora-current-release-audit.mjs | 13 + scripts/check-pingora-tls-cert-sync.mjs | 281 +++++++++++ scripts/check-production-api-deploy.mjs | 32 ++ scripts/check-production-api-release.mjs | 10 +- scripts/deploy/pingora-tls-cert-sync.mjs | 470 ++++++++++++++++++ scripts/deploy/production-api-deploy.sh | 7 + scripts/ops/pingora-current-release-audit.mjs | 1 + 13 files changed, 841 insertions(+), 12 deletions(-) create mode 100644 scripts/check-pingora-tls-cert-sync.mjs create mode 100644 scripts/deploy/pingora-tls-cert-sync.mjs diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 4000e7481..7f8e0f8d0 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -46,7 +46,8 @@ - Pingora 网关默认仍只监听本机 shadow 端口，不绑定公网 `80/443`。 - 需要评估直连入口时，显式配置 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` 和 `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`；网关只读取现有证书文件，不负责 Certbot / ACME 自动续期。 - 需要 HTTP 到 HTTPS 跳转时，再显式配置 `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`；该入口除 `/.well-known/acme-challenge/*` 外统一 301 到 HTTPS，ACME challenge 仍从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取。 -- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- 目标机如果使用 Certbot / Let’s Encrypt 证书，不要直接放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限。先用随包脚本把 root 可读的 live symlink 解析并复制到 Pingora 私有目录，例如 `sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>`；脚本默认 dry-run，`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并复核 `genarrative` 服务用户可读。随后把 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/` 下的副本。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 - 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。 diff --git a/deploy/pingora/pingora-gateway.env.example b/deploy/pingora/pingora-gateway.env.example index a60b9d456..1c69ad8cf 100644 --- a/deploy/pingora/pingora-gateway.env.example +++ b/deploy/pingora/pingora-gateway.env.example @@ -4,11 +4,13 @@ GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081 # 可选：直连 HTTPS 入口。默认关闭；启用时必须同时配置 cert/key，并把 FORWARDED_PROTO 设为 https。 -# cert/key 必须允许 genarrative 用户读取；80/443 必须先从 Nginx 或其它进程释放。 +# cert/key 必须允许 genarrative 用户读取；不要直接 chmod Let's Encrypt live/archive 原始目录。 +# 可先用随包 scripts/deploy/pingora-tls-cert-sync.mjs 把证书同步到 /etc/genarrative/pingora-tls//。 +# 80/443 必须先从 Nginx 或其它进程释放；高端口 loopback 演练可先使用 127.0.0.1:18443 / 127.0.0.1:18080。 # 如果要绑定 443/80，必须另行人工启用 systemd direct-entry drop-in 授予 CAP_NET_BIND_SERVICE。 # GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:18443 -# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/etc/letsencrypt/live/genarrative.world/fullchain.pem -# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/etc/letsencrypt/live/genarrative.world/privkey.pem +# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/etc/genarrative/pingora-tls/genarrative.world/fullchain.pem +# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/etc/genarrative/pingora-tls/genarrative.world/privkey.pem # 可选：直连 HTTP 只做 301 到 HTTPS；启用时必须已经配置 TLS_LISTEN，ACME challenge 仍从 ACME_ROOT 静态读取。 # GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:18080 # GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME=https diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index a65fc8c9e..55d6e0d82 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -20,6 +20,7 @@ - 现象：`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`，但 `genarrative-pingora-gateway.service` 启动失败，日志出现低端口绑定权限错误。 - 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 - 处理：确认真实 TLS 证书和 redirect env 已写入 `/etc/genarrative/pingora-gateway.env`、service 模板和 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含这份 env、当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都能读取证书链 / 私钥、current release 的 `pingora-gateway` 已存在且可执行、Nginx 或其它进程已释放 `80/443` 后，先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成只读 JSON runbook，并逐条审阅 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础 readiness、direct enable dry-run、direct enable apply、启用后 `--require-direct` 复核、rollback dry-run、rollback apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；runbook 只用于审阅，不修改系统。正式 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 或回退 smoke 各自验证到不同入口；同时必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检需要 Host 覆盖，再追加 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 把现场巡检入口覆盖成仓库默认值；如需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文。随后先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，再 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`，最后执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，由脚本先跑 direct preflight，再安装 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`、用 `systemctl is-active` 确认服务 active，再以 JSON 模式执行 direct live smoke，验证 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘，并要求 `direct-access-log` 结构化结果 `matchedCount == checked`、`missingCount=0`、`mismatchCount=0`；如果 direct live 退出 0 但缺少该结构化证据，也必须视为启用失败。直连启用后同步调整 `/etc/genarrative/health-patrol.env`：设置 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，本机打 `127.0.0.1` 时设置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`，否则巡检会继续按 Nginx 模式误报。验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 或 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>`；回退脚本先跑 `nginx -t`，通过后才移除 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 已移除、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`，随后 reload Nginx、确认 Nginx service 仍为 active，并用 curl smoke URL 证明公网入口已回到 Nginx；回退脚本 `--apply` 不允许省略 `--reload-nginx` 或 `--nginx-smoke-url`，当 smoke URL 指向本机地址时必须同时提供 `--nginx-smoke-host <域名>`，且 host 值不能包含 URL、路径或查询；回退后把 health patrol gateway mode 改回 `nginx`，恢复切换前 public base URL / Host，并用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url <切换前Nginx巡检入口> --require-empty-public-host` 复核；若切换前 Nginx 巡检需要 Host 覆盖，则把 `--require-empty-public-host` 换成 `--expected-public-host <切换前Host>`。若 env 已在回退命令前切回 Nginx，也可给 rollback 脚本追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让它在 Nginx smoke 后自动复核；切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若要同时证明 Pingora shadow 高端口仍活着，追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应为 `gateway=pingora-shadow`。 +- 处理补充：不要直接 chmod `/etc/letsencrypt/live` 或 `archive` 来让 Pingora 读取证书；Certbot live 路径通常是 symlink，即使 `stat -L` 看起来是普通文件，父目录权限也会让非 root `genarrative` 用户不可达。先用随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>` 把证书同步到 Pingora 私有目录，再让 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/fullchain.pem` 和 `privkey.pem`。脚本默认 dry-run，`--apply` 才写入，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并拒绝符号链接目标目录或目标文件。 - 处理补充：不要在切换窗口手工编辑 `/etc/genarrative/health-patrol.env` 的三项网关变量；使用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切到直连，回退前用同一脚本传 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>` 并按切换前记录选择 `--clear-public-host` 或 `--public-host <切换前Host>`。脚本只改 gateway mode / public base URL / public Host，并立即复用随包 env 复核脚本，减少空 Host 和旧值残留；生产巡检、env 复核和 env 切换脚本读取的布尔 env 都必须是明确布尔值，非法值直接失败，不能把拼写错误当成 false；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，直接用 `node script.mjs --env-file ...` 或 shebang 执行 `.mjs --env-file ...` 都可能让 Node 抢走业务参数；所有这类命令都必须写成 `node -- script.mjs --env-file ...`，或通过已内置 `node --` 的 npm script 执行。 - 踩坑补充：health patrol env 切换脚本必须先复核权限固定为 `0600` 的临时目标 env 再写真实文件，真实 env 原子替换时保持原文件权限和 owner/group；如果随包 env 复核脚本失败，`--apply` 应失败且真实 env 保持原样，避免“切换脚本失败但巡检配置已半改”的状态。`--apply` 的 `--env-file` 必须直接指向真实普通文件，不能传符号链接；如果 `/etc/genarrative/health-patrol.env` 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 - 踩坑补充：直连启用脚本的 `--preflight-script`、`--direct-live-script`、`--current-release-audit-script`、`--template-path`、`--service-unit-path`、`--dropin-path` 和 env 文件参数都必须使用绝对路径；不要在切换窗口传相对脚本路径，否则会把 current release、Jenkins 工作区或现场 cwd 混在一起。`--apply` 会在安装 direct-entry drop-in 前确认 current release 自审、direct preflight 和 direct live smoke 脚本存在，缺脚本时应先修发布包或复制链路，不要手工改成工作区相对路径绕过。启用脚本还会在任何自审、preflight、drop-in 写入或 systemctl 前拒绝 service、路径、URL、Host、probe token、access log、数据库名、tail 行数和 timeout 参数中的换行或 NUL 字符；遇到这类失败先修 runbook 参数来源或现场 env，不要手工绕过脚本。启用脚本还会拒绝符号链接形式的 drop-in 目录或 drop-in 目标文件，以及已存在但不是普通文件的目标；如果现场 systemd 目录被软链改写，应先修正真实路径，不要让脚本把低端口 capability 写入非预期位置。回退脚本 `--apply` 同样会在 `nginx -t` 和删除 drop-in 前拒绝 service、路径、Nginx smoke URL / Host / 响应片段、health patrol 复核参数、shadow probe URL / token 和二进制 override 中的换行或 NUL 字符，并拒绝符号链接 drop-in 目录 / 目标以及非普通 drop-in 目标；如果现场路径或参数异常，应先修正 systemd 路径、runbook 参数或现场 env，不要手工删 drop-in、绕过 `nginx -t` 或把删除 symlink 当成已回退真实低端口能力。回退脚本覆盖 `--nginx-binary` 或 `--curl-binary` 时也不要传 `./nginx`、`tools/curl` 这类相对路径；裸命令名可以走 `PATH`，路径形式必须使用绝对路径。`--nginx-smoke-url` 必须带 `http://` 或 `https://`，不要只写 host/path，否则脚本会在移除 drop-in 前失败。 @@ -58,7 +59,7 @@ - 踩坑补充：同一天多次演练或切换时，只靠“最新证据”和 24 小时窗口仍可能把两轮证据拼在一起。正式 runbook 会生成或接受 `--cutover-run-id `，并把同一 `manifest.cutoverRunId` 写入三阶段证据包、enable / rollback apply 命令证据和最终总审计；最终审计必须带 `--require-cutover-run-id <本次cutoverRunId>`，缺少该字段或 ID 不一致时必须失败。即使人工临时总审计忘记带 `--require-cutover-run-id`，标准五段时间线里只要任一证据声明了 `manifest.cutoverRunId`，五段也必须全部声明同一个值，否则总审计失败。 - 验证：先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，确认 env、drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和 80/443 已释放；`systemctl cat genarrative-pingora-gateway.service` 必须显示 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env`；启用脚本 apply 必须先通过 current release 自审，失败时不安装 direct-entry drop-in；还必须带 direct HTTPS / HTTP / Host / redirect host / SpacetimeDB database / Pingora access log 参数，并在重启后直接完成 direct live smoke 和 direct-access-log JSON 证据校验；也可用 release readiness `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 把 HTTPS、HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id 落盘、env 预检、systemd drop-in、service EnvironmentFile 一致性、当前用户和服务用户证书可读、service 二进制可执行、端口释放、显式目标库和 WSS 101 一起纳入硬门禁，并拒绝 `--direct-skip-wss`，避免 TLS 证书只按 `127.0.0.1` 误测、HTTP redirect Location 指错域名、Nginx 仍占用 80/443、service 实际读取另一份 env、root / deploy 用户可读但 systemd 服务用户不可读、current release 缺少可执行 `pingora-gateway`，或 WSS subscribe 隐式打到默认 SpacetimeDB 库。`check-pingora-release-readiness.mjs --help` 的正式直连和只生成 runbook 示例也必须带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，不要让值班人员复制示例后才被 `--require-direct` 拦截。current release 自审、状态快照和证据包的布尔 env 必须是明确布尔值，非法值会失败，不得把拼错的 run / require / fail 开关当成 false。`npm run plan:pingora-direct-cutover -- --require-direct ...` 输出必须包含 Host 与回退巡检入口确认、current release preflight、启用前不带 `--require-direct` 的基础 readiness、direct enable dry-run/apply、启用后带 `--require-direct` 的复核、rollback dry-run/apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；缺少 `--require-direct`、缺少 `--rollback-health-patrol-public-base-url`、缺少 `--direct-pingora-access-log`、redirect Host 漂移或 rollback smoke Host 漂移时必须失败，避免生成缺少正式直连硬门禁或验证不同入口的切换计划。Host 与回退巡检入口确认步骤必须展示回退后要恢复的 health patrol public base URL / Host。直连后 `genarrative-health-patrol.service` 应使用 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，状态 JSON 中 `gatewayMode` 应为 `pingora-direct`，并检查 `genarrative-pingora-gateway.service` 而不是 `nginx.service`；public probe 走 `127.0.0.1` 时应带 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`。回退后 `nginx -t` 必须先通过，`systemctl cat genarrative-pingora-gateway.service` 不应再显示这两条 capability，`systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager` 必须仍指向 current release 的 `pingora-gateway`，`systemctl is-active nginx.service` 应为 `active`，`curl --fail --max-time 5` 访问 `--nginx-smoke-url` 应成功；若 smoke URL 为本机地址必须带 `--nginx-smoke-host <域名>`，证明正式 vhost 已回到 Nginx；随后用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs ...` 复核 health patrol env，必须显示 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 且 public base URL / Host 与切换前记录一致，shadow probe 可选复核必须返回 `gateway=pingora-shadow`。本机提交前还要运行 `npm run check:pingora-direct-enable`、`npm run check:pingora-direct-rollback`、`npm run check:production-health-patrol`、`npm run check:production-api-release`、`npm run check:pingora-production-release-build` 和 `npm run check:production-api-deploy`，确保脚本默认 dry-run 不会安装或删除 drop-in、current release 自审失败时启用脚本不会安装 drop-in、direct live 退出 0 但缺少 `direct-access-log` 结构化证据时启用失败，API release 布局自包含，真实 Pingora release 二进制能构建并进入发布包，API deploy 从发布产物内执行后 current release 自包含；缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、切换命令证据脚本、env 示例目录或 direct live smoke 脚本的发布包都必须部署失败并保持维护模式。正式直连 readiness 必须带 `--direct-health-patrol-env-file /etc/genarrative/health-patrol.env`，并用 `scripts/check-production-health-patrol-env.mjs` 阻断 health patrol 仍停在 Nginx 模式或本机 direct probe 缺少正式 Host；发布包包含 `pingora-gateway` 时，`npm run check:production-api-deploy` 必须覆盖服务 active 时执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核 active，以及服务 inactive 时跳过且不主动拉起。 - 顺序补充：正式 runbook 必须先通过 health patrol env 切换脚本预置回 Nginx 和切换前 public base URL / Host，再执行 `rollback apply`；回退脚本内置 env 复核和独立 env 复核都会阻断 public base URL / Host 漂移。 -- 关联：`deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`、`deploy/env/health-patrol.env.example`、`deploy/env/pingora-direct-live.env.example`、`deploy/env/pingora-canary-live.env.example`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/jenkins-server-provision.sh`、`scripts/build-production-release.sh`、`scripts/deploy/production-api-deploy.sh`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 +- 关联：`deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`、`deploy/env/health-patrol.env.example`、`deploy/env/pingora-direct-live.env.example`、`deploy/env/pingora-canary-live.env.example`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/jenkins-server-provision.sh`、`scripts/build-production-release.sh`、`scripts/deploy/production-api-deploy.sh`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 ## 外部生成 worker 业务失败重试会撞上钱包扣退费幂等 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 0832bf487..17aab3968 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -221,7 +221,7 @@ npm run check:pingora-direct-live npm run build:production-release -- --component api-server --name --include-pingora-gateway ``` -Jenkins `Genarrative-Api-Build` 对应参数是 `INCLUDE_PINGORA_GATEWAY`，默认关闭；勾选后才会归档 `pingora-gateway` 与 `pingora-gateway.sha256`，触发后续 `Genarrative-Api-Deploy` 时也会传递该布尔参数并复制这两个可选产物。无论是否打包 Pingora 二进制，API release 都必须携带 `build//scripts/deploy/production-api-deploy.sh`、同目录的 `maintenance-on.sh` / `maintenance-off.sh`、`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh`、`pingora-direct-rollback.sh`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/`。`Genarrative-Api-Deploy` 只能从上游构建归档复制并执行 `build//scripts/deploy/production-api-deploy.sh`，不能继续执行部署工作区根部脚本；维护脚本必须与 deploy 脚本来自同一发布包同一目录，避免 Jenkins 工作区里的旧脚本掩盖 release 包布局缺陷。直连启用脚本从 current release 执行时默认读取 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，必须能自包含完成 preflight、systemd 模板读取和 direct live smoke，不依赖 Jenkins 工作区、目标机源码 checkout 或 `/etc` 参考模板；current release 自审、health patrol env 复核、canary access log 对账、canary live、direct live、状态快照、证据包、命令证据、证据验真和证据根目录审计也必须来自 current release 随包脚本与 `deploy/env/`。`/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 只作为 Server-Provision 安装的人工审阅 / 手动覆盖模板；确需使用时显式传 `--template-path` 或 `GENARRATIVE_PINGORA_DIRECT_TEMPLATE_PATH`。`npm run check:production-api-release` 会用临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 跑 `build-production-release.sh --component api-server --skip-api-build`，动态验证默认 API release 不登记 Pingora，显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时则必须包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest，同时验证 Pingora 直连依赖、current release 自审脚本、状态快照脚本、证据包脚本、命令证据脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、env 示例目录、API deploy 执行入口自包含和 `deploy/pingora/pingora-gateway.env.example` 的生产安全默认值：gzip-only、`TRUST_X_FORWARDED_FOR=false`、`TRUSTED_FRONT_PROXY_CONFIRMED=false`、接流保护默认开启、probe token 为空；该检查还会读取发布包 README，并直接运行发布包内 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover`，确认最终证据根目录总审计步骤仍带两条 `--require-command-executable ... /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` / `pingora-direct-rollback.sh` current release 脚本身份要求。`npm run check:pingora-production-release-build` 会用假 `api-server`、临时 `CARGO_TARGET_DIR` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行 `pingora-gateway`、checksum 和 manifest 登记，避免只靠假二进制布局 smoke。`production-api-deploy.sh` 对这些 Pingora 直连依赖采取 fail-fast：上游发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；`release-manifest.json` 必须存在且登记 `api-server` artifact，若发布包包含 `pingora-gateway`，manifest 也必须登记 `pingora-gateway` artifact，否则部署会在切换 current 前失败。API deploy 要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，`--version` 必须以数字或字母开头并拒绝点目录，会先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部校验和复制完成后用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，再用固定替换语义切换 current 符号链接；同版本正式 release 已存在、提升前竞态出现或 current 路径不是符号链接时都会拒绝合并 / 覆盖，失败时清理 staging 并保持维护模式。本机用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 动态验证从发布产物内执行 deploy 脚本后 current release 自洽，并覆盖缺少 current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、`deploy/env/`、release manifest、api-server manifest artifact、Pingora manifest artifact、相对 release root / current link / api env file、点目录或点开头 version、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时必须失败，同时复核 current release 内 Pingora env 示例仍保持同一组生产安全默认值。 +Jenkins `Genarrative-Api-Build` 对应参数是 `INCLUDE_PINGORA_GATEWAY`，默认关闭；勾选后才会归档 `pingora-gateway` 与 `pingora-gateway.sha256`，触发后续 `Genarrative-Api-Deploy` 时也会传递该布尔参数并复制这两个可选产物。无论是否打包 Pingora 二进制，API release 都必须携带 `build//scripts/deploy/production-api-deploy.sh`、同目录的 `maintenance-on.sh` / `maintenance-off.sh`、`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh`、`pingora-direct-rollback.sh`、`pingora-tls-cert-sync.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/`。`Genarrative-Api-Deploy` 只能从上游构建归档复制并执行 `build//scripts/deploy/production-api-deploy.sh`，不能继续执行部署工作区根部脚本；维护脚本必须与 deploy 脚本来自同一发布包同一目录，避免 Jenkins 工作区里的旧脚本掩盖 release 包布局缺陷。直连启用脚本从 current release 执行时默认读取 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，必须能自包含完成 preflight、systemd 模板读取和 direct live smoke，不依赖 Jenkins 工作区、目标机源码 checkout 或 `/etc` 参考模板；current release 自审、TLS 证书同步、health patrol env 复核、canary access log 对账、canary live、direct live、状态快照、证据包、命令证据、证据验真和证据根目录审计也必须来自 current release 随包脚本与 `deploy/env/`。`/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 只作为 Server-Provision 安装的人工审阅 / 手动覆盖模板；确需使用时显式传 `--template-path` 或 `GENARRATIVE_PINGORA_DIRECT_TEMPLATE_PATH`。`npm run check:production-api-release` 会用临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 跑 `build-production-release.sh --component api-server --skip-api-build`，动态验证默认 API release 不登记 Pingora，显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时则必须包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest，同时验证 Pingora 直连依赖、TLS 证书同步脚本、current release 自审脚本、状态快照脚本、证据包脚本、命令证据脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、env 示例目录、API deploy 执行入口自包含和 `deploy/pingora/pingora-gateway.env.example` 的生产安全默认值：gzip-only、`TRUST_X_FORWARDED_FOR=false`、`TRUSTED_FRONT_PROXY_CONFIRMED=false`、接流保护默认开启、probe token 为空；该检查还会读取发布包 README，并直接运行发布包内 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover`，确认最终证据根目录总审计步骤仍带两条 `--require-command-executable ... /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` / `pingora-direct-rollback.sh` current release 脚本身份要求。`npm run check:pingora-production-release-build` 会用假 `api-server`、临时 `CARGO_TARGET_DIR` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行 `pingora-gateway`、checksum 和 manifest 登记，避免只靠假二进制布局 smoke。`production-api-deploy.sh` 对这些 Pingora 直连依赖采取 fail-fast：上游发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；`release-manifest.json` 必须存在且登记 `api-server` artifact，若发布包包含 `pingora-gateway`，manifest 也必须登记 `pingora-gateway` artifact，否则部署会在切换 current 前失败。API deploy 要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，`--version` 必须以数字或字母开头并拒绝点目录，会先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部校验和复制完成后用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，再用固定替换语义切换 current 符号链接；同版本正式 release 已存在、提升前竞态出现或 current 路径不是符号链接时都会拒绝合并 / 覆盖，失败时清理 staging 并保持维护模式。本机用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 动态验证从发布产物内执行 deploy 脚本后 current release 自洽，并覆盖缺少 TLS 证书同步脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、`deploy/env/`、release manifest、api-server manifest artifact、Pingora manifest artifact、相对 release root / current link / api env file、点目录或点开头 version、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时必须失败，同时复核 current release 内 Pingora env 示例仍保持同一组生产安全默认值。 API release 还必须携带 `scripts/check-pingora-release-readiness.mjs` 与 `scripts/check-pingora-canary-live.mjs`。前者支撑 current release 的 `--release-runtime-only` 聚合复核，后者支撑目标 Nginx canary live smoke；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败。 @@ -229,7 +229,17 @@ API release 还必须携带 `scripts/check-pingora-release-readiness.mjs` 与 `s 也可以复制 `deploy/pingora/pingora-gateway.env.example` 到部署环境的非 Git 配置文件，由 systemd 或容器注入。仓库提供 `deploy/systemd/genarrative-pingora-gateway.service` 作为影子服务模板，默认读取 `/etc/genarrative/pingora-gateway.env`，仍只应监听本机高端口，再由 Nginx 或本机 smoke 主动访问。Server-Provision 会把主 service 安装到 `/etc/systemd/system/genarrative-pingora-gateway.service`，并把直连低端口 drop-in 模板安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为参考和手动覆盖来源；该模板不会默认生效，`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 默认使用随 current release 发布的 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是可选的 Nginx -> Pingora 前缀 canary 模板，Server-Provision 会安装到 `/etc/nginx/snippets/`，但主站配置默认不 include；启用前必须把 `__GENARRATIVE_PINGORA_PROBE_TOKEN__` 替换为真实 token，并确认 `allow/deny` 来源边界符合当次验证窗口。 -直连公网入口切换窗口如果需要让非 root 的 `genarrative` 用户绑定 `80/443`，先确认 `/etc/genarrative/pingora-gateway.env` 已显式设置 TLS / redirect 入口和证书路径，并确认 `genarrative` 用户可读取证书链和私钥；生产 `80/443` 还必须先从 Nginx 或其它进程释放，非切换窗口建议先用 `18443/18080` 这类高端口验证。随后先 dry-run 直连启用脚本： +直连公网入口切换窗口如果需要让非 root 的 `genarrative` 用户绑定 `80/443`，先确认 `/etc/genarrative/pingora-gateway.env` 已显式设置 TLS / redirect 入口和证书路径，并确认 `genarrative` 用户可读取证书链和私钥；生产 `80/443` 还必须先从 Nginx 或其它进程释放，非切换窗口建议先用 `18443/18080` 这类高端口验证。若证书来自 Certbot / Let’s Encrypt，不要放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限；先用随包 `pingora-tls-cert-sync.mjs` 把 live symlink 解析后的真实证书复制到 `/etc/genarrative/pingora-tls/<域名>/`，再把 env 中 `TLS_CERT_FILE` / `TLS_KEY_FILE` 指向该私有副本： + +```bash +sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs \ + --apply \ + --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem \ + --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem \ + --target-dir /etc/genarrative/pingora-tls/<域名> +``` + +该脚本默认 dry-run，`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并复核 `genarrative` 服务用户可读；目标目录和目标文件不能是符号链接。随后先 dry-run 直连启用脚本： ```bash npm run plan:pingora-direct-cutover -- --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --rollback-nginx-smoke-url http://127.0.0.1/healthz --rollback-health-patrol-public-base-url <切换前Nginx巡检入口> @@ -383,8 +393,8 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 | ------------------------------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------------------------------------------- | | `GENARRATIVE_PINGORA_GATEWAY_LISTEN` | `127.0.0.1:18081` | Pingora 监听地址。 | | `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` | 空 | 可选 HTTPS 监听地址；启用时必须同时设置 `TLS_CERT_FILE` 和 `TLS_KEY_FILE`。 | -| `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` | 空 | 可选 HTTPS 证书链文件；必须是网关运行用户可读取的文件。 | -| `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE` | 空 | 可选 HTTPS 私钥文件；必须是网关运行用户可读取的文件。 | +| `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` | 空 | 可选 HTTPS 证书链文件；必须是网关运行用户可读取的文件。Certbot 证书建议先同步到 `/etc/genarrative/pingora-tls/<域名>/fullchain.pem`。 | +| `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE` | 空 | 可选 HTTPS 私钥文件；必须是网关运行用户可读取的文件。Certbot 私钥建议先同步到 `/etc/genarrative/pingora-tls/<域名>/privkey.pem`。 | | `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN` | 空 | 可选 HTTP 重定向监听地址；启用时必须已配置 TLS 入口，ACME challenge 仍静态读取。 | | `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME` | `https` | HTTP 重定向目标 scheme，当前只允许 `https`。 | | `GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM` | `127.0.0.1:8082` | `api-server` 上游地址。 | @@ -462,7 +472,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 4. 使用 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 做 Nginx 前缀 canary；启用前先跑 `npm run check:nginx-pingora-canary` 和 `npm run check:pingora-canary-docker`，有 Nginx 或 Docker 的目标环境分别强制跑 `node scripts/check-nginx-pingora-canary.mjs --require-nginx` 与 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，其中 Docker handoff 会自动对账临时 Nginx 与 Pingora access log。启用后跑 `npm run check:pingora-canary-live`，再用 current release 随包 access log parity 脚本对账目标机 Nginx 与 Pingora access log。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；canary live timeout 和 access log `since-lines` 必须是正整数，非法值直接失败。 5. 前缀 canary 通过后，再使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary；它必须作为独立本机 `server` include 到 Nginx `http` 上下文，不能 include 到生产 `443` server 内。启用后跑 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 6. 目标机 canary include 后必须跑正式切换聚合门禁：源码 checkout / CI / 构建环境执行默认全量命令 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；目标机 current release 执行 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。启用真实路径 canary 时，两种命令都追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。缺少 Host 会直接失败，避免 live canary 误测默认 vhost；live smoke 后还会按 `request_id` 对账 Nginx 与 Pingora access log，缺少同一请求的 Pingora 日志、状态码、方法或 path 漂移都会失败。 -7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 +7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs` 和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 9. direct preflight / direct live timeout 必须是正整数；直连相关布尔 env 和 Pingora gateway env 里的 `TRUST_X_FORWARDED_FOR` / `TRUSTED_FRONT_PROXY_CONFIRMED` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。 10. Pingora 正式化口径固定为 gzip-only；Brotli 继续由 Nginx / 前置代理承担，直连 Pingora 不以 Brotli parity 作为切换门禁。如需多实例，开启网关接流保护时必须先引入共享限流 / 共享并发保护层并设置 `GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true`，否则保持 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1`；关闭网关保护的多实例方案必须明确由前置 Nginx / LB 承担全局限流。 diff --git a/package.json b/package.json index 82a573da4..8fe9d55e0 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "check:pingora-direct-enable": "node scripts/check-pingora-direct-enable.mjs", "check:pingora-direct-rollback": "node scripts/check-pingora-direct-rollback.mjs", "check:pingora-health-patrol-env-switch": "node scripts/check-pingora-health-patrol-env-switch.mjs", + "check:pingora-tls-cert-sync": "node scripts/check-pingora-tls-cert-sync.mjs", "check:pingora-current-release-audit": "node scripts/check-pingora-current-release-audit.mjs", "check:pingora-cutover-status-snapshot": "node scripts/check-pingora-cutover-status-snapshot.mjs", "check:pingora-cutover-evidence-bundle": "node scripts/check-pingora-cutover-evidence-bundle.mjs", diff --git a/scripts/build-production-release.sh b/scripts/build-production-release.sh index de2103e1f..d31b556e0 100644 --- a/scripts/build-production-release.sh +++ b/scripts/build-production-release.sh @@ -521,6 +521,7 @@ cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/deploy/maint cp "${SCRIPT_DIR}/deploy/pingora-direct-enable.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh" cp "${SCRIPT_DIR}/deploy/pingora-direct-rollback.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh" cp "${SCRIPT_DIR}/deploy/pingora-health-patrol-env-switch.mjs" "${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +cp "${SCRIPT_DIR}/deploy/pingora-tls-cert-sync.mjs" "${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" cp "${SCRIPT_DIR}/deploy/jenkins-inbound-agent-start.sh" "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" cp "${SCRIPT_DIR}/deploy/install-jenkins-inbound-agent.sh" "${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh" cp "${SCRIPT_DIR}/deploy/jenkins-agent-reverse-tunnel.ps1" "${TARGET_DIR}/scripts/jenkins-agent-reverse-tunnel.ps1" @@ -535,6 +536,7 @@ chmod +x \ "${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh" \ "${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh" \ "${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" \ + "${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" \ "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" \ "${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh" @@ -578,7 +580,7 @@ cat >"${TARGET_DIR}/README.md" < artifact.path === 'scripts/deploy/pingora-tls-cert-sync.mjs', + ); + if (!tlsCertSyncArtifact || tlsCertSyncArtifact.status !== 'OK') { + failures.push('完整 release 自审必须确认 TLS 证书同步脚本存在且可执行。'); + } } function assertChecksumMismatchFails() { @@ -453,6 +464,7 @@ function prepareReleaseRoot(releaseRoot, options) { 'scripts/deploy/pingora-direct-enable.sh', 'scripts/deploy/pingora-direct-rollback.sh', 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'scripts/deploy/pingora-tls-cert-sync.mjs', 'deploy/systemd/genarrative-pingora-gateway.service', 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', 'deploy/nginx/snippets/genarrative-pingora-canary.conf', @@ -476,6 +488,7 @@ function prepareReleaseRoot(releaseRoot, options) { 'scripts/deploy/pingora-direct-enable.sh', 'scripts/deploy/pingora-direct-rollback.sh', 'scripts/deploy/pingora-health-patrol-env-switch.mjs', + 'scripts/deploy/pingora-tls-cert-sync.mjs', ]) { chmodExecutable(path.join(releaseRoot, file)); } diff --git a/scripts/check-pingora-tls-cert-sync.mjs b/scripts/check-pingora-tls-cert-sync.mjs new file mode 100644 index 000000000..fb02d670b --- /dev/null +++ b/scripts/check-pingora-tls-cert-sync.mjs @@ -0,0 +1,281 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const SCRIPT = 'scripts/deploy/pingora-tls-cert-sync.mjs'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-tls-cert-sync-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-tls-cert-sync] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-tls-cert-sync] OK'); + +function main() { + assertDryRunAcceptsCertbotLiveSymlinks(); + assertApplyCopiesToPrivateTarget(); + assertRejectsRelativeAndRootPaths(); + assertRejectsControlCharactersBeforeApply(); + assertRejectsSymlinkTargetDirectory(); + assertRejectsSymlinkTargetFiles(); + assertRejectsWorldReadableModes(); +} + +function assertDryRunAcceptsCertbotLiveSymlinks() { + const fixture = prepareFixture('dry-run-live-symlink'); + const result = runSync(fixture, []); + assertStatus(result, 0, 'dry-run 应接受 Certbot live symlink 源。'); + if (existsSync(fixture.targetCert) || existsSync(fixture.targetKey)) { + failures.push('dry-run 不应写入目标证书文件。'); + } + const payload = parseJson(result.stdout, 'dry-run JSON 输出'); + assertEqual(payload.mode, 'dry-run', 'dry-run JSON mode 应为 dry-run。'); + assertEqual( + payload.files?.[0]?.source?.sourceIsSymlink, + true, + 'dry-run JSON 应标记源证书来自 symlink。', + ); +} + +function assertApplyCopiesToPrivateTarget() { + const fixture = prepareFixture('apply-copy', { symlinkSources: false }); + const result = runSync(fixture, ['--apply'], { + serviceUser: `#${process.getuid?.() ?? 0}`, + serviceGroup: `#${process.getgid?.() ?? 0}`, + }); + assertStatus(result, 0, 'apply 应复制证书到目标目录。'); + assertFileContent(fixture.targetCert, 'fixture fullchain\n'); + assertFileContent(fixture.targetKey, 'fixture private key\n'); + assertMode(fixture.targetDir, 0o750, '目标目录权限应为 0750。'); + assertMode(fixture.targetCert, 0o640, '目标证书权限应为 0640。'); + assertMode(fixture.targetKey, 0o640, '目标私钥权限应为 0640。'); +} + +function assertRejectsRelativeAndRootPaths() { + const fixture = prepareFixture('reject-paths'); + const relative = runRaw([ + '--source-cert-file', + 'relative/fullchain.pem', + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + ]); + assertNonZero(relative, '相对源证书路径必须失败。'); + assertIncludes( + `${relative.stdout}\n${relative.stderr}`, + '--source-cert-file 必须是绝对路径', + '相对路径必须给出明确错误。', + ); + + const root = runRaw([ + '--source-cert-file', + fixture.sourceCert, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + '/', + ]); + assertNonZero(root, 'target-dir 指向文件系统根目录必须失败。'); + assertIncludes( + `${root.stdout}\n${root.stderr}`, + '--target-dir 不能是文件系统根目录', + '根目录目标必须给出明确错误。', + ); +} + +function assertRejectsControlCharactersBeforeApply() { + const fixture = prepareFixture('reject-control'); + const result = runRaw([ + '--source-cert-file', + `${fixture.sourceCert}\n--apply`, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + ]); + assertNonZero(result, '带换行的源证书路径必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--source-cert-file 不能包含换行或 NUL 字符', + '控制字符路径必须给出明确错误。', + ); + if (existsSync(fixture.targetCert) || existsSync(fixture.targetKey)) { + failures.push('控制字符参数失败时不应写入目标证书。'); + } +} + +function assertRejectsSymlinkTargetDirectory() { + const fixture = prepareFixture('reject-target-dir-symlink'); + const realTarget = path.join(fixture.root, 'real-target'); + mkdirSync(realTarget, { recursive: true }); + rmSync(fixture.targetDir, { recursive: true, force: true }); + symlinkSync(realTarget, fixture.targetDir); + const result = runSync(fixture, []); + assertNonZero(result, 'target-dir 是 symlink 时必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--target-dir 已存在路径不能包含符号链接', + 'target-dir symlink 必须给出明确错误。', + ); +} + +function assertRejectsSymlinkTargetFiles() { + const fixture = prepareFixture('reject-target-file-symlink'); + mkdirSync(fixture.targetDir, { recursive: true }); + const outside = path.join(fixture.root, 'outside-fullchain.pem'); + writeFileSync(outside, 'outside\n', 'utf8'); + symlinkSync(outside, fixture.targetCert); + const result = runSync(fixture, []); + assertNonZero(result, '目标证书是 symlink 时必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '目标 fullchain.pem 不能是符号链接', + '目标文件 symlink 必须给出明确错误。', + ); +} + +function assertRejectsWorldReadableModes() { + const fixture = prepareFixture('reject-world-readable-mode'); + const result = runSync(fixture, ['--file-mode', '0644']); + assertNonZero(result, 'world-readable file mode 必须失败。'); + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--file-mode 不能允许 other 读取证书或私钥', + 'world-readable file mode 必须给出明确错误。', + ); +} + +function prepareFixture(name, options = {}) { + const root = path.join(tmpRoot, name); + const archiveDir = path.join(root, 'archive'); + const liveDir = path.join(root, 'live'); + const targetDir = path.join(root, 'target'); + mkdirSync(archiveDir, { recursive: true }); + mkdirSync(liveDir, { recursive: true }); + writeFileSync(path.join(archiveDir, 'fullchain1.pem'), 'fixture fullchain\n', 'utf8'); + writeFileSync(path.join(archiveDir, 'privkey1.pem'), 'fixture private key\n', 'utf8'); + + const symlinkSources = options.symlinkSources !== false; + const sourceCert = symlinkSources + ? path.join(liveDir, 'fullchain.pem') + : path.join(archiveDir, 'fullchain1.pem'); + const sourceKey = symlinkSources + ? path.join(liveDir, 'privkey.pem') + : path.join(archiveDir, 'privkey1.pem'); + if (symlinkSources) { + symlinkSync('../archive/fullchain1.pem', sourceCert); + symlinkSync('../archive/privkey1.pem', sourceKey); + } + + return { + root, + sourceCert, + sourceKey, + targetDir, + targetCert: path.join(targetDir, 'fullchain.pem'), + targetKey: path.join(targetDir, 'privkey.pem'), + }; +} + +function runSync(fixture, args = [], options = {}) { + return runRaw([ + '--source-cert-file', + fixture.sourceCert, + '--source-key-file', + fixture.sourceKey, + '--target-dir', + fixture.targetDir, + '--service-user', + options.serviceUser || `#${process.getuid?.() ?? 0}`, + '--service-group', + options.serviceGroup || `#${process.getgid?.() ?? 0}`, + '--json', + ...args, + ]); +} + +function runRaw(args) { + return spawnSync('node', [SCRIPT, ...args], { + cwd: process.cwd(), + encoding: 'utf8', + }); +} + +function parseJson(text, label) { + try { + return JSON.parse(text); + } catch (error) { + failures.push(`${label} 不是合法 JSON: ${error.message}\n${text}`); + return {}; + } +} + +function assertStatus(result, expected, reason) { + if ((result.status ?? 0) !== expected) { + failures.push( + `${reason} 实际退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertNonZero(result, reason) { + if ((result.status ?? 0) === 0) { + failures.push(`${reason}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } +} + +function assertIncludes(value, expected, reason) { + if (!String(value).includes(expected)) { + failures.push(`${reason} 缺少: ${expected}`); + } +} + +function assertEqual(actual, expected, reason) { + if (actual !== expected) { + failures.push(`${reason} 实际 ${JSON.stringify(actual)}，预期 ${JSON.stringify(expected)}`); + } +} + +function assertFileContent(filePath, expected) { + if (!existsSync(filePath)) { + failures.push(`缺少文件: ${filePath}`); + return; + } + const actual = readFileSync(filePath, 'utf8'); + if (actual !== expected) { + failures.push(`${filePath} 内容不符合预期。`); + } +} + +function assertMode(filePath, expected, reason) { + const mode = statSync(filePath).mode & 0o777; + if (mode !== expected) { + failures.push(`${reason} 实际 0${mode.toString(8)}，预期 0${expected.toString(8)}。`); + } +} diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 21843139c..8920d7035 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -62,6 +62,7 @@ function main() { assertMissingHealthPatrolEnvCheckFails(); assertMissingPingoraReleaseReadinessFails(); assertMissingPingoraHealthPatrolEnvSwitchFails(); + assertMissingPingoraTlsCertSyncFails(); assertMissingEnvExamplesFails(); assertMissingPingoraDirectCheckFails(); assertMissingPingoraCanaryLiveFails(); @@ -105,6 +106,10 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() { ), 'current release 必须包含 Pingora health patrol env 切换脚本。', ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'current release 必须包含 Pingora TLS 证书同步脚本。', + ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), 'current release 必须包含 Pingora 直连预检脚本。', @@ -885,6 +890,29 @@ function assertMissingPingoraHealthPatrolEnvSwitchFails() { } } +function assertMissingPingoraTlsCertSyncFails() { + const fixture = prepareFixture('missing-pingora-tls-cert-sync'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-tls-cert-sync.mjs', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora TLS 证书同步脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora TLS 证书同步脚本', + '缺少 Pingora TLS 证书同步脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + function assertMissingEnvExamplesFails() { const fixture = prepareFixture('missing-env-examples'); rmSync(path.join(fixture.sourceDir, 'deploy/env'), { @@ -1020,6 +1048,10 @@ function prepareFixture(name) { 'scripts/deploy/pingora-health-patrol-env-switch.mjs', path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), ); + copyFile( + 'scripts/deploy/pingora-tls-cert-sync.mjs', + path.join(sourceDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + ); copyFile( 'scripts/check-production-health-patrol-env.mjs', path.join(sourceDir, 'scripts/check-production-health-patrol-env.mjs'), diff --git a/scripts/check-production-api-release.mjs b/scripts/check-production-api-release.mjs index 27647404c..151ccd3f3 100644 --- a/scripts/check-production-api-release.mjs +++ b/scripts/check-production-api-release.mjs @@ -116,6 +116,14 @@ function assertApiReleaseContainsPingoraDirectDependencies() { ), 'Pingora health patrol env 切换脚本必须可执行。', ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'API release 必须包含 Pingora TLS 证书同步脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), + 'Pingora TLS 证书同步脚本必须可执行。', + ); assertFileExists( path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), 'API release 必须包含健康巡检 env 复核脚本。', @@ -246,7 +254,7 @@ function assertApiReleaseContainsPingoraDirectDependencies() { ); assertIncludes( releaseReadme, - 'Pingora release readiness 聚合门禁、直连启用 / 回退 / health patrol env 切换 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', + 'Pingora release readiness 聚合门禁、直连启用 / 回退 / health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', 'API release README 必须说明随包携带 Pingora 直连脚本。', ); assertIncludes( diff --git a/scripts/deploy/pingora-tls-cert-sync.mjs b/scripts/deploy/pingora-tls-cert-sync.mjs new file mode 100644 index 000000000..e1f7beb6f --- /dev/null +++ b/scripts/deploy/pingora-tls-cert-sync.mjs @@ -0,0 +1,470 @@ +#!/usr/bin/env node + +import { randomBytes } from 'node:crypto'; +import { + accessSync, + chmodSync, + chownSync, + constants, + copyFileSync, + existsSync, + lstatSync, + mkdirSync, + realpathSync, + renameSync, + rmSync, + statSync, +} from 'node:fs'; +import { execFileSync, spawnSync } from 'node:child_process'; +import path from 'node:path'; +import { userInfo } from 'node:os'; + +const DEFAULT_SERVICE_USER = 'genarrative'; +const DEFAULT_SERVICE_GROUP = 'genarrative'; +const DEFAULT_DIRECTORY_MODE = '0750'; +const DEFAULT_FILE_MODE = '0640'; + +const config = parseArgs(process.argv.slice(2)); +const result = run(); + +if (config.json) { + console.log(`${JSON.stringify(result, null, 2)}\n`); +} else { + const mode = config.apply ? 'apply' : 'dry-run'; + console.log(`[pingora-tls-cert-sync] ${mode} OK`); + console.log(`[pingora-tls-cert-sync] target=${result.target.directory}`); + for (const item of result.files) { + console.log( + `[pingora-tls-cert-sync] ${item.kind}: ${item.source.path} -> ${item.target.path}`, + ); + } +} + +function usage() { + console.log(`Usage: + node scripts/deploy/pingora-tls-cert-sync.mjs --source-cert-file --source-key-file --target-dir [options] + +Options: + --source-cert-file 已存在的证书链文件，通常是 Let's Encrypt live/fullchain.pem。 + --source-key-file 已存在的私钥文件，通常是 Let's Encrypt live/privkey.pem。 + --target-dir Pingora 私有 TLS 目录，例如 /etc/genarrative/pingora-tls/。 + --service-user 目标服务用户，默认 genarrative。 + --service-group + 目标服务组，默认 genarrative。 + --directory-mode 目标目录权限，默认 0750。 + --file-mode 证书和私钥权限，默认 0640。 + --apply 执行复制；默认只做 dry-run 校验。 + --json 输出 JSON。 + +该脚本只把现有证书复制到 Pingora 私有目录，不修改 Let's Encrypt / Certbot / Nginx 原始文件。 +源路径可以是 Certbot live symlink，但解析后的目标必须是普通文件；目标目录和目标文件不允许是 symlink。 +`); +} + +function parseArgs(argv) { + const result = { + sourceCertFile: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SOURCE_CERT_FILE || '', + sourceKeyFile: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SOURCE_KEY_FILE || '', + targetDir: process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_TARGET_DIR || '', + serviceUser: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SERVICE_USER || + DEFAULT_SERVICE_USER, + serviceGroup: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SERVICE_GROUP || + DEFAULT_SERVICE_GROUP, + directoryMode: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_DIRECTORY_MODE || + DEFAULT_DIRECTORY_MODE, + fileMode: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_FILE_MODE || + DEFAULT_FILE_MODE, + apply: readBoolEnv( + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_APPLY, + false, + 'GENARRATIVE_PINGORA_TLS_CERT_SYNC_APPLY', + ), + json: false, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + usage(); + process.exit(0); + break; + case '--source-cert-file': + result.sourceCertFile = requireValue(argv, ++index, arg); + break; + case '--source-key-file': + result.sourceKeyFile = requireValue(argv, ++index, arg); + break; + case '--target-dir': + result.targetDir = requireValue(argv, ++index, arg); + break; + case '--service-user': + result.serviceUser = requireValue(argv, ++index, arg); + break; + case '--service-group': + result.serviceGroup = requireValue(argv, ++index, arg); + break; + case '--directory-mode': + result.directoryMode = requireValue(argv, ++index, arg); + break; + case '--file-mode': + result.fileMode = requireValue(argv, ++index, arg); + break; + case '--apply': + result.apply = true; + break; + case '--json': + result.json = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateSafeAbsoluteFilePath( + result.sourceCertFile, + '--source-cert-file', + ); + validateSafeAbsoluteFilePath(result.sourceKeyFile, '--source-key-file'); + validateSafeAbsoluteDirectoryPath(result.targetDir, '--target-dir'); + validateIdentity(result.serviceUser, '--service-user'); + validateIdentity(result.serviceGroup, '--service-group'); + result.directoryModeNumber = parseMode( + result.directoryMode, + '--directory-mode', + ); + result.fileModeNumber = parseMode(result.fileMode, '--file-mode'); + if ((result.fileModeNumber & 0o007) !== 0) { + throw new Error('--file-mode 不能允许 other 读取证书或私钥。'); + } + if ((result.directoryModeNumber & 0o007) !== 0) { + throw new Error('--directory-mode 不能允许 other 进入 TLS 私有目录。'); + } + return result; +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + validateNoControlCharacters(raw, label); + const normalized = String(raw).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${label} 必须是布尔值 true/false 或 1/0。`); +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateSafeAbsoluteDirectoryPath(value, label) { + validateSafeAbsoluteFilePath(value, label); + if (path.basename(path.resolve(value)) === '..') { + throw new Error(`${label} 不能以 .. 结尾。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function validateIdentity(value, label) { + validateNoControlCharacters(value, label); + if (!/^[A-Za-z0-9_.#-]+$/u.test(String(value))) { + throw new Error(`${label} 只能包含字母、数字、点、下划线、短横线或 #。`); + } +} + +function parseMode(value, label) { + validateNoControlCharacters(value, label); + const text = String(value).trim(); + if (!/^0?[0-7]{3}$/u.test(text)) { + throw new Error(`${label} 必须是三位八进制权限，例如 0640。`); + } + return Number.parseInt(text, 8); +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +function run() { + const certSource = inspectSourceFile( + config.sourceCertFile, + '--source-cert-file', + ); + const keySource = inspectSourceFile(config.sourceKeyFile, '--source-key-file'); + const certTarget = path.join(config.targetDir, 'fullchain.pem'); + const keyTarget = path.join(config.targetDir, 'privkey.pem'); + const owner = config.apply + ? resolveOwner(config.serviceUser, config.serviceGroup) + : { uid: null, gid: null }; + + assertNoSymlinkAncestors(config.targetDir, '--target-dir'); + assertTargetPath(certTarget, '目标 fullchain.pem'); + assertTargetPath(keyTarget, '目标 privkey.pem'); + if (existsSync(config.targetDir)) { + assertTargetDirectory(config.targetDir); + } + + const planned = { + mode: config.apply ? 'apply' : 'dry-run', + target: { + directory: config.targetDir, + user: config.serviceUser, + group: config.serviceGroup, + uid: owner.uid, + gid: owner.gid, + directoryMode: formatMode(config.directoryModeNumber), + fileMode: formatMode(config.fileModeNumber), + }, + files: [ + { + kind: 'cert', + source: certSource, + target: { path: certTarget }, + }, + { + kind: 'key', + source: keySource, + target: { path: keyTarget }, + }, + ], + serviceUserReadable: false, + }; + + if (!config.apply) { + return planned; + } + + mkdirSync(config.targetDir, { recursive: true }); + assertTargetDirectory(config.targetDir); + chmodSync(config.targetDir, config.directoryModeNumber); + applyOwner(config.targetDir, owner); + + copyAtomic(certSource.resolvedPath, certTarget, owner); + copyAtomic(keySource.resolvedPath, keyTarget, owner); + + planned.serviceUserReadable = assertServiceUserReadable( + config.serviceUser, + [certTarget, keyTarget], + ); + return planned; +} + +function inspectSourceFile(filePath, label) { + const linkStat = lstatSync(filePath); + const sourceIsSymlink = linkStat.isSymbolicLink(); + const resolvedPath = realpathSync(filePath); + const fileStat = statSync(resolvedPath); + if (!fileStat.isFile()) { + throw new Error(`${label} 解析后必须是普通文件: ${resolvedPath}`); + } + return { + path: filePath, + resolvedPath, + sourceIsSymlink, + sizeBytes: fileStat.size, + }; +} + +function assertTargetDirectory(directory) { + const linkStat = lstatSync(directory); + if (linkStat.isSymbolicLink()) { + throw new Error(`--target-dir 不能是符号链接: ${directory}`); + } + if (!linkStat.isDirectory()) { + throw new Error(`--target-dir 已存在但不是目录: ${directory}`); + } +} + +function assertNoSymlinkAncestors(targetDir, label) { + const resolved = path.resolve(targetDir); + const parsed = path.parse(resolved); + let current = parsed.root; + for (const part of resolved.slice(parsed.root.length).split(path.sep)) { + if (!part) { + continue; + } + current = path.join(current, part); + if (!existsSync(current)) { + break; + } + const linkStat = lstatSync(current); + if (linkStat.isSymbolicLink()) { + throw new Error(`${label} 已存在路径不能包含符号链接: ${current}`); + } + if (!linkStat.isDirectory()) { + throw new Error(`${label} 已存在父路径不是目录: ${current}`); + } + } +} + +function assertTargetPath(filePath, label) { + if (!existsSync(filePath)) { + return; + } + const linkStat = lstatSync(filePath); + if (linkStat.isSymbolicLink()) { + throw new Error(`${label} 不能是符号链接: ${filePath}`); + } + if (!linkStat.isFile()) { + throw new Error(`${label} 已存在但不是普通文件: ${filePath}`); + } +} + +function resolveOwner(user, group) { + const current = currentIdentity(); + const uid = resolveUserId(user, current); + const gid = resolveGroupId(group, current, user); + if (!isCurrentUserRoot()) { + if (uid !== current.uid || gid !== current.gid) { + throw new Error( + `当前用户不是 root，不能把证书授权给 ${user}:${group}；请用 sudo -n 执行 --apply。`, + ); + } + } + return { uid, gid }; +} + +function resolveUserId(user, current) { + if (/^#?\d+$/u.test(user)) { + return Number.parseInt(user.replace(/^#/u, ''), 10); + } + if (user === current.username) { + return current.uid; + } + return Number.parseInt( + execFileSync('id', ['-u', user], { encoding: 'utf8' }).trim(), + 10, + ); +} + +function resolveGroupId(group, current, user) { + if (/^#?\d+$/u.test(group)) { + return Number.parseInt(group.replace(/^#/u, ''), 10); + } + if (group === current.groupname || group === String(current.gid)) { + return current.gid; + } + const getent = spawnSync('getent', ['group', group], { encoding: 'utf8' }); + if (getent.status === 0 && getent.stdout.trim()) { + const parts = getent.stdout.trim().split(':'); + return Number.parseInt(parts[2], 10); + } + if (group === user) { + return Number.parseInt( + execFileSync('id', ['-g', user], { encoding: 'utf8' }).trim(), + 10, + ); + } + throw new Error(`无法解析服务组: ${group}`); +} + +function currentIdentity() { + const uid = typeof process.getuid === 'function' ? process.getuid() : -1; + const gid = typeof process.getgid === 'function' ? process.getgid() : -1; + let username = ''; + try { + username = userInfo().username; + } catch { + username = process.env.USER || process.env.LOGNAME || ''; + } + return { + uid, + gid, + username, + groupname: '', + }; +} + +function isCurrentUserRoot() { + return typeof process.getuid === 'function' && process.getuid() === 0; +} + +function applyOwner(filePath, owner) { + const current = currentIdentity(); + if (!isCurrentUserRoot() && owner.uid === current.uid && owner.gid === current.gid) { + return; + } + chownSync(filePath, owner.uid, owner.gid); +} + +function copyAtomic(sourcePath, targetPath, owner) { + const tempPath = path.join( + path.dirname(targetPath), + `.${path.basename(targetPath)}.tmp-${process.pid}-${randomBytes(4).toString('hex')}`, + ); + try { + copyFileSync(sourcePath, tempPath, constants.COPYFILE_EXCL); + chmodSync(tempPath, config.fileModeNumber); + applyOwner(tempPath, owner); + renameSync(tempPath, targetPath); + chmodSync(targetPath, config.fileModeNumber); + applyOwner(targetPath, owner); + } catch (error) { + rmSync(tempPath, { force: true }); + throw error; + } +} + +function assertServiceUserReadable(serviceUser, files) { + if (/^#?\d+$/u.test(serviceUser)) { + for (const file of files) { + accessSync(file, constants.R_OK); + } + return true; + } + const current = currentIdentity(); + if (serviceUser === current.username && !isCurrentUserRoot()) { + for (const file of files) { + accessSync(file, constants.R_OK); + } + return true; + } + for (const file of files) { + const check = spawnSync('sudo', ['-n', '-u', serviceUser, 'test', '-r', file], { + encoding: 'utf8', + }); + if (check.status !== 0) { + const detail = (check.stderr || check.stdout || '').trim(); + throw new Error( + `服务用户 ${serviceUser} 不可读: ${file}${detail ? ` (${detail})` : ''}`, + ); + } + } + return true; +} + +function formatMode(mode) { + return `0${mode.toString(8).padStart(3, '0')}`; +} diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 7a017686b..142d32e45 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -620,6 +620,7 @@ PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-rel PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" PINGORA_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-live.mjs" PINGORA_CANARY_LIVE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-canary-live.mjs" @@ -707,6 +708,12 @@ if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then fi cp "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" +if [[ ! -f "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora TLS 证书同步脚本: ${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" >&2 + exit 1 +fi +cp "${PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" if [[ ! -f "${PINGORA_PREFLIGHT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连预检脚本: ${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" >&2 exit 1 diff --git a/scripts/ops/pingora-current-release-audit.mjs b/scripts/ops/pingora-current-release-audit.mjs index 1a2c7b968..e73da3f64 100644 --- a/scripts/ops/pingora-current-release-audit.mjs +++ b/scripts/ops/pingora-current-release-audit.mjs @@ -22,6 +22,7 @@ const REQUIRED_ARTIFACTS = [ { path: 'scripts/deploy/pingora-direct-enable.sh', executable: true }, { path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true }, { path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', executable: true }, + { path: 'scripts/deploy/pingora-tls-cert-sync.mjs', executable: true }, { path: 'deploy/systemd/genarrative-pingora-gateway.service' }, { path: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf' }, { path: 'deploy/nginx/snippets/genarrative-pingora-canary.conf' }, From 764f3609d0c6bfc9d92177fed6eec6c199a0cd80 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 22:44:10 +0800 Subject: [PATCH 11/22] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20Pingora=20=E8=AF=81?= =?UTF-8?q?=E4=B9=A6=E5=89=AF=E6=9C=AC=E5=BD=92=E5=B1=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 拆分 TLS 证书同步脚本的 target owner 与服务用户。 目标证书默认保持 root:genarrative，服务用户只做可读性复核。 更新证书同步测试，覆盖显式 target-user 的本机 apply 场景。 --- scripts/check-pingora-tls-cert-sync.mjs | 3 +++ scripts/deploy/pingora-tls-cert-sync.mjs | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/check-pingora-tls-cert-sync.mjs b/scripts/check-pingora-tls-cert-sync.mjs index fb02d670b..a3578fe2c 100644 --- a/scripts/check-pingora-tls-cert-sync.mjs +++ b/scripts/check-pingora-tls-cert-sync.mjs @@ -67,6 +67,7 @@ function assertApplyCopiesToPrivateTarget() { const result = runSync(fixture, ['--apply'], { serviceUser: `#${process.getuid?.() ?? 0}`, serviceGroup: `#${process.getgid?.() ?? 0}`, + targetUser: `#${process.getuid?.() ?? 0}`, }); assertStatus(result, 0, 'apply 应复制证书到目标目录。'); assertFileContent(fixture.targetCert, 'fixture fullchain\n'); @@ -215,6 +216,8 @@ function runSync(fixture, args = [], options = {}) { options.serviceUser || `#${process.getuid?.() ?? 0}`, '--service-group', options.serviceGroup || `#${process.getgid?.() ?? 0}`, + '--target-user', + options.targetUser || `#${process.getuid?.() ?? 0}`, '--json', ...args, ]); diff --git a/scripts/deploy/pingora-tls-cert-sync.mjs b/scripts/deploy/pingora-tls-cert-sync.mjs index e1f7beb6f..63533eea7 100644 --- a/scripts/deploy/pingora-tls-cert-sync.mjs +++ b/scripts/deploy/pingora-tls-cert-sync.mjs @@ -21,6 +21,7 @@ import { userInfo } from 'node:os'; const DEFAULT_SERVICE_USER = 'genarrative'; const DEFAULT_SERVICE_GROUP = 'genarrative'; +const DEFAULT_TARGET_USER = 'root'; const DEFAULT_DIRECTORY_MODE = '0750'; const DEFAULT_FILE_MODE = '0640'; @@ -50,7 +51,8 @@ Options: --target-dir Pingora 私有 TLS 目录，例如 /etc/genarrative/pingora-tls/。 --service-user 目标服务用户，默认 genarrative。 --service-group - 目标服务组，默认 genarrative。 + 目标文件组，默认 genarrative。 + --target-user 目标文件 owner，默认 root。 --directory-mode 目标目录权限，默认 0750。 --file-mode 证书和私钥权限，默认 0640。 --apply 执行复制；默认只做 dry-run 校验。 @@ -74,6 +76,9 @@ function parseArgs(argv) { serviceGroup: process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_SERVICE_GROUP || DEFAULT_SERVICE_GROUP, + targetUser: + process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_TARGET_USER || + DEFAULT_TARGET_USER, directoryMode: process.env.GENARRATIVE_PINGORA_TLS_CERT_SYNC_DIRECTORY_MODE || DEFAULT_DIRECTORY_MODE, @@ -111,6 +116,9 @@ function parseArgs(argv) { case '--service-group': result.serviceGroup = requireValue(argv, ++index, arg); break; + case '--target-user': + result.targetUser = requireValue(argv, ++index, arg); + break; case '--directory-mode': result.directoryMode = requireValue(argv, ++index, arg); break; @@ -136,6 +144,7 @@ function parseArgs(argv) { validateSafeAbsoluteDirectoryPath(result.targetDir, '--target-dir'); validateIdentity(result.serviceUser, '--service-user'); validateIdentity(result.serviceGroup, '--service-group'); + validateIdentity(result.targetUser, '--target-user'); result.directoryModeNumber = parseMode( result.directoryMode, '--directory-mode', @@ -226,7 +235,7 @@ function run() { const certTarget = path.join(config.targetDir, 'fullchain.pem'); const keyTarget = path.join(config.targetDir, 'privkey.pem'); const owner = config.apply - ? resolveOwner(config.serviceUser, config.serviceGroup) + ? resolveOwner(config.targetUser, config.serviceGroup) : { uid: null, gid: null }; assertNoSymlinkAncestors(config.targetDir, '--target-dir'); @@ -240,10 +249,11 @@ function run() { mode: config.apply ? 'apply' : 'dry-run', target: { directory: config.targetDir, - user: config.serviceUser, + user: config.targetUser, group: config.serviceGroup, uid: owner.uid, gid: owner.gid, + serviceUser: config.serviceUser, directoryMode: formatMode(config.directoryModeNumber), fileMode: formatMode(config.fileModeNumber), }, From 75b85ee5a991d590a72b72704f06cbd4a652c9e8 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 23:00:28 +0800 Subject: [PATCH 12/22] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=20Pingora=20=E9=AB=98?= =?UTF-8?q?=E7=AB=AF=E5=8F=A3=E9=87=8D=E5=AE=9A=E5=90=91=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit direct live 支持独立 redirect base URL 预期 release readiness 和切换证据链透传 redirect base URL 直连启用脚本透传高端口 rehearsal 参数 补充 direct live guard、运维门禁和文档示例 --- deploy/env/pingora-direct-live.env.example | 5 ++- deploy/nginx/README.md | 2 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 2 +- scripts/check-pingora-direct-enable.mjs | 10 +++++ scripts/check-pingora-direct-live-guard.mjs | 34 +++++++++++++++ scripts/check-pingora-direct-live.mjs | 34 +++++++++++++++ scripts/check-pingora-release-readiness.mjs | 41 +++++++++++++++++++ scripts/check-production-ops-guardrails.mjs | 18 ++++++++ scripts/deploy/pingora-direct-enable.sh | 17 +++++++- .../ops/pingora-cutover-evidence-bundle.mjs | 28 +++++++++++++ 10 files changed, 186 insertions(+), 5 deletions(-) diff --git a/deploy/env/pingora-direct-live.env.example b/deploy/env/pingora-direct-live.env.example index 87dc1369b..b3d04c609 100644 --- a/deploy/env/pingora-direct-live.env.example +++ b/deploy/env/pingora-direct-live.env.example @@ -28,9 +28,12 @@ GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL=http://127.0.0.1 # 本机直连 127.0.0.1 时必须设置正式域名；direct live 会同时用作 Host header 和 TLS SNI。release readiness --require-direct 必须提供。只填域名或 host:port，不要填 https:// URL。 GENARRATIVE_PINGORA_DIRECT_HOST= -# 单独 direct live 可选；release readiness --require-direct 必须提供，用于显式校验 HTTP redirect Location 的 host。只填域名或 host:port，非标准端口本机 smoke 可设为 127.0.0.1:18443。 +# 单独 direct live 可选；release readiness --require-direct 必须提供，用于显式校验 HTTP redirect Location 的 host。只填域名或 host:port。 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST= +# 可选：高端口 rehearsal 访问 127.0.0.1:18443、但期望 Location 指向正式域名默认 HTTPS 入口时设置；只能填 https:// base URL，不要带路径、查询或片段。 +GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL= + # 可选：配置后会额外检查 /__genarrative_pingora/healthz 内部探针。 GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN= diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 7f8e0f8d0..d5db0afec 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -50,7 +50,7 @@ - 目标机如果使用 Certbot / Let’s Encrypt 证书，不要直接放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限。先用随包脚本把 root 可读的 live symlink 解析并复制到 Pingora 私有目录，例如 `sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>`；脚本默认 dry-run，`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并复核 `genarrative` 服务用户可读。随后把 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/` 下的副本。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 -- 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。 +- 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。高端口 rehearsal 如果通过 `https://127.0.0.1:18443` 打入、但 Pingora 应返回正式域名默认 HTTPS Location，可额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于直连切换 Host 一致性约束。 - direct live 在 HTTPS 根路径返回 `200` 且 HTML 中发现 `/assets/` 或 `/admin/assets/` 引用时，会自动请求该静态资源，确认 `Cache-Control`、`ETag`、`Last-Modified`、`Accept-Ranges: bytes`、`HEAD` 头响应、`If-None-Match` / `If-Modified-Since` 的 `304` 响应和 `Range: bytes=0-0` 的 `206 + Content-Range` 响应，并纳入 Pingora access log method/path/status 对账；若首页引用 Vite 指纹资源，还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 以及指纹资源 GET / HEAD / 304 / Range access log 证据。维护模式、非 HTML 或发布包首页没有资产引用时该项标记为 skipped，不阻断维护窗口。 - 直连启用前先 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`；验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`，脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` 已从 systemd 最终配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 smoke URL 证明 Nginx 入口真实可访问；传入 `--nginx-smoke-expect-body` 时还会要求响应体包含该片段。仓库工作区可用 `npm run deploy:pingora-direct-enable -- --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log` 和 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`。两个脚本默认 dry-run；启用脚本 `--apply` 必须先通过 current release 自审，并带 env、证书可读、service EnvironmentFile 一致性、服务用户证书可读、service 二进制可执行、端口释放预检、direct live 参数和 Pingora access log 参数，回退脚本 `--apply` 必须带 `--reload-nginx` 和 `--nginx-smoke-url`；如果 health patrol env 已预先切回 Nginx，也可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让回退脚本在 Nginx smoke 后复核 `gateway mode=nginx` 且 public base URL / Host 已恢复，若切换前 Nginx 巡检本来需要 Host 覆盖，则把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`；如果还要同时证明 shadow 高端口仍健康，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。本机 `npm run check:pingora-direct-enable` / `npm run check:pingora-direct-rollback` 会验证 dry-run 不会安装或删除临时 drop-in、current release 自审失败时启用脚本不会安装 drop-in，并展示启用 / 回退后的 systemd 最终配置核验、ExecStart 指向核验、Pingora active 核验、direct live smoke、Nginx 语法检查、reload 状态核验、Nginx smoke 及响应体片段核验、可选 health patrol env 复核和可选 shadow probe 复核。 - `npm run check:pingora-current-release-audit` 会烟测 current release 自审脚本；正式直连 runbook 会先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，只读确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、`release-manifest.api-server.json` 已登记 `pingora-gateway`、`pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 17aab3968..c2404c9fa 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -472,7 +472,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 4. 使用 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 做 Nginx 前缀 canary；启用前先跑 `npm run check:nginx-pingora-canary` 和 `npm run check:pingora-canary-docker`，有 Nginx 或 Docker 的目标环境分别强制跑 `node scripts/check-nginx-pingora-canary.mjs --require-nginx` 与 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，其中 Docker handoff 会自动对账临时 Nginx 与 Pingora access log。启用后跑 `npm run check:pingora-canary-live`，再用 current release 随包 access log parity 脚本对账目标机 Nginx 与 Pingora access log。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；canary live timeout 和 access log `since-lines` 必须是正整数，非法值直接失败。 5. 前缀 canary 通过后，再使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary；它必须作为独立本机 `server` include 到 Nginx `http` 上下文，不能 include 到生产 `443` server 内。启用后跑 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 6. 目标机 canary include 后必须跑正式切换聚合门禁：源码 checkout / CI / 构建环境执行默认全量命令 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；目标机 current release 执行 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。启用真实路径 canary 时，两种命令都追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。缺少 Host 会直接失败，避免 live canary 误测默认 vhost；live smoke 后还会按 `request_id` 对账 Nginx 与 Pingora access log，缺少同一请求的 Pingora 日志、状态码、方法或 path 漂移都会失败。 -7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 +7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。高端口 rehearsal 使用 `https://127.0.0.1:<高端口>` 打入但期望 HTTP redirect Location 指向正式域名默认 HTTPS 入口时，额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于 runbook Host 一致性约束。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、redirect base URL、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs` 和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 9. direct preflight / direct live timeout 必须是正整数；直连相关布尔 env 和 Pingora gateway env 里的 `TRUST_X_FORWARDED_FOR` / `TRUSTED_FRONT_PROXY_CONFIRMED` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。 10. Pingora 正式化口径固定为 gzip-only；Brotli 继续由 Nginx / 前置代理承担，直连 Pingora 不以 Brotli parity 作为切换门禁。如需多实例，开启网关接流保护时必须先引入共享限流 / 共享并发保护层并设置 `GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true`，否则保持 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1`；关闭网关保护的多实例方案必须明确由前置 Nginx / LB 承担全局限流。 diff --git a/scripts/check-pingora-direct-enable.mjs b/scripts/check-pingora-direct-enable.mjs index 050a70a94..bc3f50d08 100644 --- a/scripts/check-pingora-direct-enable.mjs +++ b/scripts/check-pingora-direct-enable.mjs @@ -558,6 +558,11 @@ function assertScriptShape() { '--apply 必须同时提供 --direct-host', '启用脚本 apply 时必须强制正式域名 Host/SNI 参数。', ); + assertIncludes( + content, + '--direct-redirect-base-url', + '启用脚本必须支持 direct live redirect base URL 覆盖，用于高端口 rehearsal。', + ); assertIncludes( content, '--apply 必须同时提供 --direct-redirect-host', @@ -653,6 +658,11 @@ function assertScriptShape() { 'print_redacted_command node "${direct_live_args[@]}"', '启用脚本展示 direct live smoke 命令时必须隐藏 probe token。', ); + assertIncludes( + content, + 'direct_live_args+=(--redirect-base-url "${DIRECT_REDIRECT_BASE_URL}")', + '启用脚本必须把 direct redirect base URL 透传给 direct live smoke。', + ); assertIncludes( content, 'drop-in 目录不能是符号链接', diff --git a/scripts/check-pingora-direct-live-guard.mjs b/scripts/check-pingora-direct-live-guard.mjs index 1b2bcef2d..7984f2d38 100644 --- a/scripts/check-pingora-direct-live-guard.mjs +++ b/scripts/check-pingora-direct-live-guard.mjs @@ -50,6 +50,13 @@ function main() { '--redirect-host', 'example.com\nhidden', ]); + assertRejectsControlCharacter('--redirect-base-url', [ + '--https-base-url', + 'https://127.0.0.1', + '--redirect-base-url', + 'https://example.com\nhidden', + ]); + assertRejectsInvalidRedirectBaseUrl(); assertRejectsControlCharacter('--probe-token', [ '--https-base-url', 'https://127.0.0.1', @@ -87,6 +94,33 @@ function main() { ]); } +function assertRejectsInvalidRedirectBaseUrl() { + const result = spawnSync( + 'node', + [ + DIRECT_LIVE_SCRIPT, + '--https-base-url', + 'https://127.0.0.1', + '--redirect-base-url', + 'http://example.com', + ], + { + cwd: process.cwd(), + encoding: 'utf8', + env: process.env, + }, + ); + if ((result.status ?? 0) === 0) { + failures.push('--redirect-base-url 必须拒绝非 HTTPS base URL。'); + return; + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + '--redirect-base-url 必须使用 https://', + '--redirect-base-url 非 HTTPS 时必须给出明确错误。', + ); +} + function assertRejectsControlCharacter(label, args) { const result = spawnSync('node', [DIRECT_LIVE_SCRIPT, ...args], { cwd: process.cwd(), diff --git a/scripts/check-pingora-direct-live.mjs b/scripts/check-pingora-direct-live.mjs index b9a8f7659..1fb3c48fe 100644 --- a/scripts/check-pingora-direct-live.mjs +++ b/scripts/check-pingora-direct-live.mjs @@ -41,6 +41,8 @@ Options: --http-base-url Optional HTTP base URL that should 301 to HTTPS. --host Optional Host header for local --resolve style checks. --redirect-host Optional expected Location host for HTTP redirects. + --redirect-base-url + Optional expected HTTPS base URL for HTTP redirects. --probe-token Optional Pingora shadow probe token. --spacetime-database SpacetimeDB database for WSS subscribe smoke, default genarrative-prod. @@ -60,6 +62,7 @@ Environment aliases: GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL GENARRATIVE_PINGORA_DIRECT_HOST GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST + GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE @@ -77,6 +80,8 @@ function parseArgs(argv) { httpBaseUrl: process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '', host: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', redirectHost: process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + redirectBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '', probeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', spacetimeDatabase: process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || @@ -133,6 +138,9 @@ function parseArgs(argv) { case '--redirect-host': result.redirectHost = requireValue(argv, ++index, arg); break; + case '--redirect-base-url': + result.redirectBaseUrl = requireValue(argv, ++index, arg); + break; case '--probe-token': result.probeToken = requireValue(argv, ++index, arg); break; @@ -197,6 +205,9 @@ function parseArgs(argv) { if (result.redirectHost) { validateHostOption(result.redirectHost, '--redirect-host'); } + if (result.redirectBaseUrl) { + validateHttpsBaseUrl(result.redirectBaseUrl, '--redirect-base-url'); + } if (result.probeToken) { validateNoControlCharacters(result.probeToken, '--probe-token'); } @@ -255,6 +266,26 @@ function validateHostOption(value, flag) { } } +function validateHttpsBaseUrl(value, flag) { + validateNoControlCharacters(value, flag); + const parsed = new URL(value); + if (parsed.protocol !== 'https:') { + throw new Error(`${flag} 必须使用 https://`); + } + if ( + !parsed.hostname || + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error( + `${flag} 只能是 HTTPS base URL，不能包含路径、查询、片段或认证信息。`, + ); + } +} + function validateSafeAbsoluteFilePath(filePath, label) { validateNoControlCharacters(filePath, label); if (!nodePath.isAbsolute(filePath)) { @@ -1065,6 +1096,9 @@ function assertPingoraGatewayHeader(check, response) { } function directHttpsLocation(httpsBaseUrl, path) { + if (config.redirectBaseUrl) { + return joinUrl(config.redirectBaseUrl, path); + } if (!config.redirectHost) { return joinUrl(httpsBaseUrl, path); } diff --git a/scripts/check-pingora-release-readiness.mjs b/scripts/check-pingora-release-readiness.mjs index 4909e369d..bbdd1ee81 100644 --- a/scripts/check-pingora-release-readiness.mjs +++ b/scripts/check-pingora-release-readiness.mjs @@ -86,6 +86,8 @@ function parseArgs(argv) { directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', directRedirectHost: process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directRedirectBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '', directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', directPingoraAccessLog: process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '', @@ -228,6 +230,9 @@ function parseArgs(argv) { case '--direct-redirect-host': result.directRedirectHost = requireValue(argv, ++index, arg); break; + case '--direct-redirect-base-url': + result.directRedirectBaseUrl = requireValue(argv, ++index, arg); + break; case '--direct-probe-token': result.directProbeToken = requireValue(argv, ++index, arg); break; @@ -412,6 +417,24 @@ function parseArgs(argv) { if (result.directRedirectHost) { validateHostOption(result.directRedirectHost, '--direct-redirect-host'); } + if (result.directRedirectBaseUrl) { + validateHttpUrl(result.directRedirectBaseUrl, '--direct-redirect-base-url'); + const parsed = new URL(result.directRedirectBaseUrl); + if (parsed.protocol !== 'https:') { + throw new Error('--direct-redirect-base-url 必须使用 https://。'); + } + if ( + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error( + '--direct-redirect-base-url 只能是 HTTPS base URL，不能包含路径、查询、片段或认证信息。', + ); + } + } if (result.requireDirect && !result.directPingoraAccessLog) { throw new Error( '启用 --require-direct 时必须提供 --direct-pingora-access-log 或 GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG', @@ -661,6 +684,8 @@ Options: --direct-http-base-url direct HTTP redirect base URL；也可用 GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL。 --direct-host direct live Host header 和 TLS SNI；也可用 GENARRATIVE_PINGORA_DIRECT_HOST。--require-direct 时必须显式提供。 --direct-redirect-host direct redirect Location host；也可用 GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST。--require-direct 时必须显式提供，cutover runbook 中必须和 --direct-host 使用同一 hostname。 + --direct-redirect-base-url + 可选 direct redirect Location HTTPS base URL；用于高端口 rehearsal 访问 127.0.0.1:18443 但期望 Location 指向正式 HTTPS 入口。 --direct-probe-token direct live shadow probe token；也可用 GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN。 --direct-pingora-access-log direct live 后 Pingora access log 落盘校验路径；--require-direct 时必须显式提供。 @@ -971,6 +996,9 @@ function buildCutoverPlan(config) { config.directHost, '--direct-redirect-host', config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), '--direct-spacetime-database', config.directSpacetimeDatabase, '--direct-pingora-access-log', @@ -1023,6 +1051,7 @@ function buildCutoverPlan(config) { `cutover-run-id=${config.cutoverRunId}`, `direct-host=${config.directHost}`, `direct-redirect-host=${config.directRedirectHost}`, + `direct-redirect-base-url=${config.directRedirectBaseUrl || '-'}`, `health-patrol-public-host=${config.directHost}`, `rollback-nginx-smoke-host=${config.rollbackNginxSmokeHost}`, `rollback-nginx-smoke-expect-body=${config.rollbackNginxSmokeExpectBody}`, @@ -1229,6 +1258,9 @@ function buildCutoverPlan(config) { config.directHost, '--direct-redirect-host', config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), ...(config.directProbeToken ? ['--direct-probe-token', config.directProbeToken] : []), @@ -1485,6 +1517,9 @@ function readinessArgs(config, { includeDirect }) { config.directHost, '--direct-redirect-host', config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--direct-redirect-base-url', config.directRedirectBaseUrl] + : []), ...(config.directProbeToken ? ['--direct-probe-token', config.directProbeToken] : []), @@ -1718,6 +1753,9 @@ function buildSteps(config) { ...(config.directRedirectHost ? ['--redirect-host', config.directRedirectHost] : []), + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), ...(config.directProbeToken ? ['--probe-token', config.directProbeToken] : []), @@ -1910,6 +1948,9 @@ function appendTargetDirectSteps(steps, config, scriptPath) { ...(config.directRedirectHost ? ['--redirect-host', config.directRedirectHost] : []), + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), ...(config.directProbeToken ? ['--probe-token', config.directProbeToken] : []), diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index a3c75e041..4a0c65f98 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -2933,6 +2933,12 @@ const checks = [ reason: 'Pingora direct live smoke 环境示例必须暴露 HTTP redirect Location host。', }, + { + file: 'deploy/env/pingora-direct-live.env.example', + includes: 'GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL', + reason: + 'Pingora direct live smoke 环境示例必须暴露高端口 rehearsal 的 redirect Location base URL 覆盖。', + }, { file: 'deploy/env/pingora-direct-live.env.example', includes: '只填域名或 host:port', @@ -3106,6 +3112,12 @@ const checks = [ reason: 'Pingora direct live smoke 必须提前拒绝非法 redirect Location host 参数。', }, + { + file: 'scripts/check-pingora-direct-live.mjs', + includes: 'validateHttpsBaseUrl(result.redirectBaseUrl,', + reason: + 'Pingora direct live smoke 必须提前拒绝非法 redirect Location base URL 参数。', + }, { file: 'scripts/check-pingora-direct-live.mjs', includes: 'parseRequiredPositiveInt', @@ -3638,6 +3650,12 @@ const checks = [ includes: '--apply 必须同时提供 --direct-redirect-host', reason: 'Pingora 直连启用 apply 后必须验证 redirect Location host。', }, + { + file: 'scripts/deploy/pingora-direct-enable.sh', + includes: '--direct-redirect-base-url', + reason: + 'Pingora 直连启用脚本必须支持把高端口 rehearsal 的 redirect base URL 透传给 direct live smoke。', + }, { file: 'scripts/deploy/pingora-direct-enable.sh', includes: '--apply 必须同时提供 --direct-spacetime-database', diff --git a/scripts/deploy/pingora-direct-enable.sh b/scripts/deploy/pingora-direct-enable.sh index 99ca08846..55bfa52ff 100644 --- a/scripts/deploy/pingora-direct-enable.sh +++ b/scripts/deploy/pingora-direct-enable.sh @@ -26,6 +26,7 @@ DIRECT_HTTPS_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL:-}" DIRECT_HTTP_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL:-}" DIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_HOST:-}" DIRECT_REDIRECT_HOST="${GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST:-}" +DIRECT_REDIRECT_BASE_URL="${GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL:-}" DIRECT_PROBE_TOKEN="${GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN:-}" DIRECT_PINGORA_ACCESS_LOG="${GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG:-}" DIRECT_ACCESS_LOG_SINCE_LINES="${GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES:-}" @@ -39,7 +40,7 @@ usage() { cat <<'EOF' 用法: scripts/deploy/pingora-direct-enable.sh [--apply] [--preflight-env-file ] [--preflight-systemd] [--preflight-check-cert-readable] [--preflight-check-service-env-file] [--preflight-check-service-user-cert-readable] [--preflight-check-service-binary-executable] [--preflight-check-ports-free] - [--direct-https-base-url ] [--direct-http-base-url ] [--direct-host ] [--direct-redirect-host ] + [--direct-https-base-url ] [--direct-http-base-url ] [--direct-host ] [--direct-redirect-host ] [--direct-redirect-base-url ] [--direct-spacetime-database ] [--direct-probe-token ] [--direct-pingora-access-log ] [--direct-access-log-since-lines ] [--direct-timeout-ms ] [--direct-live-script ] [--no-postcheck] [--current-release-audit-script ] [--current-release-root ] [--current-release-audit-timeout-ms ] [--service ] [--service-unit-path ] [--template-path ] [--dropin-path ] @@ -211,6 +212,14 @@ while [[ $# -gt 0 ]]; do fi shift 2 ;; + --direct-redirect-base-url) + DIRECT_REDIRECT_BASE_URL="${2:-}" + if [[ -z "${DIRECT_REDIRECT_BASE_URL}" ]]; then + echo "[pingora-direct-enable] --direct-redirect-base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; --direct-probe-token) DIRECT_PROBE_TOKEN="${2:-}" if [[ -z "${DIRECT_PROBE_TOKEN}" ]]; then @@ -297,6 +306,7 @@ reject_control_characters "--direct-https-base-url" "${DIRECT_HTTPS_BASE_URL}" reject_control_characters "--direct-http-base-url" "${DIRECT_HTTP_BASE_URL}" reject_control_characters "--direct-host" "${DIRECT_HOST}" reject_control_characters "--direct-redirect-host" "${DIRECT_REDIRECT_HOST}" +reject_control_characters "--direct-redirect-base-url" "${DIRECT_REDIRECT_BASE_URL}" reject_control_characters "--direct-probe-token" "${DIRECT_PROBE_TOKEN}" reject_control_characters "--direct-pingora-access-log" "${DIRECT_PINGORA_ACCESS_LOG}" reject_control_characters "--direct-access-log-since-lines" "${DIRECT_ACCESS_LOG_SINCE_LINES}" @@ -732,6 +742,9 @@ run_direct_live_smoke_after_enable() { if [[ -n "${DIRECT_REDIRECT_HOST}" ]]; then direct_live_args+=(--redirect-host "${DIRECT_REDIRECT_HOST}") fi + if [[ -n "${DIRECT_REDIRECT_BASE_URL}" ]]; then + direct_live_args+=(--redirect-base-url "${DIRECT_REDIRECT_BASE_URL}") + fi if [[ -n "${DIRECT_PROBE_TOKEN}" ]]; then direct_live_args+=(--probe-token "${DIRECT_PROBE_TOKEN}") fi @@ -905,5 +918,5 @@ cat <} --http-base-url ${DIRECT_HTTP_BASE_URL:-} --host ${DIRECT_HOST:-} --redirect-host ${DIRECT_REDIRECT_HOST:-} --pingora-access-log ${DIRECT_PINGORA_ACCESS_LOG:-} --spacetime-database ${DIRECT_SPACETIME_DATABASE:-} --require-wss-upgrade --json 已通过，且 direct-access-log 结构化证据 matched=checked、missing=0、mismatches=0 +- node ${DIRECT_LIVE_SCRIPT} --https-base-url ${DIRECT_HTTPS_BASE_URL:-} --http-base-url ${DIRECT_HTTP_BASE_URL:-} --host ${DIRECT_HOST:-} --redirect-host ${DIRECT_REDIRECT_HOST:-} ${DIRECT_REDIRECT_BASE_URL:+--redirect-base-url ${DIRECT_REDIRECT_BASE_URL} }--pingora-access-log ${DIRECT_PINGORA_ACCESS_LOG:-} --spacetime-database ${DIRECT_SPACETIME_DATABASE:-} --require-wss-upgrade --json 已通过，且 direct-access-log 结构化证据 matched=checked、missing=0、mismatches=0 EOF diff --git a/scripts/ops/pingora-cutover-evidence-bundle.mjs b/scripts/ops/pingora-cutover-evidence-bundle.mjs index 46765da95..c9cb176a8 100644 --- a/scripts/ops/pingora-cutover-evidence-bundle.mjs +++ b/scripts/ops/pingora-cutover-evidence-bundle.mjs @@ -442,6 +442,8 @@ function parseArgs(argv) { directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '', directRedirectHost: process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '', + directRedirectBaseUrl: + process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '', directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '', directSpacetimeDatabase: process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '', @@ -521,6 +523,9 @@ function parseArgs(argv) { case '--direct-redirect-host': result.directRedirectHost = requireValue(argv, ++index, arg); break; + case '--direct-redirect-base-url': + result.directRedirectBaseUrl = requireValue(argv, ++index, arg); + break; case '--direct-probe-token': result.directProbeToken = requireValue(argv, ++index, arg); break; @@ -597,6 +602,7 @@ Options: --direct-http-base-url 透传给 direct live smoke 的 HTTP redirect / ACME 入口。 --direct-host 透传给 direct live smoke 的正式 Host/SNI。 --direct-redirect-host 透传给 direct live smoke 的 redirect Location host。 + --direct-redirect-base-url 可选，透传给 direct live smoke 的 redirect Location HTTPS base URL。 --direct-probe-token 可选，透传给 direct live smoke 的内部探针 token；证据中会脱敏。 --direct-spacetime-database 透传给 direct live smoke 的 SpacetimeDB 数据库名。 --direct-pingora-access-log @@ -681,6 +687,9 @@ function validateDirectLiveConfig(config) { throw new Error('--run-direct-live 必须提供 --direct-redirect-host。'); } validateHostOption(config.directRedirectHost, '--direct-redirect-host'); + if (config.directRedirectBaseUrl) { + validateHttpsBaseUrl(config.directRedirectBaseUrl, '--direct-redirect-base-url'); + } if (!config.directSpacetimeDatabase) { throw new Error('--run-direct-live 必须提供 --direct-spacetime-database。'); } @@ -807,6 +816,22 @@ function validateHttpsUrl(value, label) { } } +function validateHttpsBaseUrl(value, label) { + validateHttpsUrl(value, label); + const parsed = new URL(value); + if ( + parsed.pathname !== '/' || + parsed.search || + parsed.hash || + parsed.username || + parsed.password + ) { + throw new Error( + `${label} 只能是 HTTPS base URL，不能包含路径、查询、片段或认证信息。`, + ); + } +} + function validateHttpOnlyUrl(value, label) { validateHttpUrl(value, label); if (new URL(value).protocol !== 'http:') { @@ -961,6 +986,9 @@ function buildDirectLiveArgs(config) { config.directHost, '--redirect-host', config.directRedirectHost, + ...(config.directRedirectBaseUrl + ? ['--redirect-base-url', config.directRedirectBaseUrl] + : []), ...(config.directProbeToken ? ['--probe-token', config.directProbeToken] : []), From 93f085f95c22ecbcf626b6728273882cb5c84669 Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 23:21:56 +0800 Subject: [PATCH 13/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20realpath?= =?UTF-8?q?=20canary=20=E5=8A=A0=E8=BD=BD=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 realpath canary 在 conf.d 中必须晚于 log_format 加载 记录 dev realpath canary current release 真实验收结果 新增生产运维护栏防止加载顺序说明丢失 --- deploy/nginx/README.md | 1 + docs/project-memory/shared-memory/pitfalls.md | 8 ++++++++ ...开发运维】Pingora独立网关试点-2026-06-11.md | 9 ++++++++- scripts/check-production-ops-guardrails.mjs | 18 ++++++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index d5db0afec..ac9a3f253 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -64,6 +64,7 @@ - `deploy/nginx/snippets/genarrative-pingora-canary.conf` 是默认不启用的人工前缀 canary 入口，只在需要验证 Nginx -> Pingora handoff 时临时 include。它使用 `/__genarrative_pingora_canary/` 前缀改写后转发到 `127.0.0.1:18081`，并返回 `X-Genarrative-Nginx-Handoff: pingora-canary`。 - `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary。它只能在 Nginx `http` 上下文人工 include，默认监听 `127.0.0.1:18083`，覆盖 `/api/creation-entry/config`、`/v1/identity`、`/v1/database//subscribe`、`/assets/app.js` 和拒绝入口，并写入独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`；不要把它 include 到生产 `443` server 内作为 location 覆盖。 +- 真实路径 canary 片段使用 `access_log ... genarrative_upstream`，如果以独立文件放入 `/etc/nginx/conf.d/`，文件加载顺序必须晚于定义 `log_format genarrative_upstream` 的主站配置。当前目标机可使用 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`；若站点把 `log_format` 移到全局 Nginx 配置，则需确保它仍在所有 `conf.d` server include 之前加载。启用前必须先跑 `nginx -t`，不要用会早于 `genarrative.conf` 加载的文件名。 - 启用 Nginx canary 前，目标机必须已经有持久 `genarrative-pingora-gateway.service` 运行在 `127.0.0.1:18081`，且带真实 `api-server`、SpacetimeDB、静态目录和 `/var/log/genarrative/pingora-gateway.access.log` 完成本机 shadow 验收；不要从一次性 `/tmp` 网关进程直接切到 Nginx handoff。 - 启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，同时做两个 snippet 的静态护栏和真实 `nginx -t`。本机或 CI 可运行 `npm run check:pingora-canary-docker`，用 Docker Nginx、真实 Pingora 和 mock 上游复现前缀 canary 与真实路径 canary 两条 handoff 链路。 - 前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`，再用 `scripts/check-pingora-canary-access-log-parity.mjs` 对账 `/var/log/nginx/genarrative.access.log` 与 `/var/log/genarrative/pingora-gateway.access.log`。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 55d6e0d82..b87df7a6a 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -125,6 +125,14 @@ - 验证：本机或 CI 执行 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull` 时应同时完成临时 Nginx / Pingora access log 对账。目标机执行 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；本机执行 `npm run check:pingora-release-readiness-plan` 和 `npm run check:production-ops`，确认 live 门禁计划包含真实 access log 对账。 - 关联：`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`deploy/env/pingora-canary-live.env.example`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 +## Pingora realpath canary include 要晚于 log_format + +- 现象：目标机把 `genarrative-pingora-realpath-canary.conf` 放进 `/etc/nginx/conf.d/` 后，`nginx -t` 失败并报 `unknown log format "genarrative_upstream"`。 +- 原因：真实路径 canary 是独立 `server` 片段，并使用 `access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;`。Nginx 会按文件名顺序加载 `conf.d`；如果 canary 文件名早于定义 `log_format genarrative_upstream` 的主站配置，access log 行会先被解析而找不到格式。 +- 处理：真实路径 canary 要么 include 在已经定义 `log_format genarrative_upstream` 之后，要么放入晚于主站配置加载的文件，例如 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`；另一种长期做法是把 `log_format` 放到所有 `conf.d` server 之前的全局 Nginx 配置。启用前必须先跑 `nginx -t`，失败时先移除临时 canary 文件再 reload，避免保留坏配置；检查配置时不要把 probe token 原文写入记录。 +- 验证：`nginx -t` 通过后 reload Nginx，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`、`node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ...` 和 `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...`。若只启用了真实路径 canary，不要同时传 `--require-live`，否则前缀 canary 未启用时会按正式 Nginx HTTP 入口返回 301。 +- 关联：`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf`、`deploy/nginx/README.md`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`、`scripts/check-pingora-release-readiness.mjs`。 + ## Pingora release readiness 脚本不能只存在于源码 checkout - 现象：本机 runbook 能生成，但目标机切换窗口执行启用前或启用后的 release readiness 复核时，可能命中 Jenkins workspace 或源码 checkout 的 `scripts/check-pingora-release-readiness.mjs`，而不是当前发布包里的脚本。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index c2404c9fa..fdecfb491 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -369,6 +369,13 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - `node /tmp/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ... --json` 对账 `6/6 matched`，`missingCount=0`，`mismatchCount=0`；除 healthz 探针映射到 `__genarrative_pingora/healthz` 外，API config、SpacetimeDB identity、WSS subscribe、静态代表路径和 generated 拒绝路径的 Nginx path 与 Pingora path 完全一致。 - 恢复后 `grep genarrative-pingora-realpath-canary /etc/nginx/conf.d/genarrative.conf` 无匹配，`ss -ltnp | grep :18083` 无监听，证明临时 realpath canary server 已移除；`genarrative-pingora-gateway.service` 仍为 `active` 且 `NRestarts=0`。 +同日下一阶段在 current release `/opt/genarrative/releases/dev-pingora-api-20260617230221` 上重新启用 loopback realpath canary，并用正式 Host `dev.genarrative.world` 验收。第一次把独立 server 写成 `/etc/nginx/conf.d/genarrative-pingora-realpath-canary.conf` 时，因它早于 `genarrative.conf` 加载，`nginx -t` 报 `unknown log format "genarrative_upstream"`；改为 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf` 后 `nginx -t` 和 reload 均通过。随后： + +- `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host dev.genarrative.world --json` 通过：healthz `200`、API config `200`、SpacetimeDB identity `405`、静态代表路径 `404`、generated 拒绝路径 `404`，全部带 `X-Genarrative-Nginx-Handoff: pingora-realpath-canary`。 +- `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath ... --json` 对账 `12/12 matched`，`missingCount=0`，`mismatchCount=0`。 +- `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...` 通过 current release 自包含自审、realpath live smoke 和 realpath access log 对账，最终对账 `18/18 matched`。 +- 本阶段没有启用前缀 canary，因此不能同时传 `--require-live`；否则正式 HTTP 入口会按 Nginx 策略返回 `301`，并缺少 `X-Genarrative-Nginx-Handoff: pingora-canary`。当前 public `80/443` 仍由 Nginx 承接，realpath canary 只保留在 `127.0.0.1:18083` 用于后续 loopback 复核。 + ## dev API release 正式路径验收记录 2026-06-17 已在 dev 机用正式 API release 路径部署包含 Pingora 的发布包 `dev-pingora-api-20260617140915`。发布包由分支 `codex/pingoranginx` 的 `68bf3b698a885ee2cd248dcba81e4b57460000f8` 构建，包含 `api-server`、`api-server.sha256`、`pingora-gateway`、`pingora-gateway.sha256`、`release-manifest.json` 和随包部署 / 自审脚本。部署命令从上传到 `/tmp/dev-pingora-api-20260617140915` 的发布包内执行 `scripts/deploy/production-api-deploy.sh`，目标 release root 为 `/opt/genarrative/releases`，current link 为 `/opt/genarrative/current`。 @@ -470,7 +477,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 2. 涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时，同步更新 `deploy/pingora/nginx-route-parity.matrix.json`，并运行 `npm run check:pingora-route-parity` 与 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix`。 3. 容器内使用同一份 Web 产物、同一组真实上游地址跑 Pingora smoke，并继续对照 `deploy/nginx/genarrative.conf` 扩展真实上游路由 parity 自动测试。 4. 使用 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 做 Nginx 前缀 canary；启用前先跑 `npm run check:nginx-pingora-canary` 和 `npm run check:pingora-canary-docker`，有 Nginx 或 Docker 的目标环境分别强制跑 `node scripts/check-nginx-pingora-canary.mjs --require-nginx` 与 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，其中 Docker handoff 会自动对账临时 Nginx 与 Pingora access log。启用后跑 `npm run check:pingora-canary-live`，再用 current release 随包 access log parity 脚本对账目标机 Nginx 与 Pingora access log。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；canary live timeout 和 access log `since-lines` 必须是正整数，非法值直接失败。 -5. 前缀 canary 通过后，再使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary；它必须作为独立本机 `server` include 到 Nginx `http` 上下文，不能 include 到生产 `443` server 内。启用后跑 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 +5. 前缀 canary 通过后，再使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary；它必须作为独立本机 `server` include 到 Nginx `http` 上下文，不能 include 到生产 `443` server 内。该片段使用 `access_log ... genarrative_upstream`，如果放入 `/etc/nginx/conf.d/`，文件名必须保证晚于定义 `log_format genarrative_upstream` 的主站配置加载，例如 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`；否则 `nginx -t` 会报 `unknown log format "genarrative_upstream"`。启用后跑 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 6. 目标机 canary include 后必须跑正式切换聚合门禁：源码 checkout / CI / 构建环境执行默认全量命令 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；目标机 current release 执行 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。启用真实路径 canary 时，两种命令都追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。缺少 Host 会直接失败，避免 live canary 误测默认 vhost；live smoke 后还会按 `request_id` 对账 Nginx 与 Pingora access log，缺少同一请求的 Pingora 日志、状态码、方法或 path 漂移都会失败。 7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。高端口 rehearsal 使用 `https://127.0.0.1:<高端口>` 打入但期望 HTTP redirect Location 指向正式域名默认 HTTPS 入口时，额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于 runbook Host 一致性约束。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、redirect base URL、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs` 和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 4a0c65f98..0206acc8b 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -832,6 +832,24 @@ const checks = [ reason: 'Pingora 真实路径 canary 必须覆盖代表性 API 真实路径。', }, + { + file: 'deploy/nginx/README.md', + includes: 'zz-genarrative-pingora-realpath-canary.conf', + reason: + 'Pingora 真实路径 canary 运维说明必须记录 conf.d 加载顺序，避免早于 log_format 定义加载。', + }, + { + file: 'docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md', + includes: 'unknown log format "genarrative_upstream"', + reason: + 'Pingora 试点文档必须记录 realpath canary 的 Nginx log_format 加载顺序踩坑。', + }, + { + file: 'docs/project-memory/shared-memory/pitfalls.md', + includes: 'Pingora realpath canary include 要晚于 log_format', + reason: + '团队共享踩坑必须记录 realpath canary 独立 server 在 conf.d 中的加载顺序要求。', + }, { file: 'scripts/jenkins-server-provision.sh', includes: 'genarrative-health-patrol.timer', From 123d4ada01070ca3a4e1e52d4c0c30021969364e Mon Sep 17 00:00:00 2001 From: kdletters Date: Wed, 17 Jun 2026 23:45:58 +0800 Subject: [PATCH 14/22] =?UTF-8?q?=E8=84=9A=E6=9C=AC=E5=8C=96=20Pingora=20r?= =?UTF-8?q?ealpath=20canary=20=E5=90=AF=E5=81=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 realpath canary 启用和关闭脚本 将脚本纳入 release、deploy、Jenkins 和 current release 自审 补充脚本化 canary 文档和生产运维护栏 增加 realpath canary 启停自测和发布部署护栏 --- deploy/nginx/README.md | 3 +- docs/project-memory/shared-memory/pitfalls.md | 4 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 5 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- jenkins/Jenkinsfile.production-api-build | 2 +- jenkins/Jenkinsfile.production-api-deploy | 4 +- package.json | 3 + scripts/build-production-release.sh | 6 +- .../check-pingora-current-release-audit.mjs | 4 + .../check-pingora-cutover-status-snapshot.mjs | 6 + .../check-pingora-realpath-canary-toggle.mjs | 432 ++++++++++++++++++ scripts/check-production-api-deploy.mjs | 64 +++ scripts/check-production-api-release.mjs | 28 +- scripts/check-production-ops-guardrails.mjs | 113 +++++ .../deploy/pingora-realpath-canary-disable.sh | 197 ++++++++ .../deploy/pingora-realpath-canary-enable.sh | 350 ++++++++++++++ scripts/deploy/production-api-deploy.sh | 14 + scripts/ops/pingora-current-release-audit.mjs | 2 + .../ops/pingora-cutover-status-snapshot.mjs | 2 + 19 files changed, 1230 insertions(+), 11 deletions(-) create mode 100644 scripts/check-pingora-realpath-canary-toggle.mjs create mode 100644 scripts/deploy/pingora-realpath-canary-disable.sh create mode 100644 scripts/deploy/pingora-realpath-canary-enable.sh diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index ac9a3f253..4ab54ceb0 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -68,7 +68,8 @@ - 启用 Nginx canary 前，目标机必须已经有持久 `genarrative-pingora-gateway.service` 运行在 `127.0.0.1:18081`，且带真实 `api-server`、SpacetimeDB、静态目录和 `/var/log/genarrative/pingora-gateway.access.log` 完成本机 shadow 验收；不要从一次性 `/tmp` 网关进程直接切到 Nginx handoff。 - 启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，同时做两个 snippet 的静态护栏和真实 `nginx -t`。本机或 CI 可运行 `npm run check:pingora-canary-docker`，用 Docker Nginx、真实 Pingora 和 mock 上游复现前缀 canary 与真实路径 canary 两条 handoff 链路。 - 前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`，再用 `scripts/check-pingora-canary-access-log-parity.mjs` 对账 `/var/log/nginx/genarrative.access.log` 与 `/var/log/genarrative/pingora-gateway.access.log`。 -- 真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 +- 真实路径 canary 不再手工编辑 `/etc/nginx/conf.d/`。目标机从 current release 执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`；脚本会渲染到 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，保持晚于主站 `log_format genarrative_upstream` 加载，执行 `nginx -t`、reload Nginx，并默认运行 realpath live smoke。`nginx -t`、reload 或 live smoke 任一失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会删除该 `zz-...` 文件，执行 `nginx -t` 和 reload，失败时恢复删除前配置。 +- 真实路径 canary reload 后可再次运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 - 正式切换前在源码 checkout / CI / 构建环境跑 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。 - 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁：`/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证真实路径 canary 或 Pingora 直连入口时分别追加 `--require-realpath-live ...` 或 `--require-direct ...` 参数。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index b87df7a6a..8fd688ad1 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -129,8 +129,8 @@ - 现象：目标机把 `genarrative-pingora-realpath-canary.conf` 放进 `/etc/nginx/conf.d/` 后，`nginx -t` 失败并报 `unknown log format "genarrative_upstream"`。 - 原因：真实路径 canary 是独立 `server` 片段，并使用 `access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;`。Nginx 会按文件名顺序加载 `conf.d`；如果 canary 文件名早于定义 `log_format genarrative_upstream` 的主站配置，access log 行会先被解析而找不到格式。 -- 处理：真实路径 canary 要么 include 在已经定义 `log_format genarrative_upstream` 之后，要么放入晚于主站配置加载的文件，例如 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`；另一种长期做法是把 `log_format` 放到所有 `conf.d` server 之前的全局 Nginx 配置。启用前必须先跑 `nginx -t`，失败时先移除临时 canary 文件再 reload，避免保留坏配置；检查配置时不要把 probe token 原文写入记录。 -- 验证：`nginx -t` 通过后 reload Nginx，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`、`node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ...` 和 `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...`。若只启用了真实路径 canary，不要同时传 `--require-live`，否则前缀 canary 未启用时会按正式 Nginx HTTP 入口返回 301。 +- 处理：真实路径 canary 启停统一用 current release 随包脚本，不再手工写 `/etc/nginx/conf.d/`。启用执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`，脚本固定写入晚于主站配置加载的 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，并在 `nginx -t`、reload 或 live smoke 失败时恢复写入前配置。关闭执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本在 `nginx -t` 或 reload 失败时恢复删除前配置。另一种长期做法是把 `log_format` 放到所有 `conf.d` server 之前的全局 Nginx 配置。检查配置时不要把 probe token 原文写入记录。 +- 验证：启用脚本通过后，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`、`node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ...` 和 `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...`。若只启用了真实路径 canary，不要同时传 `--require-live`，否则前缀 canary 未启用时会按正式 Nginx HTTP 入口返回 301。 - 关联：`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf`、`deploy/nginx/README.md`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`、`scripts/check-pingora-release-readiness.mjs`。 ## Pingora release readiness 脚本不能只存在于源码 checkout diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index fdecfb491..7ef9331dd 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -375,6 +375,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath ... --json` 对账 `12/12 matched`，`missingCount=0`，`mismatchCount=0`。 - `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...` 通过 current release 自包含自审、realpath live smoke 和 realpath access log 对账，最终对账 `18/18 matched`。 - 本阶段没有启用前缀 canary，因此不能同时传 `--require-live`；否则正式 HTTP 入口会按 Nginx 策略返回 `301`，并缺少 `X-Genarrative-Nginx-Handoff: pingora-canary`。当前 public `80/443` 仍由 Nginx 承接，realpath canary 只保留在 `127.0.0.1:18083` 用于后续 loopback 复核。 +- 后续 realpath canary 启停统一走 current release 随包脚本：`/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083` 和 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`。启用脚本固定写入 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，并在 `nginx -t`、reload 或 live smoke 失败时回滚；关闭脚本在 `nginx -t` 或 reload 失败时恢复删除前配置。 ## dev API release 正式路径验收记录 @@ -477,10 +478,10 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 2. 涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时，同步更新 `deploy/pingora/nginx-route-parity.matrix.json`，并运行 `npm run check:pingora-route-parity` 与 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix`。 3. 容器内使用同一份 Web 产物、同一组真实上游地址跑 Pingora smoke，并继续对照 `deploy/nginx/genarrative.conf` 扩展真实上游路由 parity 自动测试。 4. 使用 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 做 Nginx 前缀 canary；启用前先跑 `npm run check:nginx-pingora-canary` 和 `npm run check:pingora-canary-docker`，有 Nginx 或 Docker 的目标环境分别强制跑 `node scripts/check-nginx-pingora-canary.mjs --require-nginx` 与 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，其中 Docker handoff 会自动对账临时 Nginx 与 Pingora access log。启用后跑 `npm run check:pingora-canary-live`，再用 current release 随包 access log parity 脚本对账目标机 Nginx 与 Pingora access log。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；canary live timeout 和 access log `since-lines` 必须是正整数，非法值直接失败。 -5. 前缀 canary 通过后，再使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary；它必须作为独立本机 `server` include 到 Nginx `http` 上下文，不能 include 到生产 `443` server 内。该片段使用 `access_log ... genarrative_upstream`，如果放入 `/etc/nginx/conf.d/`，文件名必须保证晚于定义 `log_format genarrative_upstream` 的主站配置加载，例如 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`；否则 `nginx -t` 会报 `unknown log format "genarrative_upstream"`。启用后跑 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 +5. 前缀 canary 通过后，再使用 current release 随包 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083` 启用真实路径 canary；它会把 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 渲染成独立本机 `server`，写入 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，不能 include 到生产 `443` server 内。该片段使用 `access_log ... genarrative_upstream`，文件名必须保证晚于定义 `log_format genarrative_upstream` 的主站配置加载；否则 `nginx -t` 会报 `unknown log format "genarrative_upstream"`。启用脚本会先执行 `nginx -t`、reload Nginx，再默认运行 realpath live smoke，任一阶段失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会在 `nginx -t` 或 reload 失败时恢复删除前配置。启用后跑 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 6. 目标机 canary include 后必须跑正式切换聚合门禁：源码 checkout / CI / 构建环境执行默认全量命令 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；目标机 current release 执行 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。启用真实路径 canary 时，两种命令都追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。缺少 Host 会直接失败，避免 live canary 误测默认 vhost；live smoke 后还会按 `request_id` 对账 Nginx 与 Pingora access log，缺少同一请求的 Pingora 日志、状态码、方法或 path 漂移都会失败。 7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。高端口 rehearsal 使用 `https://127.0.0.1:<高端口>` 打入但期望 HTTP redirect Location 指向正式域名默认 HTTPS 入口时，额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于 runbook Host 一致性约束。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、redirect base URL、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 -8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs` 和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 +8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs`、realpath canary 启停脚本和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 9. direct preflight / direct live timeout 必须是正整数；直连相关布尔 env 和 Pingora gateway env 里的 `TRUST_X_FORWARDED_FOR` / `TRUSTED_FRONT_PROXY_CONFIRMED` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。 10. Pingora 正式化口径固定为 gzip-only；Brotli 继续由 Nginx / 前置代理承担，直连 Pingora 不以 Brotli parity 作为切换门禁。如需多实例，开启网关接流保护时必须先引入共享限流 / 共享并发保护层并设置 `GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true`，否则保持 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1`；关闭网关保护的多实例方案必须明确由前置 Nginx / LB 承担全局限流。 10. 前缀 canary 稳定后，再评估是否做真实路径 canary；真实路径 canary 稳定后，再评估是否让 Pingora 直接承接公网入口。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index c115880e5..98cd0ff95 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -322,7 +322,7 @@ current release 随包执行的 release readiness 必须追加 `--release-runtim 涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时，必须同步更新 `deploy/pingora/nginx-route-parity.matrix.json`，并运行 `npm run check:pingora-route-parity`；Rust 侧 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix` 会读取同一份矩阵验证 `classify_path` 的路由结果、body limit 和接流保护分组。 -Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。需要证明 Docker Nginx -> Pingora -> 真实本地服务时，先启动真实 api-server 与 SpacetimeDB，再运行 `node scripts/check-pingora-canary-docker.mjs --require-docker --real-upstreams --api-upstream 127.0.0.1: --spacetime-upstream 127.0.0.1: --web-root dist`；该模式不会启动内置 mock 上游，会先检查真实 `/healthz` 和 `/v1/ping`，随后复用同一组 live smoke 与 access log 对账。真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，canary live 只把它作为路径路由代表，不要求该 GET 创建 identity。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary reload 后运行 `node scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 +Pingora canary 入口默认由 Server-Provision 安装但不接入主站配置。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是前缀 canary，人工 include 后用 `/__genarrative_pingora_canary/` rewrite 到 Pingora shadow；`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 是前缀 canary 之后、direct 直连之前的真实路径 canary，只能在 Nginx `http` 上下文 include，默认监听 `127.0.0.1:18083` 并写独立 `/var/log/nginx/genarrative-pingora-realpath-canary.access.log`，不要 include 到生产 `443` server 内覆盖正式 location。启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，强制把前缀 location 片段和真实路径 server 片段一起执行 `nginx -t`。本机或 CI 可用 `npm run check:pingora-canary-docker` 启动 Docker Nginx、真实 Pingora 和 mock 上游复现两条 Nginx -> Pingora handoff，并复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按 `request_id` 对账。需要证明 Docker Nginx -> Pingora -> 真实本地服务时，先启动真实 api-server 与 SpacetimeDB，再运行 `node scripts/check-pingora-canary-docker.mjs --require-docker --real-upstreams --api-upstream 127.0.0.1: --spacetime-upstream 127.0.0.1: --web-root dist`；该模式不会启动内置 mock 上游，会先检查真实 `/healthz` 和 `/v1/ping`，随后复用同一组 live smoke 与 access log 对账。真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，canary live 只把它作为路径路由代表，不要求该 GET 创建 identity。前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`；真实路径 canary 启停统一使用 current release 随包脚本：`/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083` 写入 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf` 并执行 `nginx -t`、reload 和 realpath live smoke，`/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply` 删除该配置并复核 Nginx；两者失败都会恢复操作前状态。真实路径 canary reload 后运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，随后用 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js` 对账；真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。canary live 的 base URL、prefix、Host、额外 path、`--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS` 不能包含换行或 NUL，脚本会在发起请求前失败；canary access log 对账脚本的日志路径、prefix、必需路径和 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也不能包含换行或 NUL，日志行里解析出的 URI / path 含控制字符时必须失败；timeout 和 access log 对账的 `--since-lines` / 对应 env 必须是正整数，非法值直接失败，不静默回默认值。 Pingora 正式切换前必须额外运行聚合门禁：`node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。该门禁串起 Rust 单测、mock smoke、路由 parity、Nginx snippet 校验、Docker Nginx handoff、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、health patrol env 切换脚本烟测、current release 自审烟测、release readiness 计划自检、生产运维护栏、API release build 烟测、Pingora production release 真实构建烟测、API deploy release 烟测、目标机 live canary 和目标机真实 access log 对账；其中 `--require-live` 只能在目标 Nginx 已人工 include canary snippet 并 reload 后执行，并会强制要求 `--live-host`，避免只访问 `127.0.0.1` 命中 Nginx 默认 vhost。live smoke 成功后会立即读取 Nginx 与 Pingora access log 尾部记录，按 `request_id` 验证 `/__genarrative_pingora_canary/healthz` 和 `/__genarrative_pingora_canary/api/creation-entry/config` 已在两边落盘且 method/status/path 没有漂移。启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，它会用独立 Nginx access log 对账真实 `/api`、`/v1` 与 `/assets` 路径。live canary、direct live 和 access log 对账的显式毫秒超时或尾部行数参数都必须是正整数，非法值应先修参数再重跑，不能把默认值兜底后的结果当成切换证据；direct live 与 release readiness 读取的直连布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，拼写错误会直接失败，避免 `REQUIRE_WSS_UPGRADE`、preflight 开关或 `SKIP_WSS` 被悄悄当成 false。若验证 Pingora 直连公网入口，还必须追加 direct preflight 参数检查目标机 env、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和端口释放：`--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free`；该模式缺少 `--direct-http-base-url`、`--direct-host`、`--direct-redirect-host`、`--direct-pingora-access-log`、`--direct-preflight-systemd`、`--direct-preflight-check-cert-readable`、`--direct-preflight-check-service-env-file`、`--direct-preflight-check-service-user-cert-readable`、`--direct-preflight-check-service-binary-executable`、`--direct-preflight-check-ports-free` 、缺少 `--direct-health-patrol-env-file` 或缺少 `--direct-spacetime-database` 会直接失败，且会拒绝 `--direct-skip-wss`，并默认要求 WSS subscribe 返回 101，证明 HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id + path/status 对账、health patrol direct 模式、systemd drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性、80/443 端口释放与目标 SpacetimeDB 长连接都能从直连入口透传。`check-pingora-release-readiness.mjs --help` 中的正式直连和只生成 runbook 示例也必须显式带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，避免值班人员复制示例后被 `--require-direct` 自身拦住。普通本机提交前可先跑 `npm run check:pingora-release-readiness`，但该默认模式不能替代目标机切换窗口的强制门禁。直连切换前还应先运行 `npm run plan:pingora-direct-cutover -- --require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free --rollback-nginx-smoke-url http://127.0.0.1/healthz --rollback-health-patrol-public-base-url <切换前Nginx巡检入口>` 生成只读 JSON runbook；若切换参数提供 `--direct-probe-token`，启用后复核会继续透传 direct probe token 检查内部探针，runbook JSON 只显示 ``。runbook 会列出 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply、用 `/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply` 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、回退前用同一脚本预置回 `nginx` 并恢复切换前 public base URL / Host、rollback apply、回退后 health patrol env Nginx 模式复核，供当班人员逐条审阅。direct enable / rollback `--apply` 都会拒绝符号链接形式的 systemd drop-in 目录或目标文件，并拒绝已存在但不是普通文件的目标，避免把低端口 capability 写入非预期位置，或在回退窗口误删 / 误判非预期 systemd 位置；health patrol env 切换脚本 `--apply` 会保留原文件权限和 owner/group，此时 `--env-file` 必须直接指向真实普通文件，不能是符号链接。如果现场 env 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 diff --git a/jenkins/Jenkinsfile.production-api-build b/jenkins/Jenkinsfile.production-api-build index 919ffcb7a..ba7a9cf2e 100644 --- a/jenkins/Jenkinsfile.production-api-build +++ b/jenkins/Jenkinsfile.production-api-build @@ -123,7 +123,7 @@ pipeline { stage('Archive') { steps { - archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true + archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true script { if (params.INCLUDE_PINGORA_GATEWAY) { archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway,build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway.sha256", fingerprint: true diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index 657d5fefa..50bee6574 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -66,7 +66,7 @@ pipeline { copyArtifacts( projectName: params.BUILD_JOB_NAME, selector: specific(params.BUILD_NUMBER_TO_DEPLOY), - filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", + filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", target: '.', fingerprintArtifacts: true ) @@ -92,7 +92,7 @@ pipeline { sh ''' bash -lc ' set -euo pipefail - chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" + chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \ --source-dir "build/${BUILD_VERSION}" \ --version "${BUILD_VERSION}" \ diff --git a/package.json b/package.json index 8fe9d55e0..2e715da20 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "check:pingora-route-parity": "node scripts/check-pingora-route-parity.mjs", "check:pingora-canary-live": "node scripts/check-pingora-canary-live.mjs", "check:pingora-canary-live-guard": "node scripts/check-pingora-canary-live-guard.mjs", + "check:pingora-realpath-canary-toggle": "node scripts/check-pingora-realpath-canary-toggle.mjs", "check:pingora-direct-preflight": "node -- scripts/check-pingora-direct-preflight.mjs && node scripts/check-pingora-direct-preflight-guard.mjs", "check:pingora-direct-live": "node scripts/check-pingora-direct-live.mjs", "check:pingora-direct-live-guard": "node scripts/check-pingora-direct-live-guard.mjs", @@ -57,6 +58,8 @@ "check:pingora-release-readiness-plan": "node scripts/check-pingora-release-readiness-plan.mjs", "check:pingora-release-readiness": "node scripts/check-pingora-release-readiness.mjs", "plan:pingora-direct-cutover": "node scripts/check-pingora-release-readiness.mjs --dry-run-cutover", + "deploy:pingora-realpath-canary-enable": "bash scripts/deploy/pingora-realpath-canary-enable.sh", + "deploy:pingora-realpath-canary-disable": "bash scripts/deploy/pingora-realpath-canary-disable.sh", "deploy:pingora-direct-enable": "bash scripts/deploy/pingora-direct-enable.sh", "deploy:pingora-direct-rollback": "bash scripts/deploy/pingora-direct-rollback.sh", "assets:child-motion-demo": "node scripts/generate-child-motion-demo-assets.mjs", diff --git a/scripts/build-production-release.sh b/scripts/build-production-release.sh index d31b556e0..b5a007db9 100644 --- a/scripts/build-production-release.sh +++ b/scripts/build-production-release.sh @@ -520,6 +520,8 @@ cp "${SCRIPT_DIR}/deploy/maintenance-on.sh" "${TARGET_DIR}/scripts/deploy/mainte cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/deploy/maintenance-off.sh" cp "${SCRIPT_DIR}/deploy/pingora-direct-enable.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh" cp "${SCRIPT_DIR}/deploy/pingora-direct-rollback.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh" +cp "${SCRIPT_DIR}/deploy/pingora-realpath-canary-enable.sh" "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +cp "${SCRIPT_DIR}/deploy/pingora-realpath-canary-disable.sh" "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" cp "${SCRIPT_DIR}/deploy/pingora-health-patrol-env-switch.mjs" "${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" cp "${SCRIPT_DIR}/deploy/pingora-tls-cert-sync.mjs" "${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" cp "${SCRIPT_DIR}/deploy/jenkins-inbound-agent-start.sh" "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" @@ -535,6 +537,8 @@ chmod +x \ "${TARGET_DIR}/scripts/deploy/maintenance-off.sh" \ "${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh" \ "${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh" \ + "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" \ + "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" \ "${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" \ "${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" \ "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" \ @@ -580,7 +584,7 @@ cat >"${TARGET_DIR}/README.md" < 0) { + console.error('[check:pingora-realpath-canary-toggle] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-realpath-canary-toggle] OK'); + +function main() { + assertScriptShape(); + assertDryRunDoesNotWrite(); + assertApplyWritesRenderedConfigAndRunsLive(); + assertEnableRollsBackWhenLiveFails(); + assertDisableRemovesConfig(); + assertDisableRollsBackWhenReloadFails(); + assertRejectsUnsafeInputs(); +} + +function assertScriptShape() { + for (const script of [ENABLE_SCRIPT, DISABLE_SCRIPT]) { + const content = readFileSync(script, 'utf8'); + assertIncludes(content, '--apply', `${script} 必须默认 dry-run 并要求显式 --apply。`); + assertIncludes(content, 'nginx -t', `${script} 帮助文案必须说明会先做 nginx -t。`); + assertIncludes( + content, + 'zz-genarrative-pingora-realpath-canary.conf', + `${script} 必须默认使用晚于主站配置加载的 conf.d 文件名。`, + ); + assertIncludes( + content, + '不能包含换行或 NUL 字符', + `${script} 必须拒绝控制字符参数。`, + ); + assertIncludes(content, '不能是文件系统根目录', `${script} 必须拒绝根路径。`); + } + + const enableContent = readFileSync(ENABLE_SCRIPT, 'utf8'); + assertIncludes( + enableContent, + 'check-pingora-canary-live.mjs', + 'enable 脚本必须默认运行 realpath live smoke。', + ); + assertIncludes( + enableContent, + '恢复写入前配置', + 'enable 脚本失败时必须恢复写入前配置。', + ); + + const disableContent = readFileSync(DISABLE_SCRIPT, 'utf8'); + assertIncludes( + disableContent, + '恢复删除前配置', + 'disable 脚本失败时必须恢复删除前配置。', + ); +} + +function assertDryRunDoesNotWrite() { + const fixture = createFixture('dry-run'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + + const result = runScript(ENABLE_SCRIPT, [ + '--probe-token', + 'dry-run-token', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ]); + + assertStatus(result, 0, 'enable dry-run 应成功。'); + if (existsSync(targetPath)) { + failures.push('enable dry-run 不应写入目标 Nginx 配置。'); + } + assertIncludes(result.stdout, 'dry-run', 'enable dry-run 应输出 dry-run 提示。'); +} + +function assertApplyWritesRenderedConfigAndRunsLive() { + const fixture = createFixture('apply-ok'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + const result = runScript(ENABLE_SCRIPT, [ + '--apply', + '--probe-token', + 'apply-real-token', + '--host', + 'dev.genarrative.world', + '--base-url', + 'http://127.0.0.1:18083', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + '--no-status', + ]); + + assertStatus(result, 0, 'enable apply 应成功。'); + const rendered = readFileSync(targetPath, 'utf8'); + assertIncludes(rendered, '"apply-real-token"', 'enable apply 应替换 probe token。'); + if (rendered.includes('__GENARRATIVE_PINGORA_PROBE_TOKEN__')) { + failures.push('enable apply 后目标配置不应保留 probe token 占位符。'); + } + const calls = readFileSync(fixture.callsPath, 'utf8'); + assertIncludes(calls, 'nginx -t', 'enable apply 必须执行 nginx -t。'); + assertIncludes(calls, 'systemctl reload nginx.service', 'enable apply 必须 reload nginx。'); + assertIncludes( + calls, + 'live --realpath --base-url http://127.0.0.1:18083 --host dev.genarrative.world', + 'enable apply 必须执行 realpath live smoke。', + ); +} + +function assertEnableRollsBackWhenLiveFails() { + const fixture = createFixture('enable-live-fails', { liveExitCode: 7 }); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'previous-config\n', 'utf8'); + + const result = runScript( + ENABLE_SCRIPT, + [ + '--apply', + '--probe-token', + 'rollback-token', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + '--no-status', + ], + fixture.env, + ); + + if (result.status === 0) { + failures.push('realpath live smoke 失败时 enable apply 必须失败。'); + } + const restored = readFileSync(targetPath, 'utf8'); + if (restored !== 'previous-config\n') { + failures.push('realpath live smoke 失败时 enable apply 必须恢复旧配置。'); + } + assertIncludes( + result.stderr, + '恢复写入前配置', + 'enable live 失败时必须说明已恢复写入前配置。', + ); +} + +function assertDisableRemovesConfig() { + const fixture = createFixture('disable-ok'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'enabled-config\n', 'utf8'); + + const result = runScript( + DISABLE_SCRIPT, + [ + '--apply', + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--no-status', + ], + fixture.env, + ); + + assertStatus(result, 0, 'disable apply 应成功。'); + if (existsSync(targetPath)) { + failures.push('disable apply 成功后必须删除 realpath canary 配置。'); + } + const calls = readFileSync(fixture.callsPath, 'utf8'); + assertIncludes(calls, 'nginx -t', 'disable apply 必须执行 nginx -t。'); + assertIncludes(calls, 'systemctl reload nginx.service', 'disable apply 必须 reload nginx。'); +} + +function assertDisableRollsBackWhenReloadFails() { + const fixture = createFixture('disable-reload-fails', { reloadExitCode: 9 }); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + writeFileSync(targetPath, 'enabled-config\n', 'utf8'); + + const result = runScript(DISABLE_SCRIPT, [ + '--apply', + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--no-status', + ]); + + if (result.status === 0) { + failures.push('reload 失败时 disable apply 必须失败。'); + } + const restored = readFileSync(targetPath, 'utf8'); + if (restored !== 'enabled-config\n') { + failures.push('reload 失败时 disable apply 必须恢复删除前配置。'); + } + assertIncludes( + result.stderr, + '恢复删除前配置', + 'disable reload 失败时必须说明已恢复删除前配置。', + ); +} + +function assertRejectsUnsafeInputs() { + const fixture = createFixture('unsafe-inputs'); + const targetPath = path.join(fixture.nginxDir, 'zz-genarrative-pingora-realpath-canary.conf'); + const cases = [ + { + name: 'missing token', + script: ENABLE_SCRIPT, + args: [ + '--apply', + '--host', + 'dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ], + expected: '--apply 必须提供 --probe-token', + }, + { + name: 'url host', + script: ENABLE_SCRIPT, + args: [ + '--probe-token', + 'unsafe-token', + '--host', + 'https://dev.genarrative.world', + '--template-path', + fixture.templatePath, + '--target-path', + targetPath, + '--nginx-binary', + fixture.nginxBinary, + '--systemctl-binary', + fixture.systemctlBinary, + '--live-script', + fixture.liveScript, + ], + expected: '不能是 URL', + }, + { + name: 'relative target', + script: DISABLE_SCRIPT, + args: ['--target-path', 'relative.conf'], + expected: '--target-path 必须是绝对路径', + }, + { + name: 'root target', + script: DISABLE_SCRIPT, + args: ['--target-path', '/'], + expected: '--target-path 不能是文件系统根目录', + }, + ]; + + for (const item of cases) { + const result = runScript(item.script, item.args); + if (result.status === 0) { + failures.push(`${item.name}: 不安全参数必须失败。`); + } + assertIncludes( + `${result.stderr}\n${result.stdout}`, + item.expected, + `${item.name}: 应输出明确错误。`, + ); + } +} + +function createFixture(name, options = {}) { + const root = path.join(tmpRoot, name); + const nginxDir = path.join(root, 'nginx-conf'); + mkdirSync(nginxDir, { recursive: true }); + const callsPath = path.join(root, 'calls.log'); + const templatePath = path.join(root, 'genarrative-pingora-realpath-canary.conf'); + const nginxBinary = path.join(root, 'nginx'); + const systemctlBinary = path.join(root, 'systemctl'); + const liveScript = path.join(root, 'live.mjs'); + + writeFileSync( + templatePath, + [ + 'server {', + ' listen 127.0.0.1:18083;', + ' access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;', + ' location = /__genarrative_pingora_realpath_canary/healthz {', + ' proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__";', + ' proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz;', + ' }', + '}', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + nginxBinary, + [ + '#!/usr/bin/env bash', + `echo "nginx $*" >> ${shellQuote(callsPath)}`, + 'if [[ "${GENARRATIVE_FAKE_NGINX_T_EXIT:-0}" != "0" ]]; then exit "${GENARRATIVE_FAKE_NGINX_T_EXIT}"; fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + systemctlBinary, + [ + '#!/usr/bin/env bash', + `echo "systemctl $*" >> ${shellQuote(callsPath)}`, + `reload_exit="\${GENARRATIVE_FAKE_RELOAD_EXIT:-${Number(options.reloadExitCode || 0)}}"`, + 'if [[ "$1" == "reload" && "${reload_exit}" != "0" ]]; then exit "${reload_exit}"; fi', + 'exit 0', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + liveScript, + [ + '#!/usr/bin/env node', + "import { appendFileSync } from 'node:fs';", + `appendFileSync(${JSON.stringify(callsPath)}, 'live ' + process.argv.slice(2).join(' ') + '\\n');`, + `process.exit(Number(process.env.GENARRATIVE_FAKE_LIVE_EXIT || ${Number(options.liveExitCode || 0)}));`, + '', + ].join('\n'), + 'utf8', + ); + chmodExecutable(nginxBinary); + chmodExecutable(systemctlBinary); + chmodExecutable(liveScript); + + return { + root, + nginxDir, + callsPath, + templatePath, + nginxBinary, + systemctlBinary, + liveScript, + env: { + GENARRATIVE_FAKE_RELOAD_EXIT: String(options.reloadExitCode || 0), + GENARRATIVE_FAKE_LIVE_EXIT: String(options.liveExitCode || 0), + }, + }; +} + +function runScript(script, args, env = {}) { + return spawnSync('bash', [script, ...args], { + cwd: process.cwd(), + env: { ...process.env, ...env }, + encoding: 'utf8', + }); +} + +function chmodExecutable(file) { + chmodSync(file, 0o755); +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +function assertStatus(result, expected, message) { + if (result.status !== expected) { + failures.push( + `${message} 实际退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + } +} + +function assertIncludes(content, needle, message) { + if (!content.includes(needle)) { + failures.push(`${message} 缺少 ${needle}`); + } +} diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 8920d7035..2fb6ea2e5 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -61,6 +61,8 @@ function main() { assertMissingPingoraCutoverEvidenceAuditFails(); assertMissingHealthPatrolEnvCheckFails(); assertMissingPingoraReleaseReadinessFails(); + assertMissingPingoraRealpathCanaryEnableFails(); + assertMissingPingoraRealpathCanaryDisableFails(); assertMissingPingoraHealthPatrolEnvSwitchFails(); assertMissingPingoraTlsCertSyncFails(); assertMissingEnvExamplesFails(); @@ -99,6 +101,14 @@ function assertDeployCopiesPingoraDirectReleaseDependencies() { path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), 'current release 必须包含 Pingora 直连回退脚本。', ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'current release 必须包含 Pingora realpath canary 启用脚本。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'current release 必须包含 Pingora realpath canary 关闭脚本。', + ); assertFileExists( path.join( releaseDir, @@ -890,6 +900,52 @@ function assertMissingPingoraHealthPatrolEnvSwitchFails() { } } +function assertMissingPingoraRealpathCanaryEnableFails() { + const fixture = prepareFixture('missing-pingora-realpath-canary-enable'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-realpath-canary-enable.sh', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora realpath canary 启用脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora realpath canary 启用脚本', + '缺少 Pingora realpath canary 启用脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + +function assertMissingPingoraRealpathCanaryDisableFails() { + const fixture = prepareFixture('missing-pingora-realpath-canary-disable'); + rmSync( + path.join( + fixture.sourceDir, + 'scripts/deploy/pingora-realpath-canary-disable.sh', + ), + ); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('发布产物缺少 Pingora realpath canary 关闭脚本时部署必须失败。'); + } + assertIncludes( + result.stderr, + '发布产物缺少 Pingora realpath canary 关闭脚本', + '缺少 Pingora realpath canary 关闭脚本时必须给出明确错误。', + ); + if (!existsSync(fixture.maintenanceFile)) { + failures.push('部署失败时必须保持维护模式。'); + } +} + function assertMissingPingoraTlsCertSyncFails() { const fixture = prepareFixture('missing-pingora-tls-cert-sync'); rmSync( @@ -1044,6 +1100,14 @@ function prepareFixture(name) { 'scripts/deploy/pingora-direct-rollback.sh', path.join(sourceDir, 'scripts/deploy/pingora-direct-rollback.sh'), ); + copyFile( + 'scripts/deploy/pingora-realpath-canary-enable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + ); + copyFile( + 'scripts/deploy/pingora-realpath-canary-disable.sh', + path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + ); copyFile( 'scripts/deploy/pingora-health-patrol-env-switch.mjs', path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), diff --git a/scripts/check-production-api-release.mjs b/scripts/check-production-api-release.mjs index 151ccd3f3..a0a3dc73a 100644 --- a/scripts/check-production-api-release.mjs +++ b/scripts/check-production-api-release.mjs @@ -102,6 +102,22 @@ function assertApiReleaseContainsPingoraDirectDependencies() { path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), 'Pingora 直连回退脚本必须可执行。', ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'API release 必须包含 Pingora realpath canary 启用脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), + 'Pingora realpath canary 启用脚本必须可执行。', + ); + assertFileExists( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'API release 必须包含 Pingora realpath canary 关闭脚本。', + ); + assertExecutable( + path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), + 'Pingora realpath canary 关闭脚本必须可执行。', + ); assertFileExists( path.join( releaseDir, @@ -254,9 +270,19 @@ function assertApiReleaseContainsPingoraDirectDependencies() { ); assertIncludes( releaseReadme, - 'Pingora release readiness 聚合门禁、直连启用 / 回退 / health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', + 'Pingora release readiness 聚合门禁、直连启用 / 回退', 'API release README 必须说明随包携带 Pingora 直连脚本。', ); + assertIncludes( + releaseReadme, + 'realpath canary 启用 / 关闭', + 'API release README 必须说明随包携带 Pingora realpath canary 启停脚本。', + ); + assertIncludes( + releaseReadme, + 'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审', + 'API release README 必须说明随包携带 Pingora 运行态复核脚本。', + ); assertIncludes( releaseReadme, '状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本', diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 0206acc8b..c7dc68928 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -71,6 +71,16 @@ const checks = [ includes: 'scripts/check-pingora-canary-live.mjs', reason: 'Pingora current release 自审必须确认 canary live smoke 脚本已随包发布。', }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'Pingora current release 自审必须确认 realpath canary 启用脚本已随包发布。', + }, + { + file: 'scripts/ops/pingora-current-release-audit.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'Pingora current release 自审必须确认 realpath canary 关闭脚本已随包发布。', + }, { file: 'scripts/ops/pingora-current-release-audit.mjs', includes: '--systemd-show', @@ -1013,6 +1023,16 @@ const checks = [ includes: 'scripts/deploy/pingora-direct-rollback.sh', reason: '生产 API release 必须携带 Pingora 直连回退脚本。', }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: '生产 API release 必须携带 Pingora realpath canary 启用脚本。', + }, + { + file: 'scripts/build-production-release.sh', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: '生产 API release 必须携带 Pingora realpath canary 关闭脚本。', + }, { file: 'scripts/build-production-release.sh', includes: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', @@ -1069,6 +1089,26 @@ const checks = [ includes: 'PINGORA_CANARY_LIVE_SCRIPT_SOURCE', reason: 'API deploy 必须从发布产物复制 Pingora canary live smoke 脚本到 current release。', }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora realpath canary 启用脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE', + reason: 'API deploy 必须从发布产物复制 Pingora realpath canary 关闭脚本到 current release。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 Pingora realpath canary 启用脚本', + reason: 'API deploy 缺少 realpath canary 启用脚本时必须 fail-fast。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: '发布产物缺少 Pingora realpath canary 关闭脚本', + reason: 'API deploy 缺少 realpath canary 关闭脚本时必须 fail-fast。', + }, { file: 'scripts/deploy/production-api-deploy.sh', includes: 'PINGORA_CURRENT_RELEASE_AUDIT_SCRIPT_SOURCE', @@ -1119,6 +1159,16 @@ const checks = [ includes: 'scripts/check-production-health-patrol-env.mjs', reason: 'API release 动态烟测必须断言健康巡检 env 复核脚本随包发布。', }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API release 动态烟测必须断言 Pingora realpath canary 启用脚本随包发布。', + }, + { + file: 'scripts/check-production-api-release.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API release 动态烟测必须断言 Pingora realpath canary 关闭脚本随包发布。', + }, { file: 'scripts/check-production-api-deploy.mjs', includes: 'deploy/env/health-patrol.env.example', @@ -1149,6 +1199,30 @@ const checks = [ reason: 'API deploy 动态烟测必须断言 Pingora 切换命令证据脚本复制到 current release。', }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-enable.sh', + reason: + 'API deploy 动态烟测必须断言 Pingora realpath canary 启用脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'scripts/deploy/pingora-realpath-canary-disable.sh', + reason: + 'API deploy 动态烟测必须断言 Pingora realpath canary 关闭脚本复制到 current release。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraRealpathCanaryEnableFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora realpath canary 启用脚本时 fail-fast。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertMissingPingoraRealpathCanaryDisableFails', + reason: + 'API deploy 动态烟测必须覆盖缺少 Pingora realpath canary 关闭脚本时 fail-fast。', + }, { file: 'scripts/check-production-api-deploy.mjs', includes: 'assertMissingPingoraCutoverCommandEvidenceFails', @@ -1171,6 +1245,21 @@ const checks = [ includes: 'deploy:pingora-direct-enable', reason: 'Pingora 直连启用必须保留可重复执行的 npm 入口。', }, + { + file: 'package.json', + includes: 'check:pingora-realpath-canary-toggle', + reason: 'Pingora realpath canary 启停脚本必须有本地自测入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-realpath-canary-enable', + reason: 'Pingora realpath canary 启用必须保留可重复执行的 npm 入口。', + }, + { + file: 'package.json', + includes: 'deploy:pingora-realpath-canary-disable', + reason: 'Pingora realpath canary 关闭必须保留可重复执行的 npm 入口。', + }, { file: 'package.json', includes: 'deploy:pingora-direct-rollback', @@ -5545,6 +5634,18 @@ const checks = [ 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs', reason: 'API Build 归档必须包含 Pingora health patrol env 切换脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API Build 归档必须包含 Pingora realpath canary 启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API Build 归档必须包含 Pingora realpath canary 关闭脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-build', includes: 'build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**', @@ -5640,6 +5741,18 @@ const checks = [ 'build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs', reason: 'API Deploy 复制上游产物时必须包含 Pingora canary access log 对账脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora realpath canary 启用脚本。', + }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh', + reason: 'API Deploy 复制上游产物时必须包含 Pingora realpath canary 关闭脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-deploy', includes: diff --git a/scripts/deploy/pingora-realpath-canary-disable.sh b/scripts/deploy/pingora-realpath-canary-disable.sh new file mode 100644 index 000000000..379048b8d --- /dev/null +++ b/scripts/deploy/pingora-realpath-canary-disable.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -euo pipefail + +TARGET_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TARGET_PATH:-/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +SYSTEMCTL_BINARY="${GENARRATIVE_SYSTEMCTL_BINARY:-systemctl}" +APPLY="false" +STATUS_AFTER="true" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-realpath-canary-disable.sh [--apply] [--target-path ] [--nginx-binary ] [--nginx-service ] [--systemctl-binary ] [--no-status] + +说明: + 关闭 Nginx -> Pingora 真实路径 canary：删除 + /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf，执行 nginx -t， + 再 reload nginx。默认 dry-run，必须显式传 --apply 才会修改系统。 + 如果 nginx -t 或 reload 失败，脚本会恢复删除前的目标文件。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --target-path) + TARGET_PATH="${2:-}" + if [[ -z "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] --target-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-realpath-canary-disable] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-realpath-canary-disable] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --systemctl-binary) + SYSTEMCTL_BINARY="${2:-}" + if [[ -z "${SYSTEMCTL_BINARY}" ]]; then + echo "[pingora-realpath-canary-disable] --systemctl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + *) + echo "[pingora-realpath-canary-disable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-realpath-canary-disable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +require_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" != /* ]]; then + echo "[pingora-realpath-canary-disable] ${label} 必须是绝对路径。" >&2 + exit 1 + fi + if is_filesystem_root_path "${value}"; then + echo "[pingora-realpath-canary-disable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +validate_command_or_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" == */* ]]; then + require_absolute_path "${label}" "${value}" + fi +} + +print_command() { + printf '+' + for arg in "$@"; do + printf ' %q' "${arg}" + done + printf '\n' +} + +run_or_print() { + if [[ "${APPLY}" == "true" ]]; then + "$@" + else + print_command "$@" + fi +} + +restore_previous() { + local backup_path="$1" + cp -p -- "${backup_path}" "${TARGET_PATH}" +} + +rollback_after_failure() { + local backup_path="$1" + local reason="$2" + echo "[pingora-realpath-canary-disable] ${reason}，恢复删除前配置。" >&2 + restore_previous "${backup_path}" + "${NGINX_BINARY}" -t >/dev/null 2>&1 || true + "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" >/dev/null 2>&1 || true +} + +reject_control_characters "--target-path" "${TARGET_PATH}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--systemctl-binary" "${SYSTEMCTL_BINARY}" +require_absolute_path "--target-path" "${TARGET_PATH}" +validate_command_or_absolute_path "--nginx-binary" "${NGINX_BINARY}" +validate_command_or_absolute_path "--systemctl-binary" "${SYSTEMCTL_BINARY}" + +TARGET_DIR="$(dirname -- "${TARGET_PATH}")" +if [[ "${APPLY}" == "true" && ( -L "${TARGET_DIR}" || -L "${TARGET_PATH}" ) ]]; then + echo "[pingora-realpath-canary-disable] 目标目录或目标文件不能是符号链接。" >&2 + exit 1 +fi + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-realpath-canary-disable] dry-run：将删除 realpath canary ${TARGET_PATH}。" + print_command rm -f -- "${TARGET_PATH}" + run_or_print "${NGINX_BINARY}" -t + run_or_print "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" + exit 0 +fi + +if [[ ! -e "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] 目标文件不存在，视为已关闭: ${TARGET_PATH}" + if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" + fi + exit 0 +fi +if [[ ! -f "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-disable] 目标存在但不是普通文件: ${TARGET_PATH}" >&2 + exit 1 +fi + +backup_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.disable.backup.XXXXXX")" +trap 'rm -f -- "${backup_path}"' EXIT +cp -p -- "${TARGET_PATH}" "${backup_path}" +rm -f -- "${TARGET_PATH}" + +if ! "${NGINX_BINARY}" -t; then + rollback_after_failure "${backup_path}" "nginx -t 失败" + exit 1 +fi +if ! "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}"; then + rollback_after_failure "${backup_path}" "reload ${NGINX_SERVICE} 失败" + exit 1 +fi + +if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" +fi + +echo "[pingora-realpath-canary-disable] 已关闭 realpath canary: ${TARGET_PATH}" diff --git a/scripts/deploy/pingora-realpath-canary-enable.sh b/scripts/deploy/pingora-realpath-canary-enable.sh new file mode 100644 index 000000000..dd22528f4 --- /dev/null +++ b/scripts/deploy/pingora-realpath-canary-enable.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." >/dev/null 2>&1 && pwd)" + +TEMPLATE_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TEMPLATE_PATH:-${REPO_ROOT}/deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf}" +TARGET_PATH="${GENARRATIVE_PINGORA_REALPATH_CANARY_TARGET_PATH:-/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf}" +PROBE_TOKEN="${GENARRATIVE_PINGORA_REALPATH_CANARY_PROBE_TOKEN:-${GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN:-}}" +BASE_URL="${GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL:-http://127.0.0.1:18083}" +HOST_HEADER="${GENARRATIVE_PINGORA_REALPATH_CANARY_HOST:-}" +NGINX_BINARY="${GENARRATIVE_NGINX_BINARY:-nginx}" +NGINX_SERVICE="${GENARRATIVE_NGINX_SERVICE:-nginx.service}" +SYSTEMCTL_BINARY="${GENARRATIVE_SYSTEMCTL_BINARY:-systemctl}" +LIVE_SCRIPT="${GENARRATIVE_PINGORA_REALPATH_CANARY_LIVE_SCRIPT:-${REPO_ROOT}/scripts/check-pingora-canary-live.mjs}" +APPLY="false" +RUN_LIVE="true" +STATUS_AFTER="true" + +PLACEHOLDER="__GENARRATIVE_PINGORA_PROBE_TOKEN__" + +usage() { + cat <<'EOF' +用法: + scripts/deploy/pingora-realpath-canary-enable.sh [--apply] [--probe-token ] [--host ] [--base-url ] [--template-path ] [--target-path ] [--nginx-binary ] [--nginx-service ] [--systemctl-binary ] [--live-script ] [--skip-live] [--no-status] + +说明: + 启用 Nginx -> Pingora 真实路径 canary。脚本会把随包 + deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf 渲染到 + /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf。 + +默认是 dry-run，只打印计划；必须显式传 --apply 才会写 Nginx 配置。 +--apply 会替换 probe token、写入目标文件、执行 nginx -t、reload nginx， +然后运行 realpath canary live smoke。任一阶段失败都会恢复写入前的目标文件状态。 +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --apply) + APPLY="true" + shift + ;; + --probe-token) + PROBE_TOKEN="${2:-}" + if [[ -z "${PROBE_TOKEN}" ]]; then + echo "[pingora-realpath-canary-enable] --probe-token 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --host) + HOST_HEADER="${2:-}" + if [[ -z "${HOST_HEADER}" ]]; then + echo "[pingora-realpath-canary-enable] --host 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --base-url) + BASE_URL="${2:-}" + if [[ -z "${BASE_URL}" ]]; then + echo "[pingora-realpath-canary-enable] --base-url 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --template-path) + TEMPLATE_PATH="${2:-}" + if [[ -z "${TEMPLATE_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] --template-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --target-path) + TARGET_PATH="${2:-}" + if [[ -z "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] --target-path 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-binary) + NGINX_BINARY="${2:-}" + if [[ -z "${NGINX_BINARY}" ]]; then + echo "[pingora-realpath-canary-enable] --nginx-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --nginx-service) + NGINX_SERVICE="${2:-}" + if [[ -z "${NGINX_SERVICE}" ]]; then + echo "[pingora-realpath-canary-enable] --nginx-service 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --systemctl-binary) + SYSTEMCTL_BINARY="${2:-}" + if [[ -z "${SYSTEMCTL_BINARY}" ]]; then + echo "[pingora-realpath-canary-enable] --systemctl-binary 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --live-script) + LIVE_SCRIPT="${2:-}" + if [[ -z "${LIVE_SCRIPT}" ]]; then + echo "[pingora-realpath-canary-enable] --live-script 缺少参数" >&2 + exit 1 + fi + shift 2 + ;; + --skip-live) + RUN_LIVE="false" + shift + ;; + --no-status) + STATUS_AFTER="false" + shift + ;; + *) + echo "[pingora-realpath-canary-enable] 未知参数: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +reject_control_characters() { + local label="$1" + local value="$2" + if [[ "${value}" == *$'\n'* || "${value}" == *$'\r'* ]]; then + echo "[pingora-realpath-canary-enable] ${label} 不能包含换行或 NUL 字符。" >&2 + exit 1 + fi +} + +is_filesystem_root_path() { + local value="$1" + local without_slashes="${value//\//}" + [[ -n "${value}" && -z "${without_slashes}" ]] +} + +require_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" != /* ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须是绝对路径。" >&2 + exit 1 + fi + if is_filesystem_root_path "${value}"; then + echo "[pingora-realpath-canary-enable] ${label} 不能是文件系统根目录。" >&2 + exit 1 + fi +} + +validate_command_or_absolute_path() { + local label="$1" + local value="$2" + if [[ "${value}" == */* ]]; then + require_absolute_path "${label}" "${value}" + fi +} + +validate_host() { + local label="$1" + local value="$2" + if [[ -z "${value}" ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须提供。" >&2 + exit 1 + fi + if [[ "${value}" == http://* || "${value}" == https://* || "${value}" == */* || "${value}" == *\?* || "${value}" =~ [[:space:]] ]]; then + echo "[pingora-realpath-canary-enable] ${label} 必须是 Host 或 host:port，不能是 URL、路径、查询或包含空白。" >&2 + exit 1 + fi +} + +validate_base_url() { + if [[ "${BASE_URL}" != http://* && "${BASE_URL}" != https://* ]]; then + echo "[pingora-realpath-canary-enable] --base-url 必须以 http:// 或 https:// 开头。" >&2 + exit 1 + fi +} + +validate_probe_token() { + if [[ "${APPLY}" != "true" ]]; then + return + fi + if [[ -z "${PROBE_TOKEN}" ]]; then + echo "[pingora-realpath-canary-enable] --apply 必须提供 --probe-token 或 GENARRATIVE_PINGORA_REALPATH_CANARY_PROBE_TOKEN。" >&2 + exit 1 + fi + if [[ "${PROBE_TOKEN}" == "${PLACEHOLDER}" || "${#PROBE_TOKEN}" -lt 8 ]]; then + echo "[pingora-realpath-canary-enable] probe token 不能是占位值或过短。" >&2 + exit 1 + fi +} + +escape_nginx_double_quoted() { + local value="$1" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s' "${value}" +} + +render_template_to() { + local output_path="$1" + local content escaped_token + content="$(<"${TEMPLATE_PATH}")" + escaped_token="$(escape_nginx_double_quoted "${PROBE_TOKEN}")" + printf '%s' "${content//${PLACEHOLDER}/${escaped_token}}" >"${output_path}" +} + +print_command() { + printf '+' + for arg in "$@"; do + printf ' %q' "${arg}" + done + printf '\n' +} + +run_or_print() { + if [[ "${APPLY}" == "true" ]]; then + "$@" + else + print_command "$@" + fi +} + +restore_previous() { + local backup_path="$1" + local had_target="$2" + if [[ "${had_target}" == "true" ]]; then + cp -p -- "${backup_path}" "${TARGET_PATH}" + else + rm -f -- "${TARGET_PATH}" + fi +} + +rollback_after_failure() { + local backup_path="$1" + local had_target="$2" + local reason="$3" + echo "[pingora-realpath-canary-enable] ${reason}，恢复写入前配置。" >&2 + restore_previous "${backup_path}" "${had_target}" + "${NGINX_BINARY}" -t >/dev/null 2>&1 || true + "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" >/dev/null 2>&1 || true +} + +reject_control_characters "--template-path" "${TEMPLATE_PATH}" +reject_control_characters "--target-path" "${TARGET_PATH}" +reject_control_characters "--probe-token" "${PROBE_TOKEN}" +reject_control_characters "--base-url" "${BASE_URL}" +reject_control_characters "--host" "${HOST_HEADER}" +reject_control_characters "--nginx-binary" "${NGINX_BINARY}" +reject_control_characters "--nginx-service" "${NGINX_SERVICE}" +reject_control_characters "--systemctl-binary" "${SYSTEMCTL_BINARY}" +reject_control_characters "--live-script" "${LIVE_SCRIPT}" + +require_absolute_path "--template-path" "${TEMPLATE_PATH}" +require_absolute_path "--target-path" "${TARGET_PATH}" +if [[ "${RUN_LIVE}" == "true" ]]; then + require_absolute_path "--live-script" "${LIVE_SCRIPT}" +fi +validate_command_or_absolute_path "--nginx-binary" "${NGINX_BINARY}" +validate_command_or_absolute_path "--systemctl-binary" "${SYSTEMCTL_BINARY}" +validate_host "--host" "${HOST_HEADER}" +validate_base_url +validate_probe_token + +if [[ ! -f "${TEMPLATE_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 未找到 realpath canary 模板: ${TEMPLATE_PATH}" >&2 + exit 1 +fi +if [[ "${RUN_LIVE}" == "true" && ! -f "${LIVE_SCRIPT}" ]]; then + echo "[pingora-realpath-canary-enable] 未找到 realpath canary live smoke 脚本: ${LIVE_SCRIPT}" >&2 + exit 1 +fi + +TARGET_DIR="$(dirname -- "${TARGET_PATH}")" +if [[ "${APPLY}" == "true" ]]; then + if [[ ! -d "${TARGET_DIR}" ]]; then + echo "[pingora-realpath-canary-enable] 目标目录不存在: ${TARGET_DIR}" >&2 + exit 1 + fi + if [[ -L "${TARGET_DIR}" || -L "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 目标目录或目标文件不能是符号链接。" >&2 + exit 1 + fi +fi + +if [[ "${APPLY}" != "true" ]]; then + echo "[pingora-realpath-canary-enable] dry-run：将渲染 realpath canary 到 ${TARGET_PATH}，probe token 输出已隐藏。" + print_command install -m 0644 "" "${TARGET_PATH}" + run_or_print "${NGINX_BINARY}" -t + run_or_print "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}" + if [[ "${RUN_LIVE}" == "true" ]]; then + run_or_print node -- "${LIVE_SCRIPT}" --realpath --base-url "${BASE_URL}" --host "${HOST_HEADER}" + fi + exit 0 +fi + +tmp_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.XXXXXX")" +backup_path="$(mktemp "${TARGET_DIR}/.genarrative-pingora-realpath-canary.backup.XXXXXX")" +had_target="false" +cleanup_paths=("${tmp_path}" "${backup_path}") +trap 'rm -f -- "${cleanup_paths[@]}"' EXIT + +render_template_to "${tmp_path}" +chmod 0644 "${tmp_path}" + +if [[ -e "${TARGET_PATH}" ]]; then + if [[ ! -f "${TARGET_PATH}" ]]; then + echo "[pingora-realpath-canary-enable] 目标已存在但不是普通文件: ${TARGET_PATH}" >&2 + exit 1 + fi + cp -p -- "${TARGET_PATH}" "${backup_path}" + had_target="true" +fi + +install -m 0644 "${tmp_path}" "${TARGET_PATH}" +if ! "${NGINX_BINARY}" -t; then + rollback_after_failure "${backup_path}" "${had_target}" "nginx -t 失败" + exit 1 +fi +if ! "${SYSTEMCTL_BINARY}" reload "${NGINX_SERVICE}"; then + rollback_after_failure "${backup_path}" "${had_target}" "reload ${NGINX_SERVICE} 失败" + exit 1 +fi + +if [[ "${RUN_LIVE}" == "true" ]]; then + if ! node -- "${LIVE_SCRIPT}" --realpath --base-url "${BASE_URL}" --host "${HOST_HEADER}"; then + rollback_after_failure "${backup_path}" "${had_target}" "realpath canary live smoke 失败" + exit 1 + fi +fi + +if [[ "${STATUS_AFTER}" == "true" ]]; then + "${SYSTEMCTL_BINARY}" is-active "${NGINX_SERVICE}" +fi + +echo "[pingora-realpath-canary-enable] 已启用 realpath canary: ${TARGET_PATH}" diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 142d32e45..e3ee985f2 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -619,6 +619,8 @@ HEALTH_PATROL_ENV_CHECK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-production-he PINGORA_RELEASE_READINESS_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-release-readiness.mjs" PINGORA_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-enable.sh" PINGORA_ROLLBACK_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-direct-rollback.sh" +PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" PINGORA_TLS_CERT_SYNC_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" PINGORA_PREFLIGHT_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/check-pingora-direct-preflight.mjs" @@ -702,6 +704,18 @@ if [[ ! -f "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" ]]; then fi cp "${PINGORA_ROLLBACK_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-direct-rollback.sh" +if [[ ! -f "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 启用脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" >&2 + exit 1 +fi +cp "${PINGORA_REALPATH_CANARY_ENABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" +if [[ ! -f "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" ]]; then + echo "[production-api-deploy] 发布产物缺少 Pingora realpath canary 关闭脚本: ${SOURCE_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" >&2 + exit 1 +fi +cp "${PINGORA_REALPATH_CANARY_DISABLE_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" +chmod 0755 "${RELEASE_CONTENT_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" if [[ ! -f "${PINGORA_HEALTH_PATROL_ENV_SWITCH_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora health patrol env 切换脚本: ${SOURCE_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" >&2 exit 1 diff --git a/scripts/ops/pingora-current-release-audit.mjs b/scripts/ops/pingora-current-release-audit.mjs index e73da3f64..0f84584a3 100644 --- a/scripts/ops/pingora-current-release-audit.mjs +++ b/scripts/ops/pingora-current-release-audit.mjs @@ -21,6 +21,8 @@ const REQUIRED_ARTIFACTS = [ { path: 'scripts/check-pingora-canary-access-log-parity.mjs' }, { path: 'scripts/deploy/pingora-direct-enable.sh', executable: true }, { path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true }, + { path: 'scripts/deploy/pingora-realpath-canary-enable.sh', executable: true }, + { path: 'scripts/deploy/pingora-realpath-canary-disable.sh', executable: true }, { path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', executable: true }, { path: 'scripts/deploy/pingora-tls-cert-sync.mjs', executable: true }, { path: 'deploy/systemd/genarrative-pingora-gateway.service' }, diff --git a/scripts/ops/pingora-cutover-status-snapshot.mjs b/scripts/ops/pingora-cutover-status-snapshot.mjs index c37ee8d69..34e001217 100644 --- a/scripts/ops/pingora-cutover-status-snapshot.mjs +++ b/scripts/ops/pingora-cutover-status-snapshot.mjs @@ -34,6 +34,8 @@ const RELEASE_ARTIFACTS = [ 'scripts/check-pingora-canary-access-log-parity.mjs', 'scripts/deploy/pingora-direct-enable.sh', 'scripts/deploy/pingora-direct-rollback.sh', + 'scripts/deploy/pingora-realpath-canary-enable.sh', + 'scripts/deploy/pingora-realpath-canary-disable.sh', 'scripts/deploy/pingora-health-patrol-env-switch.mjs', 'deploy/systemd/genarrative-pingora-gateway.service', 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', From 4bca480f3c4c795e8264b87aa3fc818f6eb81b46 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 00:31:42 +0800 Subject: [PATCH 15/22] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20Pingora=20=E7=9B=B4?= =?UTF-8?q?=E8=BF=9E=E5=BD=A9=E6=8E=92=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 Pingora 直连彩排状态脚本，核验 health patrol、端口归属、systemd、realpath canary 和 current release 自审 将彩排状态脚本纳入 API release 构建、部署、Jenkins 归档和生产运维护栏 补充 release readiness、current release audit、cutover snapshot 与发布部署检查对彩排脚本的覆盖 更新 Pingora 运维文档和共享记忆，记录 dev health patrol 真实配置漂移与修复方式 在 dev 服务器真实部署验证 Nginx 仍接 80/443、Pingora shadow 与 realpath canary 高端口正常、health patrol 和彩排状态均为 OK --- deploy/nginx/README.md | 2 +- .../shared-memory/decision-log.md | 3 +- docs/project-memory/shared-memory/pitfalls.md | 11 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 21 +- jenkins/Jenkinsfile.production-api-build | 2 +- jenkins/Jenkinsfile.production-api-deploy | 2 +- package.json | 1 + scripts/build-production-release.sh | 3 +- .../check-pingora-current-release-audit.mjs | 6 + .../check-pingora-cutover-status-snapshot.mjs | 12 + .../check-pingora-direct-rehearsal-status.mjs | 566 ++++++++++ .../check-pingora-release-readiness-plan.mjs | 17 + scripts/check-pingora-release-readiness.mjs | 5 + scripts/check-production-api-deploy.mjs | 32 + scripts/check-production-api-release.mjs | 9 +- scripts/check-production-ops-guardrails.mjs | 45 + scripts/deploy/production-api-deploy.sh | 7 + scripts/ops/pingora-current-release-audit.mjs | 1 + .../ops/pingora-cutover-status-snapshot.mjs | 1 + .../ops/pingora-direct-rehearsal-status.mjs | 966 ++++++++++++++++++ 20 files changed, 1702 insertions(+), 10 deletions(-) create mode 100644 scripts/check-pingora-direct-rehearsal-status.mjs create mode 100644 scripts/ops/pingora-direct-rehearsal-status.mjs diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 4ab54ceb0..2e400add7 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -46,7 +46,7 @@ - Pingora 网关默认仍只监听本机 shadow 端口，不绑定公网 `80/443`。 - 需要评估直连入口时，显式配置 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` 和 `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`；网关只读取现有证书文件，不负责 Certbot / ACME 自动续期。 - 需要 HTTP 到 HTTPS 跳转时，再显式配置 `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`；该入口除 `/.well-known/acme-challenge/*` 外统一 301 到 HTTPS，ACME challenge 仍从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取。 -- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连启用、health patrol env 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连彩排状态、直连启用、health patrol env 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 - 目标机如果使用 Certbot / Let’s Encrypt 证书，不要直接放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限。先用随包脚本把 root 可读的 live symlink 解析并复制到 Pingora 私有目录，例如 `sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>`；脚本默认 dry-run，`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并复核 `genarrative` 服务用户可读。随后把 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/` 下的副本。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index c62b18dbd..0a6cf7756 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -2331,8 +2331,9 @@ - 决策补充：生产健康巡检的公网入口模式必须显式区分 `nginx` 和 `pingora-direct`。默认 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 检查 API、SpacetimeDB 和 Nginx；Pingora 直连接管公网后切到 `pingora-direct`，改为检查 API、SpacetimeDB 和 `genarrative-pingora-gateway.service`，不再要求 `nginx.service` active。目标机本机探测 `127.0.0.1` 时用 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>` 保留正式 Host / vhost 语义。 - 决策补充：Pingora 影子网关产物不进入默认 API release；只有显式传 `--include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，才构建并打包 `pingora-gateway` / `pingora-gateway.sha256`。真实构建 Pingora 前必须先检查 `cmake`、C 编译器和 C++ 编译器；Jenkins 勾选 `INCLUDE_PINGORA_GATEWAY` 时也要先 fail-fast 检查这些工具，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。`production-api-deploy.sh` 仅在两者同时存在时校验并复制到 current release，避免现有 API 流水线被 Pingora 构建依赖影响。若发布包包含 Pingora 且 `genarrative-pingora-gateway.service` 已经处于 `active`，API deploy 在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制；若服务未运行则只跳过，不主动拉起 Pingora。`npm run check:production-api-release` 必须同时验证默认 API release 不登记 Pingora，以及显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest。 - 决策补充：生产健康巡检的显式 `--timeout-ms`、`--slow-ms`、`GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS` 和 `GENARRATIVE_HEALTH_PATROL_SLOW_MS` 必须是正整数，非法值直接失败，不静默回退默认 `5000ms` / `3000ms`。Pingora canary live 的 `--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS`、direct live 的 `--timeout-ms` / `GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS`、canary access log 对账的 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也必须正整数。Pingora direct live 和 release readiness 读取的直连布尔 env 必须严格解析，只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败，避免 `REQUIRE_WSS_UPGRADE`、preflight 开关或 `SKIP_WSS` 因拼写错误被当成 false。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；脚本必须在发起 canary 请求前失败，避免污染参数进入 URL、Host header 或 JSON 输出。canary access log 对账的日志路径、prefix、必需路径、tail 行数以及日志行中解析出的 URI / path 也不能包含换行或 NUL；脚本必须失败并给出对应参数或日志行诊断，不能把污染值写入 JSON 对账输出。direct live 的 HTTPS / HTTP base URL、Host、redirect Host、probe token、额外 path、SpacetimeDB 数据库名、access log 路径、timeout 和布尔 env 都不能包含换行或 NUL；脚本必须在发起 HTTPS / HTTP / WSS 请求前失败，避免污染参数进入请求头、URL、日志对账或 JSON 证据。Pingora 切换窗口调整巡检、live smoke、日志对账阈值或直连布尔开关时，把参数解析失败视为配置错误，而不是继续执行检查。 -- 决策补充：即使不打包 Pingora 二进制，API release 也必须随包携带 Pingora release readiness 聚合门禁、直连启用 / 回退 / preflight / live smoke / current release 自审脚本，以及 `deploy/systemd/`、`deploy/pingora/` 支撑配置；`pingora-direct-enable.sh` 和正式 cutover runbook 默认从 `/opt/genarrative/current` 推导这些路径，启用前 release readiness 基础门禁和启用后 `--require-direct` 复核也必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs`，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。API release 还必须携带 `build//scripts/deploy/production-api-deploy.sh` 和同目录 `maintenance-on.sh` / `maintenance-off.sh`；`Genarrative-Api-Deploy` 只能复制并执行 build 产物内的 deploy 脚本，禁止继续执行部署工作区根部脚本，避免 workspace 中的旧脚本掩盖发布包布局缺陷。`production-api-deploy.sh` 对数据库备份脚本、健康巡检脚本和 Pingora 直连依赖都执行 fail-fast，发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；API deploy 必须要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，且 `--version` 必须以数字或字母开头并拒绝点目录，再先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部复制完成后再用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，并用固定替换语义切换 current 符号链接，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，失败时清理 staging 且不留下正式 release；`npm run check:production-api-release` 与 `npm run check:production-api-deploy` 必须进入 `check:pingora-release-readiness` 聚合门禁，前者用临时 `CARGO_TARGET_DIR` 和假 `api-server` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 产物自包含，后者用临时 release 和 fake `systemctl` / `curl` 验证从发布产物内执行 deploy 脚本后 current release 自包含，并覆盖缺少备份脚本、健康巡检脚本、release readiness 聚合门禁脚本、current release 自审脚本、direct live smoke 脚本、相对 release root / current link / api env file、点目录或点开头 version、既有 release 目录、目录型 current 或提升前 release 目录竞态时的失败维护模式。 +- 决策补充：即使不打包 Pingora 二进制，API release 也必须随包携带 Pingora release readiness 聚合门禁、直连启用 / 回退 / preflight / live smoke / current release 自审脚本、直连彩排状态脚本，以及 `deploy/systemd/`、`deploy/pingora/` 支撑配置；`pingora-direct-enable.sh` 和正式 cutover runbook 默认从 `/opt/genarrative/current` 推导这些路径，启用前 release readiness 基础门禁和启用后 `--require-direct` 复核也必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs`，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。API release 还必须携带 `build//scripts/deploy/production-api-deploy.sh` 和同目录 `maintenance-on.sh` / `maintenance-off.sh`；`Genarrative-Api-Deploy` 只能复制并执行 build 产物内的 deploy 脚本，禁止继续执行部署工作区根部脚本，避免 workspace 中的旧脚本掩盖发布包布局缺陷。`production-api-deploy.sh` 对数据库备份脚本、健康巡检脚本和 Pingora 直连依赖都执行 fail-fast，发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；API deploy 必须要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，且 `--version` 必须以数字或字母开头并拒绝点目录，再先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部复制完成后再用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，并用固定替换语义切换 current 符号链接，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，失败时清理 staging 且不留下正式 release；`npm run check:production-api-release` 与 `npm run check:production-api-deploy` 必须进入 `check:pingora-release-readiness` 聚合门禁，前者用临时 `CARGO_TARGET_DIR` 和假 `api-server` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 产物自包含，后者用临时 release 和 fake `systemctl` / `curl` 验证从发布产物内执行 deploy 脚本后 current release 自包含，并覆盖缺少备份脚本、健康巡检脚本、release readiness 聚合门禁脚本、current release 自审脚本、直连彩排状态脚本、direct live smoke 脚本、相对 release root / current link / api env file、点目录或点开头 version、既有 release 目录、目录型 current 或提升前 release 目录竞态时的失败维护模式。 - 决策补充：正式直连 runbook 在采集状态快照前必须先执行 current release 自审：`/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`。该脚本只读检查发布包自包含、`pingora-gateway` 可执行，以及 systemd `ExecStart` 是否指向 current release 网关二进制；失败时应先修发布包、Jenkins 归档过滤、deploy 复制或 systemd 指向，再继续切换。 +- 决策补充：直连前的 dev / release 彩排状态使用 `/opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs --release-root /opt/genarrative/current --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`。该脚本只读读取 health patrol env、Pingora env、`systemctl`、`ss -H -ltnp`、realpath canary 配置和 current release 自审结果；`nginx` 期望模式要求公网 `80/443` 仍由 Nginx 监听，Pingora 只在 `127.0.0.1:18081` shadow，realpath canary 在 `127.0.0.1:18083`，不会写 `/etc`、reload systemd 或修改 Nginx / Pingora。 - 决策补充：Pingora 切换证据链正式纳入 `npm run check:pingora-current-release-audit`、`npm run check:pingora-cutover-status-snapshot`、`npm run check:pingora-cutover-evidence-bundle`、`npm run check:pingora-cutover-command-evidence`、`npm run check:pingora-cutover-evidence-verify`、`/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs`、`/opt/genarrative/current/scripts/ops/pingora-cutover-status-snapshot.mjs`、`/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs` 与 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs`。状态快照按 `pre-cutover`、`post-enable`、`post-rollback` 三个阶段输出只读 JSON evidence，收录 `summary`、`healthPatrolEnv`、`pingoraEnv`、`releaseArtifacts`、`systemd` 和 `checks`；直连 runbook 的三个证据包阶段都显式透传绝对路径 `--output-root` 和 `--require-pingora-gateway`，让 `checks.current-release-audit.details` 同步归档 Pingora 二进制、sha256、release manifest 和 systemd `ExecStart` 自审结果；证据包脚本把快照 JSON、stdout、stderr、命令记录和 manifest 写入 `--output-root` 下的新证据目录，manifest 对已生成 snapshot、direct live、stdout / stderr、命令记录和 parse-error 文件记录 `path`、`sizeBytes` 与 `sha256`；每个阶段证据目录生成、复制或归档后，都必须用随包 verifier 按 `manifest.files` 只读复核文件存在、大小和 sha256，路径逃逸、符号链接证据目录、非目录证据路径、缺文件、大小漂移或 sha256 漂移都应失败。命令证据脚本把 direct enable apply / rollback apply 的真实 stdout、stderr、退出码、脱敏命令记录和 manifest 写入同一证据根目录，命令记录同时保留脱敏后的可读命令和结构化 `executable` / `args[]`，manifest 对 `command.stdout.txt`、`command.stderr.txt` 和 `command-record.json` 同样记录 `path`、`sizeBytes` 与 `sha256`，便于切换窗口后复核归档文件未漂移；runbook 必须在命令证据生成后立即用随包 verifier 验真 `` / ``，再继续 health patrol 切换、回退后 env 复核或最终总审计，且 `--phase` 只允许 ASCII 字母、数字、点、下划线和短横线，非法阶段名直接失败，不做隐式清洗。自审和快照只读采集，证据包只写归档目录且不覆盖既有文件，证据验真脚本只读 manifest 和证据文件，命令证据脚本只执行 `--` 后面的真实命令并归档输出，证据目录权限固定为 `0750`，证据文件权限固定为 `0640`，这些脚本都不写 `/etc`、不 reload systemd，也不修改 Nginx 或 Pingora；probe token 和其他 env 敏感值只允许以是否存在或 `` 的形式进入证据链，聚合门禁真实执行日志、dry-run plan、cutover runbook、直连启用脚本 direct live 命令日志、直连回退脚本 shadow probe 命令日志、gateway smoke 命令日志和证据包命令记录都不得输出 token 原文，状态快照在收录健康巡检、current release 自审等子检查 stdout / stderr 前必须按 env 敏感值脱敏，状态快照、证据包、命令证据和证据验真自测必须确认 token 原文不会进入 stdout、snapshot、manifest、命令记录或子检查输出；正式 runbook 应在 `--fail-on-critical` 下把自审、快照、证据包和 manifest 验真当成阻断证据，避免把发布包 checksum / manifest 漂移、env 漂移、systemd capability 残留、巡检失败或归档文件损坏带入后续阶段。 - 决策补充：正式直连 runbook 的即时证据 verifier 和最终证据根目录总审计内部复用 verifier 时，都必须使用 `--require-summary-ok`；总审计不能退化成只验 `manifest.files` hash，还要把 `manifest.summary.status=OK` 作为底层 verifier 严格模式的一部分。 - 决策补充：Pingora 切换命令证据生成端必须在执行前约束真实命令身份：`-- ` 本身必须是绝对路径，不能是文件系统根目录；真实命令和每个真实命令参数都不能包含换行或 NUL 字符。正式 runbook 因此直接执行 current release 随包 enable / rollback 脚本绝对路径，禁止用 `node`、`bash`、脚本名或其它 PATH 裸命令名包装。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 8fd688ad1..01b2673b0 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -18,13 +18,14 @@ ## Pingora 直连 80/443 不能只改 env - 现象：`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`，但 `genarrative-pingora-gateway.service` 启动失败，日志出现低端口绑定权限错误。 -- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 +- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`pingora-direct-rehearsal-status.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 - 处理：确认真实 TLS 证书和 redirect env 已写入 `/etc/genarrative/pingora-gateway.env`、service 模板和 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含这份 env、当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都能读取证书链 / 私钥、current release 的 `pingora-gateway` 已存在且可执行、Nginx 或其它进程已释放 `80/443` 后，先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成只读 JSON runbook，并逐条审阅 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础 readiness、direct enable dry-run、direct enable apply、启用后 `--require-direct` 复核、rollback dry-run、rollback apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；runbook 只用于审阅，不修改系统。正式 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 或回退 smoke 各自验证到不同入口；同时必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检需要 Host 覆盖，再追加 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 把现场巡检入口覆盖成仓库默认值；如需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文。随后先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，再 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`，最后执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，由脚本先跑 direct preflight，再安装 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`、用 `systemctl is-active` 确认服务 active，再以 JSON 模式执行 direct live smoke，验证 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘，并要求 `direct-access-log` 结构化结果 `matchedCount == checked`、`missingCount=0`、`mismatchCount=0`；如果 direct live 退出 0 但缺少该结构化证据，也必须视为启用失败。直连启用后同步调整 `/etc/genarrative/health-patrol.env`：设置 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，本机打 `127.0.0.1` 时设置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`，否则巡检会继续按 Nginx 模式误报。验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 或 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>`；回退脚本先跑 `nginx -t`，通过后才移除 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 已移除、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`，随后 reload Nginx、确认 Nginx service 仍为 active，并用 curl smoke URL 证明公网入口已回到 Nginx；回退脚本 `--apply` 不允许省略 `--reload-nginx` 或 `--nginx-smoke-url`，当 smoke URL 指向本机地址时必须同时提供 `--nginx-smoke-host <域名>`，且 host 值不能包含 URL、路径或查询；回退后把 health patrol gateway mode 改回 `nginx`，恢复切换前 public base URL / Host，并用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url <切换前Nginx巡检入口> --require-empty-public-host` 复核；若切换前 Nginx 巡检需要 Host 覆盖，则把 `--require-empty-public-host` 换成 `--expected-public-host <切换前Host>`。若 env 已在回退命令前切回 Nginx，也可给 rollback 脚本追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让它在 Nginx smoke 后自动复核；切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若要同时证明 Pingora shadow 高端口仍活着，追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应为 `gateway=pingora-shadow`。 - 处理补充：不要直接 chmod `/etc/letsencrypt/live` 或 `archive` 来让 Pingora 读取证书；Certbot live 路径通常是 symlink，即使 `stat -L` 看起来是普通文件，父目录权限也会让非 root `genarrative` 用户不可达。先用随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>` 把证书同步到 Pingora 私有目录，再让 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/fullchain.pem` 和 `privkey.pem`。脚本默认 dry-run，`--apply` 才写入，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并拒绝符号链接目标目录或目标文件。 - 处理补充：不要在切换窗口手工编辑 `/etc/genarrative/health-patrol.env` 的三项网关变量；使用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切到直连，回退前用同一脚本传 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>` 并按切换前记录选择 `--clear-public-host` 或 `--public-host <切换前Host>`。脚本只改 gateway mode / public base URL / public Host，并立即复用随包 env 复核脚本，减少空 Host 和旧值残留；生产巡检、env 复核和 env 切换脚本读取的布尔 env 都必须是明确布尔值，非法值直接失败，不能把拼写错误当成 false；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，直接用 `node script.mjs --env-file ...` 或 shebang 执行 `.mjs --env-file ...` 都可能让 Node 抢走业务参数；所有这类命令都必须写成 `node -- script.mjs --env-file ...`，或通过已内置 `node --` 的 npm script 执行。 - 踩坑补充：health patrol env 切换脚本必须先复核权限固定为 `0600` 的临时目标 env 再写真实文件，真实 env 原子替换时保持原文件权限和 owner/group；如果随包 env 复核脚本失败，`--apply` 应失败且真实 env 保持原样，避免“切换脚本失败但巡检配置已半改”的状态。`--apply` 的 `--env-file` 必须直接指向真实普通文件，不能传符号链接；如果 `/etc/genarrative/health-patrol.env` 是链接，先确认真实目标路径后再传给脚本，避免替换链接本身或写入非预期目标。 - 踩坑补充：直连启用脚本的 `--preflight-script`、`--direct-live-script`、`--current-release-audit-script`、`--template-path`、`--service-unit-path`、`--dropin-path` 和 env 文件参数都必须使用绝对路径；不要在切换窗口传相对脚本路径，否则会把 current release、Jenkins 工作区或现场 cwd 混在一起。`--apply` 会在安装 direct-entry drop-in 前确认 current release 自审、direct preflight 和 direct live smoke 脚本存在，缺脚本时应先修发布包或复制链路，不要手工改成工作区相对路径绕过。启用脚本还会在任何自审、preflight、drop-in 写入或 systemctl 前拒绝 service、路径、URL、Host、probe token、access log、数据库名、tail 行数和 timeout 参数中的换行或 NUL 字符；遇到这类失败先修 runbook 参数来源或现场 env，不要手工绕过脚本。启用脚本还会拒绝符号链接形式的 drop-in 目录或 drop-in 目标文件，以及已存在但不是普通文件的目标；如果现场 systemd 目录被软链改写，应先修正真实路径，不要让脚本把低端口 capability 写入非预期位置。回退脚本 `--apply` 同样会在 `nginx -t` 和删除 drop-in 前拒绝 service、路径、Nginx smoke URL / Host / 响应片段、health patrol 复核参数、shadow probe URL / token 和二进制 override 中的换行或 NUL 字符，并拒绝符号链接 drop-in 目录 / 目标以及非普通 drop-in 目标；如果现场路径或参数异常，应先修正 systemd 路径、runbook 参数或现场 env，不要手工删 drop-in、绕过 `nginx -t` 或把删除 symlink 当成已回退真实低端口能力。回退脚本覆盖 `--nginx-binary` 或 `--curl-binary` 时也不要传 `./nginx`、`tools/curl` 这类相对路径；裸命令名可以走 `PATH`，路径形式必须使用绝对路径。`--nginx-smoke-url` 必须带 `http://` 或 `https://`，不要只写 host/path，否则脚本会在移除 drop-in 前失败。 - 踩坑补充：回退到 Nginx 后不要只把 `curl --fail` / HTTP 200 当作 Nginx 已接回的证据；正式 runbook 默认给 rollback dry-run / apply 传 `--nginx-smoke-expect-body '"ok":true'`，要求 `/healthz` 响应体包含预期片段。若现场把 `--rollback-nginx-smoke-url` 改成其它端点，必须同步调整 `--rollback-nginx-smoke-expect-body`，否则要么误卡回退，要么只验证到了错误入口。 +- 踩坑补充：直连彩排状态脚本不是修复动作。`node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs --release-root /opt/genarrative/current --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical` 只读检查公网端口归属、health patrol 模式、Pingora shadow、realpath canary、systemd 和 current release 自审；如果它报 `CRITICAL`，应先修发布包、端口归属、canary 配置、health patrol env 或 systemd 指向，不要把它当成会自动启用 canary、停止 Nginx 或修复 current release 的脚本。 - 踩坑补充：Pingora current release 自审和切换证据链都不是修复动作，`npm run check:pingora-current-release-audit` / `scripts/ops/pingora-current-release-audit.mjs` 只负责只读确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、release manifest 登记了当前要接流的 Pingora 产物、`pingora-gateway` 可执行和 systemd `ExecStart` 指向；`npm run check:pingora-cutover-status-snapshot` / `scripts/ops/pingora-cutover-status-snapshot.mjs` 只负责输出 `pre-cutover`、`post-enable`、`post-rollback` 三阶段只读 JSON evidence，并在直连 runbook 中通过 `--require-pingora-gateway` 把上述自审结果收录到 `checks.current-release-audit.details`；快照还必须确认 `systemctl cat genarrative-pingora-gateway.service` 的 `EnvironmentFile=` 精确包含本次 `--pingora-env-file`，否则 `systemd.pingoraUnit.environmentFileMatchesPingoraEnvFile=false` 且标记 `CRITICAL`，避免证据包读到一份 env、真实服务读另一份 env。正式切换窗口用 `scripts/ops/pingora-cutover-evidence-bundle.mjs` 把快照 JSON、stdout、stderr、命令记录和 manifest 写入 `--output-root` 下的新证据目录，证据包 manifest 必须记录已生成 snapshot、direct live、stdout / stderr、命令记录和 parse-error 文件的 `path`、`sizeBytes` 与 `sha256`，便于归档后复核；证据目录生成、复制或归档后必须用随包 `scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir ` 做只读验真，确认 `manifest.files` 登记的文件未缺失、大小未漂移、sha256 未漂移，且证据目录不是符号链接或非目录；三阶段证据分别验真后，还必须用随包 `scripts/ops/pingora-cutover-evidence-audit.mjs --evidence-root <证据根目录> --require-phase pre-cutover --require-phase post-enable --require-phase post-rollback` 做只读总审计，自动选择每个阶段最新 bundle 并复用 verifier，缺阶段、最新证据损坏、坏 manifest 或符号链接条目都应失败。direct enable apply / rollback apply 必须通过 `scripts/ops/pingora-cutover-command-evidence.mjs` 包装真实脚本，单独保存命令 stdout、stderr、退出码、脱敏命令记录和 manifest，runbook 必须显式传绝对路径 `--output-root`，且该路径不能是文件系统根目录、符号链接或包含换行 / NUL 字符；命令记录必须同时保留脱敏后的可读命令和结构化 `executable` / `args[]`，命令证据 manifest 也必须记录 `command.stdout.txt`、`command.stderr.txt` 和 `command-record.json` 的 `sizeBytes` 与 `sha256`；命令证据生成后也要立即把 stdout 中的 `bundleDir` 填入随包 verifier 的 `` 或 `` 占位符做只读验真，不能只等最终根目录总审计才发现 command-record 或 stdout/stderr 归档漂移。启用后证据包必须额外运行随包 direct live smoke，并写入 `direct-live.json`、`direct-live.stdout.txt`、`direct-live.stderr.txt`、`direct-live-command.json` 和 manifest summary 的 `directLiveStatus`，让 Pingora access log `request_id` 反查结果可复盘；`direct-live.json` 的 `direct-access-log` 结果必须保留扫描行数、匹配数量、缺失明细以及 method/path/status 漂移明细，不要只保留 count 或依赖 stderr。证据包 manifest summary 还必须包含 `directLiveAccessLog` 摘要；如果 direct live JSON 缺少 `direct-access-log` 结构化结果，整包应记为 `CRITICAL`。若 snapshot 或 direct live stdout 解析失败，必须保留 `snapshot-parse-error.txt` 或 `direct-live-parse-error.txt` 并在 manifest / 最终 stdout 中给出路径；不要只截图或复制 `pingora-direct-enable.sh` / release readiness 的终端输出当作直连证据。证据阶段名只能使用 ASCII 字母、数字、点、下划线和短横线，非法 `--phase` 会直接失败，不会被清洗后继续落盘；自审、状态快照和证据包的 `--release-root` 都不能是文件系统根目录，状态快照的 `--health-patrol-env-file` / `--pingora-env-file` 以及证据包所有显式路径参数也不能是文件系统根目录，状态快照自身还必须拒绝带换行或 NUL 的 release/env 路径，并在执行 systemctl、current release 自审、health patrol env 复核或生产巡检子命令前复核子命令参数，证据包执行状态快照或 direct live 子命令前也必须拒绝任何带换行或 NUL 字符的子命令参数，避免污染后的结构化 `args[]` 先进入正式证据再等总审计兜底，`--output-root` 及其已存在上级路径不能是符号链接，已存在的 `--output-root` 必须是真实目录，路径异常时会在执行状态快照前失败，避免把证据写入非预期软链目标；current release 自审、状态快照和证据包的显式 `--timeout-ms` 及对应 env 必须是正整数，生产健康巡检的 `--timeout-ms`、`--slow-ms`、`GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS` 和 `GENARRATIVE_HEALTH_PATROL_SLOW_MS`，canary / direct live smoke 的 `--timeout-ms` 及对应 env，canary access log 对账的 `--since-lines` 及对应 env 也必须是正整数，直连 live / release readiness 的布尔 env 也必须是明确布尔值，非法值都会失败，不再静默回退默认值或 false。自审、快照、证据包、证据验真和证据总审计脚本都不修改 `/etc`、systemd、Nginx 或 Pingora；命令证据脚本只执行 `--` 后面的真实命令并归档输出，不自行理解 systemd / Nginx；证据包和命令证据目录必须是 `0750`，证据文件必须是 `0640`，且不能覆盖既有文件；probe token 和其他 env 敏感值只能记录是否存在或显示 ``，不能把 env 原文写入终端执行日志、gateway smoke / direct live / direct rollback shadow probe 命令日志、stdout、snapshot、manifest、命令记录或子检查 stdout / stderr；如果自审、快照、direct live 或总审计证据里出现 `CRITICAL`，应先修发布包、Jenkins 归档过滤、deploy 复制、env、systemd capability、直连入口、巡检状态或证据归档，再继续下一阶段，不要把自审、快照或证据包当成可自动修复的烟测。 - 踩坑补充：直连 Pingora 后不要让静态缓存头继续依赖框架默认值。HTML、目录 index 和 SPA fallback 必须保持 `Cache-Control: no-cache`，否则旧入口页可能长期引用已经切换的 chunk；带 Vite 指纹的 `/assets/*` 和 `/admin/assets/*` 才能使用 `public, max-age=31536000, immutable`；普通非指纹静态和 ACME challenge 继续保守 `no-cache`。如果需要临时覆盖 `GENARRATIVE_PINGORA_GATEWAY_*_CACHE_CONTROL`，值不能包含换行或 NUL，修改后必须跑 `npm run check:pingora-gateway-smoke` 确认 HTML、普通静态和指纹资源三类响应头没有漂移。 - 踩坑补充：直连 Pingora 后也不能只验证整文件静态读取。浏览器、媒体探测和线上签名 URL 排障都可能使用 `Range: bytes=`；Pingora 静态文件必须支持单段 range 的 `206 + Content-Range` 和越界 range 的 `416 + Content-Range: bytes */`，同时给静态响应写入 `Accept-Ranges: bytes`。`If-Range` 不能被忽略：日期匹配才继续给局部内容，旧日期或弱 ETag 校验器应回完整 `200`，避免客户端拿旧校验器拼接错误文件片段。`206`、`304`、`416` 不应被 gzip 压缩，否则 `Content-Range` 指向的字节区间会和实际响应体不一致。多段 range 暂按完整文件处理，不要在切换窗口临时拼 multipart 响应。 @@ -2357,6 +2358,14 @@ - 验证：`npm run check:pingora-gateway-smoke` 必须覆盖静态 `HEAD`、`If-None-Match` 304、`If-Modified-Since` 304，并用 access log method/path/status 对账证明本地静态边界进入日志证据链；`cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml` 必须覆盖 ETag 构造和匹配 helper。 - 关联：`server-rs/crates/pingora-gateway/src/main.rs`、`scripts/check-pingora-gateway-smoke.mjs`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 +## dev health patrol 不能缺少公网 HTTPS 入口配置 + +- 现象：dev 上 `genarrative-health-patrol.timer` 正常 active，但 `genarrative-health-patrol.service` 最近一次运行失败；Pingora 直连彩排状态脚本只因 `/etc/genarrative/health-patrol.env` 缺失或 public probe 命中 `http://127.0.0.1` 后被 Nginx 301 而报 `CRITICAL`。 +- 原因：health patrol systemd unit 的 `EnvironmentFile=-/etc/genarrative/health-patrol.env` 允许文件缺失，脚本会退回默认 public base URL `http://127.0.0.1`；dev / release 的 Nginx 公开入口会把 HTTP 跳到 HTTPS，巡检按非 2xx 判失败。 +- 处理：目标机应创建 `/etc/genarrative/health-patrol.env`，保持 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx`，把 `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL` 指向真实 HTTPS 域名，例如 `https://dev.genarrative.world`；Pingora shadow 巡检同时配置 `GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL=http://127.0.0.1:18081` 和与 `/etc/genarrative/pingora-gateway.env` 一致的 probe token。不要为了让彩排状态变绿把缺 env 降级成 warning。 +- 验证：先运行随包 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url https://dev.genarrative.world --require-empty-public-host`，再 `systemctl start genarrative-health-patrol.service`；最后运行 `node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs --release-root /opt/genarrative/current --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`。 +- 关联：`deploy/env/health-patrol.env.example`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 + ## SpacetimeDB 连接池租约必须有 Drop 兜底，acquire 不允许无界自旋 - 现象：release 上 api-server 周期性出现全量 `spacetime_stage="pool_acquire" elapsed_ms=45000` 业务超时，`/readyz` 503（`reason=spacetime_unhealthy, stage=pool_acquire`），`/healthz` 仍 200，只有重启能恢复，过若干小时复发。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 7ef9331dd..001662fe1 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -50,6 +50,7 @@ npm run check:pingora-direct-live npm run check:pingora-direct-enable npm run check:pingora-direct-rollback npm run check:pingora-current-release-audit +npm run check:pingora-direct-rehearsal-status npm run check:pingora-cutover-status-snapshot npm run check:pingora-cutover-evidence-bundle npm run check:pingora-cutover-evidence-verify @@ -87,6 +88,20 @@ npm run check:pingora-direct-preflight -- \ `check:pingora-current-release-audit` 会烟测 `scripts/ops/pingora-current-release-audit.mjs` 的只读边界、current release 必备脚本 / 配置目录、可选 `pingora-gateway` 可执行性和 `--systemd-show` 下的 `ExecStart` 指向。正式直连 runbook 会先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，在采集状态快照前确认目标机 current release 本身自包含；显式 `--timeout-ms` 和 `GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS` 必须是正整数，非法值直接失败，不静默回退默认 `5000`。`GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY`、`GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败，不再按 false 继续执行；`--release-root` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL 字符，`--systemd-service` 同样不能包含换行或 NUL 字符；启用 `--systemd-show` 时，脚本会在执行 `systemctl show` 前复核子命令可执行文件和参数不含换行或 NUL，避免把污染参数带进只读审计命令。 +`check:pingora-direct-rehearsal-status` 会烟测 `scripts/ops/pingora-direct-rehearsal-status.mjs` 的只读彩排状态采集。目标机切换前可执行： + +```bash +node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs \ + --release-root /opt/genarrative/current \ + --expect-public-gateway nginx \ + --require-pingora-shadow \ + --require-realpath-canary \ + --require-current-release-gateway \ + --fail-on-critical +``` + +该脚本只读读取 health patrol env、Pingora env、`systemctl`、`ss -H -ltnp`、realpath canary 配置和 current release 自审结果，输出 JSON 总览；不会写 `/etc`、不会 reload systemd，也不会修改 Nginx 或 Pingora。`--expect-public-gateway nginx` 会要求公网 `80/443` 仍由 Nginx 监听，`--require-pingora-shadow` 会要求 `127.0.0.1:18081` 由 Pingora shadow 监听，`--require-realpath-canary` 会要求 `127.0.0.1:18083` 有 Nginx realpath canary，`--require-current-release-gateway` 会复用 current release 自审确认 `pingora-gateway`、checksum、manifest 和 systemd `ExecStart`。该脚本用于直连前确认现场状态已经进入“可彩排但未切公网”的安全边界，不替代后续证据包，也不做任何启用或回退动作。 + `check:pingora-cutover-status-snapshot` 会烟测 `scripts/ops/pingora-cutover-status-snapshot.mjs` 的只读边界、release artifact 采集、health patrol env 复核、systemd capability / EnvironmentFile 判断和 `--fail-on-critical` 行为。正式切换窗口由 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs` 调用随包状态快照脚本，按 `--phase pre-cutover`、`--phase post-enable`、`--phase post-rollback` 生成时间戳证据目录，保存 `manifest.json`、`snapshot.json`、`snapshot.stdout.txt`、`snapshot.stderr.txt` 和 `snapshot-command.json`；状态快照 stdout 解析失败时保存 `snapshot-parse-error.txt` 并在 manifest / 最终 stdout 中给出路径，启用后 direct live stdout 解析失败时保存 `direct-live-parse-error.txt` 并同样索引。状态快照会把 `--pingora-env-file` 与 `systemctl cat genarrative-pingora-gateway.service` 的 `EnvironmentFile=` 做精确匹配，支持 `EnvironmentFile=-/path` 和一行多个文件，但不接受路径前缀误判；未包含本次 pingora env 时 `systemd.pingoraUnit.environmentFileMatchesPingoraEnvFile=false` 并标记 `CRITICAL`。启用后 direct live 结果中的 `direct-access-log` 必须保留 `scannedLineCount`、`matchedCount`、`missing[]`、`mismatches[]` 以及每个 `request_id` 的预期 / 实际 method、path 与 status，避免证据包只留下计数或终端 stderr；静态 GET / HEAD / 304 / Range 结果还必须在 `direct-live.json` 中保留白名单响应头，字段限于 `cache-control`、`etag`、`last-modified`、`accept-ranges`、`content-range`、`content-length` 和 `content-encoding`，便于复盘缓存分档、校验器和 Range 语义，同时避免把 API / WSS 原始响应头落进正式证据。证据包还会把 `directLiveAccessLog` 和 `directLiveStaticHeaders` 摘要提升到 `manifest.summary`，前者用于快速确认 request_id 对账，后者用于快速确认普通静态 / Vite 指纹静态的 `Cache-Control`、`ETag`、`Last-Modified`、`Content-Length`、Range `Content-Range` 与 304 状态证据；缺少 `direct-access-log` 结构化结果、缺少可判定的静态头摘要，或静态摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 / Last-Modified 304 证据时，证据包都会直接记为 `CRITICAL`。维护模式、非 HTML 或首页确实没有构建资产引用时，direct live 会把静态检查标记为 skipped，此时允许证据包继续记录跳过原因。`snapshot-command.json` 与 manifest 的 `commands[]` 同时保留脱敏后的可读命令和结构化 `executable` / `args[]`，便于路径或参数包含空格时复盘；证据包在执行状态快照或 direct live 子命令前也会拒绝任何带换行或 NUL 字符的子命令参数，避免污染后的 `args[]` 先落盘再等总审计兜底。`--run-direct-live` 使用的 HTTPS / HTTP base URL、Host、redirect Host、probe token、SpacetimeDB 数据库名、Pingora access log 路径和 access log tail 行数也会在证据包配置层先拒绝换行或 NUL，且 `--direct-pingora-access-log` 必须是绝对路径并且不能是文件系统根目录，避免进入 snapshot 或 direct live 子命令后才暴露参数污染。`--phase` 只允许 ASCII 字母、数字、点、下划线和短横线，非法阶段名直接失败，不做隐式清洗，避免目录名和 manifest 阶段漂移；状态快照和证据包的显式 `--timeout-ms` 以及 `GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS` 必须是正整数，非法值直接失败，不再静默回退默认 `5000`。`GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST`、`GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL`、`GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY`、`GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败，避免 run / require / fail 开关拼写错误后静默按 false 采集证据。状态快照脚本只读采集 `summary`、`healthPatrolEnv`、`pingoraEnv`、`releaseArtifacts`、`systemd` 和 `checks`；probe token 和其他 env 敏感值只记录是否存在，不输出原文，健康巡检、current release 自审等子检查的 stdout / stderr 在进入快照 JSON 前也必须按 env 敏感值脱敏，证据包里的 manifest、snapshot、stdout 和命令记录都必须通过自测确认不落盘 token 原文；状态快照脚本的 `--release-root`、`--health-patrol-env-file` 和 `--pingora-env-file` 都必须是绝对路径、不能是文件系统根目录，也不能包含换行或 NUL 字符，且执行 systemctl、current release 自审、health patrol env 复核或生产巡检子命令前还会复核子命令参数不含换行或 NUL；证据包脚本只写 `--output-root` 下的新目录，且证据包所有显式路径参数也不能是文件系统根目录，`--output-root` 及其已存在上级路径不能是符号链接，已存在的 `--output-root` 必须是真实目录；路径异常时会在执行状态快照前失败，不写入软链目标、不覆盖既有文件，证据目录权限固定为 `0750`，证据文件权限固定为 `0640`，不写 `/etc`、不 reload systemd，也不修改 Nginx 或 Pingora。 直连 runbook 的三个证据包阶段都会透传 `--require-pingora-gateway`，让 `checks.current-release-audit.details` 同步归档 Pingora 二进制、sha256、release manifest 和 systemd `ExecStart` 自审结果，避免证据包只有巡检状态而缺少发布物可信度证明。每个证据包生成后都要用 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir <本阶段bundleDir>` 做只读验真；该脚本只接受 `schemaVersion=1` 的 manifest，并把 `manifest.files` 视为闭集，只读取 `manifest.files` 中的 `{ path, sizeBytes, sha256 }`，拒绝路径逃逸、符号链接证据目录、非目录证据目录、非元数据对象文件条目，以及任何未登记的额外普通文件、目录或符号链接，发现缺文件、大小漂移或 sha256 漂移时退出失败；`--allow-extra-files` 只用于人工排障时显式放行额外条目，正式切换归档不应使用。验真脚本不修改证据目录、不 reload systemd，也不访问 Nginx 或 Pingora。enable apply / rollback apply 的命令证据生成后也要立即用同一个随包 verifier 验真，将上一步 stdout 的 `bundleDir` 分别填入 `` / ``，确认 `command.stdout.txt`、`command.stderr.txt`、`command-record.json` 与 manifest 元数据一致，再继续后续 health patrol 或回退复核。三阶段证据包都生成并分别验真、enable / rollback apply 命令证据也即时验真后，还要执行 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-audit.mjs --evidence-root <证据根目录> --require-phase pre-cutover --require-phase post-enable --require-phase post-rollback --require-phase-direct-live-access-log post-enable --require-phase-direct-live-static-headers post-enable --require-command enable-apply:pingora-direct-enable-apply --require-command rollback-apply:pingora-direct-rollback-apply --require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --require-command-arg enable-apply:pingora-direct-enable-apply:--apply --require-command-arg rollback-apply:pingora-direct-rollback-apply:--apply --require-cutover-run-id <本次cutoverRunId> --timeline-max-span-ms 86400000`，由脚本按 `manifest.phase` 找到每个阶段最新证据目录、按 `manifest.phase + manifest.commandName` 找到 enable / rollback apply 最新命令证据，并复核两条真实切换命令都来自 current release 随包脚本；所有候选证据都必须带合法 `manifest.generatedAt`，格式必须是 `new Date().toISOString()` 产出的 UTC 毫秒格式 `YYYY-MM-DDTHH:mm:ss.sssZ`，最新选择和后续时间线证明只使用该字段，不依赖目录 mtime；随后复用随包 verifier 的 `--require-summary-ok` 严格模式再次只读验真，输出可归档的阶段与命令总表；阶段证据必须是不带 `manifest.commandName` 的状态快照证据包，命令证据不能冒充同名 phase 的阶段证据。总审计输出会把阶段 manifest 中的 `directLiveAccessLog` 与 `directLiveStaticHeaders` 一并带到对应 `phases[]` 项，启用后阶段可直接看到 request_id 对账数量、普通静态和指纹静态的缓存头、校验头、Range `Content-Range` 与 304 状态摘要，不必再逐个打开 bundle manifest；正式 runbook 还会通过 `--require-phase-direct-live-access-log post-enable` 和 `--require-phase-direct-live-static-headers post-enable` 把缺少 request_id 对账摘要、缺少静态头摘要、摘要被跳过或摘要缺少缓存头 / 校验头 / Range / 304 证据的启用后证据判为失败，避免旧 post-enable 证据包混入最终归档。总审计的证据根目录默认只能包含带 `manifest.json` 的证据目录，夹带普通文件、无 manifest 子目录或符号链接都会失败，`--allow-extra-root-entries` 只用于人工排障显式放行，正式切换归档不应使用。总审计不只验文件 hash，还要求所有候选证据 `manifest.schemaVersion=1`，阶段证据 `manifest.summary.status=OK`，命令证据 `manifest.summary.status=OK`、`manifest.summary.exitCode=0` 且无 signal；命令证据的顶层 `manifest.commandName` 和内嵌 `manifest.command.name` 只要存在就必须各自是安全非空命令名，且两者同时存在时必须一致，否则按坏 manifest 处理；并在标准五段证据都被要求时校验每段审计状态都是 `OK`，以及 `pre-cutover -> enable-apply -> post-enable -> rollback-apply -> post-rollback` 的 `manifest.generatedAt` 顺序和默认 24 小时最大跨度，防止从不同切换窗口拼出一组看似完整的最新证据；缺失、非法或非规范格式的 `manifest.generatedAt` 会直接失败，不能靠目录 mtime 兜底；最终总审计还会在 `--require-command-arg` 下要求 enable / rollback 命令证据的 `manifest.command.args` 和独立 `command-record.json.args` 都包含 `--apply`，并在 `--require-cutover-run-id` 下要求所有阶段和命令证据的 `manifest.cutoverRunId` 与本次 runbook 一致；确需跨更长维护窗口时，只能在生成 runbook 时显式传 `--cutover-evidence-timeline-max-span-ms `，让最终总审计 JSON 可见本次放宽值；证据根目录、verifier 路径、阶段名、命令名或 cutover run id 不安全时直接失败。 @@ -102,7 +117,7 @@ npm run check:pingora-direct-preflight -- \ `--warn-only`、`--allow-extra-files` 和 `--allow-extra-root-entries` 只允许值班人员在正式 runbook 之外作为独立排障命令使用；`plan:pingora-direct-cutover` 生成的正式切换计划不得携带这些放行参数。若需要放行额外证据条目或忽略 `CRITICAL`，应先在证据根目录外保存人工说明、修正现场或重新生成正式证据，而不是把排障开关写进切换 runbook。 -`check:pingora-release-readiness` 是正式切换前的聚合门禁，默认串行执行 `cargo test -p pingora-gateway`、mock 上游 smoke、路由矩阵 parity、Nginx canary snippet 校验、Docker handoff smoke、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、current release 自审烟测、release readiness 计划自检、生产运维护栏、Pingora cutover 状态快照烟测、Pingora cutover 证据包烟测、Pingora cutover 命令证据烟测、Pingora cutover 证据 manifest 验真烟测、Pingora cutover 证据根目录审计烟测、API release build 烟测、Pingora production release 真实构建烟测和 API deploy release 烟测。普通本机执行时 Docker / Nginx 能力仍按子脚本默认口径跳过；切换窗口或 CI 必须执行： +`check:pingora-release-readiness` 是正式切换前的聚合门禁，默认串行执行 `cargo test -p pingora-gateway`、mock 上游 smoke、路由矩阵 parity、Nginx canary snippet 校验、Docker handoff smoke、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、current release 自审烟测、直连彩排状态烟测、release readiness 计划自检、生产运维护栏、Pingora cutover 状态快照烟测、Pingora cutover 证据包烟测、Pingora cutover 命令证据烟测、Pingora cutover 证据 manifest 验真烟测、Pingora cutover 证据根目录审计烟测、API release build 烟测、Pingora production release 真实构建烟测和 API deploy release 烟测。普通本机执行时 Docker / Nginx 能力仍按子脚本默认口径跳过；切换窗口或 CI 必须执行： 目标机 current release 上的启用前基础门禁和启用后 `--require-direct` 复核使用同一个聚合脚本，但必须追加 `--release-runtime-only`。该模式只执行发布包内可自包含的运行时复核：current release 自审、live canary、真实 access log 对账、direct preflight、health patrol env 复核和 direct live smoke；不会运行 Cargo、npm、Docker 或 Nginx 源码 / 构建环境门禁，并会拒绝 `--require-docker`、`--pull-docker` 与 `--require-nginx`。 @@ -223,7 +238,7 @@ npm run build:production-release -- --component api-server --name --in Jenkins `Genarrative-Api-Build` 对应参数是 `INCLUDE_PINGORA_GATEWAY`，默认关闭；勾选后才会归档 `pingora-gateway` 与 `pingora-gateway.sha256`，触发后续 `Genarrative-Api-Deploy` 时也会传递该布尔参数并复制这两个可选产物。无论是否打包 Pingora 二进制，API release 都必须携带 `build//scripts/deploy/production-api-deploy.sh`、同目录的 `maintenance-on.sh` / `maintenance-off.sh`、`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh`、`pingora-direct-rollback.sh`、`pingora-tls-cert-sync.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/`。`Genarrative-Api-Deploy` 只能从上游构建归档复制并执行 `build//scripts/deploy/production-api-deploy.sh`，不能继续执行部署工作区根部脚本；维护脚本必须与 deploy 脚本来自同一发布包同一目录，避免 Jenkins 工作区里的旧脚本掩盖 release 包布局缺陷。直连启用脚本从 current release 执行时默认读取 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，必须能自包含完成 preflight、systemd 模板读取和 direct live smoke，不依赖 Jenkins 工作区、目标机源码 checkout 或 `/etc` 参考模板；current release 自审、TLS 证书同步、health patrol env 复核、canary access log 对账、canary live、direct live、状态快照、证据包、命令证据、证据验真和证据根目录审计也必须来自 current release 随包脚本与 `deploy/env/`。`/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 只作为 Server-Provision 安装的人工审阅 / 手动覆盖模板；确需使用时显式传 `--template-path` 或 `GENARRATIVE_PINGORA_DIRECT_TEMPLATE_PATH`。`npm run check:production-api-release` 会用临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 跑 `build-production-release.sh --component api-server --skip-api-build`，动态验证默认 API release 不登记 Pingora，显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时则必须包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest，同时验证 Pingora 直连依赖、TLS 证书同步脚本、current release 自审脚本、状态快照脚本、证据包脚本、命令证据脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、env 示例目录、API deploy 执行入口自包含和 `deploy/pingora/pingora-gateway.env.example` 的生产安全默认值：gzip-only、`TRUST_X_FORWARDED_FOR=false`、`TRUSTED_FRONT_PROXY_CONFIRMED=false`、接流保护默认开启、probe token 为空；该检查还会读取发布包 README，并直接运行发布包内 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover`，确认最终证据根目录总审计步骤仍带两条 `--require-command-executable ... /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` / `pingora-direct-rollback.sh` current release 脚本身份要求。`npm run check:pingora-production-release-build` 会用假 `api-server`、临时 `CARGO_TARGET_DIR` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行 `pingora-gateway`、checksum 和 manifest 登记，避免只靠假二进制布局 smoke。`production-api-deploy.sh` 对这些 Pingora 直连依赖采取 fail-fast：上游发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；`release-manifest.json` 必须存在且登记 `api-server` artifact，若发布包包含 `pingora-gateway`，manifest 也必须登记 `pingora-gateway` artifact，否则部署会在切换 current 前失败。API deploy 要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，`--version` 必须以数字或字母开头并拒绝点目录，会先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部校验和复制完成后用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，再用固定替换语义切换 current 符号链接；同版本正式 release 已存在、提升前竞态出现或 current 路径不是符号链接时都会拒绝合并 / 覆盖，失败时清理 staging 并保持维护模式。本机用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 动态验证从发布产物内执行 deploy 脚本后 current release 自洽，并覆盖缺少 TLS 证书同步脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、健康巡检 env 复核脚本、`deploy/env/`、release manifest、api-server manifest artifact、Pingora manifest artifact、相对 release root / current link / api env file、点目录或点开头 version、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时必须失败，同时复核 current release 内 Pingora env 示例仍保持同一组生产安全默认值。 -API release 还必须携带 `scripts/check-pingora-release-readiness.mjs` 与 `scripts/check-pingora-canary-live.mjs`。前者支撑 current release 的 `--release-runtime-only` 聚合复核，后者支撑目标 Nginx canary live smoke；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败。 +API release 还必须携带 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs` 与 `scripts/ops/pingora-direct-rehearsal-status.mjs`。前者支撑 current release 的 `--release-runtime-only` 聚合复核，canary live 脚本支撑目标 Nginx canary live smoke，直连彩排状态脚本支撑目标机切换前只读确认 Nginx 仍接公网、Pingora shadow / realpath canary 高端口和 current release 自审均可用；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败。 发布包会额外包含 `pingora-gateway` 与 `pingora-gateway.sha256`。`production-api-deploy.sh` 看到这两个文件时会校验并把 `pingora-gateway` / `pingora-gateway.sha256` 一起复制到 `/opt/genarrative/current`，供 shadow systemd 模板和 current release 自审使用；`api-server.sha256` 也必须随 `api-server` 一起进入 current release。没有这两个 Pingora 文件时现有 API 发布行为不变。API 发布仍只重启 `genarrative-api.service`；如果 `genarrative-pingora-gateway.service` 已经处于 `active`，部署脚本会在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制。若 Pingora 服务未运行，部署脚本只打印跳过信息，不主动拉起影子服务，也不改变默认接流边界。 @@ -481,7 +496,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 5. 前缀 canary 通过后，再使用 current release 随包 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083` 启用真实路径 canary；它会把 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 渲染成独立本机 `server`，写入 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，不能 include 到生产 `443` server 内。该片段使用 `access_log ... genarrative_upstream`，文件名必须保证晚于定义 `log_format genarrative_upstream` 的主站配置加载；否则 `nginx -t` 会报 `unknown log format "genarrative_upstream"`。启用脚本会先执行 `nginx -t`、reload Nginx，再默认运行 realpath live smoke，任一阶段失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会在 `nginx -t` 或 reload 失败时恢复删除前配置。启用后跑 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再用 current release 随包 access log parity 脚本传 `--realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log` 对账 `/api/creation-entry/config`、`/v1/identity` 和 `/assets/app.js` 等真实路径。 6. 目标机 canary include 后必须跑正式切换聚合门禁：源码 checkout / CI / 构建环境执行默认全量命令 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；目标机 current release 执行 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。启用真实路径 canary 时，两种命令都追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。缺少 Host 会直接失败，避免 live canary 误测默认 vhost；live smoke 后还会按 `request_id` 对账 Nginx 与 Pingora access log，缺少同一请求的 Pingora 日志、状态码、方法或 path 漂移都会失败。 7. 如需评估 Pingora 直连公网入口，必须显式配置 `TLS_LISTEN`、证书、私钥和 `HTTP_REDIRECT_LISTEN`；Certbot 证书先用随包 `scripts/deploy/pingora-tls-cert-sync.mjs` 同步到 `/etc/genarrative/pingora-tls/<域名>/`，不要直接 chmod Let’s Encrypt live/archive 原路径；绑定 `80/443` 时还必须人工启用 `genarrative-pingora-gateway-direct-entry.conf` drop-in 授予 `CAP_NET_BIND_SERVICE`。随后用 `npm run check:pingora-gateway-smoke` 覆盖 TLS / HTTP/2 ALPN / redirect / WSS subscribe；目标机必须先跑 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再跑 `npm run check:pingora-direct-live` 或 release readiness 的 `--require-direct`，且 `--require-direct` 必须带 direct HTTPS base URL、direct HTTP base URL、正式域名 Host/SNI、redirect Location host、Pingora access log 文件、health patrol env 文件、direct preflight env 文件、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户和 systemd 服务用户证书可读检查、service 二进制可执行检查、端口释放检查和显式 SpacetimeDB 数据库名，并会拒绝 `--skip-wss`。高端口 rehearsal 使用 `https://127.0.0.1:<高端口>` 打入但期望 HTTP redirect Location 指向正式域名默认 HTTPS 入口时，额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于 runbook Host 一致性约束。direct live 会用生成的 `request_id` 反查 Pingora access log；缺少对应日志、method 漂移、path 漂移或 status 漂移都算直连门禁失败。direct preflight 会拒绝开启网关保护但未确认共享保护层的 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT>1` 配置；`--env-file`、`--systemd-service`、服务用户和 env 中的 listen / cert / key 值都不能包含换行或 NUL，执行 `systemctl cat` 或 `sudo -u test -r ` 前还会复核子命令参数，避免污染参数进入目标机预检命令；direct live 的 URL、Host、redirect base URL、probe token、额外 path、数据库名、access log 路径、timeout 和布尔 env 也不能包含换行或 NUL，且会在发起请求前失败；direct live timeout 必须是正整数，直连相关布尔 env 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。证书申请与续期仍由 Certbot / 外部自动化承担，网关只读取现有文件。 -8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs`、realpath canary 启停脚本和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 +8. 正式直连 runbook 的启用前基础门禁和启用后 `--require-direct` 复核必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only`。该脚本、`scripts/check-pingora-canary-live.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、realpath canary 启停脚本和 `deploy/nginx/` 必须进入生产 API release、Jenkins API Build 归档、Jenkins API Deploy 复制清单和目标机 current release；缺失时部署应 fail-fast，切换窗口不能依赖源码 checkout 或 Jenkins workspace。 9. direct preflight / direct live timeout 必须是正整数；直连相关布尔 env 和 Pingora gateway env 里的 `TRUST_X_FORWARDED_FOR` / `TRUSTED_FRONT_PROXY_CONFIRMED` 只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败。 10. Pingora 正式化口径固定为 gzip-only；Brotli 继续由 Nginx / 前置代理承担，直连 Pingora 不以 Brotli parity 作为切换门禁。如需多实例，开启网关接流保护时必须先引入共享限流 / 共享并发保护层并设置 `GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=true`，否则保持 `GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1`；关闭网关保护的多实例方案必须明确由前置 Nginx / LB 承担全局限流。 10. 前缀 canary 稳定后，再评估是否做真实路径 canary；真实路径 canary 稳定后，再评估是否让 Pingora 直接承接公网入口。 diff --git a/jenkins/Jenkinsfile.production-api-build b/jenkins/Jenkinsfile.production-api-build index ba7a9cf2e..ace9ca71a 100644 --- a/jenkins/Jenkinsfile.production-api-build +++ b/jenkins/Jenkinsfile.production-api-build @@ -123,7 +123,7 @@ pipeline { stage('Archive') { steps { - archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true + archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true script { if (params.INCLUDE_PINGORA_GATEWAY) { archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway,build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway.sha256", fingerprint: true diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index 50bee6574..7f60e03a9 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -66,7 +66,7 @@ pipeline { copyArtifacts( projectName: params.BUILD_JOB_NAME, selector: specific(params.BUILD_NUMBER_TO_DEPLOY), - filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", + filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", target: '.', fingerprintArtifacts: true ) diff --git a/package.json b/package.json index 2e715da20..f8e01769f 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "check:pingora-health-patrol-env-switch": "node scripts/check-pingora-health-patrol-env-switch.mjs", "check:pingora-tls-cert-sync": "node scripts/check-pingora-tls-cert-sync.mjs", "check:pingora-current-release-audit": "node scripts/check-pingora-current-release-audit.mjs", + "check:pingora-direct-rehearsal-status": "node scripts/check-pingora-direct-rehearsal-status.mjs", "check:pingora-cutover-status-snapshot": "node scripts/check-pingora-cutover-status-snapshot.mjs", "check:pingora-cutover-evidence-bundle": "node scripts/check-pingora-cutover-evidence-bundle.mjs", "check:pingora-cutover-command-evidence": "node scripts/check-pingora-cutover-command-evidence.mjs", diff --git a/scripts/build-production-release.sh b/scripts/build-production-release.sh index b5a007db9..017af9a8b 100644 --- a/scripts/build-production-release.sh +++ b/scripts/build-production-release.sh @@ -552,6 +552,7 @@ copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TA copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本" copy_required_file "${SCRIPT_DIR}/ops/production-health-patrol.mjs" "${TARGET_DIR}/scripts/ops/production-health-patrol.mjs" "生产健康巡检脚本" copy_required_file "${SCRIPT_DIR}/ops/pingora-current-release-audit.mjs" "${TARGET_DIR}/scripts/ops/pingora-current-release-audit.mjs" "Pingora current release 自审脚本" +copy_required_file "${SCRIPT_DIR}/ops/pingora-direct-rehearsal-status.mjs" "${TARGET_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" "Pingora 直连切换彩排状态脚本" copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-status-snapshot.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" "Pingora 直连切换状态快照脚本" copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-bundle.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" "Pingora 直连切换证据包脚本" copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-command-evidence.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" "Pingora 直连切换命令证据脚本" @@ -584,7 +585,7 @@ cat >"${TARGET_DIR}/README.md" < item.path === 'scripts/ops/pingora-direct-rehearsal-status.mjs', + ); + if (!rehearsalStatusArtifact || rehearsalStatusArtifact.status !== 'OK') { + failures.push('快照必须确认 current release 已包含直连彩排状态脚本。'); + } const evidenceAuditArtifact = payload.releaseArtifacts.artifacts.find( (item) => item.path === 'scripts/ops/pingora-cutover-evidence-audit.mjs', ); @@ -727,6 +738,7 @@ function prepareReleaseRoot(releaseRoot) { for (const file of [ 'scripts/database-backup-to-oss.mjs', 'scripts/ops/pingora-current-release-audit.mjs', + 'scripts/ops/pingora-direct-rehearsal-status.mjs', 'scripts/ops/pingora-cutover-evidence-audit.mjs', 'scripts/ops/pingora-cutover-evidence-bundle.mjs', 'scripts/ops/pingora-cutover-command-evidence.mjs', diff --git a/scripts/check-pingora-direct-rehearsal-status.mjs b/scripts/check-pingora-direct-rehearsal-status.mjs new file mode 100644 index 000000000..0cbf8bbda --- /dev/null +++ b/scripts/check-pingora-direct-rehearsal-status.mjs @@ -0,0 +1,566 @@ +#!/usr/bin/env node + +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + chmodSync, + copyFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const STATUS_SCRIPT = 'scripts/ops/pingora-direct-rehearsal-status.mjs'; +const failures = []; +const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-pingora-direct-rehearsal-'), +); + +try { + main(); +} finally { + rmSync(tmpRoot, { recursive: true, force: true }); +} + +if (failures.length > 0) { + console.error('[check:pingora-direct-rehearsal-status] FAILED'); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exit(1); +} + +console.log('[check:pingora-direct-rehearsal-status] OK'); + +function main() { + assertScriptShape(); + assertNginxPublicRealpathRehearsalPasses(); + assertFailsWhenPublicPortsAlreadyOwnedByPingora(); + assertFailsWhenRealpathCanaryMissing(); + assertRejectsUnsafeArgs(); + assertRejectsInvalidBoolEnv(); +} + +function assertScriptShape() { + const content = readFileSync(STATUS_SCRIPT, 'utf8'); + assertIncludes( + content, + '该脚本只读采集', + 'rehearsal 状态脚本 usage 必须说明只读边界。', + ); + assertIncludes( + content, + 'systemctl', + 'rehearsal 状态脚本必须读取 systemd 状态。', + ); + assertIncludes( + content, + 'ss', + 'rehearsal 状态脚本必须读取端口监听状态。', + ); + assertIncludes( + content, + 'pingora-current-release-audit.mjs', + 'rehearsal 状态脚本必须复用 current release 自审。', + ); + if ( + content.includes('writeFile') || + content.includes('rmSync(') || + content.includes('systemctl reload') || + content.includes('daemon-reload') + ) { + failures.push('rehearsal 状态脚本不应写文件、删除文件或 reload systemd。'); + } +} + +function assertNginxPublicRealpathRehearsalPasses() { + const fixture = prepareFixture('nginx-public-realpath', { + ssMode: 'nginx-public-realpath', + }); + const result = runStatus(fixture, [ + '--expect-public-gateway', + 'nginx', + '--require-realpath-canary', + '--require-pingora-shadow', + '--require-current-release-gateway', + '--fail-on-critical', + ]); + + assertStatus(result, 0, 'Nginx 接公网 + Pingora shadow + realpath canary 应通过。'); + if (result.status !== 0) { + return; + } + const payload = parseJson(result.stdout, 'rehearsal 状态输出'); + assertEqual(payload.summary.status, 'OK', 'rehearsal summary 应为 OK。'); + assertEqual( + payload.publicBoundary.status, + 'OK', + 'Nginx 接公网时 public boundary 应通过。', + ); + assertEqual( + payload.ports.byPort['80'].hasNginx, + true, + '80 端口必须识别为 Nginx。', + ); + assertEqual( + payload.ports.byPort['443'].hasNginx, + true, + '443 端口必须识别为 Nginx。', + ); + assertEqual( + payload.ports.byPort['18081'].hasPingora, + true, + '18081 必须识别为 Pingora shadow。', + ); + assertEqual( + payload.ports.byPort['18083'].hasNginx, + true, + '18083 必须识别为 Nginx realpath canary。', + ); + assertEqual( + payload.realpathCanary.templateLooksValid, + true, + 'realpath canary 配置必须通过关键片段检查。', + ); + assertEqual( + payload.systemd.hasAmbientCapability, + false, + 'Nginx 接公网彩排阶段不得有 Pingora 低端口 capability。', + ); + assertEqual( + payload.healthPatrolEnv.values.gatewayMode, + 'nginx', + 'Nginx 接公网彩排阶段 health patrol gateway mode 应为 nginx。', + ); + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + assertIncludes( + commandsLog, + 'systemctl is-active nginx.service', + 'rehearsal 状态必须读取 Nginx service 状态。', + ); + assertIncludes( + commandsLog, + 'ss -H -ltnp', + 'rehearsal 状态必须读取端口监听。', + ); +} + +function assertFailsWhenPublicPortsAlreadyOwnedByPingora() { + const fixture = prepareFixture('pingora-public', { + ssMode: 'pingora-public', + }); + const result = runStatus(fixture, [ + '--expect-public-gateway', + 'nginx', + '--require-pingora-shadow', + '--fail-on-critical', + ]); + + assertStatus(result, 1, '期望 Nginx 接公网但 80/443 由 Pingora 监听时必须失败。'); + const payload = parseJson(result.stdout, 'Pingora 抢占公网端口状态输出'); + assertEqual( + payload.publicBoundary.status, + 'CRITICAL', + '公网端口归属漂移时 public boundary 必须 CRITICAL。', + ); + assertIncludes( + payload.publicBoundary.diagnostics.join('\n'), + '已被 Pingora 监听', + '公网端口归属漂移必须给出明确诊断。', + ); +} + +function assertFailsWhenRealpathCanaryMissing() { + const fixture = prepareFixture('missing-realpath', { + ssMode: 'nginx-public-shadow-only', + writeRealpathConfig: false, + }); + const result = runStatus(fixture, [ + '--expect-public-gateway', + 'nginx', + '--require-realpath-canary', + '--fail-on-critical', + ]); + + assertStatus(result, 1, '要求 realpath canary 但配置或端口缺失时必须失败。'); + const payload = parseJson(result.stdout, '缺少 realpath canary 状态输出'); + assertEqual( + payload.realpathCanary.status, + 'CRITICAL', + '缺少 realpath canary 时必须 CRITICAL。', + ); + assertIncludes( + payload.realpathCanary.diagnostics.join('\n'), + 'realpath canary 配置不存在', + '缺少 realpath canary 配置时必须给出明确诊断。', + ); +} + +function assertRejectsUnsafeArgs() { + const fixture = prepareFixture('unsafe'); + const cases = [ + { + args: ['--release-root', 'relative'], + expected: '--release-root 必须是绝对路径', + reason: 'release root 相对路径必须失败。', + }, + { + args: ['--release-root', '/'], + expected: '--release-root 不能是文件系统根目录', + reason: 'release root 为文件系统根目录必须失败。', + }, + { + args: ['--expect-public-gateway', 'apache'], + expected: '--expect-public-gateway 只支持 none / nginx / pingora-direct', + reason: '未知 public gateway 必须失败。', + }, + { + args: ['--timeout-ms', '0'], + expected: '--timeout-ms 必须是正整数', + reason: 'timeout 非正整数必须失败。', + }, + ]; + + for (const testCase of cases) { + const result = runStatus(fixture, testCase.args); + if ((result.status ?? 0) === 0) { + failures.push(testCase.reason); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + testCase.expected, + `${testCase.reason} 必须输出明确错误。`, + ); + } +} + +function assertRejectsInvalidBoolEnv() { + const fixture = prepareFixture('invalid-bool'); + const result = runStatus(fixture, [], { + GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY: 'maybe', + }); + if ((result.status ?? 0) === 0) { + failures.push('非法布尔 env 必须失败。'); + } + assertIncludes( + `${result.stdout}\n${result.stderr}`, + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY 必须是布尔值', + '非法布尔 env 必须输出明确错误。', + ); +} + +function prepareFixture(name, options = {}) { + const root = path.join(tmpRoot, name); + const releaseRoot = path.join(root, 'current'); + const fakeBin = path.join(root, 'bin'); + const commandsLog = path.join(root, 'commands.log'); + const healthPatrolEnvFile = path.join(root, 'etc', 'health-patrol.env'); + const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env'); + const realpathConfigFile = path.join( + root, + 'nginx', + 'zz-genarrative-pingora-realpath-canary.conf', + ); + + mkdirSync(releaseRoot, { recursive: true }); + mkdirSync(fakeBin, { recursive: true }); + mkdirSync(path.dirname(healthPatrolEnvFile), { recursive: true }); + mkdirSync(path.dirname(realpathConfigFile), { recursive: true }); + + prepareReleaseRoot(releaseRoot); + writeFileSync( + healthPatrolEnvFile, + [ + 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1', + 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', + 'GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN=health-secret-token', + '', + ].join('\n'), + 'utf8', + ); + writeFileSync( + pingoraEnvFile, + [ + 'GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081', + 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=pingora-secret-token', + 'GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE=/var/log/genarrative/pingora-gateway.access.log', + 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', + 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', + '', + ].join('\n'), + 'utf8', + ); + if (options.writeRealpathConfig !== false) { + writeFileSync( + realpathConfigFile, + [ + 'server {', + ' listen 127.0.0.1:18083;', + ' access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;', + ' add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;', + '}', + '', + ].join('\n'), + 'utf8', + ); + } + + writeFileSync( + path.join(fakeBin, 'systemctl'), + [ + '#!/usr/bin/env bash', + 'set -euo pipefail', + `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'if [[ "$1" == "is-active" ]]; then', + ' printf "active\\n"', + ' exit 0', + 'fi', + 'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', + ' cat <> ${shellQuote(commandsLog)}`, + `mode="${options.ssMode || 'nginx-public-realpath'}"`, + 'case "${mode}" in', + ' nginx-public-realpath)', + ' cat <&2 + exit 1 +fi +cp "${PINGORA_DIRECT_REHEARSAL_STATUS_SCRIPT_SOURCE}" "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" +chmod 0644 "${RELEASE_CONTENT_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" if [[ ! -f "${PINGORA_CUTOVER_STATUS_SNAPSHOT_SCRIPT_SOURCE}" ]]; then echo "[production-api-deploy] 发布产物缺少 Pingora 直连切换状态快照脚本: ${SOURCE_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" >&2 exit 1 diff --git a/scripts/ops/pingora-current-release-audit.mjs b/scripts/ops/pingora-current-release-audit.mjs index 0f84584a3..88f9a1e55 100644 --- a/scripts/ops/pingora-current-release-audit.mjs +++ b/scripts/ops/pingora-current-release-audit.mjs @@ -11,6 +11,7 @@ const REQUIRED_ARTIFACTS = [ { path: 'scripts/database-backup-to-oss.mjs' }, { path: 'scripts/ops/production-health-patrol.mjs' }, { path: 'scripts/ops/pingora-current-release-audit.mjs' }, + { path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' }, { path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' }, { path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' }, { path: 'scripts/check-production-health-patrol-env.mjs' }, diff --git a/scripts/ops/pingora-cutover-status-snapshot.mjs b/scripts/ops/pingora-cutover-status-snapshot.mjs index 34e001217..2ae7bb3d9 100644 --- a/scripts/ops/pingora-cutover-status-snapshot.mjs +++ b/scripts/ops/pingora-cutover-status-snapshot.mjs @@ -21,6 +21,7 @@ const SERVICES = [ const RELEASE_ARTIFACTS = [ 'scripts/ops/pingora-cutover-status-snapshot.mjs', 'scripts/ops/pingora-current-release-audit.mjs', + 'scripts/ops/pingora-direct-rehearsal-status.mjs', 'scripts/ops/pingora-cutover-evidence-audit.mjs', 'scripts/ops/pingora-cutover-evidence-bundle.mjs', 'scripts/ops/pingora-cutover-command-evidence.mjs', diff --git a/scripts/ops/pingora-direct-rehearsal-status.mjs b/scripts/ops/pingora-direct-rehearsal-status.mjs new file mode 100644 index 000000000..91dfe13bd --- /dev/null +++ b/scripts/ops/pingora-direct-rehearsal-status.mjs @@ -0,0 +1,966 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { constants as fsConstants } from 'node:fs'; +import { access, readFile, stat } from 'node:fs/promises'; +import path from 'node:path'; + +const STATUS_RANK = { + OK: 0, + WARNING: 1, + CRITICAL: 2, +}; + +const PUBLIC_GATEWAYS = new Set(['none', 'nginx', 'pingora-direct']); +const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); +const DEFAULT_PORTS = [80, 443, 18081, 18083]; +const SECRET_ENV_KEY_PATTERN = + /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu; + +const config = parseArgs(process.argv.slice(2)); +const status = await buildStatus(config); + +console.log(`${JSON.stringify(status, null, 2)}\n`); + +if (config.failOnCritical && status.summary.status === 'CRITICAL') { + process.exit(1); +} + +function parseArgs(argv) { + const result = { + releaseRoot: + process.env.GENARRATIVE_PINGORA_REHEARSAL_RELEASE_ROOT || + '/opt/genarrative/current', + healthPatrolEnvFile: + process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || + '/etc/genarrative/health-patrol.env', + pingoraEnvFile: + process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || + '/etc/genarrative/pingora-gateway.env', + realpathConfigFile: + process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_CONFIG_FILE || + '/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf', + expectedPublicGateway: + process.env.GENARRATIVE_PINGORA_REHEARSAL_EXPECT_PUBLIC_GATEWAY || + 'none', + expectedHealthPatrolGatewayMode: + process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', + requireRealpathCanary: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY', + ), + requirePingoraShadow: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_PINGORA_SHADOW', + ), + requireCurrentReleaseGateway: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_CURRENT_RELEASE_GATEWAY', + ), + timeoutMs: parseOptionalPositiveInt( + process.env.GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS, + 5000, + 'GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS', + ), + failOnCritical: readBoolEnv( + 'GENARRATIVE_PINGORA_REHEARSAL_FAIL_ON_CRITICAL', + ), + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case '-h': + case '--help': + printUsage(); + process.exit(0); + break; + case '--release-root': + result.releaseRoot = requireValue(argv, ++index, arg); + break; + case '--health-patrol-env-file': + result.healthPatrolEnvFile = requireValue(argv, ++index, arg); + break; + case '--pingora-env-file': + result.pingoraEnvFile = requireValue(argv, ++index, arg); + break; + case '--realpath-config-file': + result.realpathConfigFile = requireValue(argv, ++index, arg); + break; + case '--expect-public-gateway': + result.expectedPublicGateway = requireValue(argv, ++index, arg); + break; + case '--expected-health-patrol-gateway-mode': + result.expectedHealthPatrolGatewayMode = requireValue( + argv, + ++index, + arg, + ); + break; + case '--require-realpath-canary': + result.requireRealpathCanary = true; + break; + case '--require-pingora-shadow': + result.requirePingoraShadow = true; + break; + case '--require-current-release-gateway': + result.requireCurrentReleaseGateway = true; + break; + case '--timeout-ms': + result.timeoutMs = parseRequiredPositiveInt( + requireValue(argv, ++index, arg), + arg, + ); + break; + case '--fail-on-critical': + result.failOnCritical = true; + break; + default: + throw new Error(`未知参数: ${arg}`); + } + } + + validateConfig(result); + return result; +} + +function printUsage() { + console.log(`Usage: + node scripts/ops/pingora-direct-rehearsal-status.mjs [options] + +Options: + --release-root current release 根目录，默认 /opt/genarrative/current。 + --health-patrol-env-file health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。 + --pingora-env-file pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。 + --realpath-config-file realpath canary Nginx 配置，默认 /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf。 + --expect-public-gateway 可选，none / nginx / pingora-direct；nginx 模式会要求 80/443 仍由 Nginx 接流。 + --expected-health-patrol-gateway-mode + 可选，nginx 或 pingora-direct；不传时随 --expect-public-gateway 推导。 + --require-realpath-canary 要求 realpath canary 配置存在，且 127.0.0.1:18083 由 Nginx 监听。 + --require-pingora-shadow 要求 Pingora shadow 127.0.0.1:18081 正在监听。 + --require-current-release-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。 + --timeout-ms systemctl / ss / 子检查超时，默认 5000。 + --fail-on-critical 如果状态中出现 CRITICAL，则以退出码 1 结束。 + +该脚本只读采集 dev / release 上的 Pingora 直连切换彩排状态，不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 +`); +} + +function validateConfig(input) { + for (const [label, value] of [ + ['--release-root', input.releaseRoot], + ['--health-patrol-env-file', input.healthPatrolEnvFile], + ['--pingora-env-file', input.pingoraEnvFile], + ['--realpath-config-file', input.realpathConfigFile], + ]) { + validateSafeAbsoluteFilePath(value, label); + } + + if (!PUBLIC_GATEWAYS.has(input.expectedPublicGateway)) { + throw new Error( + `--expect-public-gateway 只支持 none / nginx / pingora-direct: ${input.expectedPublicGateway}`, + ); + } + if ( + input.expectedHealthPatrolGatewayMode && + !GATEWAY_MODES.has(input.expectedHealthPatrolGatewayMode) + ) { + throw new Error( + `--expected-health-patrol-gateway-mode 只支持 nginx 或 pingora-direct: ${input.expectedHealthPatrolGatewayMode}`, + ); + } +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${flag} 缺少参数值`); + } + return value; +} + +function readBoolEnv(name) { + const value = process.env[name]; + if (value === undefined || value === null || String(value).trim() === '') { + return false; + } + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) { + return true; + } + if (['0', 'false', 'no', 'off'].includes(normalized)) { + return false; + } + throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); +} + +function parseOptionalPositiveInt(raw, fallback, label) { + if (raw === undefined || raw === null || String(raw).trim() === '') { + return fallback; + } + return parseRequiredPositiveInt(raw, label); +} + +function parseRequiredPositiveInt(raw, label) { + const text = String(raw ?? '').trim(); + if (!/^[1-9][0-9]*$/u.test(text)) { + throw new Error(`${label} 必须是正整数。`); + } + return Number.parseInt(text, 10); +} + +function validateSafeAbsoluteFilePath(value, label) { + validateNoControlCharacters(value, label); + if (!path.isAbsolute(value)) { + throw new Error(`${label} 必须是绝对路径。`); + } + if (isFilesystemRootPath(value)) { + throw new Error(`${label} 不能是文件系统根目录。`); + } +} + +function validateNoControlCharacters(value, label) { + if (/[\0\r\n]/u.test(String(value))) { + throw new Error(`${label} 不能包含换行或 NUL 字符。`); + } +} + +function isFilesystemRootPath(value) { + const resolved = path.resolve(String(value)); + return resolved === path.parse(resolved).root; +} + +async function buildStatus(input) { + const healthPatrolEnv = await inspectHealthPatrolEnv(input); + const pingoraEnv = await inspectPingoraEnv(input); + const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv); + delete healthPatrolEnv.secretValues; + delete pingoraEnv.secretValues; + const services = await inspectServices(input); + const systemd = await inspectPingoraSystemd(input); + const ports = await inspectPorts(input); + const realpathCanary = await inspectRealpathCanary(input, ports); + const currentReleaseAudit = await inspectCurrentRelease(input, secrets); + const publicBoundary = inspectPublicBoundary(input, ports, systemd); + const checks = [ + healthPatrolEnv.status, + pingoraEnv.status, + services.status, + systemd.status, + ports.status, + realpathCanary.status, + currentReleaseAudit.status, + publicBoundary.status, + ]; + + return { + schemaVersion: 1, + generatedAt: new Date().toISOString(), + summary: summarize(checks), + releaseRoot: input.releaseRoot, + expectations: { + publicGateway: input.expectedPublicGateway, + healthPatrolGatewayMode: expectedHealthPatrolGatewayMode(input), + requireRealpathCanary: input.requireRealpathCanary, + requirePingoraShadow: input.requirePingoraShadow, + requireCurrentReleaseGateway: input.requireCurrentReleaseGateway, + }, + publicBoundary, + ports, + services, + systemd, + healthPatrolEnv, + pingoraEnv, + realpathCanary, + currentReleaseAudit, + }; +} + +async function inspectHealthPatrolEnv(input) { + const parsed = await readEnvFile(input.healthPatrolEnvFile); + if (parsed.status === 'CRITICAL') { + return parsed; + } + + const values = parsed.values; + const expectedMode = expectedHealthPatrolGatewayMode(input); + const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || ''; + const diagnostics = [...parsed.diagnostics]; + let status = parsed.status; + + if (expectedMode && gatewayMode !== expectedMode) { + diagnostics.push( + `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${expectedMode}，实际 ${gatewayMode || '(空)'}`, + ); + status = maxStatus(status, 'CRITICAL'); + } + + return { + path: input.healthPatrolEnvFile, + status, + secretValues: collectSecretValuesFromEnv(values), + values: { + gatewayMode, + publicBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '', + publicHost: values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '', + apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '', + pingoraBaseUrl: + values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', + hasPingoraProbeToken: Boolean( + values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || + values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN, + ), + }, + diagnostics, + }; +} + +function expectedHealthPatrolGatewayMode(input) { + if (input.expectedHealthPatrolGatewayMode) { + return input.expectedHealthPatrolGatewayMode; + } + if (input.expectedPublicGateway === 'nginx') { + return 'nginx'; + } + if (input.expectedPublicGateway === 'pingora-direct') { + return 'pingora-direct'; + } + return ''; +} + +async function inspectPingoraEnv(input) { + const parsed = await readEnvFile(input.pingoraEnvFile); + if (parsed.status === 'CRITICAL') { + return { + ...parsed, + values: {}, + }; + } + const values = parsed.values; + return { + path: input.pingoraEnvFile, + status: parsed.status, + secretValues: collectSecretValuesFromEnv(values), + values: { + listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '', + tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '', + httpRedirectListen: + values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '', + tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '', + hasTlsKeyFile: Boolean( + values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE, + ), + hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN), + accessLogFile: + values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '', + compressionAlgorithms: + values.GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS || '', + trustXForwardedFor: + values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '', + }, + diagnostics: parsed.diagnostics, + }; +} + +async function readEnvFile(filePath) { + let text; + try { + text = await readFile(filePath, 'utf8'); + } catch (error) { + return { + path: filePath, + status: 'CRITICAL', + values: {}, + diagnostics: [`无法读取 env 文件: ${error.message}`], + }; + } + + const values = {}; + const diagnostics = []; + for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { + let line = rawLine.trim(); + if (!line || line.startsWith('#')) { + continue; + } + if (line.startsWith('export ')) { + line = line.slice('export '.length).trim(); + } + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`); + continue; + } + const key = line.slice(0, equalsIndex).trim(); + values[key] = stripQuotes(line.slice(equalsIndex + 1).trim()); + } + + return { + path: filePath, + status: diagnostics.length > 0 ? 'WARNING' : 'OK', + values, + diagnostics, + }; +} + +function stripQuotes(value) { + if (value.length >= 2) { + const first = value[0]; + const last = value[value.length - 1]; + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return value.slice(1, -1); + } + } + return value; +} + +function collectSecretValuesFromEnv(values) { + const secrets = []; + for (const [key, value] of Object.entries(values || {})) { + if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') { + continue; + } + const trimmed = value.trim(); + if (trimmed.length > 0) { + secrets.push(trimmed); + } + } + return secrets; +} + +function collectSecretValues(...envSnapshots) { + const secrets = new Set(); + for (const snapshot of envSnapshots) { + for (const value of snapshot?.secretValues || []) { + if (typeof value === 'string' && value.trim()) { + secrets.add(value.trim()); + } + } + } + return [...secrets].sort((left, right) => right.length - left.length); +} + +async function inspectServices(input) { + const serviceNames = [ + 'nginx.service', + 'genarrative-pingora-gateway.service', + 'genarrative-api.service', + 'spacetimedb.service', + ]; + const services = []; + for (const service of serviceNames) { + services.push(await inspectService(service, input)); + } + return { + status: summarize(services.map((service) => service.status)).status, + services, + }; +} + +async function inspectService(service, input) { + const result = await runCommand('systemctl', ['is-active', service], input); + const state = result.stdout.trim() || result.stderr.trim() || result.error; + let status = 'OK'; + if (service === 'nginx.service' && input.expectedPublicGateway === 'nginx') { + status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL'; + } else if ( + service === 'genarrative-pingora-gateway.service' && + (input.requirePingoraShadow || + input.expectedPublicGateway === 'pingora-direct') + ) { + status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL'; + } else if ( + (service === 'genarrative-api.service' || + service === 'spacetimedb.service') && + result.code !== 0 + ) { + status = 'WARNING'; + } + return { + name: service, + activeState: state || 'unknown', + status, + command: result.command, + }; +} + +async function inspectPingoraSystemd(input) { + const catResult = await runCommand( + 'systemctl', + ['cat', 'genarrative-pingora-gateway.service'], + input, + ); + const showResult = await runCommand( + 'systemctl', + [ + 'show', + 'genarrative-pingora-gateway.service', + '--property=FragmentPath', + '--property=DropInPaths', + '--property=User', + '--property=ExecStart', + '--no-pager', + ], + input, + ); + const unitText = catResult.stdout; + const hasAmbientCapability = unitText.includes( + 'AmbientCapabilities=CAP_NET_BIND_SERVICE', + ); + const hasCapabilityBoundingSet = unitText.includes( + 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', + ); + const environmentFiles = [ + ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu), + ].map((match) => match[1].trim()); + const environmentFileMatchesPingoraEnvFile = environmentFilesInclude( + environmentFiles, + input.pingoraEnvFile, + ); + const show = parseSystemctlShow(showResult.stdout); + const diagnostics = []; + let status = 'OK'; + + if (catResult.code !== 0) { + diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`); + status = 'CRITICAL'; + } + if (showResult.code !== 0) { + diagnostics.push( + `systemctl show 失败: ${showResult.stderr || showResult.error}`, + ); + status = 'CRITICAL'; + } + if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) { + diagnostics.push( + `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`, + ); + status = 'CRITICAL'; + } + if ( + input.expectedPublicGateway === 'nginx' && + (hasAmbientCapability || hasCapabilityBoundingSet) + ) { + diagnostics.push( + 'Nginx 接公网阶段不应残留 Pingora 低端口 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + if ( + input.expectedPublicGateway === 'pingora-direct' && + (!hasAmbientCapability || !hasCapabilityBoundingSet) + ) { + diagnostics.push( + 'pingora-direct 接公网阶段必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。', + ); + status = 'CRITICAL'; + } + + return { + status, + hasAmbientCapability, + hasCapabilityBoundingSet, + environmentFiles, + environmentFileMatchesPingoraEnvFile, + fragmentPath: show.FragmentPath || '', + dropInPaths: show.DropInPaths || '', + user: show.User || '', + execStart: show.ExecStart || '', + diagnostics, + commands: [catResult.command, showResult.command], + }; +} + +function environmentFilesInclude(environmentFiles, expectedPath) { + return environmentFiles.some((entry) => + splitSystemdEnvironmentFileEntry(entry).includes(expectedPath), + ); +} + +function splitSystemdEnvironmentFileEntry(entry) { + const files = []; + for (const word of String(entry || '').split(/\s+/u)) { + if (!word) { + continue; + } + let normalized = word.trim(); + if (normalized.startsWith('-')) { + normalized = normalized.slice(1); + } + if ( + (normalized.startsWith('"') && normalized.endsWith('"')) || + (normalized.startsWith("'") && normalized.endsWith("'")) + ) { + normalized = normalized.slice(1, -1); + } + if (normalized) { + files.push(normalized); + } + } + return files; +} + +function parseSystemctlShow(text) { + const result = {}; + for (const line of text.split(/\r?\n/u)) { + const equalsIndex = line.indexOf('='); + if (equalsIndex <= 0) { + continue; + } + result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); + } + return result; +} + +async function inspectPorts(input) { + const result = await runCommand('ss', ['-H', '-ltnp'], input); + const listeners = result.code === 0 ? parseSsOutput(result.stdout) : []; + const byPort = {}; + const statuses = []; + for (const port of DEFAULT_PORTS) { + const portListeners = listeners.filter((item) => item.port === port); + const entry = inspectPort(port, portListeners, input); + byPort[String(port)] = entry; + statuses.push(entry.status); + } + if (result.code !== 0) { + statuses.push('CRITICAL'); + } + return { + status: summarize(statuses).status, + command: result.command, + error: result.code === 0 ? '' : trimForJson(result.stderr || result.error), + byPort, + }; +} + +function parseSsOutput(stdout) { + const listeners = []; + for (const line of String(stdout || '').split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed) { + continue; + } + const parts = trimmed.split(/\s+/u); + const local = parts[3] || ''; + const port = parsePortFromLocalAddress(local); + if (!port) { + continue; + } + const processText = parts.slice(5).join(' '); + const processNames = [ + ...new Set([...processText.matchAll(/"([^"]+)"/gu)].map((match) => match[1])), + ]; + listeners.push({ + localAddress: local, + port, + loopback: isLoopbackLocalAddress(local), + processNames, + }); + } + return listeners; +} + +function parsePortFromLocalAddress(value) { + const bracketMatch = String(value).match(/^\[[^\]]+\]:(\d+)$/u); + const plainMatch = String(value).match(/:(\d+)$/u); + const portText = bracketMatch?.[1] || plainMatch?.[1] || ''; + const port = Number.parseInt(portText, 10); + return Number.isInteger(port) && port > 0 ? port : null; +} + +function isLoopbackLocalAddress(value) { + const text = String(value || '').toLowerCase(); + return ( + text.startsWith('127.') || + text.startsWith('[::1]') || + text.startsWith('::1') || + text.startsWith('localhost:') + ); +} + +function inspectPort(port, listeners, input) { + const hasNginx = listeners.some((listener) => + listener.processNames.some((name) => name.toLowerCase().includes('nginx')), + ); + const hasPingora = listeners.some((listener) => + listener.processNames.some((name) => name.toLowerCase().includes('pingora')), + ); + const hasUnknownProcess = + listeners.length > 0 && + listeners.some((listener) => listener.processNames.length === 0); + const loopbackOnly = + listeners.length > 0 && listeners.every((listener) => listener.loopback); + const diagnostics = []; + let status = 'OK'; + + if ([80, 443].includes(port)) { + if (input.expectedPublicGateway === 'nginx') { + if (listeners.length === 0) { + diagnostics.push(`${port} 未监听，Nginx 公网入口未就绪。`); + status = 'CRITICAL'; + } + if (!hasNginx) { + diagnostics.push(`${port} 未看到 Nginx 监听。`); + status = 'CRITICAL'; + } + if (hasPingora) { + diagnostics.push(`${port} 已被 Pingora 监听，不能作为未切公网彩排状态。`); + status = 'CRITICAL'; + } + } + if (input.expectedPublicGateway === 'pingora-direct') { + if (!hasPingora) { + diagnostics.push(`${port} 未看到 Pingora 监听。`); + status = 'CRITICAL'; + } + if (hasNginx) { + diagnostics.push(`${port} 仍由 Nginx 监听。`); + status = 'CRITICAL'; + } + } + } + + if (port === 18081 && input.requirePingoraShadow) { + if (!hasPingora) { + diagnostics.push('18081 未看到 Pingora shadow 监听。'); + status = 'CRITICAL'; + } + if (!loopbackOnly) { + diagnostics.push('Pingora shadow 18081 必须只监听 loopback。'); + status = 'CRITICAL'; + } + } + + if (port === 18083 && input.requireRealpathCanary) { + if (!hasNginx) { + diagnostics.push('18083 未看到 Nginx realpath canary 监听。'); + status = 'CRITICAL'; + } + if (!loopbackOnly) { + diagnostics.push('Nginx realpath canary 18083 必须只监听 loopback。'); + status = 'CRITICAL'; + } + } + + if ( + hasUnknownProcess && + ([80, 443].includes(port) || + (port === 18081 && input.requirePingoraShadow) || + (port === 18083 && input.requireRealpathCanary)) + ) { + diagnostics.push('ss 未返回进程名，无法确认监听归属。'); + status = maxStatus(status, 'WARNING'); + } + + return { + port, + status, + listening: listeners.length > 0, + loopbackOnly, + hasNginx, + hasPingora, + hasUnknownProcess, + listeners, + diagnostics, + }; +} + +async function inspectRealpathCanary(input, ports) { + const diagnostics = []; + let status = 'OK'; + let exists = false; + let templateLooksValid = false; + try { + const fileStat = await stat(input.realpathConfigFile); + exists = fileStat.isFile(); + } catch { + exists = false; + } + + if (!exists) { + diagnostics.push(`realpath canary 配置不存在: ${input.realpathConfigFile}`); + status = input.requireRealpathCanary ? 'CRITICAL' : 'WARNING'; + } else { + const content = await readFile(input.realpathConfigFile, 'utf8'); + const requiredSnippets = [ + 'listen 127.0.0.1:18083', + 'genarrative-pingora-realpath-canary.access.log', + 'X-Genarrative-Nginx-Handoff pingora-realpath-canary', + ]; + const missing = requiredSnippets.filter((snippet) => !content.includes(snippet)); + templateLooksValid = missing.length === 0; + if (missing.length > 0) { + diagnostics.push(`realpath canary 配置缺少关键片段: ${missing.join(', ')}`); + status = 'CRITICAL'; + } + } + + const portStatus = ports.byPort['18083']; + if (input.requireRealpathCanary && portStatus?.status === 'CRITICAL') { + status = 'CRITICAL'; + } + + return { + path: input.realpathConfigFile, + status, + exists, + templateLooksValid, + port: portStatus, + diagnostics, + }; +} + +async function inspectCurrentRelease(input, secrets) { + const script = path.join( + input.releaseRoot, + 'scripts/ops/pingora-current-release-audit.mjs', + ); + try { + await access(script, fsConstants.R_OK); + } catch (error) { + return { + status: 'CRITICAL', + script, + code: null, + diagnostics: [`无法读取 current release 自审脚本: ${error.message}`], + }; + } + + const args = [ + '--', + script, + '--release-root', + input.releaseRoot, + '--systemd-show', + '--timeout-ms', + String(input.timeoutMs), + ]; + if (input.requireCurrentReleaseGateway) { + args.push('--require-pingora-gateway'); + } + const result = await runCommand('node', args, input); + const parsed = parseJsonObject(result.stdout); + let status = result.code === 0 ? 'OK' : 'CRITICAL'; + const diagnostics = []; + if (!parsed.ok) { + diagnostics.push(`无法解析 current release 自审 JSON: ${parsed.error}`); + status = 'CRITICAL'; + } + return { + status, + script, + code: result.code, + command: result.command, + summary: parsed.ok ? parsed.value?.summary || null : null, + pingoraGateway: parsed.ok ? parsed.value?.pingoraGateway || null : null, + releaseManifest: parsed.ok ? parsed.value?.releaseManifest || null : null, + stdout: trimForJson(redactSecrets(result.stdout, secrets)), + stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)), + diagnostics, + }; +} + +function inspectPublicBoundary(input, ports, systemd) { + const diagnostics = []; + let status = 'OK'; + if (input.expectedPublicGateway === 'none') { + return { + status, + expectedPublicGateway: input.expectedPublicGateway, + diagnostics, + }; + } + + for (const port of [80, 443]) { + const portStatus = ports.byPort[String(port)]; + if (portStatus?.status === 'CRITICAL') { + status = 'CRITICAL'; + diagnostics.push(...portStatus.diagnostics); + } + } + if (systemd.status === 'CRITICAL') { + status = 'CRITICAL'; + diagnostics.push(...systemd.diagnostics); + } + + return { + status, + expectedPublicGateway: input.expectedPublicGateway, + diagnostics, + }; +} + +function runCommand(command, args, input, env = process.env) { + validateNoControlCharacters(command, '子命令可执行文件'); + for (const arg of args) { + validateNoControlCharacters(arg, '子命令参数'); + } + return new Promise((resolve) => { + execFile( + command, + args, + { + env, + timeout: input.timeoutMs, + windowsHide: true, + maxBuffer: 1024 * 1024, + }, + (error, stdout, stderr) => { + resolve({ + command: formatCommand(command, args), + code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, + stdout: String(stdout || ''), + stderr: String(stderr || ''), + timedOut: Boolean(error?.killed), + error: error ? error.message : '', + }); + }, + ); + }); +} + +function formatCommand(command, args) { + return [command, ...args].join(' '); +} + +function redactSecrets(value, secrets) { + let text = String(value || ''); + for (const secret of secrets) { + if (!secret) { + continue; + } + text = text.split(secret).join(''); + } + return text; +} + +function trimForJson(value) { + const text = String(value || '').trim(); + if (text.length <= 1000) { + return text; + } + return `${text.slice(0, 1000)}...`; +} + +function parseJsonObject(value) { + try { + return { ok: true, value: JSON.parse(value) }; + } catch (error) { + return { ok: false, error: error.message }; + } +} + +function summarize(statuses) { + const normalized = statuses.filter(Boolean); + const status = normalized.reduce( + (current, item) => maxStatus(current, item), + 'OK', + ); + return { + status, + criticalCount: normalized.filter((item) => item === 'CRITICAL').length, + warningCount: normalized.filter((item) => item === 'WARNING').length, + }; +} + +function maxStatus(left, right) { + return STATUS_RANK[right] > STATUS_RANK[left] ? right : left; +} From 10e559701f601c78601f25778d3022a2b2bff35b Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 14:33:54 +0800 Subject: [PATCH 16/22] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20Pingora=20runtime-on?= =?UTF-8?q?ly=20=E5=BD=A9=E6=8E=92=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 current release runtime-only 基础门禁接入直连彩排状态复核 保留 /opt/genarrative/current 调用路径，避免 symlink 展开导致 systemd ExecStart 自审误判 补充 release readiness plan 自测和 Pingora 运维文档 在 dev 服务器真实验证 runtime-only 门禁、Nginx 公网边界、Pingora shadow 和 realpath canary --- deploy/nginx/README.md | 2 +- .../shared-memory/decision-log.md | 2 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 4 +- .../check-pingora-release-readiness-plan.mjs | 146 +++++++++++++++++- scripts/check-pingora-release-readiness.mjs | 30 +++- 5 files changed, 178 insertions(+), 6 deletions(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 2e400add7..55b572897 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -71,7 +71,7 @@ - 真实路径 canary 不再手工编辑 `/etc/nginx/conf.d/`。目标机从 current release 执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`；脚本会渲染到 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，保持晚于主站 `log_format genarrative_upstream` 加载，执行 `nginx -t`、reload Nginx，并默认运行 realpath live smoke。`nginx -t`、reload 或 live smoke 任一失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会删除该 `zz-...` 文件，执行 `nginx -t` 和 reload，失败时恢复删除前配置。 - 真实路径 canary reload 后可再次运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 - 正式切换前在源码 checkout / CI / 构建环境跑 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。 -- 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁：`/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证真实路径 canary 或 Pingora 直连入口时分别追加 `--require-realpath-live ...` 或 `--require-direct ...` 参数。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 +- 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁：`/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证真实路径 canary 或 Pingora 直连入口时分别追加 `--require-realpath-live ...` 或 `--require-direct ...` 参数。未带 `--require-direct` 的 runtime-only 基础门禁会自动执行直连彩排状态复核，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过；启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 ## 验证 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 0a6cf7756..d85d12761 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -2348,7 +2348,7 @@ - 决策补充：Pingora Nginx canary snippet 变更必须运行 `npm run check:nginx-pingora-canary`；该脚本静态校验本机来源限制、handoff 响应头、probe token 占位、前缀 rewrite、低缓冲和 SpacetimeDB WebSocket Upgrade。目标 agent 或 CI 有 Nginx 时必须运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，把 snippet 包进临时 `server {}` 强制执行 `nginx -t`。 - 决策补充：Pingora 与 Nginx 的核心路由 parity 以 `deploy/pingora/nginx-route-parity.matrix.json` 为共享检查输入；涉及 Nginx 模板、Pingora 路由、限流分组或路由文档时必须同步更新矩阵，并运行 `npm run check:pingora-route-parity` 与 `cargo test -p pingora-gateway --manifest-path server-rs/Cargo.toml matches_nginx_route_parity_matrix`。Rust 单测读取同一份矩阵验证 `classify_path`、body limit 和接流保护分组，Node 检查同时覆盖生产 / 开发 Nginx 模板和试点文档片段。 - 决策补充：Pingora 前缀 canary 在目标 Nginx 中人工 include 并 reload 后，必须运行 `npm run check:pingora-canary-live`；该脚本只读访问 `__genarrative_pingora_canary` 前缀下的 healthz、代表性 API、SpacetimeDB identity、静态资源和拒绝入口，并强制校验 `X-Genarrative-Nginx-Handoff: pingora-canary`，避免只通过 snippet 静态检查却没有证明真实 handoff 链路可用。 -- 决策补充：Pingora release readiness 分为源码全量门禁和 current release runtime-only 门禁。源码 checkout / CI / 构建环境继续运行默认 `check-pingora-release-readiness.mjs`，覆盖 Cargo、npm、Docker、Nginx 静态 / 真机校验和发布包构建烟测；目标机 `/opt/genarrative/current` 的启用前基础门禁和启用后 `--require-direct` 复核必须调用随包 `scripts/check-pingora-release-readiness.mjs --release-runtime-only`，只执行 current release 自审、live canary、access log 对账、direct preflight、health patrol env 复核和 direct live smoke。API release / current release 必须随包携带 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、canary access log 对账脚本和 direct preflight / live 子脚本；runtime-only 模式不得依赖源码 checkout、npm project root、Docker 或目标机 Nginx 静态校验。 +- 决策补充：Pingora release readiness 分为源码全量门禁和 current release runtime-only 门禁。源码 checkout / CI / 构建环境继续运行默认 `check-pingora-release-readiness.mjs`，覆盖 Cargo、npm、Docker、Nginx 静态 / 真机校验和发布包构建烟测；目标机 `/opt/genarrative/current` 的启用前基础门禁和启用后 `--require-direct` 复核必须调用随包 `scripts/check-pingora-release-readiness.mjs --release-runtime-only`，只执行 current release 自审、启用前直连彩排状态、live canary、access log 对账、direct preflight、health patrol env 复核和 direct live smoke。未带 `--require-direct` 时 runtime-only 必须自动运行随包 `scripts/ops/pingora-direct-rehearsal-status.mjs --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过；启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态，改为检查 direct preflight、health patrol 直连模式和 direct live smoke。API release / current release 必须随包携带 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、canary access log 对账脚本、直连彩排状态脚本和 direct preflight / live 子脚本；runtime-only 模式不得依赖源码 checkout、npm project root、Docker 或目标机 Nginx 静态校验。 - 决策补充：Pingora 直连静态响应必须显式写入缓存头。HTML、目录 index 和 SPA fallback 默认 `Cache-Control: no-cache`；`/assets/*` 与 `/admin/assets/*` 中带 Vite 指纹文件名的资源默认 `Cache-Control: public, max-age=31536000, immutable`；非指纹静态和 ACME challenge 默认 `no-cache`。三档由 `GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL`、`GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL` 和 `GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL` 覆盖，配置值不能包含换行或 NUL；`npm run check:pingora-gateway-smoke` 必须覆盖这些缓存头，避免直连后入口 HTML 被长期缓存或指纹资源失去长期缓存收益。 ## 2026-06-11 资产计费边界改为 fail-closed 并补偿退款 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 001662fe1..debc09d7f 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -119,7 +119,7 @@ node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs `check:pingora-release-readiness` 是正式切换前的聚合门禁，默认串行执行 `cargo test -p pingora-gateway`、mock 上游 smoke、路由矩阵 parity、Nginx canary snippet 校验、Docker handoff smoke、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、current release 自审烟测、直连彩排状态烟测、release readiness 计划自检、生产运维护栏、Pingora cutover 状态快照烟测、Pingora cutover 证据包烟测、Pingora cutover 命令证据烟测、Pingora cutover 证据 manifest 验真烟测、Pingora cutover 证据根目录审计烟测、API release build 烟测、Pingora production release 真实构建烟测和 API deploy release 烟测。普通本机执行时 Docker / Nginx 能力仍按子脚本默认口径跳过；切换窗口或 CI 必须执行： -目标机 current release 上的启用前基础门禁和启用后 `--require-direct` 复核使用同一个聚合脚本，但必须追加 `--release-runtime-only`。该模式只执行发布包内可自包含的运行时复核：current release 自审、live canary、真实 access log 对账、direct preflight、health patrol env 复核和 direct live smoke；不会运行 Cargo、npm、Docker 或 Nginx 源码 / 构建环境门禁，并会拒绝 `--require-docker`、`--pull-docker` 与 `--require-nginx`。 +目标机 current release 上的启用前基础门禁和启用后 `--require-direct` 复核使用同一个聚合脚本，但必须追加 `--release-runtime-only`。该模式只执行发布包内可自包含的运行时复核：current release 自审、启用前直连彩排状态复核、live canary、真实 access log 对账、direct preflight、health patrol env 复核和 direct live smoke；不会运行 Cargo、npm、Docker 或 Nginx 源码 / 构建环境门禁，并会拒绝 `--require-docker`、`--pull-docker` 与 `--require-nginx`。未带 `--require-direct` 的启用前基础门禁会自动执行 `scripts/ops/pingora-direct-rehearsal-status.mjs --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow 与 realpath canary 高端口在线；启用后 `--require-direct` 复核不再要求 Nginx 接公网的彩排状态，改为检查 direct preflight、health patrol 直连模式和 direct live smoke。 正式 runbook 的 direct enable apply / rollback apply 命令证据必须额外传 `--expected-executable ` 和 `--require-arg --apply`；命令证据脚本会在执行前拒绝与预期脚本不一致、缺少必需 `--apply` 参数，或任一真实命令参数包含换行 / NUL 字符的真实命令，并把 `expectedExecutable` 写入 manifest / command-record。命令名只用于审计分类，不能替代真实脚本身份和真实 apply 参数校验。 @@ -196,7 +196,7 @@ npm run plan:pingora-direct-cutover -- \ runbook 会列出 Host 与回退巡检入口确认、切换前 current release 自包含自审、切换前状态快照证据包、切换前证据 manifest 只读验真、current release 随包 preflight、启用前基础门禁、direct enable dry-run、direct enable apply、启用命令证据 manifest 只读验真、用 current release 随包 `scripts/deploy/pingora-health-patrol-env-switch.mjs --apply` 把 health patrol 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后状态快照证据包、启用后证据 manifest 只读验真、启用后 `--require-direct` 复核、rollback dry-run、用同一脚本在 rollback apply 前把 health patrol 预置回 `nginx`、rollback apply、回退命令证据 manifest 只读验真、回退后 health patrol env Nginx 模式复核、回退后状态快照证据包、回退后证据 manifest 只读验真和切换证据根目录三阶段总审计。rollback dry-run / apply 默认会传 `--nginx-smoke-expect-body '"ok":true'`，因此回退 smoke 不只接受 HTTP 200，还要求 `/healthz` 响应体包含预期片段；若现场回退 smoke URL 换成其它端点，必须显式调整 `--rollback-nginx-smoke-expect-body`。启用前 current release 自审使用 `--require-pingora-gateway --systemd-show`，先确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、`release-manifest.api-server.json` 已登记 `pingora-gateway`、`pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release；启用前基础门禁不带 `--require-direct`，因为此时 systemd direct-entry drop-in 尚未生效；启用后复核必须带 `--require-direct`，若切换参数提供 `--direct-probe-token`，启用后复核也会继续透传该 token 检查内部探针，JSON runbook 只显示 ``。三个状态快照证据包都使用 current release 随包脚本，显式传 `--output-root <证据根目录>` 和同一个 `--cutover-run-id`，并带 `--require-pingora-gateway --run-health-patrol --fail-on-critical`，用于把切换前基线、启用后 direct 状态、回退后 Nginx 状态和当前发布物自审结果留成可归档证据；其中启用后证据包还会额外传 `--run-direct-live`，把 direct live smoke 的 `direct-live.json`、stdout / stderr、命令记录和 Pingora access log `request_id` 反查结果一起写入同一证据目录，且 `direct-access-log` JSON 要保留匹配数量、缺失明细和 method/path/status 漂移明细，避免只在终端输出里保留直连证据；证据包 manifest 会给已生成的 snapshot、direct live、stdout / stderr、命令记录和 parse-error 文件记录 `path`、`sizeBytes` 与 `sha256`，direct enable apply / rollback apply 通过命令证据脚本归档时，命令证据 manifest 也会给 `command.stdout.txt`、`command.stderr.txt` 和 `command-record.json` 同步记录 `sizeBytes` 与 `sha256`。每次复制或归档证据目录后，都要把对应阶段 stdout 中的 `bundleDir` 填入 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir ` 做只读验真，确认文件未在归档过程中损坏或被替换；enable / rollback apply 命令证据生成后也要分别把 stdout 中的 `bundleDir` 填入 verifier 的 `` / `` 占位符，先验真命令证据，再继续 health patrol 切换或回退后复核；三阶段证据和 enable / rollback apply 命令证据都完成并即时验真后，再运行 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-audit.mjs --evidence-root <证据根目录> --require-phase pre-cutover --require-phase post-enable --require-phase post-rollback --require-command enable-apply:pingora-direct-enable-apply --require-command rollback-apply:pingora-direct-rollback-apply --require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --require-command-arg enable-apply:pingora-direct-enable-apply:--apply --require-command-arg rollback-apply:pingora-direct-rollback-apply:--apply --require-cutover-run-id <本次cutoverRunId> --timeline-max-span-ms 86400000` 生成总审计 JSON，并在 JSON 中记录 `requiredCutoverRunId`、`requiredCommandExecutables`、`requiredCommandArgs`、`timeline.maxSpanMs` 与 `timeline.spanMs`。若 snapshot 或 direct live stdout 解析失败，证据包会保留对应 `*-parse-error.txt` 并写入 manifest 索引。`--cutover-release-root` 与 `--cutover-evidence-output-root` 必须是绝对路径，且不能是文件系统根目录，证据根目录默认 `/var/log/genarrative/pingora-cutover-evidence`；`--cutover-evidence-timeline-max-span-ms` 默认 `86400000`，仅用于明确批准超过 24 小时的维护窗口。 -runbook 中“启用前基础门禁”和“启用后 `--require-direct` 复核”都调用 current release 随包 `scripts/check-pingora-release-readiness.mjs --release-runtime-only`。前者不带 `--require-direct`，因为 direct-entry drop-in 尚未生效；后者必须带 `--require-direct`，并复核 direct preflight、health patrol env 直连模式和 direct live smoke。 +runbook 中“启用前基础门禁”和“启用后 `--require-direct` 复核”都调用 current release 随包 `scripts/check-pingora-release-readiness.mjs --release-runtime-only`。前者不带 `--require-direct`，因为 direct-entry drop-in 尚未生效，并会额外跑直连彩排状态复核，确认 Nginx 仍接公网、Pingora shadow / realpath canary 高端口和 current release 自审均可用；后者必须带 `--require-direct`，不再要求 Nginx 接公网彩排状态，并复核 direct preflight、health patrol env 直连模式和 direct live smoke。 证据 verifier / 总审计的入口路径、verifier 脚本路径和 manifest 登记文件名都不能包含换行或 NUL 字符，避免污染 JSON 证据、终端输出或归档复盘；遇到此类失败应重新生成证据或修正 runbook 参数，不要手工改 manifest 兜底。 diff --git a/scripts/check-pingora-release-readiness-plan.mjs b/scripts/check-pingora-release-readiness-plan.mjs index 89b9c20bd..b7e434af2 100644 --- a/scripts/check-pingora-release-readiness-plan.mjs +++ b/scripts/check-pingora-release-readiness-plan.mjs @@ -1,7 +1,14 @@ #!/usr/bin/env node import { spawnSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -89,6 +96,8 @@ function main() { assertDefaultPlanIncludesCanaryAccessLogParitySmoke(); assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke(); assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts(); + assertReleaseRuntimeOnlyPlanIncludesDirectRehearsalStatus(); + assertReleaseRuntimeOnlyKeepsInvokedCurrentSymlinkPath(); assertReleaseRuntimeOnlyRejectsSourceOnlyFlags(); assertDryRunCutoverPlanIncludesDirectEnableAndRollbackRunbook(); assertDryRunCutoverDefaultRollbackBodyIgnoresProcessEnv(); @@ -2194,6 +2203,141 @@ function assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts() { } } +function assertReleaseRuntimeOnlyPlanIncludesDirectRehearsalStatus() { + const basePlan = readPlan(['--release-runtime-only', '--dry-run-plan']); + const baseStep = findStep( + basePlan, + '目标 Pingora direct rehearsal 状态复核', + ); + if (!baseStep) { + failures.push( + 'release runtime-only 基础计划必须包含直连彩排状态复核。', + ); + return; + } + assertAbsoluteScriptArg( + baseStep.args, + 'scripts/ops/pingora-direct-rehearsal-status.mjs', + 'release runtime-only 直连彩排状态复核必须使用 current release 随包脚本。', + ); + assertIncludes( + baseStep.args, + '--release-root', + 'release runtime-only 直连彩排状态复核必须显式传 current release 根目录。', + ); + assertIncludes( + baseStep.args, + '--expect-public-gateway', + 'release runtime-only 直连彩排状态复核必须声明公网入口期望。', + ); + assertIncludes( + baseStep.args, + 'nginx', + 'release runtime-only 直连彩排状态复核必须确认 Nginx 仍接公网。', + ); + assertIncludes( + baseStep.args, + '--require-pingora-shadow', + 'release runtime-only 直连彩排状态复核必须要求 Pingora shadow 高端口在线。', + ); + assertIncludes( + baseStep.args, + '--require-realpath-canary', + 'release runtime-only 直连彩排状态复核必须要求 realpath canary 高端口在线。', + ); + assertIncludes( + baseStep.args, + '--require-current-release-gateway', + 'release runtime-only 直连彩排状态复核必须复用 current release Pingora 自审。', + ); + assertIncludes( + baseStep.args, + '--fail-on-critical', + 'release runtime-only 直连彩排状态复核出现 CRITICAL 必须阻断门禁。', + ); + + const directPlan = readPlan([ + ...requireDirectBaseArgs, + '--release-runtime-only', + '--dry-run-plan', + ]); + if (findStep(directPlan, '目标 Pingora direct rehearsal 状态复核')) { + failures.push( + 'release runtime-only --require-direct 阶段不能继续要求 Nginx 接公网的彩排状态。', + ); + } +} + +function assertReleaseRuntimeOnlyKeepsInvokedCurrentSymlinkPath() { + const tmpRoot = mkdtempSync( + path.join(tmpdir(), 'genarrative-release-readiness-current-'), + ); + try { + const releaseDir = path.join(tmpRoot, 'releases', 'test-release'); + const currentLink = path.join(tmpRoot, 'current'); + const scriptDir = path.join(releaseDir, 'scripts'); + mkdirSync(scriptDir, { recursive: true }); + symlinkSync(releaseDir, currentLink, 'dir'); + const invokedScript = path.join( + currentLink, + 'scripts', + 'check-pingora-release-readiness.mjs', + ); + const realScript = path.join( + scriptDir, + 'check-pingora-release-readiness.mjs', + ); + symlinkSync( + path.join(process.cwd(), 'scripts/check-pingora-release-readiness.mjs'), + realScript, + ); + + const result = spawnSync( + 'node', + [invokedScript, '--release-runtime-only', '--dry-run-plan'], + { + cwd: process.cwd(), + encoding: 'utf8', + env: readinessPlanEnv(), + }, + ); + if ((result.status ?? 0) !== 0) { + failures.push( + `通过 current symlink 读取 runtime-only plan 失败，退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + return; + } + const plan = JSON.parse(result.stdout); + const audit = findStep(plan, 'current release 自包含自审'); + const rehearsal = findStep(plan, '目标 Pingora direct rehearsal 状态复核'); + for (const [label, step] of [ + ['current release 自审', audit], + ['直连彩排状态复核', rehearsal], + ]) { + if (!step) { + failures.push( + `通过 current symlink 生成的 runtime-only plan 缺少${label}步骤。`, + ); + continue; + } + assertIncludes( + step.args, + currentLink, + `${label}必须保留调用方 current symlink 作为 release root，而不是展开到真实 release 目录。`, + ); + if (step.args.includes(releaseDir)) { + failures.push( + `${label}不应把 current symlink 展开成真实 release 目录: ${releaseDir}`, + ); + } + } + } catch (error) { + failures.push(`current symlink runtime-only plan 自测异常: ${error.message}`); + } finally { + rmSync(tmpRoot, { recursive: true, force: true }); + } +} + function assertReleaseRuntimeOnlyRejectsSourceOnlyFlags() { for (const flag of ['--require-docker', '--pull-docker', '--require-nginx']) { const result = runReadinessExpectFailure([ diff --git a/scripts/check-pingora-release-readiness.mjs b/scripts/check-pingora-release-readiness.mjs index 61491e811..126e95a18 100644 --- a/scripts/check-pingora-release-readiness.mjs +++ b/scripts/check-pingora-release-readiness.mjs @@ -6,7 +6,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; const repoRoot = process.cwd(); -const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const scriptDir = path.dirname(resolveInvokedScriptPath()); const releaseRoot = path.resolve(scriptDir, '..'); const DEFAULT_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; const DEFAULT_CUTOVER_RUN_ID_PREFIX = 'pingora-direct-'; @@ -46,6 +46,14 @@ if (failures.length > 0) { console.log('\n[pingora-release-readiness] 通过'); +function resolveInvokedScriptPath() { + const invoked = process.argv[1]; + if (invoked) { + return path.resolve(invoked); + } + return fileURLToPath(import.meta.url); +} + function parseArgs(argv) { const result = { requireDocker: false, @@ -1802,6 +1810,26 @@ function buildReleaseRuntimeSteps(config) { }, ]; + if (!config.requireDirect) { + steps.push({ + name: '目标 Pingora direct rehearsal 状态复核', + command: 'node', + args: [ + '--', + releaseScriptPath('scripts/ops/pingora-direct-rehearsal-status.mjs'), + '--release-root', + releaseRoot, + '--expect-public-gateway', + 'nginx', + '--require-pingora-shadow', + '--require-realpath-canary', + '--require-current-release-gateway', + '--fail-on-critical', + ], + cwd: releaseRoot, + }); + } + appendTargetLiveSteps(steps, config, releaseScriptPath); appendTargetRealpathLiveSteps(steps, config, releaseScriptPath); appendTargetDirectSteps(steps, config, releaseScriptPath); From 0cc2a79c7bff9743c940579f0688a81dfc2d9166 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 14:53:03 +0800 Subject: [PATCH 17/22] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E6=8B=89=E8=B5=B7=20Pi?= =?UTF-8?q?ngora=20shadow=20=E5=8F=91=E5=B8=83=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 让 API deploy 在发布包包含 Pingora 时先校验 shadow systemd 和 env 安全边界 在 shadow 配置安全时自动启动或重启 genarrative-pingora-gateway.service 拒绝 direct-entry capability、公网监听 env，并在 current 切换前失败 补充 production-api-deploy 动态烟测、生产运维护栏和 Pingora 运维文档 在 dev 服务器真实验证 shadow inactive 场景可由发布脚本自动拉起 --- .../shared-memory/decision-log.md | 2 +- docs/project-memory/shared-memory/pitfalls.md | 4 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 4 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 2 +- scripts/check-production-api-deploy.mjs | 161 +++++++++++++++--- scripts/check-production-ops-guardrails.mjs | 40 ++++- scripts/deploy/production-api-deploy.sh | 109 ++++++++++-- 7 files changed, 278 insertions(+), 44 deletions(-) diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index d85d12761..35f2461da 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -2329,7 +2329,7 @@ - 决策补充：`/api` 通用路由必须同时检查 `Content-Length` 与实际流式请求体累计字节数；缺少长度头时超过上限也返回统一 `PAYLOAD_TOO_LARGE` JSON。影子部署模板使用 `deploy/systemd/genarrative-pingora-gateway.service`，默认读取 `/etc/genarrative/pingora-gateway.env`，仍只监听本机高端口；`/__genarrative_pingora/healthz` 只在配置并匹配 `X-Genarrative-Pingora-Probe` token 时返回 shadow JSON。 - 决策补充：生产 `genarrative-health-patrol.service` 只在显式配置 `GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL` 与 `GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN` 时检查 Pingora shadow probe；未配置时巡检口径不变。Pingora shadow 日志必须保留 request/route/upstream/body 字段，方便和 Nginx access log 做 canary 对照。 - 决策补充：生产健康巡检的公网入口模式必须显式区分 `nginx` 和 `pingora-direct`。默认 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 检查 API、SpacetimeDB 和 Nginx；Pingora 直连接管公网后切到 `pingora-direct`，改为检查 API、SpacetimeDB 和 `genarrative-pingora-gateway.service`，不再要求 `nginx.service` active。目标机本机探测 `127.0.0.1` 时用 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>` 保留正式 Host / vhost 语义。 -- 决策补充：Pingora 影子网关产物不进入默认 API release；只有显式传 `--include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，才构建并打包 `pingora-gateway` / `pingora-gateway.sha256`。真实构建 Pingora 前必须先检查 `cmake`、C 编译器和 C++ 编译器；Jenkins 勾选 `INCLUDE_PINGORA_GATEWAY` 时也要先 fail-fast 检查这些工具，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。`production-api-deploy.sh` 仅在两者同时存在时校验并复制到 current release，避免现有 API 流水线被 Pingora 构建依赖影响。若发布包包含 Pingora 且 `genarrative-pingora-gateway.service` 已经处于 `active`，API deploy 在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制；若服务未运行则只跳过，不主动拉起 Pingora。`npm run check:production-api-release` 必须同时验证默认 API release 不登记 Pingora，以及显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest。 +- 决策补充：Pingora 影子网关产物不进入默认 API release；只有显式传 `--include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，才构建并打包 `pingora-gateway` / `pingora-gateway.sha256`。真实构建 Pingora 前必须先检查 `cmake`、C 编译器和 C++ 编译器；Jenkins 勾选 `INCLUDE_PINGORA_GATEWAY` 时也要先 fail-fast 检查这些工具，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。`production-api-deploy.sh` 仅在两者同时存在时校验并复制到 current release，避免现有 API 流水线被 Pingora 构建依赖影响。发布包包含 Pingora 时，API deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载同一份 current release 网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。`npm run check:production-api-release` 必须同时验证默认 API release 不登记 Pingora，以及显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 并写入 manifest。 - 决策补充：生产健康巡检的显式 `--timeout-ms`、`--slow-ms`、`GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS` 和 `GENARRATIVE_HEALTH_PATROL_SLOW_MS` 必须是正整数，非法值直接失败，不静默回退默认 `5000ms` / `3000ms`。Pingora canary live 的 `--timeout-ms` / `GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS`、direct live 的 `--timeout-ms` / `GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS`、canary access log 对账的 `--since-lines` / `GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES` 也必须正整数。Pingora direct live 和 release readiness 读取的直连布尔 env 必须严格解析，只接受 `true/false`、`1/0`、`yes/no`、`on/off` 或空值，非法值直接失败，避免 `REQUIRE_WSS_UPGRADE`、preflight 开关或 `SKIP_WSS` 因拼写错误被当成 false。canary live 的 base URL、prefix、Host、额外 path 和 timeout 不能包含换行或 NUL；脚本必须在发起 canary 请求前失败，避免污染参数进入 URL、Host header 或 JSON 输出。canary access log 对账的日志路径、prefix、必需路径、tail 行数以及日志行中解析出的 URI / path 也不能包含换行或 NUL；脚本必须失败并给出对应参数或日志行诊断，不能把污染值写入 JSON 对账输出。direct live 的 HTTPS / HTTP base URL、Host、redirect Host、probe token、额外 path、SpacetimeDB 数据库名、access log 路径、timeout 和布尔 env 都不能包含换行或 NUL；脚本必须在发起 HTTPS / HTTP / WSS 请求前失败，避免污染参数进入请求头、URL、日志对账或 JSON 证据。Pingora 切换窗口调整巡检、live smoke、日志对账阈值或直连布尔开关时，把参数解析失败视为配置错误，而不是继续执行检查。 - 决策补充：即使不打包 Pingora 二进制，API release 也必须随包携带 Pingora release readiness 聚合门禁、直连启用 / 回退 / preflight / live smoke / current release 自审脚本、直连彩排状态脚本，以及 `deploy/systemd/`、`deploy/pingora/` 支撑配置；`pingora-direct-enable.sh` 和正式 cutover runbook 默认从 `/opt/genarrative/current` 推导这些路径，启用前 release readiness 基础门禁和启用后 `--require-direct` 复核也必须调用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs`，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。API release 还必须携带 `build//scripts/deploy/production-api-deploy.sh` 和同目录 `maintenance-on.sh` / `maintenance-off.sh`；`Genarrative-Api-Deploy` 只能复制并执行 build 产物内的 deploy 脚本，禁止继续执行部署工作区根部脚本，避免 workspace 中的旧脚本掩盖发布包布局缺陷。`production-api-deploy.sh` 对数据库备份脚本、健康巡检脚本和 Pingora 直连依赖都执行 fail-fast，发布产物缺失时保持维护模式并停止部署，不再从部署工作区兜底复制；API deploy 必须要求 `--release-root`、`--current-link`、`--api-env-file` 使用绝对路径，且 `--version` 必须以数字或字母开头并拒绝点目录，再先写 `${RELEASE_ROOT}/.${VERSION}.staging.$`，全部复制完成后再用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}`，并用固定替换语义切换 current 符号链接，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，失败时清理 staging 且不留下正式 release；`npm run check:production-api-release` 与 `npm run check:production-api-deploy` 必须进入 `check:pingora-release-readiness` 聚合门禁，前者用临时 `CARGO_TARGET_DIR` 和假 `api-server` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 产物自包含，后者用临时 release 和 fake `systemctl` / `curl` 验证从发布产物内执行 deploy 脚本后 current release 自包含，并覆盖缺少备份脚本、健康巡检脚本、release readiness 聚合门禁脚本、current release 自审脚本、直连彩排状态脚本、direct live smoke 脚本、相对 release root / current link / api env file、点目录或点开头 version、既有 release 目录、目录型 current 或提升前 release 目录竞态时的失败维护模式。 - 决策补充：正式直连 runbook 在采集状态快照前必须先执行 current release 自审：`/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`。该脚本只读检查发布包自包含、`pingora-gateway` 可执行，以及 systemd `ExecStart` 是否指向 current release 网关二进制；失败时应先修发布包、Jenkins 归档过滤、deploy 复制或 systemd 指向，再继续切换。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 01b2673b0..df4a1823d 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -18,7 +18,7 @@ ## Pingora 直连 80/443 不能只改 env - 现象：`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`，但 `genarrative-pingora-gateway.service` 启动失败，日志出现低端口绑定权限错误。 -- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`pingora-direct-rehearsal-status.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 且服务已 active 时，deploy 必须在 current 链接切换后 `try-restart` 并复核 active，服务未运行时只跳过，不主动拉起。 +- 原因：默认 service 用非 root `genarrative` 用户运行，并且主模板为了保持 shadow 安全边界不带 `CAP_NET_BIND_SERVICE`。低端口直连必须通过显式 systemd drop-in 单独授予 capability；同时 Certbot 私钥默认未必允许 `genarrative` 读取，Nginx 也可能仍占用 `80/443`。另一个常见误区是 API release 只带 `pingora-direct-enable.sh` / rollback 壳脚本，却漏带 `pingora-current-release-audit.mjs`、`pingora-direct-rehearsal-status.mjs`、`check-pingora-direct-preflight.mjs`、`check-pingora-direct-live.mjs`、`deploy/systemd/`、`deploy/env/` 或 `deploy/pingora/`，导致从 `/opt/genarrative/current` 启用时依赖 Jenkins 工作区、源码 checkout 或 `/etc` 里某份参考模板；或者 release 已经包含新版 `pingora-gateway`，但已运行的 shadow / canary / direct service 没有随 `current` 链接切换重启，仍在跑旧二进制。Server-Provision 安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 的 drop-in 只用于人工审阅和显式覆盖；直连启用脚本默认必须读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，否则旧 `/etc` 模板会掩盖发布包缺失。API deploy 脚本本身也不能继续用部署工作区根部的 `scripts/deploy/production-api-deploy.sh`，否则 Jenkins workspace 里的脚本会掩盖 `build/` 发布包缺少 deploy / maintenance 同目录脚本的问题；备份脚本、健康巡检脚本和 env 示例目录同样不能从部署工作区兜底，切换命令证据脚本也不能从部署工作区兜底，否则 current release 会和上游构建归档漂移。Pingora 直连依赖、备份脚本、巡检脚本、env 示例目录和 API deploy 执行入口都必须来自上游发布产物；随包 `api-server.sha256` 和可选 `pingora-gateway.sha256` 也必须复制进 current release，供随包 current release 自审校验二进制；随包 `deploy/pingora/pingora-gateway.env.example` 也不能只检查存在，还要保持 gzip-only、不信任 XFF、前置代理确认关闭、接流保护开启和空 probe token 这些生产安全默认值；`production-api-deploy.sh` 发现缺失时应 fail-fast 并保持维护模式，不应从部署工作区兜底补齐；所有 API 发布包都必须携带 `release-manifest.json` 且登记 `api-server` artifact，发布包包含 Pingora 时还必须登记 `pingora-gateway` artifact，否则 deploy 应在切换 current 前失败；deploy 必须要求 release root、current link 和 api env file 都是绝对路径，release version 以数字或字母开头并拒绝点目录，再先写 staging release，全部复制完成后用非合并语义提升为正式 release，失败时清理 staging 且不留下正式 release，同版本 release 已存在、提升前竞态出现或 current 路径不是符号链接时拒绝覆盖 / 合并，避免旧文件混入 current；发布包包含 Pingora 时，deploy 必须先确认 systemd 最终配置没有 direct-entry `CAP_NET_BIND_SERVICE`、env 仍是 `127.0.0.1:18081` shadow 且未配置 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，再提升 release、切换 current 并 `restart` Pingora shadow；配置不安全时必须在切换 current 前失败并保持维护模式。 - 处理：确认真实 TLS 证书和 redirect env 已写入 `/etc/genarrative/pingora-gateway.env`、service 模板和 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含这份 env、当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都能读取证书链 / 私钥、current release 的 `pingora-gateway` 已存在且可执行、Nginx 或其它进程已释放 `80/443` 后，先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成只读 JSON runbook，并逐条审阅 Host 与回退巡检入口确认、current release 自包含自审、current release preflight、启用前基础 readiness、direct enable dry-run、direct enable apply、启用后 `--require-direct` 复核、rollback dry-run、rollback apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；runbook 只用于审阅，不修改系统。正式 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 或回退 smoke 各自验证到不同入口；同时必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检需要 Host 覆盖，再追加 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 把现场巡检入口覆盖成仓库默认值；如需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文。随后先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，再 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`，最后执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，由脚本先跑 direct preflight，再安装 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`、用 `systemctl is-active` 确认服务 active，再以 JSON 模式执行 direct live smoke，验证 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘，并要求 `direct-access-log` 结构化结果 `matchedCount == checked`、`missingCount=0`、`mismatchCount=0`；如果 direct live 退出 0 但缺少该结构化证据，也必须视为启用失败。直连启用后同步调整 `/etc/genarrative/health-patrol.env`：设置 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，本机打 `127.0.0.1` 时设置 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`，否则巡检会继续按 Nginx 模式误报。验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 或 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>`；回退脚本先跑 `nginx -t`，通过后才移除 drop-in、reload systemd、重启 Pingora，并用 `systemctl cat` 核验 capability 已移除、用 `systemctl show ... ExecStart` 核验最终 service 仍指向随包主 service 模板里的 current release `pingora-gateway`，随后 reload Nginx、确认 Nginx service 仍为 active，并用 curl smoke URL 证明公网入口已回到 Nginx；回退脚本 `--apply` 不允许省略 `--reload-nginx` 或 `--nginx-smoke-url`，当 smoke URL 指向本机地址时必须同时提供 `--nginx-smoke-host <域名>`，且 host 值不能包含 URL、路径或查询；回退后把 health patrol gateway mode 改回 `nginx`，恢复切换前 public base URL / Host，并用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url <切换前Nginx巡检入口> --require-empty-public-host` 复核；若切换前 Nginx 巡检需要 Host 覆盖，则把 `--require-empty-public-host` 换成 `--expected-public-host <切换前Host>`。若 env 已在回退命令前切回 Nginx，也可给 rollback 脚本追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让它在 Nginx smoke 后自动复核；切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若要同时证明 Pingora shadow 高端口仍活着，追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应为 `gateway=pingora-shadow`。 - 处理补充：不要直接 chmod `/etc/letsencrypt/live` 或 `archive` 来让 Pingora 读取证书；Certbot live 路径通常是 symlink，即使 `stat -L` 看起来是普通文件，父目录权限也会让非 root `genarrative` 用户不可达。先用随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>` 把证书同步到 Pingora 私有目录，再让 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/fullchain.pem` 和 `privkey.pem`。脚本默认 dry-run，`--apply` 才写入，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并拒绝符号链接目标目录或目标文件。 - 处理补充：不要在切换窗口手工编辑 `/etc/genarrative/health-patrol.env` 的三项网关变量；使用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切到直连，回退前用同一脚本传 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>` 并按切换前记录选择 `--clear-public-host` 或 `--public-host <切换前Host>`。脚本只改 gateway mode / public base URL / public Host，并立即复用随包 env 复核脚本，减少空 Host 和旧值残留；生产巡检、env 复核和 env 切换脚本读取的布尔 env 都必须是明确布尔值，非法值直接失败，不能把拼写错误当成 false；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，直接用 `node script.mjs --env-file ...` 或 shebang 执行 `.mjs --env-file ...` 都可能让 Node 抢走业务参数；所有这类命令都必须写成 `node -- script.mjs --env-file ...`，或通过已内置 `node --` 的 npm script 执行。 @@ -58,7 +58,7 @@ - 踩坑补充：证据根目录总审计选择“每类最新证据”后，还必须证明这些证据来自同一次切换时间线。最新证据选择和标准五段时间线证明只接受 `schemaVersion=1` 且带合法、规范 UTC 毫秒格式 `manifest.generatedAt` 的 manifest，命令记录 `startedAt` / `finishedAt` 也必须是 `new Date().toISOString()` 形式；缺失、非法、省略毫秒、本地时区或其它宽松可解析格式都会直接失败，不能用目录 mtime 兜底；证据目录被复制、归档或恢复后，也必须以 manifest 时间为准。同一阶段或同一命令如果出现多个候选共享最新 `manifest.generatedAt`，总审计会以 `AMBIGUOUS_LATEST` 失败并列出重复目录，不能按目录名排序打平；应重新归档该阶段 / 命令证据，或把旧证据移出正式证据根目录后再审计。标准五段证据都被要求时，每段审计状态都必须是 `OK`，`manifest.generatedAt` 必须满足 `pre-cutover -> enable-apply -> post-enable -> rollback-apply -> post-rollback`，且默认五段跨度不能超过 24 小时；非 OK、倒序或跨度过大都代表可能混入不同切换窗口遗留证据或现场状态未达标，必须失败后重新归档或清理证据根目录。任何证据 manifest 只要显式写入 `cutoverRunId` 字段，就必须是安全非空 ID，不能用空字符串伪装成缺省字段。确需跨更长维护窗口时，只能在生成 runbook 时显式传 `--cutover-evidence-timeline-max-span-ms `，让最终总审计 JSON 记录本次放宽后的 `timeline.maxSpanMs` 与实际 `timeline.spanMs`。 - 踩坑补充：标准五段时间线失败时不要只看顶层 `ok=false` 或 `diagnostics` 文本。`timeline.failedCount` 会按具体失败项累计，`timeline.failureBreakdown` 会把非 OK 证据、缺少时间、cutoverRunId 混入、时间倒序和跨度超限拆开计数；同一次审计可能同时暴露多个证据问题，应逐项修复后重新归档。 - 踩坑补充：同一天多次演练或切换时，只靠“最新证据”和 24 小时窗口仍可能把两轮证据拼在一起。正式 runbook 会生成或接受 `--cutover-run-id `，并把同一 `manifest.cutoverRunId` 写入三阶段证据包、enable / rollback apply 命令证据和最终总审计；最终审计必须带 `--require-cutover-run-id <本次cutoverRunId>`，缺少该字段或 ID 不一致时必须失败。即使人工临时总审计忘记带 `--require-cutover-run-id`，标准五段时间线里只要任一证据声明了 `manifest.cutoverRunId`，五段也必须全部声明同一个值，否则总审计失败。 -- 验证：先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，确认 env、drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和 80/443 已释放；`systemctl cat genarrative-pingora-gateway.service` 必须显示 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env`；启用脚本 apply 必须先通过 current release 自审，失败时不安装 direct-entry drop-in；还必须带 direct HTTPS / HTTP / Host / redirect host / SpacetimeDB database / Pingora access log 参数，并在重启后直接完成 direct live smoke 和 direct-access-log JSON 证据校验；也可用 release readiness `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 把 HTTPS、HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id 落盘、env 预检、systemd drop-in、service EnvironmentFile 一致性、当前用户和服务用户证书可读、service 二进制可执行、端口释放、显式目标库和 WSS 101 一起纳入硬门禁，并拒绝 `--direct-skip-wss`，避免 TLS 证书只按 `127.0.0.1` 误测、HTTP redirect Location 指错域名、Nginx 仍占用 80/443、service 实际读取另一份 env、root / deploy 用户可读但 systemd 服务用户不可读、current release 缺少可执行 `pingora-gateway`，或 WSS subscribe 隐式打到默认 SpacetimeDB 库。`check-pingora-release-readiness.mjs --help` 的正式直连和只生成 runbook 示例也必须带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，不要让值班人员复制示例后才被 `--require-direct` 拦截。current release 自审、状态快照和证据包的布尔 env 必须是明确布尔值，非法值会失败，不得把拼错的 run / require / fail 开关当成 false。`npm run plan:pingora-direct-cutover -- --require-direct ...` 输出必须包含 Host 与回退巡检入口确认、current release preflight、启用前不带 `--require-direct` 的基础 readiness、direct enable dry-run/apply、启用后带 `--require-direct` 的复核、rollback dry-run/apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；缺少 `--require-direct`、缺少 `--rollback-health-patrol-public-base-url`、缺少 `--direct-pingora-access-log`、redirect Host 漂移或 rollback smoke Host 漂移时必须失败，避免生成缺少正式直连硬门禁或验证不同入口的切换计划。Host 与回退巡检入口确认步骤必须展示回退后要恢复的 health patrol public base URL / Host。直连后 `genarrative-health-patrol.service` 应使用 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，状态 JSON 中 `gatewayMode` 应为 `pingora-direct`，并检查 `genarrative-pingora-gateway.service` 而不是 `nginx.service`；public probe 走 `127.0.0.1` 时应带 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`。回退后 `nginx -t` 必须先通过，`systemctl cat genarrative-pingora-gateway.service` 不应再显示这两条 capability，`systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager` 必须仍指向 current release 的 `pingora-gateway`，`systemctl is-active nginx.service` 应为 `active`，`curl --fail --max-time 5` 访问 `--nginx-smoke-url` 应成功；若 smoke URL 为本机地址必须带 `--nginx-smoke-host <域名>`，证明正式 vhost 已回到 Nginx；随后用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs ...` 复核 health patrol env，必须显示 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 且 public base URL / Host 与切换前记录一致，shadow probe 可选复核必须返回 `gateway=pingora-shadow`。本机提交前还要运行 `npm run check:pingora-direct-enable`、`npm run check:pingora-direct-rollback`、`npm run check:production-health-patrol`、`npm run check:production-api-release`、`npm run check:pingora-production-release-build` 和 `npm run check:production-api-deploy`，确保脚本默认 dry-run 不会安装或删除 drop-in、current release 自审失败时启用脚本不会安装 drop-in、direct live 退出 0 但缺少 `direct-access-log` 结构化证据时启用失败，API release 布局自包含，真实 Pingora release 二进制能构建并进入发布包，API deploy 从发布产物内执行后 current release 自包含；缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、切换命令证据脚本、env 示例目录或 direct live smoke 脚本的发布包都必须部署失败并保持维护模式。正式直连 readiness 必须带 `--direct-health-patrol-env-file /etc/genarrative/health-patrol.env`，并用 `scripts/check-production-health-patrol-env.mjs` 阻断 health patrol 仍停在 Nginx 模式或本机 direct probe 缺少正式 Host；发布包包含 `pingora-gateway` 时，`npm run check:production-api-deploy` 必须覆盖服务 active 时执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核 active，以及服务 inactive 时跳过且不主动拉起。 +- 验证：先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，确认 env、drop-in、service EnvironmentFile 一致性、当前用户证书权限、服务用户证书权限、service 二进制可执行性和 80/443 已释放；`systemctl cat genarrative-pingora-gateway.service` 必须显示 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env`；启用脚本 apply 必须先通过 current release 自审，失败时不安装 direct-entry drop-in；还必须带 direct HTTPS / HTTP / Host / redirect host / SpacetimeDB database / Pingora access log 参数，并在重启后直接完成 direct live smoke 和 direct-access-log JSON 证据校验；也可用 release readiness `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 把 HTTPS、HTTP redirect / ACME、正式域名 Host/SNI、redirect Location host、Pingora access log request_id 落盘、env 预检、systemd drop-in、service EnvironmentFile 一致性、当前用户和服务用户证书可读、service 二进制可执行、端口释放、显式目标库和 WSS 101 一起纳入硬门禁，并拒绝 `--direct-skip-wss`，避免 TLS 证书只按 `127.0.0.1` 误测、HTTP redirect Location 指错域名、Nginx 仍占用 80/443、service 实际读取另一份 env、root / deploy 用户可读但 systemd 服务用户不可读、current release 缺少可执行 `pingora-gateway`，或 WSS subscribe 隐式打到默认 SpacetimeDB 库。`check-pingora-release-readiness.mjs --help` 的正式直连和只生成 runbook 示例也必须带 `--direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`，不要让值班人员复制示例后才被 `--require-direct` 拦截。current release 自审、状态快照和证据包的布尔 env 必须是明确布尔值，非法值会失败，不得把拼错的 run / require / fail 开关当成 false。`npm run plan:pingora-direct-cutover -- --require-direct ...` 输出必须包含 Host 与回退巡检入口确认、current release preflight、启用前不带 `--require-direct` 的基础 readiness、direct enable dry-run/apply、启用后带 `--require-direct` 的复核、rollback dry-run/apply、回退后 health patrol 切回 Nginx 并恢复切换前 public base URL / Host、回退后 health patrol env 复核；缺少 `--require-direct`、缺少 `--rollback-health-patrol-public-base-url`、缺少 `--direct-pingora-access-log`、redirect Host 漂移或 rollback smoke Host 漂移时必须失败，避免生成缺少正式直连硬门禁或验证不同入口的切换计划。Host 与回退巡检入口确认步骤必须展示回退后要恢复的 health patrol public base URL / Host。直连后 `genarrative-health-patrol.service` 应使用 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct`，状态 JSON 中 `gatewayMode` 应为 `pingora-direct`，并检查 `genarrative-pingora-gateway.service` 而不是 `nginx.service`；public probe 走 `127.0.0.1` 时应带 `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>`。回退后 `nginx -t` 必须先通过，`systemctl cat genarrative-pingora-gateway.service` 不应再显示这两条 capability，`systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager` 必须仍指向 current release 的 `pingora-gateway`，`systemctl is-active nginx.service` 应为 `active`，`curl --fail --max-time 5` 访问 `--nginx-smoke-url` 应成功；若 smoke URL 为本机地址必须带 `--nginx-smoke-host <域名>`，证明正式 vhost 已回到 Nginx；随后用 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs ...` 复核 health patrol env，必须显示 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx` 且 public base URL / Host 与切换前记录一致，shadow probe 可选复核必须返回 `gateway=pingora-shadow`。本机提交前还要运行 `npm run check:pingora-direct-enable`、`npm run check:pingora-direct-rollback`、`npm run check:production-health-patrol`、`npm run check:production-api-release`、`npm run check:pingora-production-release-build` 和 `npm run check:production-api-deploy`，确保脚本默认 dry-run 不会安装或删除 drop-in、current release 自审失败时启用脚本不会安装 drop-in、direct live 退出 0 但缺少 `direct-access-log` 结构化证据时启用失败，API release 布局自包含，真实 Pingora release 二进制能构建并进入发布包，API deploy 从发布产物内执行后 current release 自包含；缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、切换命令证据脚本、env 示例目录或 direct live smoke 脚本的发布包都必须部署失败并保持维护模式。正式直连 readiness 必须带 `--direct-health-patrol-env-file /etc/genarrative/health-patrol.env`，并用 `scripts/check-production-health-patrol-env.mjs` 阻断 health patrol 仍停在 Nginx 模式或本机 direct probe 缺少正式 Host；发布包包含 `pingora-gateway` 时，`npm run check:production-api-deploy` 必须覆盖服务 active / inactive 都会在 shadow 配置安全时执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，同时覆盖 direct-entry capability 或公网监听 env 下不会提升 release、不会切 current、不会自动 restart。 - 顺序补充：正式 runbook 必须先通过 health patrol env 切换脚本预置回 Nginx 和切换前 public base URL / Host，再执行 `rollback apply`；回退脚本内置 env 复核和独立 env 复核都会阻断 public base URL / Host 漂移。 - 关联：`deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`、`deploy/env/health-patrol.env.example`、`deploy/env/pingora-direct-live.env.example`、`deploy/env/pingora-canary-live.env.example`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/jenkins-server-provision.sh`、`scripts/build-production-release.sh`、`scripts/deploy/production-api-deploy.sh`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index debc09d7f..8e8476b0a 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -240,7 +240,7 @@ Jenkins `Genarrative-Api-Build` 对应参数是 `INCLUDE_PINGORA_GATEWAY`，默 API release 还必须携带 `scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs` 与 `scripts/ops/pingora-direct-rehearsal-status.mjs`。前者支撑 current release 的 `--release-runtime-only` 聚合复核，canary live 脚本支撑目标 Nginx canary live smoke，直连彩排状态脚本支撑目标机切换前只读确认 Nginx 仍接公网、Pingora shadow / realpath canary 高端口和 current release 自审均可用；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败。 -发布包会额外包含 `pingora-gateway` 与 `pingora-gateway.sha256`。`production-api-deploy.sh` 看到这两个文件时会校验并把 `pingora-gateway` / `pingora-gateway.sha256` 一起复制到 `/opt/genarrative/current`，供 shadow systemd 模板和 current release 自审使用；`api-server.sha256` 也必须随 `api-server` 一起进入 current release。没有这两个 Pingora 文件时现有 API 发布行为不变。API 发布仍只重启 `genarrative-api.service`；如果 `genarrative-pingora-gateway.service` 已经处于 `active`，部署脚本会在 `current` 链接切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载同一份 current release 网关二进制。若 Pingora 服务未运行，部署脚本只打印跳过信息，不主动拉起影子服务，也不改变默认接流边界。 +发布包会额外包含 `pingora-gateway` 与 `pingora-gateway.sha256`。`production-api-deploy.sh` 看到这两个文件时会校验并把 `pingora-gateway` / `pingora-gateway.sha256` 一起复制到 `/opt/genarrative/current`，供 shadow systemd 模板和 current release 自审使用；`api-server.sha256` 也必须随 `api-server` 一起进入 current release。没有这两个 Pingora 文件时现有 API 发布行为不变。API 发布仍只重启 `genarrative-api.service`；包含 Pingora 时，部署脚本会在提升 release 和切换 `current` 前读取 `systemctl cat genarrative-pingora-gateway.service` 与其 `EnvironmentFile`，拒绝已出现 `CAP_NET_BIND_SERVICE` direct-entry capability、拒绝 `GENARRATIVE_PINGORA_GATEWAY_LISTEN` 不是 `127.0.0.1:18081`、拒绝已配置 `TLS_LISTEN` 或 `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current，并执行 `systemctl restart genarrative-pingora-gateway.service` 后复核 active。该自动拉起只覆盖 shadow 服务，不会启用公网 `80/443` 直连入口；若现场已经处于 direct-entry 状态，应走正式直连 runbook 或先回退到 shadow 后再执行 API deploy。 也可以复制 `deploy/pingora/pingora-gateway.env.example` 到部署环境的非 Git 配置文件，由 systemd 或容器注入。仓库提供 `deploy/systemd/genarrative-pingora-gateway.service` 作为影子服务模板，默认读取 `/etc/genarrative/pingora-gateway.env`，仍只应监听本机高端口，再由 Nginx 或本机 smoke 主动访问。Server-Provision 会把主 service 安装到 `/etc/systemd/system/genarrative-pingora-gateway.service`，并把直连低端口 drop-in 模板安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为参考和手动覆盖来源；该模板不会默认生效，`/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 默认使用随 current release 发布的 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`。`deploy/nginx/snippets/genarrative-pingora-canary.conf` 是可选的 Nginx -> Pingora 前缀 canary 模板，Server-Provision 会安装到 `/etc/nginx/snippets/`，但主站配置默认不 include；启用前必须把 `__GENARRATIVE_PINGORA_PROBE_TOKEN__` 替换为真实 token，并确认 `allow/deny` 来源边界符合当次验证窗口。 @@ -399,7 +399,7 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 本阶段验收只证明 API release 的正式打包、复制、current 切换、服务重启和 shadow Pingora 随 current 重启可用，不切 `80/443` 到 Pingora。验收结果： - `production-api-deploy.sh` 进入维护模式后完成 `api-server.sha256` 和 `pingora-gateway.sha256` 校验，复制二进制、checksum、manifest 和 Pingora 直连依赖到新 release，再把 `/opt/genarrative/current` 切到 `/opt/genarrative/releases/dev-pingora-api-20260617140915`，最后退出维护模式。 -- 部署时 `genarrative-pingora-gateway.service` 已为 `active`，脚本在 current 切换后执行 `systemctl try-restart genarrative-pingora-gateway.service` 并复核 active；`genarrative-api.service`、外部生成 worker 和 worker controller 也完成重启和 active 等待。 +- 部署时 `genarrative-pingora-gateway.service` 仍是 shadow 配置，脚本在 current 切换前拒绝 direct-entry capability 和公网监听 env，切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active；`genarrative-api.service`、外部生成 worker 和 worker controller 也完成重启和 active 等待。 - `/opt/genarrative/current/release-manifest.api-server.json` 记录 `api-server` 和 `pingora-gateway` 两个 artifact，`cd /opt/genarrative/current && sha256sum -c api-server.sha256 && sha256sum -c pingora-gateway.sha256` 均为 `OK`。 - `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show` 返回 `summary.status=OK`，确认 current release 自包含、checksum 匹配、`pingora-gateway` 可执行，且 systemd `ExecStart` 指向 `/opt/genarrative/current/pingora-gateway`。 - `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only` 通过，证明目标机可以只依赖 current release 运行 runtime-only 发布自审，不需要源码 checkout 或 Jenkins 工作区。 diff --git a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md index 98cd0ff95..3e2750dcc 100644 --- a/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md +++ b/docs/【开发运维】本地开发验证与生产运维-2026-05-15.md @@ -351,7 +351,7 @@ cat /var/lib/genarrative/health-patrol/status.json `Genarrative-Web-Build` 会把 `build//web.tar.gz`、`web.tar.gz.sha256`、`release-manifest.json` 和 `scripts/deploy/production-web-deploy.sh` 直接归档为 Jenkins 构建产物；`Genarrative-Web-Deploy` 只通过 `copyArtifacts` 从指定上游构建复制这些产物和部署脚本，不再在目标机器 checkout Git，再执行随构建归档的 `scripts/deploy/production-web-deploy.sh`。Web 发布不再读取构建机本地缓存目录，也不再通过 release agent `rsync` 回构建机拉取大包；如果 deploy 找不到 `web.tar.gz`，应先检查上游 Web Build 是否按同一 `BUILD_VERSION` 成功归档产物。 -`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会保留维护模式并停止部署；失败会清理 staging 目录且不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会失败且保持维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时拒绝覆盖 / 合并。Pingora 影子网关不是默认 API 归档物；只有显式用 `npm run build:production-release -- --component api-server --include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，发布包才包含 `pingora-gateway` / `pingora-gateway.sha256`，API deploy 会在两者同时存在且 manifest 登记 `pingora-gateway` 时校验并复制到 current release；此时 build 脚本和 Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。若 `genarrative-pingora-gateway.service` 已处于 `active`，deploy 会在 current 链接切换后 `try-restart` 并复核仍为 `active`，让已启用的 shadow / canary / direct 机器加载新网关二进制；若服务未运行则只跳过，不主动拉起 Pingora。 +`Genarrative-Api-Build` 的 Jenkins 归档产物必须包含 `build//api-server`、`api-server.sha256`、`release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`scripts/database-backup-to-oss.mjs`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/ops/pingora-cutover-command-evidence.mjs`、`scripts/ops/pingora-cutover-evidence-verify.mjs`、`scripts/ops/pingora-cutover-evidence-audit.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-direct-enable.sh`、`scripts/deploy/pingora-direct-rollback.sh`、`deploy/systemd/**`、`deploy/env/**` 和 `deploy/pingora/**`。`deploy/systemd/genarrative-database-backup.service` 从 `/opt/genarrative/current/scripts/database-backup-to-oss.mjs` 执行冷备份，`deploy/systemd/genarrative-health-patrol.service` 从 `/opt/genarrative/current/scripts/ops/production-health-patrol.mjs` 执行巡检；`Genarrative-Api-Deploy` 会从上游 API 构建产物复制并执行 `build//scripts/deploy/production-api-deploy.sh`，同目录的 `maintenance-on.sh` / `maintenance-off.sh` 也必须来自同一 build 产物；部署脚本会先写入 `${RELEASE_ROOT}/.${VERSION}.staging.$$`，把 `release-manifest.json` 校验后复制为 current release 的 `release-manifest.api-server.json`，并把备份脚本、巡检脚本、Pingora 直连启用 / 回退 / 预检 / live smoke / canary access log 对账 / health patrol env 复核 / current release 自审 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本，以及 `deploy/systemd`、`deploy/env`、`deploy/pingora` 支撑配置全部复制完成后，才用非合并语义提升为 `${RELEASE_ROOT}/${VERSION}` 并用固定替换语义切换 `current` 符号链接，不再在目标机器 checkout Git，也不再执行部署工作区根部脚本。Pingora 直连启用脚本必须能从 `/opt/genarrative/current` 独立执行 preflight 和 direct live smoke，并默认读取 current release 随包 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 Jenkins 工作区、源码 checkout 或 `/etc` 参考模板；`plan:pingora-direct-cutover` 必须能用同一组参数生成 current release 切换 / 回退 runbook。`production-api-deploy.sh` 对 release manifest、备份脚本、巡检脚本、env 示例目录和 Pingora 直连依赖都执行 fail-fast，且 `--release-root`、`--current-link`、`--api-env-file` 必须是绝对路径，`--version` 必须以数字或字母开头并只能包含数字、字母、点、下划线和短横线，禁止 `.` / `..` 点目录；发布产物缺少 manifest、manifest 未登记 `api-server`、缺少脚本 / 配置目录、同版本 release 目录已存在、current 路径不是符号链接、提升前 release 目录竞态出现或 staging 构建中失败时会保留维护模式并停止部署；失败会清理 staging 目录且不会留下正式 release 目录，不再从部署机工作区兜底补文件，也不把旧同名 release 目录和新文件混合。如果 API 发布后 current release 中缺少这些脚本或目录，应先检查 `Genarrative-Api-Build` 的 `archiveArtifacts` 和 `Genarrative-Api-Deploy` 的 `copyArtifacts` 过滤器是否仍包含 `build//release-manifest.json`、`build//scripts/deploy/production-api-deploy.sh`、`build//scripts/deploy/maintenance-on.sh`、`build//scripts/deploy/maintenance-off.sh`、`build//scripts/database-backup-to-oss.mjs`、`build//scripts/ops/production-health-patrol.mjs`、`build//scripts/ops/pingora-current-release-audit.mjs`、`build//scripts/ops/pingora-cutover-status-snapshot.mjs`、`build//scripts/ops/pingora-cutover-evidence-bundle.mjs`、`build//scripts/ops/pingora-cutover-command-evidence.mjs`、`build//scripts/ops/pingora-cutover-evidence-verify.mjs`、`build//scripts/ops/pingora-cutover-evidence-audit.mjs`、`build//scripts/check-pingora-direct-preflight.mjs`、`build//scripts/check-pingora-direct-live.mjs`、`build//scripts/check-pingora-canary-access-log-parity.mjs`、`build//scripts/check-production-health-patrol-env.mjs`、`build//scripts/deploy/pingora-direct-enable.sh`、`build//scripts/deploy/pingora-direct-rollback.sh`、`build//deploy/systemd/**`、`build//deploy/env/**` 与 `build//deploy/pingora/**`，不要只在部署机工作区手工补文件。本机用 `npm run check:production-api-release` 通过临时 `CARGO_TARGET_DIR` 和假 `api-server` / `pingora-gateway` release binary 验证 `build-production-release.sh --component api-server --skip-api-build` 会把这些文件打进 API release，并验证显式 `--include-pingora-gateway --skip-pingora-gateway-build` 时发布包包含 `pingora-gateway`、`pingora-gateway.sha256` 和 manifest 登记；`npm run check:pingora-production-release-build` 则用假 `api-server` 和真实 `cargo build -p pingora-gateway --release --target x86_64-unknown-linux-gnu` 验证显式 include 路径能构出可执行网关二进制、checksum 和 manifest 登记；再用 `npm run check:production-api-deploy` 通过临时 release、fake `systemctl` / `curl` 验证从发布产物内执行 `production-api-deploy.sh` 会把这些文件复制到 current release，并验证缺少 release manifest、manifest 未登记 `api-server`、缺少数据库备份脚本、健康巡检脚本、健康巡检 env 复核脚本、current release 自审脚本、状态快照脚本、证据包脚本、证据验真脚本、证据根目录审计脚本、canary access log 对账脚本、env 示例目录或 direct live smoke 脚本时都会失败且保持维护模式，还会验证相对 release root / current link / api env file、点目录或点开头 version 被拒绝、失败时不留下 staging / 正式 release 目录、同版本 release 目录已存在、current 路径不是符号链接或提升前 release 目录竞态出现时拒绝覆盖 / 合并。Pingora 影子网关不是默认 API 归档物；只有显式用 `npm run build:production-release -- --component api-server --include-pingora-gateway` 或在 `Genarrative-Api-Build` 勾选 `INCLUDE_PINGORA_GATEWAY` 时，发布包才包含 `pingora-gateway` / `pingora-gateway.sha256`，API deploy 会在两者同时存在且 manifest 登记 `pingora-gateway` 时校验并复制到 current release；此时 build 脚本和 Jenkins 会先检查 `cmake`、C 编译器和 C++ 编译器，避免进入 Cargo 后才因 `libz-ng-sys` 构建依赖缺失失败。发布包包含 Pingora 时，deploy 会在提升 release 前读取 `systemctl cat genarrative-pingora-gateway.service` 和其 `EnvironmentFile`，拒绝 direct-entry `CAP_NET_BIND_SERVICE`、拒绝非 `127.0.0.1:18081` 的 shadow listen、拒绝 `TLS_LISTEN` / `HTTP_REDIRECT_LISTEN`，确认仍是本机 shadow 高端口后才切换 current；切换后执行 `systemctl restart genarrative-pingora-gateway.service` 并复核 active，让 shadow / canary 机器加载新网关二进制。该自动拉起不会启用公网 `80/443` 直连入口；已经进入 direct-entry 状态的机器应走正式直连 runbook 或先回退到 shadow。 Pingora current release 自审脚本 `scripts/ops/pingora-current-release-audit.mjs`、直连切换状态快照脚本 `scripts/ops/pingora-cutover-status-snapshot.mjs`、证据包脚本 `scripts/ops/pingora-cutover-evidence-bundle.mjs`、命令证据脚本 `scripts/ops/pingora-cutover-command-evidence.mjs`、证据验真脚本 `scripts/ops/pingora-cutover-evidence-verify.mjs`、证据根目录审计脚本 `scripts/ops/pingora-cutover-evidence-audit.mjs` 和 canary access log 对账脚本 `scripts/check-pingora-canary-access-log-parity.mjs` 都属于 API release 的强制随包依赖；缺少任一脚本时 `check:production-api-release`、`check:production-api-deploy` 和生产运维护栏都必须失败，避免切换窗口只能靠 Jenkins 工作区或源码 checkout 临时补自审、证据或日志对账脚本。 diff --git a/scripts/check-production-api-deploy.mjs b/scripts/check-production-api-deploy.mjs index 2a6df0648..378a442a2 100644 --- a/scripts/check-production-api-deploy.mjs +++ b/scripts/check-production-api-deploy.mjs @@ -37,7 +37,9 @@ console.log('[check:production-api-deploy] OK'); function main() { assertDeployCopiesPingoraDirectReleaseDependencies(); assertDeployRestartsActivePingoraWhenArtifactIncluded(); - assertDeploySkipsInactivePingoraWhenArtifactIncluded(); + assertDeployStartsInactivePingoraWhenArtifactIncluded(); + assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded(); + assertDeployRejectsPingoraPublicListenWhenArtifactIncluded(); assertDeployRejectsPingoraArtifactMissingManifestEntry(); assertMissingReleaseManifestFails(); assertReleaseManifestMissingApiArtifactFails(); @@ -286,22 +288,22 @@ function assertDeployRestartsActivePingoraWhenArtifactIncluded() { const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, - 'systemctl is-active --quiet genarrative-pingora-gateway.service', - '部署脚本看到 Pingora 产物后必须先检查 shadow service 是否 active。', + 'systemctl cat genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', ); assertIncludes( commandsLog, - 'systemctl try-restart genarrative-pingora-gateway.service', - 'Pingora shadow service 已 active 时必须随 current release 切换 try-restart。', + 'systemctl restart genarrative-pingora-gateway.service', + 'Pingora shadow service 已 active 时必须随 current release 切换 restart。', ); assertIncludes( result.stdout, - '发布包包含 Pingora，重启已运行的影子服务', + '发布包包含 Pingora，启动或重启 shadow 影子服务', 'Pingora shadow service 自动重启时必须输出明确提示。', ); } -function assertDeploySkipsInactivePingoraWhenArtifactIncluded() { +function assertDeployStartsInactivePingoraWhenArtifactIncluded() { const fixture = prepareFixture('with-inactive-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture, { pingoraActive: false }); @@ -309,7 +311,7 @@ function assertDeploySkipsInactivePingoraWhenArtifactIncluded() { assertStatus( result, 0, - '包含 Pingora 但 shadow service inactive 时应部署成功。', + '包含 Pingora 且 shadow service inactive 但仍是安全 shadow 配置时应部署成功。', ); if (result.status !== 0) { return; @@ -318,20 +320,86 @@ function assertDeploySkipsInactivePingoraWhenArtifactIncluded() { const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, - 'systemctl is-active --quiet genarrative-pingora-gateway.service', - '部署脚本看到 Pingora 产物后必须检查 shadow service 是否 active。', + 'systemctl cat genarrative-pingora-gateway.service', + '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', + ); + assertIncludes( + commandsLog, + 'systemctl restart genarrative-pingora-gateway.service', + 'Pingora shadow service inactive 且 shadow 配置安全时必须随 current release 启动。', + ); + assertIncludes( + commandsLog, + 'systemctl is-active --quiet genarrative-pingora-gateway.service', + '启动或重启 Pingora shadow 后必须复核 active。', ); - if ( - commandsLog.includes( - 'systemctl try-restart genarrative-pingora-gateway.service', - ) - ) { - failures.push('Pingora shadow service inactive 时不能主动 try-restart。'); - } assertIncludes( result.stdout, - 'Pingora 影子服务未处于 active，跳过自动重启', - 'Pingora shadow service inactive 时必须明确说明不会主动拉起。', + '发布包包含 Pingora，启动或重启 shadow 影子服务', + 'Pingora shadow service 自动启动时必须输出明确提示。', + ); +} + +function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() { + const fixture = prepareFixture('with-direct-entry-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + const result = runDeploy(fixture, { pingoraDirectEntry: true }); + + if (result.status === 0) { + failures.push('包含 Pingora 但 systemd 已启用 direct-entry capability 时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'Pingora systemd 已包含 CAP_NET_BIND_SERVICE', + 'direct-entry capability 存在时必须给出明确错误。', + ); + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + if ( + commandsLog.includes( + 'systemctl restart genarrative-pingora-gateway.service', + ) + ) { + failures.push('direct-entry capability 存在时不能自动 restart Pingora。'); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('direct-entry capability 导致部署失败时必须保持维护模式。'); + } + assertNoReleasePromoted( + fixture, + 'direct-entry capability 导致部署失败时不能提升正式 release。', + ); +} + +function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() { + const fixture = prepareFixture('with-public-listen-pingora-artifact'); + addPingoraGatewayArtifact(fixture); + writePingoraEnv(fixture, { + listen: '0.0.0.0:443', + }); + const result = runDeploy(fixture); + + if (result.status === 0) { + failures.push('包含 Pingora 但 env 已配置公网监听时部署必须失败。'); + } + assertIncludes( + result.stderr, + 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', + '公网监听 env 存在时必须给出明确错误。', + ); + const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); + if ( + commandsLog.includes( + 'systemctl restart genarrative-pingora-gateway.service', + ) + ) { + failures.push('公网监听 env 存在时不能自动 restart Pingora。'); + } + if (!existsSync(fixture.maintenanceFile)) { + failures.push('公网监听 env 导致部署失败时必须保持维护模式。'); + } + assertNoReleasePromoted( + fixture, + '公网监听 env 导致部署失败时不能提升正式 release。', ); } @@ -1024,10 +1092,12 @@ function prepareFixture(name) { const releaseRoot = path.join(root, 'releases'); const currentLink = path.join(root, 'current'); const apiEnvFile = path.join(root, 'etc', 'api-server.env'); + const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env'); const maintenanceFile = path.join(root, 'maintenance', 'enabled'); const fakeBin = path.join(root, 'bin'); const commandsLog = path.join(root, 'commands.log'); const workerStateFile = path.join(root, 'worker-service-enabled'); + const pingoraStateFile = path.join(root, 'pingora-service-active'); const version = `20260614-${name}`; mkdirSync(sourceDir, { recursive: true }); @@ -1054,6 +1124,7 @@ function prepareFixture(name) { ].join('\n'), 'utf8', ); + writePingoraEnv({ pingoraEnvFile }); chmodExecutable(path.join(sourceDir, 'api-server')); writeSha256(sourceDir, 'api-server'); writeFileSync( @@ -1226,6 +1297,17 @@ function prepareFixture(name) { 'set -euo pipefail', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'worker_state_file="${FAKE_WORKER_STATE_FILE}"', + 'pingora_state_file="${FAKE_PINGORA_STATE_FILE}"', + 'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', + ' printf "[Service]\\n"', + ' printf "EnvironmentFile=%s\\n" "${FAKE_PINGORA_ENV_FILE}"', + ' printf "ExecStart=/opt/genarrative/current/pingora-gateway\\n"', + ' if [[ "${FAKE_PINGORA_DIRECT_ENTRY:-false}" == "true" ]]; then', + ' printf "AmbientCapabilities=CAP_NET_BIND_SERVICE\\n"', + ' printf "CapabilityBoundingSet=CAP_NET_BIND_SERVICE\\n"', + ' fi', + ' exit 0', + 'fi', 'if [[ "$1" == "list-units" ]]; then', ' pattern="${@: -1}"', ' if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" && -f "${worker_state_file}" ]]; then', @@ -1237,9 +1319,16 @@ function prepareFixture(name) { ' printf "enabled\\n" > "${worker_state_file}"', ' exit 0', 'fi', - 'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then', + 'if [[ "$1 $2 ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" ]]; then', + ' if [[ "${FAKE_PINGORA_ACTIVE:-true}" == "true" || -f "${pingora_state_file}" ]]; then', + ' exit 0', + ' fi', ' exit 3', 'fi', + 'if [[ "$1" == "restart" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', + ' printf "active\\n" > "${pingora_state_file}"', + ' exit 0', + 'fi', 'exit 0', '', ].join('\n'), @@ -1283,14 +1372,32 @@ function prepareFixture(name) { releaseRoot, currentLink, apiEnvFile, + pingoraEnvFile, maintenanceFile, fakeBin, commandsLog, workerStateFile, + pingoraStateFile, version, }; } +function writePingoraEnv(fixture, options = {}) { + const filePath = fixture.pingoraEnvFile; + const lines = [ + `GENARRATIVE_PINGORA_GATEWAY_LISTEN=${options.listen ?? '127.0.0.1:18081'}`, + ]; + if (options.tlsListen) { + lines.push(`GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${options.tlsListen}`); + } + if (options.redirectListen) { + lines.push( + `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${options.redirectListen}`, + ); + } + writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); +} + function addPingoraGatewayArtifact(fixture, options = {}) { writeFileSync( path.join(fixture.sourceDir, 'pingora-gateway'), @@ -1351,6 +1458,10 @@ function runDeploy(fixture, options = {}) { PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile, FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true', + FAKE_PINGORA_DIRECT_ENTRY: + options.pingoraDirectEntry === true ? 'true' : 'false', + FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile, + FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile, FAKE_CREATE_RELEASE_DURING_COPY: options.createReleaseDuringCopy === true ? 'true' : 'false', FAKE_RELEASE_ROOT: fixture.releaseRoot, @@ -1397,6 +1508,16 @@ function assertFileExists(filePath, reason) { } } +function assertNoReleasePromoted(fixture, reason) { + const releaseDir = path.join(fixture.releaseRoot, fixture.version); + if (existsSync(releaseDir)) { + failures.push(`${reason} 已存在: ${releaseDir}`); + } + if (existsSync(fixture.currentLink)) { + failures.push(`${reason} 不应切换 current: ${fixture.currentLink}`); + } +} + function readDirNames(directory) { return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], { cwd: process.cwd(), diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 66734666a..f011adf78 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -5317,21 +5317,33 @@ const checks = [ }, { file: 'scripts/deploy/production-api-deploy.sh', - includes: 'systemctl is-active --quiet "${service_name}"', + includes: 'ensure_pingora_shadow_service', reason: - 'API deploy 看到 Pingora 影子网关产物时必须先确认 shadow service 已 active。', + 'API deploy 看到 Pingora 影子网关产物时必须先进入受控 shadow service 启动流程。', }, { file: 'scripts/deploy/production-api-deploy.sh', - includes: 'systemctl try-restart "${service_name}"', + includes: 'systemctl cat "${service_name}"', reason: - 'API deploy 看到 Pingora 影子网关产物且 shadow service 已 active 时必须随 current release 切换重启。', + 'API deploy 自动启动或重启 Pingora 前必须读取 systemd 最终配置。', }, { file: 'scripts/deploy/production-api-deploy.sh', - includes: 'Pingora 影子服务未处于 active，跳过自动重启', + includes: 'CAP_NET_BIND_SERVICE', reason: - 'API deploy 不能在 Pingora shadow service 未运行时主动拉起新公网入口候选服务。', + 'API deploy 自动启动或重启 Pingora 前必须拒绝疑似 direct-entry 低端口 capability。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', + reason: + 'API deploy 自动启动 Pingora 时必须限制为本机 shadow 高端口，避免误拉起公网直连入口。', + }, + { + file: 'scripts/deploy/production-api-deploy.sh', + includes: 'systemctl restart "${service_name}"', + reason: + 'API deploy 看到 Pingora 影子网关产物且 shadow 配置安全时必须随 current release 启动或重启。', }, { file: 'scripts/check-production-api-deploy.mjs', @@ -5341,9 +5353,21 @@ const checks = [ }, { file: 'scripts/check-production-api-deploy.mjs', - includes: 'assertDeploySkipsInactivePingoraWhenArtifactIncluded', + includes: 'assertDeployStartsInactivePingoraWhenArtifactIncluded', reason: - 'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 时不会被主动拉起。', + 'API deploy 动态烟测必须覆盖 Pingora shadow service inactive 但配置安全时会被主动拉起。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖 direct-entry capability 下不会自动拉起 Pingora。', + }, + { + file: 'scripts/check-production-api-deploy.mjs', + includes: 'assertDeployRejectsPingoraPublicListenWhenArtifactIncluded', + reason: + 'API deploy 动态烟测必须覆盖公网监听 env 下不会自动拉起 Pingora。', }, { file: 'scripts/check-production-api-release.mjs', diff --git a/scripts/deploy/production-api-deploy.sh b/scripts/deploy/production-api-deploy.sh index 5019e64c5..14742cdcd 100644 --- a/scripts/deploy/production-api-deploy.sh +++ b/scripts/deploy/production-api-deploy.sh @@ -11,7 +11,7 @@ usage() { 进入维护模式，校验并发布 api-server 单文件，更新 current 链接，重启 systemd 服务并执行 readiness 检查。 默认同时重启外部生成 worker controller 和已加载的 worker 实例；未启用 worker 单元时会自动跳过。 若传入 --database，会在重启前把 GENARRATIVE_SPACETIME_DATABASE 写入 api-server 环境文件，避免服务继续读取旧库。 - 若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后仅对已 active 的 Pingora 影子服务执行 try-restart 并复核 active；未运行时不会主动拉起。 + 若发布包包含 pingora-gateway，部署脚本会在 current 链接切换后先复核 systemd/env 仍是本机高端口 shadow 配置，再启动或重启 Pingora 影子服务并复核 active。 失败时保留维护模式。 EOF } @@ -235,18 +235,101 @@ ensure_runtime_env_and_dirs() { fi } -restart_pingora_if_active() { +extract_pingora_env_files_from_unit() { local service_name="$1" + local unit_content - if ! systemctl is-active --quiet "${service_name}"; then - echo "[production-api-deploy] Pingora 影子服务未处于 active，跳过自动重启: ${service_name}" - return + if ! unit_content="$(systemctl cat "${service_name}")"; then + echo "[production-api-deploy] 无法读取 Pingora systemd 最终配置: ${service_name}" >&2 + exit 1 fi - echo "[production-api-deploy] 发布包包含 Pingora，重启已运行的影子服务: ${service_name}" - systemctl try-restart "${service_name}" + if printf "%s\n" "${unit_content}" | grep -Eq '^[[:space:]]*(AmbientCapabilities|CapabilityBoundingSet)=.*CAP_NET_BIND_SERVICE'; then + echo "[production-api-deploy] Pingora systemd 已包含 CAP_NET_BIND_SERVICE，疑似直连入口配置；API deploy 不会自动启动或重启直连服务: ${service_name}" >&2 + exit 1 + fi + + printf "%s\n" "${unit_content}" | while IFS= read -r raw_line; do + local line value token env_file + line="${raw_line#"${raw_line%%[![:space:]]*}"}" + [[ "${line}" == EnvironmentFile=* ]] || continue + value="${line#EnvironmentFile=}" + for token in ${value}; do + env_file="${token#-}" + env_file="${env_file%\"}" + env_file="${env_file#\"}" + env_file="${env_file%\'}" + env_file="${env_file#\'}" + [[ -n "${env_file}" ]] && printf "%s\n" "${env_file}" + done + done +} + +find_pingora_gateway_env_file() { + local service_name="$1" + local env_file listen + + while IFS= read -r env_file; do + if [[ "${env_file}" != /* ]]; then + echo "[production-api-deploy] Pingora EnvironmentFile 必须使用绝对路径: ${env_file}" >&2 + exit 1 + fi + listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")" + if [[ -n "${listen}" ]]; then + printf "%s\n" "${env_file}" + return + fi + done < <(extract_pingora_env_files_from_unit "${service_name}") + + echo "[production-api-deploy] Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile: ${service_name}" >&2 + exit 1 +} + +require_pingora_shadow_env() { + local env_file="$1" + local listen tls_listen redirect_listen + + listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_LISTEN")" + tls_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN")" + redirect_listen="$(read_env_value "${env_file}" "GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN")" + + if [[ "${listen}" != "127.0.0.1:18081" ]]; then + echo "[production-api-deploy] Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081，当前 GENARRATIVE_PINGORA_GATEWAY_LISTEN=${listen:-}" >&2 + exit 1 + fi + if [[ -n "${tls_listen}" ]]; then + echo "[production-api-deploy] Pingora 自动启动不允许启用 TLS_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${tls_listen}" >&2 + exit 1 + fi + if [[ -n "${redirect_listen}" ]]; then + echo "[production-api-deploy] Pingora 自动启动不允许启用 HTTP_REDIRECT_LISTEN，当前 GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${redirect_listen}" >&2 + exit 1 + fi +} + +check_pingora_shadow_service_config() { + local service_name="$1" + local env_file + + env_file="$(find_pingora_gateway_env_file "${service_name}")" + require_pingora_shadow_env "${env_file}" + printf "%s\n" "${env_file}" +} + +ensure_pingora_shadow_service() { + local service_name="$1" + local env_file="${2:-}" + + if [[ -z "${env_file}" ]]; then + env_file="$(check_pingora_shadow_service_config "${service_name}")" + else + require_pingora_shadow_env "${env_file}" + fi + + echo "[production-api-deploy] 发布包包含 Pingora，启动或重启 shadow 影子服务: ${service_name} (${env_file})" + systemctl restart "${service_name}" if ! systemctl is-active --quiet "${service_name}"; then - echo "[production-api-deploy] Pingora 影子服务重启后不是 active: ${service_name}" >&2 + echo "[production-api-deploy] Pingora shadow 影子服务启动或重启后不是 active: ${service_name}" >&2 exit 1 fi } @@ -604,7 +687,7 @@ if [[ -f "${SOURCE_DIR}/pingora-gateway" ]]; then cp "${SOURCE_DIR}/pingora-gateway" "${RELEASE_CONTENT_DIR}/pingora-gateway" cp "${SOURCE_DIR}/pingora-gateway.sha256" "${RELEASE_CONTENT_DIR}/pingora-gateway.sha256" chmod +x "${RELEASE_CONTENT_DIR}/pingora-gateway" - echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核并重启已运行的 ${PINGORA_SERVICE_NAME}" + echo "[production-api-deploy] 已复制 Pingora 影子网关；current 链接切换后将复核 shadow 配置并启动或重启 ${PINGORA_SERVICE_NAME}" fi BACKUP_SCRIPT_SOURCE="${SOURCE_DIR}/scripts/database-backup-to-oss.mjs" @@ -786,6 +869,8 @@ cp -R "${ENV_DEPLOY_DIR_SOURCE}" "${RELEASE_CONTENT_DIR}/deploy/env" cp "${SOURCE_DIR}/release-manifest.json" "${RELEASE_CONTENT_DIR}/release-manifest.api-server.json" +PINGORA_SHADOW_ENV_FILE="" + if [[ -n "${DATABASE}" ]]; then echo "[production-api-deploy] 写入 api-server SpacetimeDB database: ${DATABASE} -> ${API_ENV_FILE}" write_env_value "${API_ENV_FILE}" "GENARRATIVE_SPACETIME_DATABASE" "${DATABASE}" @@ -798,6 +883,10 @@ fi ensure_runtime_env_and_dirs "${API_ENV_FILE}" +if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then + PINGORA_SHADOW_ENV_FILE="$(check_pingora_shadow_service_config "${PINGORA_SERVICE_NAME}")" +fi + mkdir -p "$(dirname "${CURRENT_LINK}")" if [[ -e "${RELEASE_DIR}" ]]; then echo "[production-api-deploy] 目标 release 在发布过程中出现，拒绝合并 staging: ${RELEASE_DIR}" >&2 @@ -808,7 +897,7 @@ STAGING_RELEASE_DIR="" ln -sfnT "${RELEASE_DIR}" "${CURRENT_LINK}" if [[ "${PINGORA_INCLUDED}" -eq 1 ]]; then - restart_pingora_if_active "${PINGORA_SERVICE_NAME}" + ensure_pingora_shadow_service "${PINGORA_SERVICE_NAME}" "${PINGORA_SHADOW_ENV_FILE}" fi echo "[production-api-deploy] 重启服务: ${SERVICE_NAME}" From e79cc5bd0909291431892f5dc7e3e68f839ae52a Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 15:07:43 +0800 Subject: [PATCH 18/22] =?UTF-8?q?=E7=BA=B3=E5=85=A5=20Pingora=20realpath?= =?UTF-8?q?=20canary=20=E5=90=AF=E5=81=9C=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 realpath canary 启停烟测纳入 Pingora release readiness 默认门禁 补齐 readiness plan 对 realpath canary toggle 的自检 修正生产健康巡检测试夹具的 worker 实例模拟 同步 Pingora realpath canary 运维文档和项目共享记忆 --- deploy/nginx/README.md | 1 + .../shared-memory/decision-log.md | 4 ++-- docs/project-memory/shared-memory/pitfalls.md | 2 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 2 +- .../check-pingora-release-readiness-plan.mjs | 17 +++++++++++++++++ scripts/check-pingora-release-readiness.mjs | 5 +++++ scripts/check-production-health-patrol.mjs | 4 ++++ 7 files changed, 31 insertions(+), 4 deletions(-) diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 55b572897..4a1bd5700 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -69,6 +69,7 @@ - 启用前必须运行 `npm run check:nginx-pingora-canary`；目标 agent 有 Nginx 时运行 `node scripts/check-nginx-pingora-canary.mjs --require-nginx`，同时做两个 snippet 的静态护栏和真实 `nginx -t`。本机或 CI 可运行 `npm run check:pingora-canary-docker`，用 Docker Nginx、真实 Pingora 和 mock 上游复现前缀 canary 与真实路径 canary 两条 handoff 链路。 - 前缀 canary reload 后运行 `GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1 GENARRATIVE_PINGORA_CANARY_HOST=<域名> npm run check:pingora-canary-live`，再用 `scripts/check-pingora-canary-access-log-parity.mjs` 对账 `/var/log/nginx/genarrative.access.log` 与 `/var/log/genarrative/pingora-gateway.access.log`。 - 真实路径 canary 不再手工编辑 `/etc/nginx/conf.d/`。目标机从 current release 执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`；脚本会渲染到 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，保持晚于主站 `log_format genarrative_upstream` 加载，执行 `nginx -t`、reload Nginx，并默认运行 realpath live smoke。`nginx -t`、reload 或 live smoke 任一失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会删除该 `zz-...` 文件，执行 `nginx -t` 和 reload，失败时恢复删除前配置。 +- `npm run check:pingora-release-readiness` 默认包含 `npm run check:pingora-realpath-canary-toggle`，用临时 Nginx 目录和 fake `nginx` / `systemctl` / live smoke 验证 realpath canary 启停脚本默认 dry-run、apply 写入、失败回滚和 disable 恢复逻辑；不再把这条脚本护栏留成单独可选项。 - 真实路径 canary reload 后可再次运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 - 正式切换前在源码 checkout / CI / 构建环境跑 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。 - 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁：`/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证真实路径 canary 或 Pingora 直连入口时分别追加 `--require-realpath-live ...` 或 `--require-direct ...` 参数。未带 `--require-direct` 的 runtime-only 基础门禁会自动执行直连彩排状态复核，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过；启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 35f2461da..4591282bc 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -92,9 +92,9 @@ - 背景：Pingora 影子网关已覆盖核心 Nginx 路由 parity，但只靠静态 snippet 检查和目标机手工 live canary，无法在本机 / CI 中复现真实 Nginx -> Pingora handoff 链路。 - 决策：新增 `npm run check:pingora-canary-docker` 作为正式切换前的本机容器验收：脚本启动 Docker Nginx、真实 `pingora-gateway`、mock `api-server` 和 mock SpacetimeDB，渲染同一份 `deploy/nginx/snippets/genarrative-pingora-canary.conf` 后复用 live canary 断言。Docker Nginx 必须写入生产同口径 access log，并在 live smoke 后复用 `scripts/check-pingora-canary-access-log-parity.mjs` 按同一 `request_id` 对账 canary healthz、代表性 API、SpacetimeDB identity 和静态资源路径，避免本机 / CI 只验证 handoff 响应头。默认 Docker 或镜像缺失时跳过，CI / 目标 agent 用 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull` 强制执行；目标机人工 include 后仍必须跑 `npm run check:pingora-canary-live`。同时新增 `npm run check:pingora-release-readiness` 作为正式切换聚合门禁，默认适合本机提交前检查；切换窗口必须用 `--require-docker --pull-docker --require-nginx --require-live` 强制 Docker handoff、目标机 `nginx -t` 和 live canary 全部通过，且 `--require-live` 必须显式提供 `--live-host`，避免只打到 Nginx 默认 vhost。 -- 真实路径补充：前缀 canary 通过后、Pingora direct 直连前，使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary。该 snippet 必须作为独立本机 `server` include 到 Nginx `http` 上下文，默认监听 `127.0.0.1:18083` 并写独立 `genarrative-pingora-realpath-canary.access.log`，不能 include 到生产 `443` server 内覆盖正式 location。`check-pingora-canary-live.mjs --realpath` 和 `check-pingora-canary-access-log-parity.mjs --realpath` 负责验证真实 `/api`、`/v1`、`/assets` 路径；release readiness 用 `--require-realpath-live` 把该阶段纳入目标机 runtime-only 门禁。 +- 真实路径补充：前缀 canary 通过后、Pingora direct 直连前，使用 `deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf` 做真实路径 canary。该 snippet 必须作为独立本机 `server` include 到 Nginx `http` 上下文，默认监听 `127.0.0.1:18083` 并写独立 `genarrative-pingora-realpath-canary.access.log`，不能 include 到生产 `443` server 内覆盖正式 location。`check-pingora-canary-live.mjs --realpath` 和 `check-pingora-canary-access-log-parity.mjs --realpath` 负责验证真实 `/api`、`/v1`、`/assets` 路径；release readiness 默认执行 `check:pingora-realpath-canary-toggle`，验证 realpath canary 启停脚本 dry-run、apply、失败回滚和 disable 恢复逻辑，目标机 runtime-only 用 `--require-realpath-live` 把已启用真实路径 canary 纳入门禁。 - 影响范围：`scripts/check-pingora-canary-docker.mjs`、`scripts/check-pingora-release-readiness.mjs`、`package.json`、生产运维护栏、Nginx README、Pingora 试点文档和生产运维文档。 -- 验证方式：`node --check scripts/check-pingora-canary-docker.mjs scripts/check-pingora-release-readiness.mjs`、`npm run check:pingora-canary-docker`、`npm run check:pingora-release-readiness`、`npm run check:nginx-pingora-canary`、`npm run check:production-ops`；有 Docker 镜像或允许拉取时追加 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，目标机切换窗口追加 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名>`。 +- 验证方式：`node --check scripts/check-pingora-canary-docker.mjs scripts/check-pingora-release-readiness.mjs scripts/check-pingora-release-readiness-plan.mjs scripts/check-pingora-realpath-canary-toggle.mjs`、`npm run check:pingora-realpath-canary-toggle`、`npm run check:pingora-canary-docker`、`npm run check:pingora-release-readiness`、`npm run check:nginx-pingora-canary`、`npm run check:production-ops`；有 Docker 镜像或允许拉取时追加 `node scripts/check-pingora-canary-docker.mjs --require-docker --pull`，目标机切换窗口追加 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名>`。 - 关联文档：`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`、`deploy/nginx/README.md`。 ## 2026-06-15 SpacetimeDB 本地 skills 只保留 CLI / Concepts / Rust diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index df4a1823d..f6d061cc4 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -131,7 +131,7 @@ - 现象：目标机把 `genarrative-pingora-realpath-canary.conf` 放进 `/etc/nginx/conf.d/` 后，`nginx -t` 失败并报 `unknown log format "genarrative_upstream"`。 - 原因：真实路径 canary 是独立 `server` 片段，并使用 `access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;`。Nginx 会按文件名顺序加载 `conf.d`；如果 canary 文件名早于定义 `log_format genarrative_upstream` 的主站配置，access log 行会先被解析而找不到格式。 - 处理：真实路径 canary 启停统一用 current release 随包脚本，不再手工写 `/etc/nginx/conf.d/`。启用执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`，脚本固定写入晚于主站配置加载的 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，并在 `nginx -t`、reload 或 live smoke 失败时恢复写入前配置。关闭执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本在 `nginx -t` 或 reload 失败时恢复删除前配置。另一种长期做法是把 `log_format` 放到所有 `conf.d` server 之前的全局 Nginx 配置。检查配置时不要把 probe token 原文写入记录。 -- 验证：启用脚本通过后，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`、`node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ...` 和 `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...`。若只启用了真实路径 canary，不要同时传 `--require-live`，否则前缀 canary 未启用时会按正式 Nginx HTTP 入口返回 301。 +- 验证：提交前运行 `npm run check:pingora-realpath-canary-toggle` 或默认聚合门禁 `npm run check:pingora-release-readiness`，确认启停脚本的 dry-run、apply、失败回滚和 disable 恢复逻辑仍被覆盖。启用脚本通过后，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`、`node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log ...` 和 `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...`。若只启用了真实路径 canary，不要同时传 `--require-live`，否则前缀 canary 未启用时会按正式 Nginx HTTP 入口返回 301。 - 关联：`deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf`、`deploy/nginx/README.md`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`、`scripts/check-pingora-release-readiness.mjs`。 ## Pingora release readiness 脚本不能只存在于源码 checkout diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index 8e8476b0a..e4c8d7940 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -117,7 +117,7 @@ node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs `--warn-only`、`--allow-extra-files` 和 `--allow-extra-root-entries` 只允许值班人员在正式 runbook 之外作为独立排障命令使用；`plan:pingora-direct-cutover` 生成的正式切换计划不得携带这些放行参数。若需要放行额外证据条目或忽略 `CRITICAL`，应先在证据根目录外保存人工说明、修正现场或重新生成正式证据，而不是把排障开关写进切换 runbook。 -`check:pingora-release-readiness` 是正式切换前的聚合门禁，默认串行执行 `cargo test -p pingora-gateway`、mock 上游 smoke、路由矩阵 parity、Nginx canary snippet 校验、Docker handoff smoke、canary access log 对账烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、current release 自审烟测、直连彩排状态烟测、release readiness 计划自检、生产运维护栏、Pingora cutover 状态快照烟测、Pingora cutover 证据包烟测、Pingora cutover 命令证据烟测、Pingora cutover 证据 manifest 验真烟测、Pingora cutover 证据根目录审计烟测、API release build 烟测、Pingora production release 真实构建烟测和 API deploy release 烟测。普通本机执行时 Docker / Nginx 能力仍按子脚本默认口径跳过；切换窗口或 CI 必须执行： +`check:pingora-release-readiness` 是正式切换前的聚合门禁，默认串行执行 `cargo test -p pingora-gateway`、mock 上游 smoke、路由矩阵 parity、Nginx canary snippet 校验、Docker handoff smoke、canary access log 对账烟测、realpath canary 启停烟测、直连入口静态预检、直连启用 / 回退 dry-run 行为检查、current release 自审烟测、直连彩排状态烟测、release readiness 计划自检、生产运维护栏、Pingora cutover 状态快照烟测、Pingora cutover 证据包烟测、Pingora cutover 命令证据烟测、Pingora cutover 证据 manifest 验真烟测、Pingora cutover 证据根目录审计烟测、API release build 烟测、Pingora production release 真实构建烟测和 API deploy release 烟测。普通本机执行时 Docker / Nginx 能力仍按子脚本默认口径跳过；切换窗口或 CI 必须执行： 目标机 current release 上的启用前基础门禁和启用后 `--require-direct` 复核使用同一个聚合脚本，但必须追加 `--release-runtime-only`。该模式只执行发布包内可自包含的运行时复核：current release 自审、启用前直连彩排状态复核、live canary、真实 access log 对账、direct preflight、health patrol env 复核和 direct live smoke；不会运行 Cargo、npm、Docker 或 Nginx 源码 / 构建环境门禁，并会拒绝 `--require-docker`、`--pull-docker` 与 `--require-nginx`。未带 `--require-direct` 的启用前基础门禁会自动执行 `scripts/ops/pingora-direct-rehearsal-status.mjs --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow 与 realpath canary 高端口在线；启用后 `--require-direct` 复核不再要求 Nginx 接公网的彩排状态，改为检查 direct preflight、health patrol 直连模式和 direct live smoke。 diff --git a/scripts/check-pingora-release-readiness-plan.mjs b/scripts/check-pingora-release-readiness-plan.mjs index b7e434af2..adc5241f0 100644 --- a/scripts/check-pingora-release-readiness-plan.mjs +++ b/scripts/check-pingora-release-readiness-plan.mjs @@ -94,6 +94,7 @@ function main() { assertDefaultPlanIncludesPingoraCutoverEvidenceVerifySmoke(); assertDefaultPlanIncludesPingoraCutoverEvidenceAuditSmoke(); assertDefaultPlanIncludesCanaryAccessLogParitySmoke(); + assertDefaultPlanIncludesRealpathCanaryToggleSmoke(); assertDefaultPlanIncludesPingoraProductionReleaseBuildSmoke(); assertReleaseRuntimeOnlyPlanUsesCurrentReleaseScripts(); assertReleaseRuntimeOnlyPlanIncludesDirectRehearsalStatus(); @@ -366,6 +367,22 @@ function assertDefaultPlanIncludesCanaryAccessLogParitySmoke() { ); } +function assertDefaultPlanIncludesRealpathCanaryToggleSmoke() { + const plan = readPlan(['--dry-run-plan']); + const step = findStep(plan, 'Pingora realpath canary 启停烟测'); + if (!step) { + failures.push( + '默认 release readiness 计划必须包含 Pingora realpath canary 启停烟测。', + ); + return; + } + assertIncludes( + step.args, + 'check:pingora-realpath-canary-toggle', + 'Pingora realpath canary 启停烟测必须调用 check:pingora-realpath-canary-toggle。', + ); +} + function assertDefaultPlanIncludesProductionApiReleaseSmoke() { const plan = readPlan(['--dry-run-plan']); const step = findStep(plan, 'API release build 烟测'); diff --git a/scripts/check-pingora-release-readiness.mjs b/scripts/check-pingora-release-readiness.mjs index 126e95a18..28c7abc95 100644 --- a/scripts/check-pingora-release-readiness.mjs +++ b/scripts/check-pingora-release-readiness.mjs @@ -1601,6 +1601,11 @@ function buildSteps(config) { command: 'npm', args: ['run', 'check:pingora-canary-access-log-parity'], }, + { + name: 'Pingora realpath canary 启停烟测', + command: 'npm', + args: ['run', 'check:pingora-realpath-canary-toggle'], + }, { name: 'Pingora canary live 参数护栏', command: 'node', diff --git a/scripts/check-production-health-patrol.mjs b/scripts/check-production-health-patrol.mjs index abc2fe718..b61f12da5 100644 --- a/scripts/check-production-health-patrol.mjs +++ b/scripts/check-production-health-patrol.mjs @@ -306,6 +306,10 @@ async function prepareFixture(name) { [ '#!/usr/bin/env bash', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, + 'if [[ "$1" == "list-units" && "$2" == "genarrative-external-generation-worker@*.service" ]]; then', + ' echo "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker 1"', + ' exit 0', + 'fi', 'echo active', 'exit 0', '', From 7fa1c6d79138ee2268186eca674f9ac1b3af7166 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 15:42:38 +0800 Subject: [PATCH 19/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20=E9=AB=98?= =?UTF-8?q?=E7=AB=AF=E5=8F=A3=E7=9B=B4=E8=BF=9E=E6=BC=94=E7=BB=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 dev 高端口直连真实演练结果 记录临时 systemd 演练与 env source 踩坑 确认公网入口仍由 Nginx 承接 --- docs/project-memory/shared-memory/pitfalls.md | 8 ++++++ ...开发运维】Pingora独立网关试点-2026-06-11.md | 27 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index f6d061cc4..bce05c706 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -2366,6 +2366,14 @@ - 验证：先运行随包 `node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs --env-file /etc/genarrative/health-patrol.env --expected-gateway-mode nginx --expected-public-base-url https://dev.genarrative.world --require-empty-public-host`，再 `systemctl start genarrative-health-patrol.service`；最后运行 `node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs --release-root /opt/genarrative/current --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical`。 - 关联：`deploy/env/health-patrol.env.example`、`scripts/ops/production-health-patrol.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 +## Pingora 高端口直连演练不要 source env 文件 + +- 现象：在 dev 上用临时 env 启动高端口 Pingora direct 演练时，shell 报 `/tmp/pingora-direct-highport-*.env: line ...: max-age=31536000,: command not found`，或者临时演练进程启动后没有按预期监听 `18443/18080`。 +- 原因：`pingora-gateway.env` 是 systemd EnvironmentFile 口径，允许 `GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable` 这类带空格的值；它不是可安全 `source` 的 shell 脚本。用 shell `source` 会把空格后的内容拆成命令或参数。另一个容易误判的点是 Pingora 默认优雅退出窗口较长，停止临时 systemd unit 后可能短暂停在 `stop-sigterm`，即使监听端口已经释放。 +- 处理：高端口真实演练优先用临时 systemd unit 启动 current release 的 `/opt/genarrative/current/pingora-gateway`，通过 `systemd-run --property=EnvironmentFile=/tmp/.env --property=User=genarrative --property=WorkingDirectory=/opt/genarrative/current ...` 让 systemd 解析 env；或使用显式安全 env 解析器，禁止直接 `source`。临时 env 要把正式 shadow 端口改到独立 loopback 端口，例如 `127.0.0.1:18084`，HTTPS / HTTP redirect 用 `127.0.0.1:18443` / `127.0.0.1:18080`，access log 写独立文件。演练结束先 `systemctl stop <临时unit>`，再用 `ss -ltnp` 确认高端口已释放；若临时 unit 仍停在 `deactivating/stop-sigterm` 且只剩演练进程，可对该临时 unit 执行 `systemctl kill -s SIGKILL <临时unit>` 收尾，不要碰正式 `genarrative-pingora-gateway.service`。 +- 验证：先跑 `check-pingora-direct-preflight.mjs --env-file <临时env> --require-live-env --check-cert-readable --check-service-user-cert-readable --check-ports-free --allow-loopback-only`；启动临时 unit 后跑 `check-pingora-direct-live.mjs --https-base-url https://127.0.0.1:18443 --http-base-url http://127.0.0.1:18080 --host <域名> --redirect-host <域名> --redirect-base-url https://<域名> --require-wss-upgrade --pingora-access-log <临时log> --insecure-tls --json`，要求 `OK` 且 `direct-access-log matchedCount == checked`。收尾后复核 `80/443` 仍由 Nginx 监听，正式 Pingora shadow 仍为 `127.0.0.1:18081`。 +- 关联：`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`deploy/pingora/pingora-gateway.env.example`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 + ## SpacetimeDB 连接池租约必须有 Drop 兜底，acquire 不允许无界自旋 - 现象：release 上 api-server 周期性出现全量 `spacetime_stage="pool_acquire" elapsed_ms=45000` 业务超时，`/readyz` 503（`reason=spacetime_unhealthy, stage=pool_acquire`），`/healthz` 仍 200，只有重启能恢复，过若干小时复发。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index e4c8d7940..b4a3894d4 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -410,6 +410,33 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - Nginx 仍监听公网 `80/443`，Pingora 仍只监听 `127.0.0.1:18081`；`127.0.0.1:18083` 无监听。`https://dev.genarrative.world/` 与 `https://dev.genarrative.world/api/creation-entry/config` 通过本机 `--resolve` 验收均为 `200`，证明正常公网入口仍由 Nginx 承接。 - 上传到 `/tmp/dev-pingora-api-20260617140915` 的临时发布包已删除；新 release 目录约 `57M`，清理后 dev 根盘约 `92%` 使用率。后续正式证据归档前仍需复核磁盘余量。 +2026-06-18 已在 dev 机把 current release 提升到 `dev-pingora-realpath-readiness-20260618151219`，发布包由分支 `codex/pingoranginx` 的 `e79cc5bd0909291431892f5dc7e3e68f839ae52a` 构建，包含最新 `check:pingora-realpath-canary-toggle` 默认门禁。部署后： + +- `/opt/genarrative/current/release-manifest.api-server.json` 记录 `source_commit=e79cc5bd0909291431892f5dc7e3e68f839ae52a`，并登记 `api-server` 与 `pingora-gateway` 两个 artifact。 +- `grep 'Pingora realpath canary 启停烟测' /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs` 可确认 realpath canary 启停烟测已进入随包聚合门禁。 +- `nginx`、`genarrative-api`、`genarrative-pingora-gateway`、`spacetimedb`、`genarrative-external-generation-worker@1.service` 和 `genarrative-external-generation-controller.service` 均为 `active`；`/healthz` 与 `/readyz` 返回正常。 +- 公网 `80/443` 仍由 Nginx 监听，Pingora shadow 仍只监听 `127.0.0.1:18081`，真实路径 canary 仍由 Nginx 监听 `127.0.0.1:18083`，本阶段没有执行 `pingora-direct-enable.sh --apply`。 +- `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live ...` 通过 current release 自审、直连彩排状态、realpath live smoke 和 realpath access log 对账，最终对账 `47/47 matched`。 +- 部署前 dev 根盘约 `99%`，先清理旧 `/tmp` 发布包，再删除不再作为当前回滚点的旧数字 release `52`、`53`、`54`、`55`、`57`，清理后 `df -h /` 约剩 `3.2G`，避免证据包阶段被磁盘空间干扰。 + +同日进入正式直连前证据链彩排，但仍未执行公网直连切换： + +- 使用 cutover run id `pingora-direct-dev-20260618T1520-realpath-readiness` 生成 `pre-cutover` 证据包：`/var/log/genarrative/pingora-cutover-evidence/20260618T071853Z-pre-cutover`。 +- `node -- /opt/genarrative/current/scripts/ops/pingora-cutover-evidence-verify.mjs --bundle-dir /var/log/genarrative/pingora-cutover-evidence/20260618T071853Z-pre-cutover --require-summary-ok` 返回 `ok=true`，`checkedCount=4`，`failedCount=0`；manifest summary 为 `OK`，`criticalCount=0`，`warningCount=0`。 +- `node -- /opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --dry-run-cutover --require-direct ... --cutover-run-id pingora-direct-dev-20260618T1520-realpath-readiness ...` 生成 22 步 JSON runbook，覆盖 Host 一致性确认、current release 自审、pre-cutover / post-enable / post-rollback 三阶段证据包、enable / rollback 命令证据、health patrol env 切换、启用后 direct live 证据和最终证据根目录总审计；最终总审计包含两条 current release 脚本身份护栏：`--require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 与 `--require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh`。 +- 当前 dev 的正式 `/etc/genarrative/pingora-gateway.env` 还没有进入 direct 模式：缺少 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE`、`GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`，`GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO` 仍为 `http`，且 systemd 最终配置未包含 `AmbientCapabilities=CAP_NET_BIND_SERVICE` / `CapabilityBoundingSet=CAP_NET_BIND_SERVICE`。`check-pingora-direct-preflight.mjs --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free` 因上述缺口失败，符合正式直连前应阻断的预期。 +- dev 已有证书副本 `/etc/genarrative/pingora-tls/dev.genarrative.world/fullchain.pem` 与 `privkey.pem`，权限为 `root:genarrative 0640`，可用于后续 loopback 高端口 direct rehearsal 或正式切换前 env 准备；不要直接放宽 `/etc/letsencrypt/live` / `archive` 权限。 + +同日已完成不触碰公网 `80/443` 的高端口直连真实演练： + +- 基于 `/etc/genarrative/pingora-gateway.env` 生成临时 env `/tmp/pingora-direct-highport-20260618T1535.env`，把 shadow 监听改到 `127.0.0.1:18084`，直连 HTTPS / HTTP redirect 分别设为 `127.0.0.1:18443` / `127.0.0.1:18080`，证书指向 `/etc/genarrative/pingora-tls/dev.genarrative.world/`，`FORWARDED_PROTO=https`，access log 独立写入 `/var/log/genarrative/pingora-direct-highport-20260618T1535.access.log`。 +- `node -- /opt/genarrative/current/scripts/check-pingora-direct-preflight.mjs --env-file /tmp/pingora-direct-highport-20260618T1535.env --require-live-env --check-cert-readable --check-service-user-cert-readable --check-ports-free --allow-loopback-only --json` 通过，确认两个高端口可绑定，当前用户和 `genarrative` 用户都可读 TLS cert/key。 +- 独立演练进程通过临时 systemd unit `genarrative-pingora-direct-highport-pingora-direct-highport-20260618T1535.service` 启动 `/opt/genarrative/current/pingora-gateway`，只读取临时 env，不改正式 `genarrative-pingora-gateway.service`、不安装 direct-entry drop-in、不重启 Nginx。 +- `node -- /opt/genarrative/current/scripts/check-pingora-direct-live.mjs --https-base-url https://127.0.0.1:18443 --http-base-url http://127.0.0.1:18080 --host dev.genarrative.world --redirect-host dev.genarrative.world --redirect-base-url https://dev.genarrative.world --spacetime-database genarrative-prod --pingora-access-log /var/log/genarrative/pingora-direct-highport-20260618T1535.access.log --access-log-since-lines 4000 --require-wss-upgrade --insecure-tls --json` 返回 `OK`：HTTPS 根路径 `200`、API config `200`、SpacetimeDB identity `405`、HTTP redirect `301` 到正式域名、ACME 静态 `404`、HTTP/2 ALPN `h2`、WSS subscribe `101`，静态资源 `Cache-Control/ETag/Last-Modified/Accept-Ranges`、`HEAD`、`Range 206`、ETag / Last-Modified `304` 全部通过，access log 对账 `19/19 matched`。 +- 演练结束后停止并清理临时 unit；因 Pingora 默认优雅退出窗口较长，临时 unit 曾短暂停在 `stop-sigterm`，确认高端口无监听后对该临时 unit 执行 `systemctl kill -s SIGKILL` 收尾。收尾后 `127.0.0.1:18080`、`127.0.0.1:18084`、`127.0.0.1:18443` 均无监听，正式公网 `80/443` 仍由 Nginx 监听，正式 Pingora shadow 仍为 `127.0.0.1:18081`，realpath canary 仍为 Nginx `127.0.0.1:18083`。 +- 复核 `http://127.0.0.1:8082/healthz` 与 `/readyz` 正常，`curl --resolve dev.genarrative.world:443:127.0.0.1 https://dev.genarrative.world/` 返回 Nginx `HTTP/2 200`，`http://127.0.0.1:18081/__genarrative_pingora/healthz` 带 probe token 返回 `{"ok":true,"gateway":"pingora-shadow","maintenance":false}`。 +- 经验：临时 env 含 `GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable` 这类带空格值，不能用 shell `source` 直接加载；演练进程应交给 systemd `EnvironmentFile=` 或使用安全 env 解析器，否则 shell 会把 `max-age=31536000,` 当命令执行。 + ## 环境变量 | 变量 | 默认值 | 说明 | From 3c1fd004a757170959c6772eb227ee5699173148 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 15:47:53 +0800 Subject: [PATCH 20/22] =?UTF-8?q?=E8=AE=B0=E5=BD=95=20Pingora=20=E7=9B=B4?= =?UTF-8?q?=E8=BF=9E=E7=AA=97=E5=8F=A3=E5=87=86=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补充 dev final-prep 审计结果 记录 direct env 候选与 dry-run runbook 明确 active env 提升前不得执行公网切换 --- docs/project-memory/shared-memory/pitfalls.md | 1 + .../【开发运维】Pingora独立网关试点-2026-06-11.md | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index bce05c706..fa45cfe79 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -2371,6 +2371,7 @@ - 现象：在 dev 上用临时 env 启动高端口 Pingora direct 演练时，shell 报 `/tmp/pingora-direct-highport-*.env: line ...: max-age=31536000,: command not found`，或者临时演练进程启动后没有按预期监听 `18443/18080`。 - 原因：`pingora-gateway.env` 是 systemd EnvironmentFile 口径，允许 `GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable` 这类带空格的值；它不是可安全 `source` 的 shell 脚本。用 shell `source` 会把空格后的内容拆成命令或参数。另一个容易误判的点是 Pingora 默认优雅退出窗口较长，停止临时 systemd unit 后可能短暂停在 `stop-sigterm`，即使监听端口已经释放。 - 处理：高端口真实演练优先用临时 systemd unit 启动 current release 的 `/opt/genarrative/current/pingora-gateway`，通过 `systemd-run --property=EnvironmentFile=/tmp/.env --property=User=genarrative --property=WorkingDirectory=/opt/genarrative/current ...` 让 systemd 解析 env；或使用显式安全 env 解析器，禁止直接 `source`。临时 env 要把正式 shadow 端口改到独立 loopback 端口，例如 `127.0.0.1:18084`，HTTPS / HTTP redirect 用 `127.0.0.1:18443` / `127.0.0.1:18080`，access log 写独立文件。演练结束先 `systemctl stop <临时unit>`，再用 `ss -ltnp` 确认高端口已释放；若临时 unit 仍停在 `deactivating/stop-sigterm` 且只剩演练进程，可对该临时 unit 执行 `systemctl kill -s SIGKILL <临时unit>` 收尾，不要碰正式 `genarrative-pingora-gateway.service`。 +- 处理补充：正式 `plan:pingora-direct-cutover` / `check-pingora-release-readiness.mjs --dry-run-cutover --require-direct` 生成的 runbook 默认读取 active `/etc/genarrative/pingora-gateway.env`，不会自动使用 `/tmp` 候选 env。若只生成了候选 direct env，必须先在维护窗口内把候选 env 提升为 active env，并确认 Nginx 已释放 `80/443`，再执行 runbook 的 direct preflight、enable dry-run 和 enable apply；否则 runbook 第 5 步仍会按 shadow env 报缺 `TLS_LISTEN`、`HTTP_REDIRECT_LISTEN`、cert/key、`FORWARDED_PROTO=https` 以及 direct-entry capability。不要把候选 env 的 loopback / 高端口预检通过误解为 active env 已满足正式直连门禁。 - 验证：先跑 `check-pingora-direct-preflight.mjs --env-file <临时env> --require-live-env --check-cert-readable --check-service-user-cert-readable --check-ports-free --allow-loopback-only`；启动临时 unit 后跑 `check-pingora-direct-live.mjs --https-base-url https://127.0.0.1:18443 --http-base-url http://127.0.0.1:18080 --host <域名> --redirect-host <域名> --redirect-base-url https://<域名> --require-wss-upgrade --pingora-access-log <临时log> --insecure-tls --json`，要求 `OK` 且 `direct-access-log matchedCount == checked`。收尾后复核 `80/443` 仍由 Nginx 监听，正式 Pingora shadow 仍为 `127.0.0.1:18081`。 - 关联：`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`deploy/pingora/pingora-gateway.env.example`、`docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md`。 diff --git a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md index b4a3894d4..934f6b28c 100644 --- a/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md +++ b/docs/technical/【开发运维】Pingora独立网关试点-2026-06-11.md @@ -437,6 +437,14 @@ dev 根盘空间在安装后曾接近满盘；2026-06-17 进入 canary 前已清 - 复核 `http://127.0.0.1:8082/healthz` 与 `/readyz` 正常，`curl --resolve dev.genarrative.world:443:127.0.0.1 https://dev.genarrative.world/` 返回 Nginx `HTTP/2 200`，`http://127.0.0.1:18081/__genarrative_pingora/healthz` 带 probe token 返回 `{"ok":true,"gateway":"pingora-shadow","maintenance":false}`。 - 经验：临时 env 含 `GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable` 这类带空格值，不能用 shell `source` 直接加载；演练进程应交给 systemd `EnvironmentFile=` 或使用安全 env 解析器，否则 shell 会把 `max-age=31536000,` 当命令执行。 +同日继续进入正式窗口前 final-prep，不执行公网切换： + +- `node -- /opt/genarrative/current/scripts/ops/pingora-direct-rehearsal-status.mjs --release-root /opt/genarrative/current --expect-public-gateway nginx --require-pingora-shadow --require-realpath-canary --require-current-release-gateway --fail-on-critical` 返回 `summary.status=OK`：公网 `80/443` 为 Nginx，Pingora shadow 为 `127.0.0.1:18081`，realpath canary 为 Nginx `127.0.0.1:18083`，health patrol env 为 `nginx` 模式且 public base URL 为 `https://dev.genarrative.world`。 +- `node -- /opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show` 返回 `summary.status=OK`，确认 current release 自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、manifest 登记 `pingora-gateway`，且 `genarrative-pingora-gateway.service` 的 `ExecStart` 指向 `/opt/genarrative/current/pingora-gateway`。 +- 生成正式 direct env 候选 `/tmp/pingora-direct-dev-20260618T1548-final-prep.pingora-gateway.env`，与正式 `/etc/genarrative/pingora-gateway.env` 的差异仅为 `FORWARDED_PROTO=https`，追加 `TLS_LISTEN=0.0.0.0:443`、`HTTP_REDIRECT_LISTEN=0.0.0.0:80`、`TLS_CERT_FILE=/etc/genarrative/pingora-tls/dev.genarrative.world/fullchain.pem`、`TLS_KEY_FILE=/etc/genarrative/pingora-tls/dev.genarrative.world/privkey.pem`、`HTTP_REDIRECT_TARGET_SCHEME=https`。该候选 env 的 `check-pingora-direct-preflight.mjs --require-live-env --check-cert-readable --check-service-user-cert-readable` 通过，证明 direct env 内容和证书权限就绪。 +- 生成正式 dry-run cutover runbook `/tmp/pingora-direct-dev-20260618T1548-final-prep.runbook.json`，共 22 步，cutover run id 为 `pingora-direct-dev-20260618T1548-final-prep`，覆盖 pre-cutover 证据、enable/rollback 命令证据、post-enable/post-rollback 证据和最终根目录总审计。 +- 注意：该 runbook 仍按正式切换窗口读取 active `/etc/genarrative/pingora-gateway.env`。因此正式执行前必须在维护窗口内先把已审阅的 direct env 提升为 `/etc/genarrative/pingora-gateway.env`，确认 Nginx 释放 `80/443`，再进入 runbook 第 5 步 preflight；否则第 5 步会继续按当前 shadow env 失败。当前阶段未替换 active env、未安装 direct-entry drop-in、未释放 Nginx 端口、未执行 `pingora-direct-enable.sh --apply`。 + ## 环境变量 | 变量 | 默认值 | 说明 | From e3258f6f1d3b502eae64124cc8f8c1466f413c76 Mon Sep 17 00:00:00 2001 From: kdletters Date: Thu, 18 Jun 2026 21:10:45 +0800 Subject: [PATCH 21/22] =?UTF-8?q?=E5=AE=8C=E5=96=84=20Pingora=20=E7=9B=B4?= =?UTF-8?q?=E8=BF=9E=E5=88=87=E6=8D=A2=E9=97=A8=E7=A6=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 Pingora shadow env 回切脚本与对应检查。 补齐直连证据包时间线和 cutoverRunId 审计门禁。 支持 Gitea Host 透传并更新直连多域名文档。 修复百分号编码静态图标路径并补 smoke 覆盖。 更新生产发布与运维护栏对 Pingora 发布包的校验。 --- deploy/nginx/README.md | 18 +- deploy/pingora/pingora-gateway.env.example | 5 + .../shared-memory/decision-log.md | 21 +- docs/project-memory/shared-memory/pitfalls.md | 38 +- ...开发运维】Pingora独立网关试点-2026-06-11.md | 65 +- ...发运维】本地开发验证与生产运维-2026-05-15.md | 28 +- jenkins/Jenkinsfile.production-api-build | 2 +- jenkins/Jenkinsfile.production-api-deploy | 4 +- package.json | 1 + scripts/build-production-release.sh | 2 + scripts/check-nginx-pingora-canary.mjs | 15 +- .../check-pingora-current-release-audit.mjs | 43 +- .../check-pingora-cutover-evidence-audit.mjs | 385 ++++++++-- .../check-pingora-cutover-evidence-bundle.mjs | 140 +++- .../check-pingora-cutover-status-snapshot.mjs | 177 ++++- scripts/check-pingora-direct-enable.mjs | 53 +- .../check-pingora-direct-rehearsal-status.mjs | 2 + ...heck-pingora-gateway-env-shadow-switch.mjs | 352 ++++++++++ scripts/check-pingora-gateway-smoke.mjs | 260 ++++++- .../check-pingora-release-readiness-plan.mjs | 662 +++++++++++++----- scripts/check-pingora-release-readiness.mjs | 217 +++++- scripts/check-production-api-deploy.mjs | 37 + scripts/check-production-api-release.mjs | 18 +- scripts/check-production-ops-guardrails.mjs | 427 ++++++++++- .../pingora-gateway-env-shadow-switch.mjs | 254 +++++++ scripts/deploy/production-api-deploy.sh | 7 + scripts/ops/pingora-current-release-audit.mjs | 39 +- .../ops/pingora-cutover-evidence-audit.mjs | 118 +++- .../ops/pingora-cutover-evidence-bundle.mjs | 100 +++ .../ops/pingora-cutover-status-snapshot.mjs | 136 +++- server-rs/crates/pingora-gateway/src/main.rs | 363 +++++++++- 31 files changed, 3591 insertions(+), 398 deletions(-) create mode 100644 scripts/check-pingora-gateway-env-shadow-switch.mjs create mode 100644 scripts/deploy/pingora-gateway-env-shadow-switch.mjs diff --git a/deploy/nginx/README.md b/deploy/nginx/README.md index 4a1bd5700..98cddbc0d 100644 --- a/deploy/nginx/README.md +++ b/deploy/nginx/README.md @@ -46,18 +46,20 @@ - Pingora 网关默认仍只监听本机 shadow 端口，不绑定公网 `80/443`。 - 需要评估直连入口时，显式配置 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN`、`GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` 和 `GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE`；网关只读取现有证书文件，不负责 Certbot / ACME 自动续期。 - 需要 HTTP 到 HTTPS 跳转时，再显式配置 `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN`；该入口除 `/.well-known/acme-challenge/*` 外统一 301 到 HTTPS，ACME challenge 仍从 `GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT` 精确读取。 -- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连彩排状态、直连启用、health patrol env 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 +- 同一公网 IP 上还承载 `git.genarrative.world` 时，直连前必须配置 `GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS=git.genarrative.world` 与 `GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM=127.0.0.1:3000`，让命中 Gitea Host 的请求整站转发到 Gitea；该路由不走应用维护页、API body limit 或网关接流保护，避免影响 git clone / push。当前 Pingora TLS listener 只加载一组 cert/key；如果直连 443 同时服务 `dev.genarrative.world` 和 `git.genarrative.world`，证书必须同时覆盖两个域名，不能使用只有单域名 SAN 的证书。 +- API release 必须把 `api-server.sha256`、可选 `pingora-gateway.sha256`、`scripts/check-pingora-release-readiness.mjs`、`scripts/check-pingora-canary-live.mjs`、`scripts/check-pingora-canary-access-log-parity.mjs`、`scripts/ops/pingora-current-release-audit.mjs`、`scripts/ops/pingora-direct-rehearsal-status.mjs`、`scripts/ops/pingora-cutover-status-snapshot.mjs`、`scripts/ops/pingora-cutover-evidence-bundle.mjs`、`scripts/check-pingora-direct-preflight.mjs`、`scripts/check-pingora-direct-live.mjs`、`scripts/check-production-health-patrol-env.mjs`、`scripts/deploy/pingora-health-patrol-env-switch.mjs`、`scripts/deploy/pingora-gateway-env-shadow-switch.mjs`、`scripts/deploy/pingora-tls-cert-sync.mjs`、`deploy/systemd/`、`deploy/env/` 和 `deploy/pingora/` 一起复制到 `/opt/genarrative/current`；current release 自审、直连彩排状态、直连启用、health patrol env 切换、Pingora gateway env shadow 切换、TLS 证书同步、canary live、access log 对账、cutover 状态快照和证据包脚本都从 current release 推导依赖路径，切换窗口不得依赖 Jenkins 工作区或目标机源码 checkout。 - 目标机如果使用 Certbot / Let’s Encrypt 证书，不要直接放宽 `/etc/letsencrypt/live` 或 `archive` 的目录 / 私钥权限。先用随包脚本把 root 可读的 live symlink 解析并复制到 Pingora 私有目录，例如 `sudo -n node -- /opt/genarrative/current/scripts/deploy/pingora-tls-cert-sync.mjs --apply --source-cert-file /etc/letsencrypt/live/<域名>/fullchain.pem --source-key-file /etc/letsencrypt/live/<域名>/privkey.pem --target-dir /etc/genarrative/pingora-tls/<域名>`；脚本默认 dry-run，`--apply` 才原子写入 `fullchain.pem` / `privkey.pem`，目标目录默认 `root:genarrative 0750`，文件默认 `root:genarrative 0640`，并复核 `genarrative` 服务用户可读。随后把 `GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE` / `TLS_KEY_FILE` 指向 `/etc/genarrative/pingora-tls/<域名>/` 下的副本。 - 默认 systemd service 以 `genarrative` 非 root 用户运行，不具备绑定 `80/443` 的能力。Server-Provision 只安装 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用模板；切直连低端口时必须先确认 `genarrative` 可读证书文件、Nginx 已释放 `80/443`，再执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；脚本会先执行随包 current release 自审，确认 `pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release，失败时不会安装 drop-in。自审通过后才安装 drop-in、`systemctl daemon-reload` 并重启 Pingora；脚本会用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` drop-in 已进入 systemd 最终配置，用 `systemctl show ... ExecStart` 确认最终 service 指向随包主 service 模板中的 current release `pingora-gateway`，用 `systemctl is-active` 确认 Pingora active，并执行 direct live smoke，强制覆盖 HTTPS / HTTP redirect / ACME / WSS 101 和 Pingora access log request_id 落盘。 - `npm run check:pingora-gateway-smoke` 会用临时自签证书覆盖 TLS 直连、HTTP redirect 和 WSS subscribe 行为；正式公网切换前仍必须先走 Nginx canary 和 release readiness 门禁。 -- 目标机直连入口启用后先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable --direct-preflight-check-ports-free` 强制纳入同一条门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查、service 二进制可执行检查或 80/443 端口释放检查都会直接失败。高端口 rehearsal 如果通过 `https://127.0.0.1:18443` 打入、但 Pingora 应返回正式域名默认 HTTPS Location，可额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于直连切换 Host 一致性约束。 +- 目标机直连入口启用前先运行 `npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free`，证明 TLS 文件、service env、current release 二进制和低端口释放都满足切流条件；启用后再运行 `npm run check:pingora-direct-live`，检查 HTTPS 根路径、HTTP/2 ALPN、代表性 API、SpacetimeDB identity、WSS subscribe 握手、generated 拒绝、公开 `/healthz` 拒绝、可选 shadow probe，以及 HTTP 301 / ACME challenge；真实 SpacetimeDB 对 `GET /v1/identity` 返回 `405 Method Not Allowed` 属于可接受语义，direct live 只把它作为路径转发代表，不要求该 GET 创建 identity；release readiness 可用 `--require-direct --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log --direct-health-patrol-env-file /etc/genarrative/health-patrol.env --direct-preflight-env-file /etc/genarrative/pingora-gateway.env --direct-preflight-systemd --direct-preflight-check-cert-readable --direct-preflight-check-service-env-file --direct-preflight-check-service-user-cert-readable --direct-preflight-check-service-binary-executable` 强制纳入同一条启用后门禁，漏掉 direct HTTP base URL、正式域名 Host/SNI、redirect Location host、目标 SpacetimeDB 库、Pingora access log request_id 落盘、health patrol env direct 模式复核、systemd drop-in 生效检查、service EnvironmentFile 一致性检查、当前用户 / systemd 服务用户证书可读检查或 service 二进制可执行检查都会直接失败；启用后 `--require-direct` 复核不再要求端口空闲，因为此时 `80/443` 应由 Pingora 占用。高端口 rehearsal 如果通过 `https://127.0.0.1:18443` 打入、但 Pingora 应返回正式域名默认 HTTPS Location，可额外传 `--direct-redirect-base-url https://<域名>`；`--direct-redirect-host` 仍必须保留，用于直连切换 Host 一致性约束。 - direct live 在 HTTPS 根路径返回 `200` 且 HTML 中发现 `/assets/` 或 `/admin/assets/` 引用时，会自动请求该静态资源，确认 `Cache-Control`、`ETag`、`Last-Modified`、`Accept-Ranges: bytes`、`HEAD` 头响应、`If-None-Match` / `If-Modified-Since` 的 `304` 响应和 `Range: bytes=0-0` 的 `206 + Content-Range` 响应，并纳入 Pingora access log method/path/status 对账；若首页引用 Vite 指纹资源，还会额外确认 `Cache-Control: public, max-age=31536000, immutable` 以及指纹资源 GET / HEAD / 304 / Range access log 证据。维护模式、非 HTML 或发布包首页没有资产引用时该项标记为 skipped，不阻断维护窗口。 -- 直连启用前先 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`；验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`，脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` 已从 systemd 最终配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 smoke URL 证明 Nginx 入口真实可访问；传入 `--nginx-smoke-expect-body` 时还会要求响应体包含该片段。仓库工作区可用 `npm run deploy:pingora-direct-enable -- --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log` 和 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名> --nginx-smoke-expect-body '"ok":true'`。两个脚本默认 dry-run；启用脚本 `--apply` 必须先通过 current release 自审，并带 env、证书可读、service EnvironmentFile 一致性、服务用户证书可读、service 二进制可执行、端口释放预检、direct live 参数和 Pingora access log 参数，回退脚本 `--apply` 必须带 `--reload-nginx` 和 `--nginx-smoke-url`；如果 health patrol env 已预先切回 Nginx，也可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让回退脚本在 Nginx smoke 后复核 `gateway mode=nginx` 且 public base URL / Host 已恢复，若切换前 Nginx 巡检本来需要 Host 覆盖，则把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`；如果还要同时证明 shadow 高端口仍健康，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。本机 `npm run check:pingora-direct-enable` / `npm run check:pingora-direct-rollback` 会验证 dry-run 不会安装或删除临时 drop-in、current release 自审失败时启用脚本不会安装 drop-in，并展示启用 / 回退后的 systemd 最终配置核验、ExecStart 指向核验、Pingora active 核验、direct live smoke、Nginx 语法检查、reload 状态核验、Nginx smoke 及响应体片段核验、可选 health patrol env 复核和可选 shadow probe 复核。 +- 直连启用前先 dry-run `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status`；验证失败时执行 `/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''`，脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，用 `systemctl cat` 确认 `CAP_NET_BIND_SERVICE` 已从 systemd 最终配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 smoke URL 证明 Nginx 入口真实可访问；传入 `--nginx-smoke-expect-body` 时还会要求响应体包含该片段。回退 smoke 的 URL 与 body 必须来自切换前真实 Nginx 入口，不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`；若 smoke URL 打本机地址，再显式追加 `--nginx-smoke-host <域名>` 避免命中默认 vhost。仓库工作区可用 `npm run deploy:pingora-direct-enable -- --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log` 和 `npm run deploy:pingora-direct-rollback -- --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''`。两个脚本默认 dry-run；启用脚本 `--apply` 必须先通过 current release 自审，并带 env、证书可读、service EnvironmentFile 一致性、服务用户证书可读、service 二进制可执行、端口释放预检、direct live 参数和 Pingora access log 参数，回退脚本 `--apply` 必须带 `--reload-nginx` 和 `--nginx-smoke-url`；如果 health patrol env 已预先切回 Nginx，也可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 让回退脚本在 Nginx smoke 后复核 `gateway mode=nginx` 且 public base URL / Host 已恢复，若切换前 Nginx 巡检本来需要 Host 覆盖，则把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`；如果还要同时证明 shadow 高端口仍健康，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。本机 `npm run check:pingora-direct-enable` / `npm run check:pingora-direct-rollback` 会验证 dry-run 不会安装或删除临时 drop-in、current release 自审失败时启用脚本不会安装 drop-in，并展示启用 / 回退后的 systemd 最终配置核验、ExecStart 指向核验、Pingora active 核验、direct live smoke、Nginx 语法检查、reload 状态核验、Nginx smoke 及响应体片段核验、可选 health patrol env 复核和可选 shadow probe 复核。 +- rollback apply 前必须先执行 `node -- /opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs --apply --env-file /etc/genarrative/pingora-gateway.env`，把 active Pingora env 恢复为 `GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081`，并清空 `TLS_LISTEN`、`HTTP_REDIRECT_LISTEN`、`TLS_CERT_FILE` 和 `TLS_KEY_FILE`；只移除 systemd direct-entry drop-in 而不恢复 env，会让回退后的 Pingora shadow 继续按 direct 低端口或半 TLS 配置启动。 - `npm run check:pingora-current-release-audit` 会烟测 current release 自审脚本；正式直连 runbook 会先执行 `/opt/genarrative/current/scripts/ops/pingora-current-release-audit.mjs --release-root /opt/genarrative/current --require-pingora-gateway --systemd-show`，只读确认发布包自包含、`api-server.sha256` / `pingora-gateway.sha256` 匹配、`release-manifest.api-server.json` 已登记 `pingora-gateway`、`pingora-gateway` 可执行且 systemd `ExecStart` 指向 current release。 -- `npm run check:pingora-cutover-status-snapshot` 会烟测状态快照脚本，`npm run check:pingora-cutover-evidence-bundle` 会烟测证据包脚本；正式切换窗口与 release readiness 使用 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs` 归档切换前 / 启用后 / 回退后三个阶段的 `manifest.json`、`snapshot.json`、stdout / stderr 和命令记录。直连 runbook 的三个证据包阶段都会向状态快照透传 `--require-pingora-gateway`，在 `checks.current-release-audit` 中保存 Pingora 二进制、sha256、release manifest 和 systemd `ExecStart` 自审结果；启用后证据包还会通过 `--run-direct-live` 保存 `direct-live.json`、direct live stdout / stderr、命令记录、Pingora access log `request_id` 反查结果，以及静态响应头白名单证据。`manifest.summary.directLiveAccessLog` 会提升 checked / matched / missing / mismatch 对账摘要，`manifest.summary.directLiveStaticHeaders` 会提升普通静态和 Vite 指纹静态的缓存头、校验头、Range `Content-Range` 和 304 状态摘要，便于值班人员不打开 `direct-live.json` 也能先判断证据是否完整；如果 direct live 已输出静态资产结果但摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 或 Last-Modified 304 证据，证据包会记为 `CRITICAL`。最终总审计还会带 `--require-phase-direct-live-access-log post-enable --require-phase-direct-live-static-headers post-enable`，防止缺少 access log 对账摘要或静态头摘要的旧启用后证据包混入正式归档。快照脚本只读采集状态，证据包脚本只写指定证据目录，不写 `/etc`、不 reload systemd，也不修改 Nginx 或 Pingora。 +- `npm run check:pingora-cutover-status-snapshot` 会烟测状态快照脚本，`npm run check:pingora-cutover-evidence-bundle` 会烟测证据包脚本；正式切换窗口与 release readiness 使用 `/opt/genarrative/current/scripts/ops/pingora-cutover-evidence-bundle.mjs` 归档切换前 / 启用后 / 回退后三个阶段的 `manifest.json`、`snapshot.json`、stdout / stderr 和命令记录。直连 runbook 的三个证据包阶段都会向状态快照透传 `--require-pingora-gateway`，在 `checks.current-release-audit` 中保存 Pingora 二进制、sha256、release manifest 和 systemd `ExecStart` 自审结果；启用后证据包还会通过 `--run-direct-live` 保存 `direct-live.json`、direct live stdout / stderr、命令记录、Pingora access log `request_id` 反查结果，以及静态响应头白名单证据。`post-enable` 证据包必须传 `--expected-pingora-env-mode direct`，`post-rollback` 证据包必须传 `--expected-pingora-env-mode shadow`。`manifest.summary.directLiveAccessLog` 会提升 checked / matched / missing / mismatch 对账摘要，`manifest.summary.directLiveStaticHeaders` 会提升普通静态和 Vite 指纹静态的缓存头、校验头、Range `Content-Range` 和 304 状态摘要，`manifest.summary.pingoraEnvShadow` 会提升 `listen`、`tlsListen`、`httpRedirectListen`、`tlsCertFile`、`tlsKeyFile`、`mode` 和 `shadowReady`，便于值班人员不打开 `direct-live.json` 或 `snapshot.json` 也能先判断证据是否完整；如果 direct live 已输出静态资产结果但摘要缺少缓存头、校验头、Range `206 + Content-Range`、ETag 304 或 Last-Modified 304 证据，证据包会记为 `CRITICAL`。最终总审计还会带 `--require-phase-direct-live-access-log post-enable --require-phase-direct-live-static-headers post-enable --require-phase-pingora-env-shadow post-rollback`，防止缺少 access log 对账摘要、静态头摘要或回退后 shadow env 摘要的旧证据包混入正式归档。快照脚本只读采集状态，证据包脚本只写指定证据目录，不写 `/etc`、不 reload systemd，也不修改 Nginx 或 Pingora。 - 最终证据根目录总审计 JSON 先看 `summary.status`、`summary.failedItems[]` 和 `summary.directLiveEvidence[]`：失败时这里会直接指出阶段 / 命令 / 根目录问题，以及 `post-enable` access log 或静态头摘要的短 reason；`phases[]`、`commands[]` 和 `timeline` 继续作为完整机器归档。 -- 最终证据根目录总审计必须同时带 `--require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 和 `--require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh`，避免只证明有命令证据，却没有证明真实执行的是 current release 随包 enable / rollback 脚本。 -- `npm run check:production-api-release` 会动态生成 API release，并检查发布包 README 与随包 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover` 输出的最终总审计步骤都保留上述两条 `--require-command-executable` 要求，避免源码文档正确但发布包操作口径漂移。 +- 最终证据根目录总审计必须同时带五条 `--require-command-executable`：`enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh`、`post-enable:pingora-health-patrol-direct-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs`、`rollback-prep:pingora-gateway-shadow-env-switch:/opt/genarrative/current/scripts/deploy/pingora-gateway-env-shadow-switch.mjs`、`rollback-prep:pingora-health-patrol-nginx-env-switch:/opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs` 和 `rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh`，避免只证明有命令证据，却没有证明真实执行的是 current release 随包切换脚本。 +- `npm run check:production-api-release` 会动态生成 API release，并检查发布包 README 与随包 `scripts/check-pingora-release-readiness.mjs --dry-run-cutover` 输出的最终总审计步骤都保留上述五条 `--require-command-executable` 要求，避免源码文档正确但发布包操作口径漂移。 - 直连接管后用 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply --env-file /etc/genarrative/health-patrol.env --gateway-mode pingora-direct --public-base-url <直连HTTPS入口> --public-host <域名>` 切换 health patrol env；该脚本只改 gateway mode、public base URL 和 public Host，并先用临时目标 env 调用随包 `check-production-health-patrol-env.mjs` 复核，复核通过后才原子替换真实 env，复核失败不会落盘。回退到 Nginx 前用同一脚本改回 `--gateway-mode nginx --public-base-url <切换前Nginx巡检入口>`，若切换前 Nginx 巡检不需要 Host 覆盖则传 `--clear-public-host`，需要 Host 覆盖则传 `--public-host <切换前Host>`；随后 rollback 脚本和独立 env 复核会再次阻断 public base URL / Host 漂移。`plan:pingora-direct-cutover` 会在 JSON runbook 中列出 direct / nginx 两个方向的 env 切换脚本和 env 复核步骤，并先输出 Host 与回退巡检入口确认；正式切换 runbook 会拒绝 `--direct-redirect-host` 或 `--rollback-nginx-smoke-host` 与 `--direct-host` 使用不同 hostname，且必须提供 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退计划把现场巡检入口覆盖成仓库默认值。若切换计划希望回退后同时证明 Pingora shadow 高端口仍健康，可给 runbook 追加 `--rollback-pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --rollback-pingora-shadow-probe-token `；JSON 输出会隐藏 token 原文，并把对应参数传给 rollback dry-run / apply 步骤。 ## Pingora canary @@ -71,8 +73,8 @@ - 真实路径 canary 不再手工编辑 `/etc/nginx/conf.d/`。目标机从 current release 执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-enable.sh --apply --probe-token --host <域名> --base-url http://127.0.0.1:18083`；脚本会渲染到 `/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf`，保持晚于主站 `log_format genarrative_upstream` 加载，执行 `nginx -t`、reload Nginx，并默认运行 realpath live smoke。`nginx -t`、reload 或 live smoke 任一失败都会恢复写入前配置。关闭时执行 `/opt/genarrative/current/scripts/deploy/pingora-realpath-canary-disable.sh --apply`，脚本会删除该 `zz-...` 文件，执行 `nginx -t` 和 reload，失败时恢复删除前配置。 - `npm run check:pingora-release-readiness` 默认包含 `npm run check:pingora-realpath-canary-toggle`，用临时 Nginx 目录和 fake `nginx` / `systemctl` / live smoke 验证 realpath canary 启停脚本默认 dry-run、apply 写入、失败回滚和 disable 恢复逻辑；不再把这条脚本护栏留成单独可选项。 - 真实路径 canary reload 后可再次运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-live.mjs --realpath --base-url http://127.0.0.1:18083 --host <域名>`，再运行 `node -- /opt/genarrative/current/scripts/check-pingora-canary-access-log-parity.mjs --realpath --nginx-log-file /var/log/nginx/genarrative-pingora-realpath-canary.access.log --pingora-log-file /var/log/genarrative/pingora-gateway.access.log --path /__genarrative_pingora_realpath_canary/healthz --path /api/creation-entry/config --path /v1/identity --path /assets/app.js`。真实路径模式除 healthz 探针外要求 Nginx path 与 Pingora path 完全一致。 -- 正式切换前在源码 checkout / CI / 构建环境跑 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；启用真实路径 canary 时追加 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。 -- 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁：`/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证真实路径 canary 或 Pingora 直连入口时分别追加 `--require-realpath-live ...` 或 `--require-direct ...` 参数。未带 `--require-direct` 的 runtime-only 基础门禁会自动执行直连彩排状态复核，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过；启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 +- 正式切换前在源码 checkout / CI / 构建环境跑聚合门禁，并按目标机已启用的 canary 入口选择参数：前缀 canary 已启用时使用 `node scripts/check-pingora-release-readiness.mjs --require-docker --pull-docker --require-nginx --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；真实路径 canary 已启用时追加或单独使用 `--require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`。如果现场只启用了真实路径 canary，不要同时传 `--require-live`。 +- 目标机 `/opt/genarrative/current` 只能跑随包 runtime-only 门禁，并同样按现场已启用入口选择：前缀 canary 用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-live --live-base-url http://127.0.0.1 --live-host <域名> --live-nginx-access-log /var/log/nginx/genarrative.access.log --live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；真实路径 canary 用 `/opt/genarrative/current/scripts/check-pingora-release-readiness.mjs --release-runtime-only --require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名> --realpath-live-nginx-access-log /var/log/nginx/genarrative-pingora-realpath-canary.access.log --realpath-live-pingora-access-log /var/log/genarrative/pingora-gateway.access.log`；验证 Pingora 直连入口时再追加 `--require-direct ...` 参数。未带 `--require-direct` 的 runtime-only 基础门禁会自动执行直连彩排状态复核，确认公网 `80/443` 仍由 Nginx 接流、Pingora shadow `127.0.0.1:18081`、realpath canary `127.0.0.1:18083` 和 current release 自审均通过；启用后 `--require-direct` 复核不再要求 Nginx 接公网彩排状态。生成正式直连切换 runbook 时，`--direct-redirect-host` 和回退 smoke Host 必须与 `--direct-host` 同 hostname，只允许端口不同。 ## 验证 diff --git a/deploy/pingora/pingora-gateway.env.example b/deploy/pingora/pingora-gateway.env.example index 1c69ad8cf..67b1bc3ff 100644 --- a/deploy/pingora/pingora-gateway.env.example +++ b/deploy/pingora/pingora-gateway.env.example @@ -16,6 +16,11 @@ GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081 # GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME=https GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM=127.0.0.1:8082 GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM=127.0.0.1:3101 +# 可选：同一公网 IP 上还承载 Gitea 时，按 Host 整站转发到 Gitea。 +# 命中这些 Host 的请求不走应用维护页、API body limit 或网关接流保护，避免影响 git clone / push。 +# 直连 443 如果要同时服务这些域名，当前单 listener TLS 配置必须使用覆盖全部 Host 的同一张证书。 +# GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS=git.genarrative.world +# GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM=127.0.0.1:3000 GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT=/srv/genarrative/web GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT=/var/www/html diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 4591282bc..1fa9243e6 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -19,10 +19,11 @@ ## 2026-06-13 Pingora 低端口直连只通过显式 systemd drop-in 启用 - 背景：`genarrative-pingora-gateway.service` 默认以 `genarrative` 非 root 用户运行，shadow 阶段只监听本机高端口；如果正式评估让 Pingora 直接绑定公网 `80/443`，需要低端口绑定能力，但不能让 Server-Provision 或默认 service 自动改变接流边界。 -- 决策：主 systemd service 保持 shadow 口径，不携带 `CAP_NET_BIND_SERVICE`。仓库提供 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用 drop-in 模板，Server-Provision 只安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 备查和手动覆盖；正式切换窗口从 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 执行时默认读取 current release 随包的 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 `/etc` 参考模板、Jenkins 工作区或源码 checkout。直连切换前先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成 JSON runbook，逐条审阅 Host 与回退巡检入口确认、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply（通过命令证据脚本归档 stdout / stderr / 退出码）、切换后 health patrol 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、rollback apply（通过命令证据脚本归档 stdout / stderr / 退出码）、回退后 health patrol 切回 `nginx` 并恢复切换前 public base URL / Host、回退后 health patrol env Nginx 模式复核；启用前基础门禁不带 `--require-direct`，因为 systemd drop-in 尚未生效，启用后复核必须带 `--require-direct`。正式切换 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 和回退 smoke 分别验证到不同入口；还必须显式传 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，切换前 Nginx 巡检需要 Host 覆盖时再传 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 覆盖现场原有巡检入口。若需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文并把参数传给 rollback dry-run / apply。只有切换窗口通过 `pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名>` 先跑 current release 自审，确认发布包自包含、`pingora-gateway` 可执行且 systemd `ExecStart` 指向随包网关，失败时不安装 drop-in；随后跑 direct preflight，确认当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都可读取证书链 / 私钥，确认 service 模板与 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含本次 `/etc/genarrative/pingora-gateway.env`，并确认 service `ExecStart=` 指向的 current release `pingora-gateway` 存在且可执行，再安装到 `/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf`、执行 `systemctl daemon-reload`、重启 Pingora，并用 `systemctl cat` 核验 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl is-active` 确认服务 active、用 direct live smoke 验证 HTTPS / HTTP redirect / ACME / WSS 101 后，才视为授予低端口能力成功。启用前还必须显式配置 TLS / redirect env 和真实证书，并确认 current release 已落盘可执行 `pingora-gateway`、Nginx 或其它进程已释放 `80/443`；启用后仍必须跑 release readiness 门禁；验证失败时统一执行 `pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>` 回到 shadow / Nginx 入口，回退脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，再用 `systemctl cat` 确认两条低端口 capability 均已从最终 unit 配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 curl smoke URL 证明 Nginx 入口真实可访问。回退脚本 `--apply` 必须同时带 `--reload-nginx` 和 `--nginx-smoke-url`，避免只撤掉 Pingora 低端口能力却没有证明 Nginx 已重新接流；本机打 `127.0.0.1`、`localhost` 或 `::1` 时，`--apply` 必须带 `--nginx-smoke-host <域名>`，且该值只能是 host 或 `host:port`，避免命中默认 vhost。回退后必须复核 health patrol env 已切回 `nginx` 且 public base URL / Host 恢复为切换前记录值；如果 env 已预先修正，rollback 脚本可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 自动执行这项复核，切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若需证明 Pingora 仍以 shadow 高端口存活，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。 +- 决策：主 systemd service 保持 shadow 口径，不携带 `CAP_NET_BIND_SERVICE`。仓库提供 `deploy/systemd/genarrative-pingora-gateway-direct-entry.conf` 作为人工启用 drop-in 模板，Server-Provision 只安装到 `/etc/genarrative/pingora/genarrative-pingora-gateway-direct-entry.conf` 备查和手动覆盖；正式切换窗口从 `/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh` 执行时默认读取 current release 随包的 `/opt/genarrative/current/deploy/systemd/genarrative-pingora-gateway-direct-entry.conf`，不依赖 `/etc` 参考模板、Jenkins 工作区或源码 checkout。直连切换前先用 `npm run plan:pingora-direct-cutover -- --require-direct ...` 生成 JSON runbook，逐条审阅 Host 与回退巡检入口确认、current release preflight、启用前基础门禁、direct enable dry-run、direct enable apply（通过命令证据脚本归档 stdout / stderr / 退出码）、切换后 health patrol 切到 `pingora-direct`、启用后 health patrol env 直连复核、启用后 `--require-direct` 复核、rollback dry-run、rollback apply（通过命令证据脚本归档 stdout / stderr / 退出码）、回退后 health patrol 切回 `nginx` 并恢复切换前 public base URL / Host、回退后 health patrol env Nginx 模式复核；启用前基础门禁不带 `--require-direct`，因为 systemd drop-in 尚未生效，启用后复核必须带 `--require-direct`。正式切换 runbook 中 `--direct-redirect-host`、`--rollback-nginx-smoke-host` 和 `--direct-host` 必须使用同一 hostname，只允许端口不同，避免 redirect 和回退 smoke 分别验证到不同入口；还必须显式传 `--rollback-health-patrol-public-base-url <切换前Nginx巡检入口>`，切换前 Nginx 巡检需要 Host 覆盖时再传 `--rollback-health-patrol-public-host <切换前Host>`，确认步骤会展示回退后要恢复的 public base URL / Host，避免回退 runbook 覆盖现场原有巡检入口。若需把回退后 Pingora shadow 探针复核纳入 runbook，追加 `--rollback-pingora-shadow-probe-url` / `--rollback-pingora-shadow-probe-token`，JSON 输出会隐藏 token 原文并把参数传给 rollback dry-run / apply。只有切换窗口通过 `pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名>` 先跑 current release 自审，确认发布包自包含、`pingora-gateway` 可执行且 systemd `ExecStart` 指向随包网关，失败时不安装 drop-in；随后跑 direct preflight，确认当前执行用户和 `genarrative-pingora-gateway.service` 的 `User=` 服务用户都可读取证书链 / 私钥，确认 service 模板与 `systemctl cat` 最终配置读取的 `EnvironmentFile=` 都包含本次 `/etc/genarrative/pingora-gateway.env`，并确认 service `ExecStart=` 指向的 current release `pingora-gateway` 存在且可执行，再安装到 `/etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf`、执行 `systemctl daemon-reload`、重启 Pingora，并用 `systemctl cat` 核验 `AmbientCapabilities=CAP_NET_BIND_SERVICE`、`CapabilityBoundingSet=CAP_NET_BIND_SERVICE` 和 `EnvironmentFile=/etc/genarrative/pingora-gateway.env` 已生效、用 `systemctl is-active` 确认服务 active、用 direct live smoke 验证 HTTPS / HTTP redirect / ACME / WSS 101 后，才视为授予低端口能力成功。启用前还必须显式配置 TLS / redirect env 和真实证书，并确认 current release 已落盘可执行 `pingora-gateway`、Nginx 或其它进程已释放 `80/443`；启用后仍必须跑 release readiness 门禁；验证失败时统一执行 `pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url https://<域名>/ --nginx-smoke-expect-body ''` 回到 shadow / Nginx 入口，回退脚本会先运行 `nginx -t`，通过后移除 direct-entry drop-in、重启 Pingora，再用 `systemctl cat` 确认两条低端口 capability 均已从最终 unit 配置中移除，用 `systemctl show ... ExecStart` 确认最终 service 仍指向随包主 service 模板中的 current release `pingora-gateway`，并 reload Nginx、确认 Nginx service 仍为 `active`，最后用 curl smoke URL 证明 Nginx 入口真实可访问；回退 smoke URL/body 必须来自切换前真实 Nginx 入口，不要继续用固定 `http://127.0.0.1/healthz` 与 `"ok":true`。回退脚本 `--apply` 必须同时带 `--reload-nginx` 和 `--nginx-smoke-url`，避免只撤掉 Pingora 低端口能力却没有证明 Nginx 已重新接流；本机打 `127.0.0.1`、`localhost` 或 `::1` 时，`--apply` 必须带 `--nginx-smoke-host <域名>`，且该值只能是 host 或 `host:port`，避免命中默认 vhost。回退后必须复核 health patrol env 已切回 `nginx` 且 public base URL / Host 恢复为切换前记录值；如果 env 已预先修正，rollback 脚本可追加 `--health-patrol-env-file /etc/genarrative/health-patrol.env --health-patrol-expected-public-base-url <切换前Nginx巡检入口> --health-patrol-require-empty-public-host` 自动执行这项复核，切换前 Nginx 巡检需要 Host 覆盖时把最后一项换成 `--health-patrol-expected-public-host <切换前Host>`。若需证明 Pingora 仍以 shadow 高端口存活，可追加 `--pingora-shadow-probe-url http://127.0.0.1:18081/__genarrative_pingora/healthz --pingora-shadow-probe-token `，脚本会隐藏 token 并要求响应包含 `gateway=pingora-shadow`。 - 决策补充：health patrol env 的直连/回退切换不再靠人工编辑三行变量；正式 runbook 使用 current release 随包 `node -- /opt/genarrative/current/scripts/deploy/pingora-health-patrol-env-switch.mjs --apply`，只更新 `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL`、`GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST` 并立即调用随包 `check-production-health-patrol-env.mjs` 复核。Pingora direct 使用本机 public base URL 时脚本必须带 `--public-host <域名>`；回退到 Nginx 时根据切换前记录传 `--clear-public-host` 或 `--public-host <切换前Host>`。生产巡检、health patrol env 复核和 env 切换脚本读取的布尔 env 必须严格解析，非法值直接失败，不得静默按 false 继续；env 复核脚本的 `--env-file` 与 env 切换脚本的 `--env-file` / `--check-script` 必须是绝对路径且不能是文件系统根目录，也不能包含换行或 NUL；env 切换脚本写入的 public base URL / Host 同样不能包含换行或 NUL。Node 22 已内置 `--env-file` 启动参数，凡是用 Node 启动项目脚本且要把业务 `--env-file` 传给脚本时，必须写成 `node --