Merge remote-tracking branch 'origin/codex/pingoranginx'

This commit is contained in:
2026-06-19 10:50:09 +08:00
89 changed files with 58953 additions and 499 deletions
+19
View File
@@ -0,0 +1,19 @@
# 复制到 /etc/genarrative/health-patrol.env 后按生产环境调整。
# 默认不启用 Pingora shadow 巡检;只有同时配置 base URL 与 probe token 才会检查。
GENARRATIVE_HEALTH_PATROL_API_BASE_URL=http://127.0.0.1:8082
GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL=http://127.0.0.1:3101
GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=http://127.0.0.1
# 默认公网入口仍按 Nginx 巡检;Pingora 直连切换后改为 pingora-direct。
GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx
# 本机打 127.0.0.1 时可填正式域名,确保命中正确 vhost / 证书语义。
GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=
GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS=5000
GENARRATIVE_HEALTH_PATROL_SLOW_MS=3000
GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING=false
GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL=false
GENARRATIVE_HEALTH_PATROL_WEBHOOK_URL=
# Pingora 影子网关巡检;probe token 应与 /etc/genarrative/pingora-gateway.env 保持一致。
GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL=
GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN=
+35
View File
@@ -0,0 +1,35 @@
# 复制到本机临时 shell 或 CI secret 环境后运行:
# npm run check:pingora-canary-live
# 该检查只读访问已经人工启用的 Nginx -> Pingora canary,不会修改 Nginx。
# 例如 http://127.0.0.1 或 https://genarrative.world。
GENARRATIVE_PINGORA_CANARY_BASE_URL=http://127.0.0.1
# 默认前缀必须和 deploy/nginx/snippets/genarrative-pingora-canary.conf 保持一致。
GENARRATIVE_PINGORA_CANARY_PREFIX=/__genarrative_pingora_canary
# 可选:真实路径 canary 使用 deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf。
# 启用后运行:
# GENARRATIVE_PINGORA_CANARY_MODE=realpath \
# GENARRATIVE_PINGORA_CANARY_BASE_URL=${GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL} \
# npm run check:pingora-canary-live
# 或使用 release readiness:
# npm run check:pingora-release-readiness -- --require-realpath-live --realpath-live-base-url http://127.0.0.1:18083 --realpath-live-host <域名>
GENARRATIVE_PINGORA_CANARY_MODE=prefix
GENARRATIVE_PINGORA_REALPATH_CANARY_BASE_URL=http://127.0.0.1:18083
GENARRATIVE_PINGORA_REALPATH_CANARY_HOST=
# 本机直连 127.0.0.1 时必须设置正式域名,避免只命中 Nginx 默认 vhostrelease readiness --require-live 必须提供。只填域名或 host:port,不要填 https:// URL。
GENARRATIVE_PINGORA_CANARY_HOST=
GENARRATIVE_PINGORA_CANARY_TIMEOUT_MS=5000
# live smoke 后必须继续按 request_id 对照 Nginx handoff access log 与 Pingora access log。
GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG=/var/log/nginx/genarrative.access.log
GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log
GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES=2000
# 真实路径 canary 使用独立 Nginx access log,避免和生产真实用户请求混在一起。
GENARRATIVE_PINGORA_REALPATH_CANARY_NGINX_ACCESS_LOG=/var/log/nginx/genarrative-pingora-realpath-canary.access.log
GENARRATIVE_PINGORA_REALPATH_CANARY_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log
GENARRATIVE_PINGORA_REALPATH_CANARY_ACCESS_LOG_SINCE_LINES=2000
+68
View File
@@ -0,0 +1,68 @@
# 复制到本机临时 shell 或 CI secret 环境后运行:
# npm run check:pingora-direct-live
# 该检查只读访问已经显式配置 TLS_LISTEN / HTTP_REDIRECT_LISTEN 的 pingora-gateway。
# 切换窗口建议先运行:
# npm run check:pingora-direct-preflight -- --env-file /etc/genarrative/pingora-gateway.env --require-live-env --systemd-cat --check-cert-readable --check-service-env-file --check-service-user-cert-readable --check-service-binary-executable --check-ports-free
# release readiness --require-direct 必须确认 TLS/HTTP redirect 端口已释放,复核 health patrol env 已切到 direct 模式,并提供 Pingora access log 路径用于 request_id 落盘校验。
# 直连启用前:
# /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --no-status
# /opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh --apply --preflight-env-file /etc/genarrative/pingora-gateway.env --preflight-check-cert-readable --preflight-check-service-env-file --preflight-check-service-user-cert-readable --preflight-check-service-binary-executable --preflight-check-ports-free --direct-https-base-url https://127.0.0.1 --direct-http-base-url http://127.0.0.1 --direct-host <域名> --direct-redirect-host <域名或host:port> --direct-spacetime-database <库名> --direct-pingora-access-log /var/log/genarrative/pingora-gateway.access.log
# 启用脚本会先执行 current release 自审,确认 pingora-gateway 可执行且 systemd ExecStart 指向 current release;自审失败时不会安装 direct-entry drop-in。
# 自审和 preflight 都通过后,启用脚本才会安装 drop-in、重启 Pingora、执行 direct live smoke,并强制 WSS subscribe 返回 101,同时用生成的 request_id 反查 Pingora access log。
# 直连接管后同步更新 /etc/genarrative/health-patrol.env
# GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct
# GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://127.0.0.1
# GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=<域名>
# 然后用 node -- /opt/genarrative/current/scripts/check-production-health-patrol-env.mjs 复核。
# 直连验证失败时:
# /opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh --apply --reload-nginx --nginx-smoke-url http://127.0.0.1/healthz --nginx-smoke-host <域名>
# 回退脚本会先执行 nginx -treload 后确认 nginx.service 仍为 active,并用 smoke URL 证明 Nginx 入口可访问。
# 回退后把 GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 改回 nginx。
# 例如 https://127.0.0.1 或 https://genarrative.world。
GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL=https://127.0.0.1
# 单独 direct live 可选;release readiness --require-direct 必须提供,用于校验 HTTP 入口 301 到 HTTPS,并确认 ACME challenge 没被重定向吞掉。
GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL=http://127.0.0.1
# 本机直连 127.0.0.1 时必须设置正式域名;direct live 会同时用作 Host header 和 TLS SNI。release readiness --require-direct 必须提供。只填域名或 host:port,不要填 https:// URL。
GENARRATIVE_PINGORA_DIRECT_HOST=
# 单独 direct live 可选;release readiness --require-direct 必须提供,用于显式校验 HTTP redirect Location 的 host。只填域名或 host:port。
GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST=
# 可选:高端口 rehearsal 访问 127.0.0.1:18443、但期望 Location 指向正式域名默认 HTTPS 入口时设置;只能填 https:// base URL,不要带路径、查询或片段。
GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL=
# 可选:配置后会额外检查 /__genarrative_pingora/healthz 内部探针。
GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN=
# direct live 默认会对 wss://<HTTPS>/v1/database/<database>/subscribe 做 SpacetimeDB 握手检查;release readiness --require-direct 必须按目标环境显式设置。
GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE=genarrative-prod
# release readiness --require-direct 必须显式提供;direct live 会用每个请求生成的 X-Request-Id 反查 Pingora access log,证明直连流量真实进入 Pingora。
GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG=/var/log/genarrative/pingora-gateway.access.log
GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES=2000
# direct live 单独运行时可设 true,要求 WSS subscribe 必须返回 101release readiness --require-direct 会自动强制该口径。
GENARRATIVE_PINGORA_DIRECT_REQUIRE_WSS_UPGRADE=true
# 只允许单独 direct live 临时排障时跳过 WSSrelease readiness --require-direct 会直接拒绝,正式切流必须保持 false。
GENARRATIVE_PINGORA_DIRECT_SKIP_WSS=false
# 仅本机自签证书 smoke 使用;生产必须保持 false。
GENARRATIVE_PINGORA_DIRECT_INSECURE_TLS=false
GENARRATIVE_PINGORA_DIRECT_TIMEOUT_MS=5000
# release readiness --require-direct 会先跑 direct-entry preflight;提供 env 文件后会强制检查 TLS/redirect/cert/key 配置。
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE=/etc/genarrative/pingora-gateway.env
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_SYSTEMD_CAT=true
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_CERT_READABLE=true
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_ENV_FILE=true
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_USER_CERT_READABLE=true
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_SERVICE_BINARY_EXECUTABLE=true
GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_CHECK_PORTS_FREE=true
# release readiness --require-direct 必须复核健康巡检 env,避免切直连后巡检仍按 Nginx 模式误报。
GENARRATIVE_PINGORA_DIRECT_HEALTH_PATROL_ENV_FILE=/etc/genarrative/health-patrol.env
@@ -0,0 +1,10 @@
/var/log/genarrative/pingora-gateway.access.log {
daily
rotate 14
missingok
notifempty
compress
delaycompress
copytruncate
create 0640 genarrative genarrative
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,64 @@
# Pingora canary handoff snippet.
# 中文注释:本文件只提供人工 include 后的 Nginx -> Pingora 试跑入口,不默认承接公网流量。
# 启用前复制到目标机并替换 __GENARRATIVE_PINGORA_PROBE_TOKEN__,确认只允许可信来源访问。
location = /__genarrative_pingora_canary/healthz {
allow 127.0.0.1;
allow ::1;
deny all;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__";
add_header X-Genarrative-Nginx-Handoff pingora-canary always;
proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz;
}
location ~ ^/__genarrative_pingora_canary/v1/database/[^/]+/subscribe$ {
allow 127.0.0.1;
allow ::1;
deny all;
rewrite ^/__genarrative_pingora_canary(/.*)$ $1 break;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_read_timeout 3600s;
add_header X-Genarrative-Nginx-Handoff pingora-canary always;
proxy_pass http://127.0.0.1:18081;
}
location /__genarrative_pingora_canary/ {
allow 127.0.0.1;
allow ::1;
deny all;
rewrite ^/__genarrative_pingora_canary(/.*)$ $1 break;
proxy_http_version 1.1;
proxy_buffering off;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_set_header X-Genarrative-Pingora-Canary "nginx-prefixed";
add_header X-Accel-Buffering no always;
add_header X-Genarrative-Nginx-Handoff pingora-canary always;
proxy_pass http://127.0.0.1:18081;
}
@@ -0,0 +1,90 @@
# Pingora real-path canary handoff server.
# 中文注释:本文件只提供人工 include 后的本机真实路径 canary,不默认承接公网流量。
# 启用前复制到目标机并替换 __GENARRATIVE_PINGORA_PROBE_TOKEN__,只能在 nginx http 上下文 include。
server {
listen 127.0.0.1:18083;
server_name genarrative-pingora-realpath-canary.local;
access_log /var/log/nginx/genarrative-pingora-realpath-canary.access.log genarrative_upstream;
error_log /var/log/nginx/genarrative-pingora-realpath-canary.error.log warn;
allow 127.0.0.1;
allow ::1;
deny all;
location = /__genarrative_pingora_realpath_canary/healthz {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_set_header X-Genarrative-Pingora-Probe "__GENARRATIVE_PINGORA_PROBE_TOKEN__";
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081/__genarrative_pingora/healthz;
}
location = /api/creation-entry/config {
default_type application/json;
proxy_http_version 1.1;
proxy_buffering off;
proxy_read_timeout 3600s;
proxy_send_timeout 3600s;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_set_header X-Genarrative-Pingora-Canary "nginx-realpath";
add_header X-Accel-Buffering no always;
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081;
}
location ~ ^/v1/database/[^/]+/subscribe$ {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $host;
proxy_set_header X-Request-Id $request_id;
proxy_read_timeout 3600s;
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081;
}
location ^~ /v1/identity {
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
proxy_set_header Host $host;
proxy_set_header X-Request-Id $request_id;
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081;
}
location = /assets/app.js {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Request-Id $request_id;
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081;
}
location = /generated-pingora-canary-smoke {
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Request-Id $request_id;
add_header X-Genarrative-Nginx-Handoff pingora-realpath-canary always;
proxy_pass http://127.0.0.1:18081;
}
location / {
return 404;
}
}
@@ -0,0 +1,375 @@
{
"version": 1,
"description": "Pingora shadow gateway routes that must stay aligned with the current Genarrative Nginx production/development templates before any public cutover.",
"routes": [
{
"id": "acme_challenge",
"samplePath": "/.well-known/acme-challenge/token",
"expect": {
"kind": "static",
"root": "acme",
"mode": "exact"
},
"nginx": {
"production": [
"location /.well-known/acme-challenge/",
"root /var/www/html;"
]
},
"docs": ["`/.well-known/acme-challenge/*`"]
},
{
"id": "shadow_probe",
"samplePath": "/__genarrative_pingora/healthz",
"expect": {
"kind": "shadow_probe"
},
"docs": ["`/__genarrative_pingora/healthz`"]
},
{
"id": "admin_redirect",
"samplePath": "/admin",
"expect": {
"kind": "redirect_permanent",
"location": "/admin/"
},
"nginx": {
"production": ["location = /admin", "return 301 /admin/;"],
"development": ["location = /admin", "return 301 /admin/;"]
},
"docs": ["`/admin`", "301 到 `/admin/`"]
},
{
"id": "admin_api_proxy",
"samplePath": "/admin/api/users",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": null,
"protectionClass": "admin_api"
},
"nginx": {
"production": [
"location ^~ /admin/api/",
"proxy_pass http://genarrative_api/admin/api/;",
"limit_conn genarrative_api_conn 64;",
"limit_req zone=genarrative_admin_rps burst=16 nodelay;"
],
"development": [
"location ^~ /admin/api/",
"proxy_pass http://genarrative_api/admin/api/;",
"limit_conn genarrative_api_conn 64;",
"limit_req zone=genarrative_admin_rps burst=16 nodelay;"
]
},
"docs": ["`/admin/api/*`"]
},
{
"id": "admin_assets",
"samplePath": "/admin/assets/index.js",
"expect": {
"kind": "static",
"root": "web",
"mode": "exact"
},
"nginx": {
"production": ["location ^~ /admin/assets/", "try_files $uri =404;"],
"development": ["location ^~ /admin/assets/", "try_files $uri =404;"]
},
"docs": ["`/admin/assets/*`"]
},
{
"id": "admin_spa_fallback",
"samplePath": "/admin/settings",
"expect": {
"kind": "static",
"root": "web",
"mode": "spa_fallback"
},
"nginx": {
"production": [
"location ^~ /admin/",
"try_files $uri $uri/ /admin/index.html;"
],
"development": [
"location ^~ /admin/",
"try_files $uri $uri/ /admin/index.html;"
]
},
"docs": ["`/admin/*`"]
},
{
"id": "web_assets",
"samplePath": "/assets/app.js",
"expect": {
"kind": "static",
"root": "web",
"mode": "exact"
},
"nginx": {
"production": ["location ^~ /assets/", "try_files $uri =404;"],
"development": ["location ^~ /assets/", "try_files $uri =404;"]
},
"docs": ["`/assets/*`"]
},
{
"id": "puzzle_gallery_list",
"samplePath": "/api/runtime/puzzle/gallery",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": null,
"protectionClass": "gallery_list"
},
"nginx": {
"production": [
"location = /api/runtime/puzzle/gallery",
"limit_conn genarrative_api_conn 320;",
"limit_req zone=genarrative_gallery_rps burst=4096 nodelay;",
"add_header X-Accel-Buffering no always;"
],
"development": [
"location = /api/runtime/puzzle/gallery",
"limit_conn genarrative_api_conn 320;",
"limit_req zone=genarrative_gallery_rps burst=4096 nodelay;",
"add_header X-Accel-Buffering no always;"
]
},
"docs": ["`/api/runtime/puzzle/gallery`"]
},
{
"id": "custom_world_gallery_list",
"samplePath": "/api/runtime/custom-world-gallery",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": null,
"protectionClass": "gallery_list"
},
"nginx": {
"production": [
"location = /api/runtime/custom-world-gallery",
"limit_conn genarrative_api_conn 320;",
"limit_req zone=genarrative_gallery_rps burst=4096 nodelay;",
"add_header X-Accel-Buffering no always;"
],
"development": [
"location = /api/runtime/custom-world-gallery",
"limit_conn genarrative_api_conn 320;",
"limit_req zone=genarrative_gallery_rps burst=4096 nodelay;",
"add_header X-Accel-Buffering no always;"
]
},
"docs": ["`/api/runtime/custom-world-gallery`"]
},
{
"id": "puzzle_gallery_detail",
"samplePath": "/api/runtime/puzzle/gallery/work-a",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": null,
"protectionClass": "gallery_detail"
},
"nginx": {
"production": [
"location ~ ^/api/runtime/puzzle/gallery/[^/]+$",
"limit_conn genarrative_api_conn 32;",
"limit_req zone=genarrative_api_rps burst=32 nodelay;",
"add_header X-Accel-Buffering no always;"
],
"development": [
"location ~ ^/api/runtime/puzzle/gallery/[^/]+$",
"limit_conn genarrative_api_conn 32;",
"limit_req zone=genarrative_api_rps burst=32 nodelay;",
"add_header X-Accel-Buffering no always;"
]
},
"docs": ["`/api/runtime/puzzle/gallery/{id}`"]
},
{
"id": "custom_world_gallery_detail",
"samplePath": "/api/runtime/custom-world-gallery/profile-a/owner-b",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": null,
"protectionClass": "gallery_detail"
},
"nginx": {
"production": [
"location ~ ^/api/runtime/custom-world-gallery/[^/]+/[^/]+$",
"limit_conn genarrative_api_conn 32;",
"limit_req zone=genarrative_api_rps burst=32 nodelay;",
"add_header X-Accel-Buffering no always;"
],
"development": [
"location ~ ^/api/runtime/custom-world-gallery/[^/]+/[^/]+$",
"limit_conn genarrative_api_conn 32;",
"limit_req zone=genarrative_api_rps burst=32 nodelay;",
"add_header X-Accel-Buffering no always;"
]
},
"docs": ["`/api/runtime/custom-world-gallery/{profile}/{owner}`"]
},
{
"id": "generic_api_proxy",
"samplePath": "/api/creation-entry/config",
"expect": {
"kind": "proxy",
"target": "api",
"bodyLimit": "default",
"protectionClass": "api"
},
"nginx": {
"production": [
"location ~ ^/api(?:/|$)",
"client_max_body_size 64m;",
"limit_conn genarrative_api_conn 64;",
"limit_req zone=genarrative_api_rps burst=64 nodelay;",
"add_header X-Accel-Buffering no always;"
],
"development": [
"location ~ ^/api(?:/|$)",
"client_max_body_size 64m;",
"limit_conn genarrative_api_conn 64;",
"limit_req zone=genarrative_api_rps burst=64 nodelay;",
"add_header X-Accel-Buffering no always;"
]
},
"docs": ["`/api`、`/api/*`"]
},
{
"id": "spacetime_subscribe",
"samplePath": "/v1/database/genarrative/subscribe",
"expect": {
"kind": "proxy",
"target": "spacetime",
"bodyLimit": null,
"protectionClass": "spacetime"
},
"nginx": {
"production": [
"location ~ ^/v1/database/[^/]+/subscribe$",
"proxy_pass http://127.0.0.1:3101;",
"proxy_set_header Upgrade $http_upgrade;",
"proxy_set_header Connection \"Upgrade\";",
"proxy_read_timeout 3600s;"
],
"development": [
"location ~ ^/v1/database/[^/]+/subscribe$",
"proxy_pass http://127.0.0.1:3101;",
"proxy_set_header Upgrade $http_upgrade;",
"proxy_set_header Connection \"Upgrade\";",
"proxy_read_timeout 3600s;"
]
},
"docs": ["`/v1/database/{db}/subscribe`"]
},
{
"id": "spacetime_identity",
"samplePath": "/v1/identity",
"expect": {
"kind": "proxy",
"target": "spacetime",
"bodyLimit": null,
"protectionClass": "spacetime"
},
"nginx": {
"production": [
"location ^~ /v1/identity",
"proxy_pass http://127.0.0.1:3101;",
"proxy_set_header Upgrade $http_upgrade;",
"proxy_set_header Connection \"Upgrade\";"
],
"development": [
"location ^~ /v1/identity",
"proxy_pass http://127.0.0.1:3101;",
"proxy_set_header Upgrade $http_upgrade;",
"proxy_set_header Connection \"Upgrade\";"
]
},
"docs": ["`/v1/identity*`"]
},
{
"id": "v1_forbidden",
"samplePath": "/v1/ping",
"expect": {
"kind": "not_found"
},
"nginx": {
"production": ["location ^~ /v1/", "return 404;"],
"development": ["location ^~ /v1/", "return 404;"]
},
"docs": ["`/v1/*`"]
},
{
"id": "healthz_forbidden",
"samplePath": "/healthz",
"expect": {
"kind": "not_found"
},
"nginx": {
"production": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
],
"development": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
]
},
"docs": ["`/healthz*`"]
},
{
"id": "readyz_forbidden",
"samplePath": "/readyz",
"expect": {
"kind": "not_found"
},
"nginx": {
"production": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
],
"development": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
]
},
"docs": ["`/readyz*`"]
},
{
"id": "generated_assets_forbidden",
"samplePath": "/generated-puzzle-assets/foo.webp",
"expect": {
"kind": "not_found"
},
"nginx": {
"production": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
],
"development": [
"location ~ ^/(generated-|healthz|readyz)",
"return 404;"
]
},
"docs": ["`/generated-*`"]
},
{
"id": "web_spa_fallback",
"samplePath": "/some/spa/path",
"expect": {
"kind": "static",
"root": "web",
"mode": "spa_fallback"
},
"nginx": {
"production": ["location /", "try_files $uri $uri/ /index.html;"],
"development": ["location /", "try_files $uri $uri/ /index.html;"]
},
"docs": ["其它路径", "失败回退 `/index.html`"]
}
]
}
@@ -0,0 +1,81 @@
# Pingora 独立网关试点配置示例。
# 当前只用于本机影子验证,不绑定 80/443,也不替代生产 Nginx。
# systemd 模板默认读取 /etc/genarrative/pingora-gateway.env。
GENARRATIVE_PINGORA_GATEWAY_LISTEN=127.0.0.1:18081
# 可选:直连 HTTPS 入口。默认关闭;启用时必须同时配置 cert/key,并把 FORWARDED_PROTO 设为 https。
# cert/key 必须允许 genarrative 用户读取;不要直接 chmod Let's Encrypt live/archive 原始目录。
# 可先用随包 scripts/deploy/pingora-tls-cert-sync.mjs 把证书同步到 /etc/genarrative/pingora-tls/<host>/。
# 80/443 必须先从 Nginx 或其它进程释放;高端口 loopback 演练可先使用 127.0.0.1:18443 / 127.0.0.1:18080。
# 如果要绑定 443/80,必须另行人工启用 systemd direct-entry drop-in 授予 CAP_NET_BIND_SERVICE。
# GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:18443
# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=/etc/genarrative/pingora-tls/genarrative.world/fullchain.pem
# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=/etc/genarrative/pingora-tls/genarrative.world/privkey.pem
# 可选:直连 HTTP 只做 301 到 HTTPS;启用时必须已经配置 TLS_LISTENACME challenge 仍从 ACME_ROOT 静态读取。
# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:18080
# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_TARGET_SCHEME=https
GENARRATIVE_PINGORA_GATEWAY_API_UPSTREAM=127.0.0.1:8082
GENARRATIVE_PINGORA_GATEWAY_SPACETIME_UPSTREAM=127.0.0.1:3101
# 可选:同一公网 IP 上还承载 Gitea 时,按 Host 整站转发到 Gitea。
# 命中这些 Host 的请求不走应用维护页、API body limit 或网关接流保护,避免影响 git clone / push。
# 直连 443 如果要同时服务这些域名,当前单 listener TLS 配置必须使用覆盖全部 Host 的同一张证书。
# GENARRATIVE_PINGORA_GATEWAY_GITEA_HOSTS=git.genarrative.world
# GENARRATIVE_PINGORA_GATEWAY_GITEA_UPSTREAM=127.0.0.1:3000
GENARRATIVE_PINGORA_GATEWAY_WEB_ROOT=/srv/genarrative/web
GENARRATIVE_PINGORA_GATEWAY_ACME_ROOT=/var/www/html
GENARRATIVE_PINGORA_GATEWAY_MAINTENANCE_FILE=/var/lib/genarrative/maintenance/enabled
GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO=http
GENARRATIVE_PINGORA_GATEWAY_MAX_API_BODY_BYTES=67108864
# gzip 默认开启;等级和最小响应长度对齐 Nginx gzip_comp_level 5 / gzip_min_length 1024。
# Pingora 正式化口径固定为 gzip-onlybr / zstd 不进入当前网关,Brotli 继续由 Nginx / 前置代理承担。
GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip
GENARRATIVE_PINGORA_GATEWAY_GZIP_ENABLED=true
GENARRATIVE_PINGORA_GATEWAY_GZIP_LEVEL=5
GENARRATIVE_PINGORA_GATEWAY_GZIP_MIN_LENGTH_BYTES=1024
# 静态响应缓存头:HTML / SPA fallback 保持 no-cacheVite 指纹资源长缓存,其它静态资源保守 no-cache。
# 网关会额外写入 ETag / Last-Modified 并支持 304 协商缓存;下面三项只控制 Cache-Control。
GENARRATIVE_PINGORA_GATEWAY_HTML_CACHE_CONTROL=no-cache
GENARRATIVE_PINGORA_GATEWAY_ASSET_CACHE_CONTROL=public, max-age=31536000, immutable
GENARRATIVE_PINGORA_GATEWAY_STATIC_CACHE_CONTROL=no-cache
# 上游超时默认对齐当前 Nginx 模板:普通无显式长超时路由 60 秒,/api、公开列表/详情和 subscribe 为 3600 秒。
GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_CONNECT_TIMEOUT_MS=3000
GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_DEFAULT_READ_TIMEOUT_SECONDS=60
GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_API_READ_TIMEOUT_SECONDS=3600
GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_LONG_READ_TIMEOUT_SECONDS=3600
GENARRATIVE_PINGORA_GATEWAY_UPSTREAM_WRITE_TIMEOUT_SECONDS=3600
# 默认只信任 TCP 对端 IP;只有确认前置 Nginx / LB 会清洗 X-Forwarded-For 后才打开。
GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false
# 打开 TRUST_X_FORWARDED_FOR 前必须显式确认;否则网关会拒绝启动。
GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false
# 单进程内接流保护;默认值对齐当前 Nginx limit_conn / limit_req 口径。
# INSTANCE_COUNT 默认 1;若横向运行多个 Pingora 实例且仍启用网关接流保护,必须先落地共享保护层并显式确认。
GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT=1
GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED=false
GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true
GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_MAX_CONCURRENT=64
GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_RATE_PER_SECOND=30
GENARRATIVE_PINGORA_GATEWAY_ADMIN_API_BURST=16
GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_MAX_CONCURRENT=320
GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_RATE_PER_SECOND=5000
GENARRATIVE_PINGORA_GATEWAY_GALLERY_LIST_BURST=4096
GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_MAX_CONCURRENT=32
GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_RATE_PER_SECOND=300
GENARRATIVE_PINGORA_GATEWAY_GALLERY_DETAIL_BURST=32
GENARRATIVE_PINGORA_GATEWAY_API_MAX_CONCURRENT=64
GENARRATIVE_PINGORA_GATEWAY_API_RATE_PER_SECOND=300
GENARRATIVE_PINGORA_GATEWAY_API_BURST=64
GENARRATIVE_PINGORA_GATEWAY_SPACETIME_MAX_CONCURRENT=256
GENARRATIVE_PINGORA_GATEWAY_SPACETIME_RATE_PER_SECOND=1000
GENARRATIVE_PINGORA_GATEWAY_SPACETIME_BURST=256
# 仅供本机影子探针使用;为空时 /__genarrative_pingora/healthz 一律返回 404。
GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=
GENARRATIVE_PINGORA_GATEWAY_LOG=info,pingora=info,pingora_gateway=info
GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE=/var/log/genarrative/pingora-gateway.access.log
GENARRATIVE_PINGORA_GATEWAY_OTEL_ENABLED=false
@@ -9,7 +9,7 @@ User=root
Group=root
WorkingDirectory=/opt/genarrative/current
EnvironmentFile=/etc/genarrative/api-server.env
ExecStart=/usr/bin/node /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service
ExecStart=/usr/bin/node -- /opt/genarrative/current/scripts/database-backup-to-oss.mjs --env-file /etc/genarrative/api-server.env --stop-service spacetimedb.service --restart-service-after genarrative-api.service
# 备份需要停止 / 启动 spacetimedb.service,并读取 /stdb、写入 /var/lib/genarrative/database-backups。
PrivateTmp=true
@@ -0,0 +1,17 @@
# Pingora 直连公网入口的 systemd drop-in 模板。
# 默认 provision 只把本文件安装到 /etc/genarrative/pingora/ 作为人工启用材料;
# 需要低端口 80/443 时,才复制到:
# /etc/systemd/system/genarrative-pingora-gateway.service.d/direct-entry.conf
#
# 启用前必须同时在 /etc/genarrative/pingora-gateway.env 中显式配置:
# GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=0.0.0.0:443
# GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=0.0.0.0:80
# GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE=...
# GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE=...
# 证书链和私钥必须允许 genarrative 用户读取;80/443 必须先从 Nginx 或其它进程释放。
[Service]
# 仅授予非 root genarrative 用户绑定 80/443 所需的最小 capability。
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE
NoNewPrivileges=true
@@ -0,0 +1,31 @@
[Unit]
Description=Genarrative Pingora Shadow Gateway
After=network-online.target genarrative-api.service spacetimedb.service
Wants=network-online.target
Requires=genarrative-api.service spacetimedb.service
[Service]
Type=simple
User=genarrative
Group=genarrative
WorkingDirectory=/opt/genarrative/current
EnvironmentFile=/etc/genarrative/pingora-gateway.env
ExecStartPre=/usr/bin/test -d /srv/genarrative/web
ExecStartPre=/usr/bin/test -d /var/www/html
ExecStart=/opt/genarrative/current/pingora-gateway
Restart=always
RestartSec=5
KillSignal=SIGINT
TimeoutStopSec=60
LimitNOFILE=65535
TasksMax=2048
# 影子网关只监听本机高端口,公网入口继续由 Nginx 承接。
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ReadOnlyPaths=/srv/genarrative/web /var/www/html
ReadWritePaths=/var/lib/genarrative /var/log/genarrative
[Install]
WantedBy=multi-user.target
+1 -1
View File
@@ -23,7 +23,7 @@
本地通过 SSH alias 管理多台服务器、查看硬件 / systemd / HTTP 健康状态并执行受控服务启停的 egui 桌面工具见 [【开发运维】本地SSH服务器管理面板技术方案-2026-06-11.md](./technical/【开发运维】本地SSH服务器管理面板技术方案-2026-06-11.md)。
生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md),该文档也是当前生产 Jenkinsfile 的唯一入口。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md);后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。
生产部署切换到 systemd + Nginx + SpacetimeDB 自托管的总方案见 [PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md](./technical/PRODUCTION_DEPLOYMENT_PLAN_2026-05-02.md),该文档也是当前生产 Jenkinsfile 的唯一入口。Pingora 只作为独立二进制影子网关试点时,边界、路由口径与替换前验收见 [【开发运维】Pingora独立网关试点-2026-06-11.md](./technical/【开发运维】Pingora独立网关试点-2026-06-11.md)。SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)private 表迁移 JSON 导入导出、HTTP 413 分片导入和旧数据库迁移流水线经验见 [SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md](./technical/SPACETIMEDB_JSON_STRING_MIGRATION_PROCEDURE_2026-04-27.md) 与 [JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md](./technical/JENKINS_SPACETIMEDB_DATABASE_MIGRATION_PIPELINES_2026-04-29.md);后台管理独立前端工程技术方案见 [ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md](./technical/ADMIN_WEB_CONSOLE_TECHNICAL_SOLUTION_2026-04-30.md)。
SpacetimeDB 表结构变更、自动迁移边界和保留旧数据的分阶段迁移流程见 [SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md](./technical/SPACETIMEDB_SCHEMA_CHANGE_CONSTRAINTS.md)。
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+28 -2
View File
@@ -23,6 +23,7 @@ pipeline {
string(name: 'COMMIT_HASH', defaultValue: '', description: '可选,指定属于 SOURCE_BRANCH 的 Git commit')
string(name: 'BUILD_VERSION', defaultValue: '', description: '发布版本号,留空则使用 Jenkins BUILD_NUMBER')
string(name: 'NOTIFICATION_EMAILS', defaultValue: '', description: '本次运行追加通知邮箱;会与 Jenkins Secret Text 凭据 genarrative-notification-emails 合并发送')
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '是否额外构建并归档 Pingora 影子网关二进制')
booleanParam(name: 'PUBLISH_AFTER_BUILD', defaultValue: false, description: '构建成功后是否触发 API 发布')
string(name: 'DEPLOY_JOB_NAME', defaultValue: 'Genarrative-Api-Deploy', description: 'API 发布流水线作业名')
choice(name: 'DEPLOY_TARGET', choices: ['development', 'release'], description: 'PUBLISH_AFTER_BUILD=true 时的逻辑部署目标;development 使用当前 Linux 开发/构建/开发部署 agent')
@@ -91,12 +92,30 @@ pipeline {
echo "[api-build] 缺少 clang/lld。请在 genarrative-build 节点预先安装 Linux 构建依赖。" >&2
exit 1
fi
if [[ "${INCLUDE_PINGORA_GATEWAY:-false}" == "true" ]]; then
if ! command -v cmake >/dev/null 2>&1; then
echo "[api-build] 构建 Pingora 影子网关缺少 cmake;请在 genarrative-build 节点预先安装 cmake。" >&2
exit 1
fi
if ! command -v cc >/dev/null 2>&1 && ! command -v clang >/dev/null 2>&1 && ! command -v gcc >/dev/null 2>&1; then
echo "[api-build] 构建 Pingora 影子网关缺少 C 编译器;请预先安装 clang 或 gcc。" >&2
exit 1
fi
if ! command -v c++ >/dev/null 2>&1 && ! command -v clang++ >/dev/null 2>&1 && ! command -v g++ >/dev/null 2>&1; then
echo "[api-build] 构建 Pingora 影子网关缺少 C++ 编译器;请预先安装 clang++ 或 g++。" >&2
exit 1
fi
fi
if ! command -v sccache >/dev/null 2>&1; then
echo "[api-build] 未找到 sccache,改用 rustc 直接构建。"
unset RUSTC_WRAPPER
fi
pingora_args=()
if [[ "${INCLUDE_PINGORA_GATEWAY:-false}" == "true" ]]; then
pingora_args+=(--include-pingora-gateway)
fi
SOURCE_BRANCH="${SOURCE_BRANCH}" SOURCE_COMMIT="${SOURCE_COMMIT}" \
npm run build:production-release -- --component api-server --name "${EFFECTIVE_BUILD_VERSION}"
npm run build:production-release -- --component api-server --name "${EFFECTIVE_BUILD_VERSION}" "${pingora_args[@]}"
'
'''
}
@@ -104,7 +123,12 @@ pipeline {
stage('Archive') {
steps {
archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,scripts/deploy/production-api-deploy.sh,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh", fingerprint: true
archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true
script {
if (params.INCLUDE_PINGORA_GATEWAY) {
archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway,build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway.sha256", fingerprint: true
}
}
}
}
@@ -122,6 +146,7 @@ pipeline {
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION),
string(name: 'NOTIFICATION_EMAILS', value: params.NOTIFICATION_EMAILS ?: ''),
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET),
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', value: params.INCLUDE_PINGORA_GATEWAY),
booleanParam(name: 'CONFIRM_RELEASE_DEPLOY_AGENT', value: params.CONFIRM_RELEASE_DEPLOY_AGENT),
string(name: 'BUILD_JOB_NAME', value: env.JOB_NAME),
string(name: 'BUILD_NUMBER_TO_DEPLOY', value: env.BUILD_NUMBER),
@@ -143,6 +168,7 @@ pipeline {
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION ?: (params.BUILD_VERSION ?: '')),
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET ?: ''),
string(name: 'DATABASE', value: params.DATABASE ?: ''),
string(name: 'INCLUDE_PINGORA_GATEWAY', value: String.valueOf(params.INCLUDE_PINGORA_GATEWAY)),
string(name: 'SUMMARY', value: 'API 构建流水线结束'),
]
def notificationRecipients = params.NOTIFICATION_EMAILS?.trim()
+16 -3
View File
@@ -16,6 +16,7 @@ pipeline {
string(name: 'BUILD_VERSION', defaultValue: '', description: '待发布版本号')
string(name: 'BUILD_JOB_NAME', defaultValue: 'Genarrative-Api-Build', description: 'API 构建流水线作业名')
string(name: 'BUILD_NUMBER_TO_DEPLOY', defaultValue: '', description: '要复制归档产物的上游构建号')
booleanParam(name: 'INCLUDE_PINGORA_GATEWAY', defaultValue: false, description: '上游构建是否包含 Pingora 影子网关产物')
string(name: 'RELEASE_ROOT', defaultValue: '/opt/genarrative/releases', description: '生产 release 根目录')
string(name: 'CURRENT_LINK', defaultValue: '/opt/genarrative/current', description: '当前版本软链接')
string(name: 'SERVICE_NAME', defaultValue: 'genarrative-api.service', description: 'systemd 服务名')
@@ -65,10 +66,21 @@ pipeline {
copyArtifacts(
projectName: params.BUILD_JOB_NAME,
selector: specific(params.BUILD_NUMBER_TO_DEPLOY),
filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,scripts/deploy/production-api-deploy.sh,scripts/deploy/maintenance-on.sh,scripts/deploy/maintenance-off.sh",
filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**",
target: '.',
fingerprintArtifacts: true
)
script {
if (params.INCLUDE_PINGORA_GATEWAY) {
copyArtifacts(
projectName: params.BUILD_JOB_NAME,
selector: specific(params.BUILD_NUMBER_TO_DEPLOY),
filter: "build/${params.BUILD_VERSION}/pingora-gateway,build/${params.BUILD_VERSION}/pingora-gateway.sha256",
target: '.',
fingerprintArtifacts: true
)
}
}
}
}
@@ -80,8 +92,8 @@ pipeline {
sh '''
bash -lc '
set -euo pipefail
chmod +x scripts/deploy/production-api-deploy.sh scripts/deploy/maintenance-on.sh scripts/deploy/maintenance-off.sh
scripts/deploy/production-api-deploy.sh \
chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
"build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \
--source-dir "build/${BUILD_VERSION}" \
--version "${BUILD_VERSION}" \
--release-root "${RELEASE_ROOT}" \
@@ -110,6 +122,7 @@ pipeline {
string(name: 'BUILD_VERSION', value: env.EFFECTIVE_BUILD_VERSION ?: (params.BUILD_VERSION ?: '')),
string(name: 'DEPLOY_TARGET', value: params.DEPLOY_TARGET ?: ''),
string(name: 'DATABASE', value: params.DATABASE ?: ''),
string(name: 'INCLUDE_PINGORA_GATEWAY', value: String.valueOf(params.INCLUDE_PINGORA_GATEWAY)),
string(name: 'SUMMARY', value: 'API 发布流水线结束'),
]
def notificationRecipients = params.NOTIFICATION_EMAILS?.trim()
+35
View File
@@ -29,6 +29,41 @@
"check:encoding": "node scripts/check-encoding.mjs",
"check:spacetime-schema": "node scripts/check-spacetime-schema-guard.mjs",
"check:production-ops": "node scripts/check-production-ops-guardrails.mjs",
"check:production-health-patrol": "node scripts/check-production-health-patrol.mjs",
"check:production-health-patrol-env": "node scripts/check-production-health-patrol-env-check.mjs",
"check:production-api-release": "node scripts/check-production-api-release.mjs",
"check:production-api-deploy": "node scripts/check-production-api-deploy.mjs",
"check:pingora-gateway-smoke": "node scripts/check-pingora-gateway-smoke.mjs",
"check:nginx-pingora-canary": "node scripts/check-nginx-pingora-canary.mjs",
"check:pingora-route-parity": "node scripts/check-pingora-route-parity.mjs",
"check:pingora-canary-live": "node scripts/check-pingora-canary-live.mjs",
"check:pingora-canary-live-guard": "node scripts/check-pingora-canary-live-guard.mjs",
"check:pingora-realpath-canary-toggle": "node scripts/check-pingora-realpath-canary-toggle.mjs",
"check:pingora-direct-preflight": "node -- scripts/check-pingora-direct-preflight.mjs && node scripts/check-pingora-direct-preflight-guard.mjs",
"check:pingora-direct-live": "node scripts/check-pingora-direct-live.mjs",
"check:pingora-direct-live-guard": "node scripts/check-pingora-direct-live-guard.mjs",
"check:pingora-direct-enable": "node scripts/check-pingora-direct-enable.mjs",
"check:pingora-direct-rollback": "node scripts/check-pingora-direct-rollback.mjs",
"check:pingora-health-patrol-env-switch": "node scripts/check-pingora-health-patrol-env-switch.mjs",
"check:pingora-gateway-env-shadow-switch": "node scripts/check-pingora-gateway-env-shadow-switch.mjs",
"check:pingora-tls-cert-sync": "node scripts/check-pingora-tls-cert-sync.mjs",
"check:pingora-current-release-audit": "node scripts/check-pingora-current-release-audit.mjs",
"check:pingora-direct-rehearsal-status": "node scripts/check-pingora-direct-rehearsal-status.mjs",
"check:pingora-cutover-status-snapshot": "node scripts/check-pingora-cutover-status-snapshot.mjs",
"check:pingora-cutover-evidence-bundle": "node scripts/check-pingora-cutover-evidence-bundle.mjs",
"check:pingora-cutover-command-evidence": "node scripts/check-pingora-cutover-command-evidence.mjs",
"check:pingora-cutover-evidence-verify": "node scripts/check-pingora-cutover-evidence-verify.mjs",
"check:pingora-cutover-evidence-audit": "node scripts/check-pingora-cutover-evidence-audit.mjs",
"check:pingora-canary-access-log-parity": "node scripts/check-pingora-canary-access-log-parity-check.mjs",
"check:pingora-canary-docker": "node scripts/check-pingora-canary-docker.mjs",
"check:pingora-production-release-build": "node scripts/check-pingora-production-release-build.mjs",
"check:pingora-release-readiness-plan": "node scripts/check-pingora-release-readiness-plan.mjs",
"check:pingora-release-readiness": "node scripts/check-pingora-release-readiness.mjs",
"plan:pingora-direct-cutover": "node scripts/check-pingora-release-readiness.mjs --dry-run-cutover",
"deploy:pingora-realpath-canary-enable": "bash scripts/deploy/pingora-realpath-canary-enable.sh",
"deploy:pingora-realpath-canary-disable": "bash scripts/deploy/pingora-realpath-canary-disable.sh",
"deploy:pingora-direct-enable": "bash scripts/deploy/pingora-direct-enable.sh",
"deploy:pingora-direct-rollback": "bash scripts/deploy/pingora-direct-rollback.sh",
"assets:child-motion-demo": "node scripts/generate-child-motion-demo-assets.mjs",
"assets:match3d-style-references": "node scripts/generate-match3d-style-references.mjs",
"check:visual-novel-vn11": "node scripts/check-visual-novel-vn11-negative-scan.mjs",
+114 -3
View File
@@ -18,6 +18,10 @@ usage() {
--skip-web-build 跳过主站与后台构建,仅复制已有 dist 产物
--skip-api-build 跳过 api-server 构建,仅复制已有 release 二进制
--skip-spacetime-build 跳过 spacetime-module 构建,仅复制已有 wasm
--include-pingora-gateway
额外构建并打包 Pingora 影子网关二进制,默认不启用
--skip-pingora-gateway-build
跳过 Pingora 构建,仅复制已有 release 二进制
--no-migration-bootstrap-secret
构建不带迁移引导密钥的 spacetime-module wasm
EOF
@@ -32,6 +36,28 @@ require_command() {
fi
}
require_any_command() {
local label="$1"
shift
local command_name
for command_name in "$@"; do
if command -v "${command_name}" >/dev/null 2>&1; then
return
fi
done
echo "[production-release] 缺少 ${label}: $*" >&2
exit 1
}
require_pingora_build_toolchain() {
require_command cargo
require_command cmake
require_any_command "C 编译器" cc clang gcc
require_any_command "C++ 编译器" c++ clang++ g++
}
copy_required_file() {
local source_path="$1"
local target_path="$2"
@@ -140,6 +166,7 @@ write_release_manifest() {
RELEASE_INCLUDE_WEB="${BUILD_WEB}" \
RELEASE_INCLUDE_API="${BUILD_API}" \
RELEASE_INCLUDE_SPACETIME="${BUILD_SPACETIME}" \
RELEASE_INCLUDE_PINGORA_GATEWAY="${INCLUDE_PINGORA_GATEWAY}" \
RELEASE_INCLUDE_MIGRATION_BOOTSTRAP_SECRET="${MIGRATION_BOOTSTRAP_SECRET_ARTIFACT}" \
node <<'NODE'
const fs = require('fs');
@@ -166,6 +193,13 @@ if (process.env.RELEASE_INCLUDE_SPACETIME === '1') {
checksum_path: 'spacetime_module.wasm.sha256',
});
}
if (process.env.RELEASE_INCLUDE_PINGORA_GATEWAY === '1') {
artifacts.push({
component: 'pingora-gateway',
path: 'pingora-gateway',
checksum_path: 'pingora-gateway.sha256',
});
}
if (process.env.RELEASE_INCLUDE_MIGRATION_BOOTSTRAP_SECRET === '1') {
artifacts.push({
component: 'spacetime-module',
@@ -196,6 +230,8 @@ COMPONENT="all"
SKIP_WEB_BUILD=0
SKIP_API_BUILD=0
SKIP_SPACETIME_BUILD=0
SKIP_PINGORA_GATEWAY_BUILD=0
INCLUDE_PINGORA_GATEWAY=0
MIGRATION_BOOTSTRAP_SECRET=""
MIGRATION_BOOTSTRAP_SECRET_ARTIFACT=0
MIGRATION_BOOTSTRAP_SECRET_MODE="auto"
@@ -227,6 +263,14 @@ while [[ $# -gt 0 ]]; do
SKIP_SPACETIME_BUILD=1
shift
;;
--include-pingora-gateway)
INCLUDE_PINGORA_GATEWAY=1
shift
;;
--skip-pingora-gateway-build)
SKIP_PINGORA_GATEWAY_BUILD=1
shift
;;
--no-migration-bootstrap-secret)
MIGRATION_BOOTSTRAP_SECRET=""
MIGRATION_BOOTSTRAP_SECRET_MODE="disabled"
@@ -276,6 +320,7 @@ WEB_DIR="${TARGET_DIR}/web"
ADMIN_WEB_DIR="${WEB_DIR}/admin"
CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${SERVER_RS_DIR}/target}"
API_BINARY_SOURCE="${CARGO_TARGET_DIR}/x86_64-unknown-linux-gnu/release/api-server"
PINGORA_GATEWAY_BINARY_SOURCE="${CARGO_TARGET_DIR}/x86_64-unknown-linux-gnu/release/pingora-gateway"
WASM_SOURCE="${CARGO_TARGET_DIR}/wasm32-unknown-unknown/release/spacetime_module.wasm"
RELEASE_SOURCE_BRANCH="${SOURCE_BRANCH:-${GIT_BRANCH:-}}"
RELEASE_SOURCE_BRANCH="${RELEASE_SOURCE_BRANCH#origin/}"
@@ -310,7 +355,6 @@ if [[ -e "${TARGET_DIR}" ]]; then
exit 1
fi
require_command node
require_command node
require_command sha256sum
@@ -322,6 +366,10 @@ if [[ "${BUILD_SPACETIME}" -eq 1 && "${SKIP_SPACETIME_BUILD}" -ne 1 ]]; then
require_command cargo
fi
if [[ "${INCLUDE_PINGORA_GATEWAY}" -eq 1 && "${SKIP_PINGORA_GATEWAY_BUILD}" -ne 1 ]]; then
require_pingora_build_toolchain
fi
if [[ "${BUILD_WEB}" -eq 1 && "${SKIP_WEB_BUILD}" -ne 1 ]]; then
require_command node
require_command npm
@@ -427,6 +475,24 @@ if [[ "${BUILD_API}" -eq 1 ]]; then
write_sha256_file "${TARGET_DIR}/api-server"
fi
if [[ "${INCLUDE_PINGORA_GATEWAY}" -eq 1 && "${SKIP_PINGORA_GATEWAY_BUILD}" -ne 1 ]]; then
echo "[production-release] 构建 pingora-gateway -> x86_64-unknown-linux-gnu"
(
cd "${SERVER_RS_DIR}"
cargo build \
-p pingora-gateway \
--release \
--target x86_64-unknown-linux-gnu \
--manifest-path "${SERVER_RS_DIR}/Cargo.toml"
)
fi
if [[ "${INCLUDE_PINGORA_GATEWAY}" -eq 1 ]]; then
copy_required_file "${PINGORA_GATEWAY_BINARY_SOURCE}" "${TARGET_DIR}/pingora-gateway" "pingora-gateway release binary"
chmod +x "${TARGET_DIR}/pingora-gateway"
write_sha256_file "${TARGET_DIR}/pingora-gateway"
fi
if [[ "${BUILD_SPACETIME}" -eq 1 && "${SKIP_SPACETIME_BUILD}" -ne 1 ]]; then
echo "[production-release] 构建 spacetime-module -> wasm32-unknown-unknown"
(
@@ -445,10 +511,20 @@ if [[ "${BUILD_SPACETIME}" -eq 1 ]]; then
write_migration_bootstrap_secret_file
fi
mkdir -p "${TARGET_DIR}/scripts" "${TARGET_DIR}/scripts/ops" "${TARGET_DIR}/deploy"
mkdir -p "${TARGET_DIR}/scripts" "${TARGET_DIR}/scripts/deploy" "${TARGET_DIR}/scripts/ops" "${TARGET_DIR}/deploy"
cp "${SCRIPT_DIR}/deploy/maintenance-on.sh" "${TARGET_DIR}/scripts/maintenance-on.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/maintenance-off.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-status.sh" "${TARGET_DIR}/scripts/maintenance-status.sh"
cp "${SCRIPT_DIR}/deploy/production-api-deploy.sh" "${TARGET_DIR}/scripts/deploy/production-api-deploy.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-on.sh" "${TARGET_DIR}/scripts/deploy/maintenance-on.sh"
cp "${SCRIPT_DIR}/deploy/maintenance-off.sh" "${TARGET_DIR}/scripts/deploy/maintenance-off.sh"
cp "${SCRIPT_DIR}/deploy/pingora-direct-enable.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh"
cp "${SCRIPT_DIR}/deploy/pingora-direct-rollback.sh" "${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh"
cp "${SCRIPT_DIR}/deploy/pingora-realpath-canary-enable.sh" "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh"
cp "${SCRIPT_DIR}/deploy/pingora-realpath-canary-disable.sh" "${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh"
cp "${SCRIPT_DIR}/deploy/pingora-health-patrol-env-switch.mjs" "${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs"
cp "${SCRIPT_DIR}/deploy/pingora-gateway-env-shadow-switch.mjs" "${TARGET_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs"
cp "${SCRIPT_DIR}/deploy/pingora-tls-cert-sync.mjs" "${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs"
cp "${SCRIPT_DIR}/deploy/jenkins-inbound-agent-start.sh" "${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh"
cp "${SCRIPT_DIR}/deploy/install-jenkins-inbound-agent.sh" "${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh"
cp "${SCRIPT_DIR}/deploy/jenkins-agent-reverse-tunnel.ps1" "${TARGET_DIR}/scripts/jenkins-agent-reverse-tunnel.ps1"
@@ -457,6 +533,16 @@ chmod +x \
"${TARGET_DIR}/scripts/maintenance-on.sh" \
"${TARGET_DIR}/scripts/maintenance-off.sh" \
"${TARGET_DIR}/scripts/maintenance-status.sh" \
"${TARGET_DIR}/scripts/deploy/production-api-deploy.sh" \
"${TARGET_DIR}/scripts/deploy/maintenance-on.sh" \
"${TARGET_DIR}/scripts/deploy/maintenance-off.sh" \
"${TARGET_DIR}/scripts/deploy/pingora-direct-enable.sh" \
"${TARGET_DIR}/scripts/deploy/pingora-direct-rollback.sh" \
"${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-enable.sh" \
"${TARGET_DIR}/scripts/deploy/pingora-realpath-canary-disable.sh" \
"${TARGET_DIR}/scripts/deploy/pingora-health-patrol-env-switch.mjs" \
"${TARGET_DIR}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" \
"${TARGET_DIR}/scripts/deploy/pingora-tls-cert-sync.mjs" \
"${TARGET_DIR}/scripts/jenkins-inbound-agent-start.sh" \
"${TARGET_DIR}/scripts/install-jenkins-inbound-agent.sh"
@@ -467,10 +553,24 @@ copy_required_file "${SCRIPT_DIR}/spacetime-authorize-migration-operator.mjs" "$
copy_required_file "${SCRIPT_DIR}/spacetime-revoke-migration-operator.mjs" "${TARGET_DIR}/scripts/spacetime-revoke-migration-operator.mjs" "数据库迁移撤权脚本"
copy_required_file "${SCRIPT_DIR}/database-backup-to-oss.mjs" "${TARGET_DIR}/scripts/database-backup-to-oss.mjs" "数据库 OSS 备份脚本"
copy_required_file "${SCRIPT_DIR}/ops/production-health-patrol.mjs" "${TARGET_DIR}/scripts/ops/production-health-patrol.mjs" "生产健康巡检脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-current-release-audit.mjs" "${TARGET_DIR}/scripts/ops/pingora-current-release-audit.mjs" "Pingora current release 自审脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-direct-rehearsal-status.mjs" "${TARGET_DIR}/scripts/ops/pingora-direct-rehearsal-status.mjs" "Pingora 直连切换彩排状态脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-status-snapshot.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-status-snapshot.mjs" "Pingora 直连切换状态快照脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-bundle.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-bundle.mjs" "Pingora 直连切换证据包脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-command-evidence.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-command-evidence.mjs" "Pingora 直连切换命令证据脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-verify.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-verify.mjs" "Pingora 直连切换证据验真脚本"
copy_required_file "${SCRIPT_DIR}/ops/pingora-cutover-evidence-audit.mjs" "${TARGET_DIR}/scripts/ops/pingora-cutover-evidence-audit.mjs" "Pingora 直连切换证据根目录审计脚本"
copy_required_file "${SCRIPT_DIR}/check-production-health-patrol-env.mjs" "${TARGET_DIR}/scripts/check-production-health-patrol-env.mjs" "生产健康巡检 env 复核脚本"
copy_required_file "${SCRIPT_DIR}/check-pingora-release-readiness.mjs" "${TARGET_DIR}/scripts/check-pingora-release-readiness.mjs" "Pingora release readiness 聚合门禁脚本"
copy_required_file "${SCRIPT_DIR}/check-pingora-direct-preflight.mjs" "${TARGET_DIR}/scripts/check-pingora-direct-preflight.mjs" "Pingora 直连预检脚本"
copy_required_file "${SCRIPT_DIR}/check-pingora-direct-live.mjs" "${TARGET_DIR}/scripts/check-pingora-direct-live.mjs" "Pingora 直连 live smoke 脚本"
copy_required_file "${SCRIPT_DIR}/check-pingora-canary-live.mjs" "${TARGET_DIR}/scripts/check-pingora-canary-live.mjs" "Pingora canary live smoke 脚本"
copy_required_file "${SCRIPT_DIR}/check-pingora-canary-access-log-parity.mjs" "${TARGET_DIR}/scripts/check-pingora-canary-access-log-parity.mjs" "Pingora canary access log 对账脚本"
copy_required_dir "${REPO_ROOT}/deploy/systemd" "${TARGET_DIR}/deploy/systemd" "systemd 配置"
copy_required_dir "${REPO_ROOT}/deploy/nginx" "${TARGET_DIR}/deploy/nginx" "Nginx 配置"
copy_required_dir "${REPO_ROOT}/deploy/env" "${TARGET_DIR}/deploy/env" "生产环境示例"
copy_required_dir "${REPO_ROOT}/deploy/pingora" "${TARGET_DIR}/deploy/pingora" "Pingora 配置"
cat >"${TARGET_DIR}/README.md" <<EOF
# Genarrative Production Release
@@ -482,13 +582,24 @@ cat >"${TARGET_DIR}/README.md" <<EOF
- \`web/\`:主站静态资源,\`web/admin/\` 为后台静态资源,\`web/maintenance.html\` 为维护页。
- \`web.tar.gz\` / \`web.tar.gz.sha256\`Web 发布流水线使用的静态资源压缩包与校验文件。
- \`api-server\`:生产 Linux release 可执行文件。
- \`pingora-gateway\`:可选 Pingora 影子网关可执行文件,仅在显式 \`--include-pingora-gateway\` 时包含。
- \`spacetime_module.wasm\`SpacetimeDB 模块 wasm。
- \`migration-bootstrap-secret.txt\`:构建 \`spacetime_module.wasm\` 时注入的迁移引导密钥,仅用于创建首个迁移操作员;请作为敏感文件保存到 Jenkins Secret Text,授权完成后不要长期留在公开归档中。
- \`*.sha256\`:发布产物 checksum,用于部署前校验。
- \`release-manifest.json\`:发布版本、源码 commit 与产物清单。
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`scripts/\`:维护模式脚本、数据库导入导出脚本、数据库 OSS 备份脚本、生产健康巡检脚本、Pingora release readiness 聚合门禁、直连启用 / 回退、realpath canary 启用 / 关闭、health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态 / 状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本、迁移授权脚本和 Jenkins inbound agent systemd 安装脚本。
- \`scripts/deploy/production-api-deploy.sh\`API Deploy 执行入口;同目录的 \`maintenance-on.sh\` / \`maintenance-off.sh\` 必须来自同一发布包。
- \`deploy/\`systemd、Nginx 和生产环境变量示例;\`deploy/nginx/genarrative-dev-http.conf\` 仅供无域名开发服初始化使用。
## Pingora 直连证据总审计
正式切换 runbook 的最终证据根目录总审计必须使用 current release 随包脚本,并同时保留两条真实切换脚本身份要求:
- \`--require-command-executable enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh\`
- \`--require-command-executable rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh\`
这两项用于证明 enable / rollback apply 命令证据里真实执行的是 \`/opt/genarrative/current\` 下的随包脚本,而不是 Jenkins 工作区、源码 checkout 或旧发布目录中的同名脚本。
## 生产部署口径
本发布包不包含旧一体化 \`start.sh\`、\`stop.sh\` 或 \`web-server.mjs\`。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,493 @@
#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
const PARITY_SCRIPT = 'scripts/check-pingora-canary-access-log-parity.mjs';
const failures = [];
const tmpRoot = mkdtempSync(
path.join(tmpdir(), 'genarrative-pingora-canary-log-parity-'),
);
try {
main();
} finally {
rmSync(tmpRoot, { recursive: true, force: true });
}
if (failures.length > 0) {
console.error('[check:pingora-canary-access-log-parity] FAILED');
for (const failure of failures) {
console.error(`- ${failure}`);
}
process.exit(1);
}
console.log('[check:pingora-canary-access-log-parity] OK');
function main() {
assertScriptShape();
assertParitySucceeds();
assertRealpathParitySucceeds();
assertMissingPingoraRecordFails();
assertStatusMismatchFails();
assertRequiredPathFails();
assertRejectsRelativeLogPaths();
assertRejectsFilesystemRootLogPaths();
assertRejectsLogPathControlCharacters();
assertRejectsPrefixAndPathControlCharacters();
assertRejectsParsedLogPathControlCharacters();
assertRejectsInvalidSinceLines();
}
function assertScriptShape() {
const content = readFileSync(PARITY_SCRIPT, 'utf8');
assertIncludes(
content,
'该脚本只读比较 Nginx canary handoff access log 和 Pingora access log',
'canary access log parity 脚本 usage 必须说明只读边界。',
);
assertIncludes(
content,
'request_id',
'canary access log parity 必须按 request_id 对照。',
);
assertIncludes(
content,
'realpath',
'canary access log parity 必须支持真实路径 canary 对账模式。',
);
if (
content.includes('writeFile') ||
content.includes('rmSync(') ||
content.includes('nginx -s reload')
) {
failures.push('canary access log parity 脚本不应写文件、删除文件或 reload Nginx。');
}
}
function assertParitySucceeds() {
const fixture = prepareFixture('ok');
writeLogs(fixture, [
nginxLine('rid-health', 'GET', '/__genarrative_pingora_canary/healthz', 200),
nginxLine(
'rid-api',
'GET',
'/__genarrative_pingora_canary/api/creation-entry/config',
200,
),
], [
pingoraLine('rid-health', 'GET', '/__genarrative_pingora/healthz', 200, {
route: 'shadow_probe',
}),
pingoraLine('rid-api', 'GET', '/api/creation-entry/config', 200, {
route: 'api_proxy',
proxyTarget: 'api-server',
}),
]);
const result = runParity(fixture, [
'--path',
'/__genarrative_pingora_canary/healthz',
'--path',
'/__genarrative_pingora_canary/api/creation-entry/config',
'--json',
]);
assertStatus(result, 0, '日志对照完整时必须通过。');
if (result.status !== 0) {
return;
}
const payload = parseJson(result.stdout, '日志对照 JSON 输出');
assertEqual(payload.summary.matchedCount, 2, '应匹配两条 canary 请求。');
assertEqual(payload.summary.missingCount, 0, '不应缺少 Pingora 对应日志。');
}
function assertRealpathParitySucceeds() {
const fixture = prepareFixture('realpath-ok');
writeLogs(fixture, [
nginxLine(
'rid-real-health',
'GET',
'/__genarrative_pingora_realpath_canary/healthz',
200,
),
nginxLine('rid-real-api', 'GET', '/api/creation-entry/config', 200),
nginxLine('rid-real-asset', 'GET', '/assets/app.js', 200),
], [
pingoraLine('rid-real-health', 'GET', '/__genarrative_pingora/healthz', 200, {
route: 'shadow_probe',
}),
pingoraLine('rid-real-api', 'GET', '/api/creation-entry/config', 200, {
route: 'api_proxy',
proxyTarget: 'api-server',
}),
pingoraLine('rid-real-asset', 'GET', '/assets/app.js', 200, {
route: 'static',
}),
]);
const result = runParity(fixture, [
'--realpath',
'--path',
'/__genarrative_pingora_realpath_canary/healthz',
'--path',
'/api/creation-entry/config',
'--path',
'/assets/app.js',
'--json',
]);
assertStatus(result, 0, '真实路径日志对照完整时必须通过。');
if (result.status !== 0) {
return;
}
const payload = parseJson(result.stdout, '真实路径日志对照 JSON 输出');
assertEqual(payload.mode, 'realpath', '真实路径对账 JSON 必须标记 realpath 模式。');
assertEqual(payload.summary.matchedCount, 3, '应匹配三条真实路径 canary 请求。');
assertEqual(payload.summary.missingCount, 0, '真实路径对账不应缺少 Pingora 对应日志。');
}
function assertMissingPingoraRecordFails() {
const fixture = prepareFixture('missing-pingora');
writeLogs(fixture, [
nginxLine('rid-missing', 'GET', '/__genarrative_pingora_canary/v1/identity', 200),
], []);
const result = runParity(fixture);
assertStatus(result, 1, '缺少同 request_id Pingora 日志时必须失败。');
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'缺少对应 Pingora access log',
'缺少 Pingora 日志时必须给出明确错误。',
);
}
function assertStatusMismatchFails() {
const fixture = prepareFixture('status-mismatch');
writeLogs(fixture, [
nginxLine('rid-status', 'GET', '/__genarrative_pingora_canary/api/test', 200),
], [
pingoraLine('rid-status', 'GET', '/api/test', 503),
]);
const result = runParity(fixture);
assertStatus(result, 1, 'Nginx/Pingora 状态码不一致时必须失败。');
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'status 200 != 503',
'状态码不一致必须给出明确错误。',
);
}
function assertRequiredPathFails() {
const fixture = prepareFixture('missing-required-path');
writeLogs(fixture, [
nginxLine('rid-health', 'GET', '/__genarrative_pingora_canary/healthz', 200),
], [
pingoraLine('rid-health', 'GET', '/__genarrative_pingora/healthz', 200),
]);
const result = runParity(fixture, [
'--path',
'/__genarrative_pingora_canary/api/creation-entry/config',
]);
assertStatus(result, 1, '必需路径未出现在 Nginx canary 日志时必须失败。');
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'Nginx canary 日志缺少必需路径',
'必需路径缺失必须给出明确错误。',
);
}
function assertRejectsRelativeLogPaths() {
const fixture = prepareFixture('relative-path');
const result = spawnSync(
'node',
[
PARITY_SCRIPT,
'--nginx-log-file',
'nginx.log',
'--pingora-log-file',
fixture.pingoraLogFile,
],
{
cwd: process.cwd(),
encoding: 'utf8',
},
);
if ((result.status ?? 0) === 0) {
failures.push('日志路径为相对路径时必须失败。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--nginx-log-file 必须是绝对路径',
'相对 Nginx 日志路径必须给出明确错误。',
);
}
function assertRejectsFilesystemRootLogPaths() {
const fixture = prepareFixture('filesystem-root-path');
const result = spawnSync(
'node',
[
PARITY_SCRIPT,
'--nginx-log-file',
'/',
'--pingora-log-file',
fixture.pingoraLogFile,
],
{
cwd: process.cwd(),
encoding: 'utf8',
},
);
if ((result.status ?? 0) === 0) {
failures.push('日志路径指向文件系统根目录时必须失败。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--nginx-log-file 不能是文件系统根目录',
'文件系统根目录 Nginx 日志路径必须给出明确错误。',
);
}
function assertRejectsLogPathControlCharacters() {
const fixture = prepareFixture('log-path-control-character');
const result = spawnSync(
'node',
[
PARITY_SCRIPT,
'--nginx-log-file',
`${fixture.nginxLogFile}\nspoofed`,
'--pingora-log-file',
fixture.pingoraLogFile,
],
{
cwd: process.cwd(),
encoding: 'utf8',
},
);
if ((result.status ?? 0) === 0) {
failures.push('日志路径包含换行时必须失败。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--nginx-log-file 不能包含换行或 NUL 字符',
'带控制字符的 Nginx 日志路径必须给出明确错误。',
);
}
function assertRejectsPrefixAndPathControlCharacters() {
const fixture = prepareFixture('prefix-path-control-character');
writeLogs(fixture, [
nginxLine('rid-health', 'GET', '/__genarrative_pingora_canary/healthz', 200),
], [
pingoraLine('rid-health', 'GET', '/__genarrative_pingora/healthz', 200),
]);
const prefixResult = runParity(fixture, [
'--prefix',
'/__genarrative_pingora_canary\nspoofed',
]);
if ((prefixResult.status ?? 0) === 0) {
failures.push('canary prefix 包含换行时必须失败。');
}
assertIncludes(
`${prefixResult.stdout}\n${prefixResult.stderr}`,
'--prefix 不能包含换行或 NUL 字符',
'带控制字符的 canary prefix 必须给出明确错误。',
);
const pathResult = runParity(fixture, [
'--path',
'/__genarrative_pingora_canary/healthz\nspoofed',
]);
if ((pathResult.status ?? 0) === 0) {
failures.push('必需 canary path 包含换行时必须失败。');
}
assertIncludes(
`${pathResult.stdout}\n${pathResult.stderr}`,
'--path 不能包含换行或 NUL 字符',
'带控制字符的必需路径必须给出明确错误。',
);
}
function assertRejectsParsedLogPathControlCharacters() {
const fixture = prepareFixture('parsed-log-path-control-character');
writeLogs(fixture, [
nginxLine('rid-health', 'GET', '/__genarrative_pingora_canary/healthz', 200),
], [
pingoraLine('rid-health', 'GET', '/__genarrative_pingora/healthz%0Aspoofed', 200),
]);
const result = runParity(fixture);
if ((result.status ?? 0) === 0) {
failures.push('Pingora 日志 path 解析后包含换行时必须失败。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'Pingora access log 第 1 行 path 不能包含换行或 NUL 字符',
'带控制字符的 Pingora 日志 path 必须给出明确错误。',
);
}
function assertRejectsInvalidSinceLines() {
const fixture = prepareFixture('invalid-since-lines');
writeLogs(fixture, [
nginxLine('rid-health', 'GET', '/__genarrative_pingora_canary/healthz', 200),
], [
pingoraLine('rid-health', 'GET', '/__genarrative_pingora/healthz', 200),
]);
const cliResult = runParity(fixture, ['--since-lines', '0']);
if ((cliResult.status ?? 0) === 0) {
failures.push('非正数 --since-lines 必须失败。');
}
assertIncludes(
`${cliResult.stdout}\n${cliResult.stderr}`,
'--since-lines 必须是正整数',
'非法 --since-lines 必须给出明确错误。',
);
const controlResult = runParity(fixture, ['--since-lines', '10\nspoofed']);
if ((controlResult.status ?? 0) === 0) {
failures.push('带控制字符的 --since-lines 必须失败。');
}
assertIncludes(
`${controlResult.stdout}\n${controlResult.stderr}`,
'--since-lines 不能包含换行或 NUL 字符',
'带控制字符的 --since-lines 必须给出明确错误。',
);
const envResult = spawnSync(
'node',
[
PARITY_SCRIPT,
'--nginx-log-file',
fixture.nginxLogFile,
'--pingora-log-file',
fixture.pingoraLogFile,
],
{
cwd: process.cwd(),
encoding: 'utf8',
env: {
...process.env,
GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES: 'abc',
},
},
);
if ((envResult.status ?? 0) === 0) {
failures.push('非法 env since-lines 必须失败。');
}
assertIncludes(
`${envResult.stdout}\n${envResult.stderr}`,
'GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES 必须是正整数',
'非法 env since-lines 必须给出明确错误。',
);
}
function prepareFixture(name) {
const root = path.join(tmpRoot, name);
mkdirSync(root, { recursive: true });
return {
root,
nginxLogFile: path.join(root, 'nginx.access.log'),
pingoraLogFile: path.join(root, 'pingora.access.log'),
};
}
function writeLogs(fixture, nginxLines, pingoraLines) {
writeFileSync(fixture.nginxLogFile, `${nginxLines.join('\n')}\n`, 'utf8');
writeFileSync(fixture.pingoraLogFile, `${pingoraLines.join('\n')}\n`, 'utf8');
}
function nginxLine(requestId, method, uri, status) {
return [
'127.0.0.1 - - [16/Jun/2026:02:00:00 +0800]',
`"${method} ${uri} HTTP/1.1"`,
`${status} 12 "-" "agent"`,
'request_time=0.001 upstream_connect_time=0.000',
'upstream_header_time=0.001 upstream_response_time=0.001',
`upstream_status=${status} request_id=${requestId}`,
].join(' ');
}
function pingoraLine(
requestId,
method,
requestPath,
status,
options = {},
) {
return [
`request_id=${requestId}`,
`method=${method}`,
`path=${requestPath}`,
`uri=${requestPath}`,
'host=example.com',
'client_ip=127.0.0.1',
`status=${status}`,
`route=${options.route || 'api_proxy'}`,
`proxy_target=${options.proxyTarget || '-'}`,
'upstream=127.0.0.1:8082',
'content_length=-',
'body_bytes_seen=0',
'protection_class=api',
'protection_client=127.0.0.1',
'elapsed_ms=2',
'error=-',
].join('\t');
}
function runParity(fixture, args = []) {
return spawnSync(
'node',
[
PARITY_SCRIPT,
'--nginx-log-file',
fixture.nginxLogFile,
'--pingora-log-file',
fixture.pingoraLogFile,
...args,
],
{
cwd: process.cwd(),
encoding: 'utf8',
},
);
}
function parseJson(text, label) {
try {
return JSON.parse(text);
} catch (error) {
failures.push(`${label} 不是合法 JSON: ${error.message}`);
return {};
}
}
function assertStatus(result, expected, reason) {
if ((result.status ?? 0) !== expected) {
failures.push(
`${reason} 实际退出码 ${result.status}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
);
}
}
function assertEqual(actual, expected, reason) {
if (actual !== expected) {
failures.push(`${reason} 实际 ${actual},预期 ${expected}`);
}
}
function assertIncludes(value, expected, reason) {
const haystack = Array.isArray(value) ? value.join('\n') : String(value);
if (!haystack.includes(expected)) {
failures.push(`${reason} 缺少: ${expected}`);
}
}
@@ -0,0 +1,427 @@
#!/usr/bin/env node
import { readFileSync } from 'node:fs';
import path from 'node:path';
const DEFAULT_CANARY_PREFIX = '/__genarrative_pingora_canary';
const REALPATH_HEALTHZ_PATH = '/__genarrative_pingora_realpath_canary/healthz';
const SHADOW_HEALTHZ_PATH = '/__genarrative_pingora/healthz';
const config = parseArgs(process.argv.slice(2));
const failures = [];
const nginxRecords = readLogRecords(config.nginxLogFile, parseNginxAccessLine);
const pingoraRecords = readLogRecords(
config.pingoraLogFile,
parseKeyValueAccessLine,
);
const parity = buildParity(nginxRecords, pingoraRecords);
if (config.json) {
console.log(`${JSON.stringify(parity, null, 2)}\n`);
}
if (failures.length > 0) {
console.error('[pingora-canary-access-log-parity] FAILED');
for (const failure of failures) {
console.error(`- ${failure}`);
}
process.exit(1);
}
if (!config.json) {
console.log(
`[pingora-canary-access-log-parity] OK (${parity.summary.matchedCount}/${parity.summary.nginxCanaryCount} matched)`,
);
}
function parseArgs(argv) {
const result = {
nginxLogFile:
process.env.GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG ||
'/var/log/nginx/genarrative.access.log',
pingoraLogFile:
process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG ||
'/var/log/genarrative/pingora-gateway.access.log',
prefix:
process.env.GENARRATIVE_PINGORA_CANARY_PREFIX || DEFAULT_CANARY_PREFIX,
mode: normalizeMode(
process.env.GENARRATIVE_PINGORA_CANARY_MODE || 'prefix',
'GENARRATIVE_PINGORA_CANARY_MODE',
),
sinceLines: parseOptionalPositiveInt(
process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES,
2000,
'GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES',
),
requiredPaths: [],
json: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
switch (arg) {
case '-h':
case '--help':
usage();
process.exit(0);
break;
case '--nginx-log-file':
result.nginxLogFile = requireValue(argv, ++index, arg);
break;
case '--pingora-log-file':
result.pingoraLogFile = requireValue(argv, ++index, arg);
break;
case '--prefix':
result.prefix = normalizePrefix(requireValue(argv, ++index, arg), arg);
break;
case '--mode':
result.mode = normalizeMode(requireValue(argv, ++index, arg), arg);
break;
case '--realpath':
result.mode = 'realpath';
break;
case '--since-lines':
result.sinceLines = parseRequiredPositiveInt(
requireValue(argv, ++index, arg),
'--since-lines',
);
break;
case '--path':
result.requiredPaths.push(
normalizeRequestPath(requireValue(argv, ++index, arg), arg),
);
break;
case '--json':
result.json = true;
break;
default:
throw new Error(`未知参数: ${arg}`);
}
}
result.prefix = normalizePrefix(result.prefix, '--prefix');
for (const [label, file] of [
['--nginx-log-file', result.nginxLogFile],
['--pingora-log-file', result.pingoraLogFile],
]) {
validateSafeAbsoluteFilePath(file, label);
}
return result;
}
function usage() {
console.log(`Usage:
node scripts/check-pingora-canary-access-log-parity.mjs [options]
Options:
--nginx-log-file <path> Nginx access log,默认 /var/log/nginx/genarrative.access.log。
--pingora-log-file <path> Pingora access log,默认 /var/log/genarrative/pingora-gateway.access.log。
--prefix <path> canary 前缀,默认 /__genarrative_pingora_canary。
--mode <prefix|realpath> 对账模式,默认 prefixrealpath 要求 Nginx path 与 Pingora path 对齐。
--realpath Shortcut for --mode realpath。
--since-lines <count> 只读取日志尾部行数,默认 2000。
--path <path> 必须出现并完成对照的原始 canary 路径,可重复。
--json 输出 JSON。
该脚本只读比较 Nginx canary handoff access log 和 Pingora access log,不修改日志、不 reload Nginx 或 Pingora。
Nginx canary exact healthz 会转发到 Pingora shadow healthz,其余前缀路径按 rewrite 后路径对照。
真实路径 canary 使用独立 Nginx access log;除 realpath healthz 探针外,Nginx 与 Pingora path 必须一致。
`);
}
function requireValue(argv, index, flag) {
const value = argv[index];
if (!value || value.startsWith('--')) {
throw new Error(`${flag} 缺少参数值`);
}
return value;
}
function parseOptionalPositiveInt(raw, fallback, label) {
if (raw === undefined || raw === null || String(raw).trim() === '') {
return fallback;
}
return parseRequiredPositiveInt(raw, label);
}
function parseRequiredPositiveInt(raw, label) {
const rawText = String(raw ?? '');
validateNoControlCharacters(rawText, label);
const text = rawText.trim();
if (!/^[1-9]\d*$/.test(text)) {
throw new Error(`${label} 必须是正整数。`);
}
return Number.parseInt(text, 10);
}
function normalizeMode(raw, label) {
validateNoControlCharacters(raw, label);
const value = String(raw || '').trim();
if (!value || value === 'prefix') {
return 'prefix';
}
if (value === 'realpath') {
return 'realpath';
}
throw new Error(`${label} 必须是 prefix 或 realpath。`);
}
function validateSafeAbsoluteFilePath(value, flag) {
validateNoControlCharacters(value, flag);
if (!path.isAbsolute(value)) {
throw new Error(`${flag} 必须是绝对路径。`);
}
if (isFilesystemRootPath(value)) {
throw new Error(`${flag} 不能是文件系统根目录。`);
}
}
function isFilesystemRootPath(value) {
const resolved = path.resolve(String(value));
return resolved === path.parse(resolved).root;
}
function normalizePrefix(prefix, label) {
validateNoControlCharacters(prefix, label);
if (!prefix || prefix === '/') {
return DEFAULT_CANARY_PREFIX;
}
const withLeadingSlash = prefix.startsWith('/') ? prefix : `/${prefix}`;
return withLeadingSlash.endsWith('/')
? withLeadingSlash.slice(0, -1)
: withLeadingSlash;
}
function normalizeRequestPath(value, label = '--path') {
validateNoControlCharacters(value, label);
const raw = String(value || '').trim();
if (!raw) {
return '/';
}
const pathOnly = raw.includes('://') ? new URL(raw).pathname : raw.split('?')[0];
return pathOnly.startsWith('/') ? pathOnly : `/${pathOnly}`;
}
function validateNoControlCharacters(value, label) {
if (/[\0\r\n]/u.test(String(value ?? ''))) {
throw new Error(`${label} 不能包含换行或 NUL 字符。`);
}
}
function readLogRecords(filePath, parser) {
let text;
try {
text = readFileSync(filePath, 'utf8');
} catch (error) {
failures.push(`无法读取日志文件 ${filePath}: ${error.message}`);
return [];
}
return text
.split(/\r?\n/u)
.filter(Boolean)
.slice(-config.sinceLines)
.map((line, index) => parseLogRecord(parser, line, index + 1, filePath))
.filter(Boolean);
}
function parseLogRecord(parser, line, lineNumber, filePath) {
try {
return parser(line, lineNumber);
} catch (error) {
failures.push(
`日志文件 ${filePath}${lineNumber} 行解析失败: ${error.message}`,
);
return null;
}
}
function parseNginxAccessLine(line, lineNumber) {
const requestMatch = line.match(/"([A-Z]+)\s+([^"\s]+)\s+HTTP\/[^"]+"/u);
const statusMatch = line.match(/"\s+(\d{3})\s+/u);
const requestIdMatch = line.match(/(?:^|\s)request_id=([^\s]+)/u);
if (!requestMatch || !statusMatch || !requestIdMatch) {
return null;
}
const uri = requestMatch[2];
return {
source: 'nginx',
lineNumber,
requestId: requestIdMatch[1],
method: requestMatch[1],
uri,
path: normalizeRequestPath(uri, `Nginx access log 第 ${lineNumber} 行 URI`),
status: Number.parseInt(statusMatch[1], 10),
raw: line,
};
}
function parseKeyValueAccessLine(line, lineNumber) {
const fields = {};
for (const part of line.split(/\t/u)) {
const separator = part.indexOf('=');
if (separator <= 0) {
continue;
}
fields[part.slice(0, separator)] = unescapeAccessLogValue(
part.slice(separator + 1),
);
}
if (!fields.request_id || !fields.path || !fields.status) {
return null;
}
return {
source: 'pingora',
lineNumber,
requestId: fields.request_id,
method: fields.method || '',
uri: fields.uri || fields.path,
path: normalizeRequestPath(
fields.path,
`Pingora access log 第 ${lineNumber} 行 path`,
),
status: Number.parseInt(fields.status, 10),
route: fields.route || '',
proxyTarget: fields.proxy_target || '',
upstream: fields.upstream || '',
raw: line,
};
}
function unescapeAccessLogValue(value) {
return String(value)
.replaceAll('%09', '\t')
.replaceAll('%0A', '\n')
.replaceAll('%0D', '\r');
}
function buildParity(nginxRecords, pingoraRecords) {
const pingoraByRequestId = new Map();
for (const record of pingoraRecords) {
if (!pingoraByRequestId.has(record.requestId)) {
pingoraByRequestId.set(record.requestId, []);
}
pingoraByRequestId.get(record.requestId).push(record);
}
const nginxCanary = nginxRecords.filter((record) => isCanaryRecord(record));
if (nginxCanary.length === 0) {
failures.push(noCanaryRecordsMessage());
}
const matches = [];
const missing = [];
const mismatches = [];
const seenCanaryPaths = new Set();
for (const nginxRecord of nginxCanary) {
seenCanaryPaths.add(nginxRecord.path);
const expectedPath = stripCanaryPrefix(nginxRecord.path);
const candidates = pingoraByRequestId.get(nginxRecord.requestId) || [];
const match = candidates.find((record) => record.path === expectedPath);
if (!match) {
missing.push({
requestId: nginxRecord.requestId,
nginxPath: nginxRecord.path,
expectedPingoraPath: expectedPath,
});
continue;
}
const mismatch = {
requestId: nginxRecord.requestId,
nginxPath: nginxRecord.path,
pingoraPath: match.path,
problems: [],
};
if (nginxRecord.method && match.method && nginxRecord.method !== match.method) {
mismatch.problems.push(`method ${nginxRecord.method} != ${match.method}`);
}
if (nginxRecord.status !== match.status) {
mismatch.problems.push(`status ${nginxRecord.status} != ${match.status}`);
}
if (mismatch.problems.length > 0) {
mismatches.push(mismatch);
continue;
}
matches.push({
requestId: nginxRecord.requestId,
nginxPath: nginxRecord.path,
pingoraPath: match.path,
status: nginxRecord.status,
route: match.route,
proxyTarget: match.proxyTarget,
});
}
for (const requiredPath of config.requiredPaths) {
if (!seenCanaryPaths.has(requiredPath)) {
failures.push(`Nginx canary 日志缺少必需路径: ${requiredPath}`);
}
}
for (const item of missing) {
failures.push(
`缺少对应 Pingora access log: request_id=${item.requestId} nginx_path=${item.nginxPath} expected_pingora_path=${item.expectedPingoraPath}`,
);
}
for (const item of mismatches) {
failures.push(
`Nginx/Pingora access log 不一致: request_id=${item.requestId} nginx_path=${item.nginxPath} pingora_path=${item.pingoraPath} ${item.problems.join(', ')}`,
);
}
return {
ok: failures.length === 0,
mode: config.mode,
prefix: config.prefix,
summary: {
nginxCanaryCount: nginxCanary.length,
pingoraCount: pingoraRecords.length,
matchedCount: matches.length,
missingCount: missing.length,
mismatchCount: mismatches.length,
},
matches,
missing,
mismatches,
};
}
function isCanaryRecord(record) {
if (config.mode === 'realpath') {
return (
record.path === REALPATH_HEALTHZ_PATH ||
record.path === '/api/creation-entry/config' ||
record.path.startsWith('/v1/database/') ||
record.path.startsWith('/v1/identity') ||
record.path === '/assets/app.js' ||
record.path === '/generated-pingora-canary-smoke'
);
}
return (
record.path.startsWith(`${config.prefix}/`) || record.path === config.prefix
);
}
function noCanaryRecordsMessage() {
if (config.mode === 'realpath') {
return `Nginx 日志尾部 ${config.sinceLines} 行中没有真实路径 canary 请求`;
}
return `Nginx 日志尾部 ${config.sinceLines} 行中没有 canary 前缀请求: ${config.prefix}`;
}
function stripCanaryPrefix(canaryPath) {
if (config.mode === 'realpath') {
if (canaryPath === REALPATH_HEALTHZ_PATH) {
return SHADOW_HEALTHZ_PATH;
}
return canaryPath;
}
if (canaryPath === `${config.prefix}/healthz`) {
return SHADOW_HEALTHZ_PATH;
}
if (canaryPath === config.prefix) {
return '/';
}
const stripped = canaryPath.slice(config.prefix.length);
return stripped || '/';
}
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More