diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 71f280e06..23854b85c 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -15,6 +15,14 @@ - 关联:相关文件、文档、提交或 Issue ``` +## API Build / Deploy 归档清单不能漏掉随包 Pingora 脚本 + +- 现象:`Genarrative-Api-Deploy` 在发布阶段报 `发布产物缺少 Pingora TLS 证书同步脚本: build//scripts/deploy/pingora-tls-cert-sync.mjs`。 +- 原因:`scripts/build-production-release.sh` 已经把脚本复制进 `build//scripts/deploy/`,但 Jenkins API Build 的 `archiveArtifacts` 和 API Deploy 的 `copyArtifacts` 过滤清单仍可能漏掉新增随包脚本,导致 Deploy 工作区拿到的是残缺发布包。 +- 处理:新增随包部署脚本时,必须同时更新 `jenkins/Jenkinsfile.production-api-build` 的归档清单、`jenkins/Jenkinsfile.production-api-deploy` 的复制清单和 `scripts/check-production-ops-guardrails.mjs` 的字符串门禁;不要在 Deploy Job 里从工作区根目录或源码 checkout 兜底补脚本。 +- 验证:运行 `npm run check:production-ops`、`npm run check:production-api-release` 和 `npm run check:production-api-deploy`,确认构建包、Jenkins 归档链路和 deploy fail-fast 检查口径一致。 +- 关联:`jenkins/Jenkinsfile.production-api-build`、`jenkins/Jenkinsfile.production-api-deploy`、`scripts/deploy/production-api-deploy.sh`、`scripts/check-production-ops-guardrails.mjs`。 + ## Pingora 直连 80/443 不能只改 env - 现象:`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`,但 `genarrative-pingora-gateway.service` 启动失败,日志出现低端口绑定权限错误。 diff --git a/jenkins/Jenkinsfile.production-api-build b/jenkins/Jenkinsfile.production-api-build index 92591ec19..0a1b0c84a 100644 --- a/jenkins/Jenkinsfile.production-api-build +++ b/jenkins/Jenkinsfile.production-api-build @@ -138,7 +138,7 @@ pipeline { stage('Archive') { steps { - archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true + archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true script { if (params.INCLUDE_PINGORA_GATEWAY) { archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway,build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway.sha256", fingerprint: true diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index 39fc7e70b..e116474b1 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -66,7 +66,7 @@ pipeline { copyArtifacts( projectName: params.BUILD_JOB_NAME, selector: specific(params.BUILD_NUMBER_TO_DEPLOY), - filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", + filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", target: '.', fingerprintArtifacts: true ) @@ -92,7 +92,7 @@ pipeline { sh ''' bash -lc ' set -euo pipefail - chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" + chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs" "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \ --source-dir "build/${BUILD_VERSION}" \ --version "${BUILD_VERSION}" \ diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 4ab1e3ff1..948de009a 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -6098,6 +6098,12 @@ const checks = [ 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', reason: 'API Build 归档必须包含 Pingora gateway env shadow 切换脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs', + reason: 'API Build 归档必须包含 Pingora TLS 证书同步脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-build', includes: @@ -6246,6 +6252,12 @@ const checks = [ 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', reason: 'API Deploy 必须复制 build 产物内的 Pingora gateway env shadow 切换脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs', + reason: 'API Deploy 必须复制 build 产物内的 Pingora TLS 证书同步脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-deploy', includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh',