diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 2da702408..b38971d58 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -15,6 +15,14 @@ - 关联:相关文件、文档、提交或 Issue ``` +## API Build / Deploy 归档清单不能漏掉随包 Pingora 脚本 + +- 现象:`Genarrative-Api-Deploy` 在发布阶段报 `发布产物缺少 Pingora TLS 证书同步脚本: build//scripts/deploy/pingora-tls-cert-sync.mjs`。 +- 原因:`scripts/build-production-release.sh` 已经把脚本复制进 `build//scripts/deploy/`,但 Jenkins API Build 的 `archiveArtifacts` 和 API Deploy 的 `copyArtifacts` 过滤清单仍可能漏掉新增随包脚本,导致 Deploy 工作区拿到的是残缺发布包。 +- 处理:新增随包部署脚本时,必须同时更新 `jenkins/Jenkinsfile.production-api-build` 的归档清单、`jenkins/Jenkinsfile.production-api-deploy` 的复制清单和 `scripts/check-production-ops-guardrails.mjs` 的字符串门禁;不要在 Deploy Job 里从工作区根目录或源码 checkout 兜底补脚本。 +- 验证:运行 `npm run check:production-ops`、`npm run check:production-api-release` 和 `npm run check:production-api-deploy`,确认构建包、Jenkins 归档链路和 deploy fail-fast 检查口径一致。 +- 关联:`jenkins/Jenkinsfile.production-api-build`、`jenkins/Jenkinsfile.production-api-deploy`、`scripts/deploy/production-api-deploy.sh`、`scripts/check-production-ops-guardrails.mjs`。 + ## 图片画布角色动作结果不能把首帧当主媒体 - 现象:画板里 `生成角色动作` 返回后显示成一张不可播放图片,点击下载或素材 ZIP 导出时拿到的也是 PNG,而不是动作视频。 @@ -158,7 +166,6 @@ - 处理:Windows 本地 dev 脚本应把 `RUSTC_WRAPPER` 和 `CARGO_BUILD_RUSTC_WRAPPER` 显式设为空字符串,让 Cargo 覆盖项目配置并直连真实 rustc;Linux 保持 `/usr/bin/env` 绕过 sccache。 - 验证:`npm run test -- scripts/dev.test.ts -t "Windows 下本地 dev Rust env 用空 wrapper 覆盖项目 sccache"`,并用 `npm run dev:api-server` 拉起后访问实际 api 端口的 `/healthz` 返回 200。 - 关联:`scripts/dev.mjs`、`scripts/dev.test.ts`、`docs/【开发运维】本地开发验证与生产运维-2026-05-15.md`。 - ## Pingora 直连 80/443 不能只改 env - 现象:`/etc/genarrative/pingora-gateway.env` 已把 `GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN` / `HTTP_REDIRECT_LISTEN` 改到 `0.0.0.0:443` / `0.0.0.0:80`,但 `genarrative-pingora-gateway.service` 启动失败,日志出现低端口绑定权限错误。 diff --git a/jenkins/Jenkinsfile.production-api-build b/jenkins/Jenkinsfile.production-api-build index 92591ec19..0a1b0c84a 100644 --- a/jenkins/Jenkinsfile.production-api-build +++ b/jenkins/Jenkinsfile.production-api-build @@ -138,7 +138,7 @@ pipeline { stage('Archive') { steps { - archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true + archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/api-server,build/${env.EFFECTIVE_BUILD_VERSION}/api-server.sha256,build/${env.EFFECTIVE_BUILD_VERSION}/release-manifest.json,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/systemd/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/nginx/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/env/**,build/${env.EFFECTIVE_BUILD_VERSION}/deploy/pingora/**", fingerprint: true script { if (params.INCLUDE_PINGORA_GATEWAY) { archiveArtifacts artifacts: "build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway,build/${env.EFFECTIVE_BUILD_VERSION}/pingora-gateway.sha256", fingerprint: true diff --git a/jenkins/Jenkinsfile.production-api-deploy b/jenkins/Jenkinsfile.production-api-deploy index 39fc7e70b..e116474b1 100644 --- a/jenkins/Jenkinsfile.production-api-deploy +++ b/jenkins/Jenkinsfile.production-api-deploy @@ -66,7 +66,7 @@ pipeline { copyArtifacts( projectName: params.BUILD_JOB_NAME, selector: specific(params.BUILD_NUMBER_TO_DEPLOY), - filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", + filter: "build/${params.BUILD_VERSION}/api-server,build/${params.BUILD_VERSION}/api-server.sha256,build/${params.BUILD_VERSION}/release-manifest.json,build/${params.BUILD_VERSION}/scripts/database-backup-to-oss.mjs,build/${params.BUILD_VERSION}/scripts/ops/production-health-patrol.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-current-release-audit.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-direct-rehearsal-status.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-status-snapshot.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-bundle.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-command-evidence.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-verify.mjs,build/${params.BUILD_VERSION}/scripts/ops/pingora-cutover-evidence-audit.mjs,build/${params.BUILD_VERSION}/scripts/check-production-health-patrol-env.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-release-readiness.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-preflight.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-direct-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-live.mjs,build/${params.BUILD_VERSION}/scripts/check-pingora-canary-access-log-parity.mjs,build/${params.BUILD_VERSION}/scripts/deploy/production-api-deploy.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-on.sh,build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh,build/${params.BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs,build/${params.BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs,build/${params.BUILD_VERSION}/deploy/systemd/**,build/${params.BUILD_VERSION}/deploy/nginx/**,build/${params.BUILD_VERSION}/deploy/env/**,build/${params.BUILD_VERSION}/deploy/pingora/**", target: '.', fingerprintArtifacts: true ) @@ -92,7 +92,7 @@ pipeline { sh ''' bash -lc ' set -euo pipefail - chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" + chmod +x "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-on.sh" "build/${BUILD_VERSION}/scripts/deploy/maintenance-off.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-direct-rollback.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-enable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-realpath-canary-disable.sh" "build/${BUILD_VERSION}/scripts/deploy/pingora-health-patrol-env-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs" "build/${BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs" "build/${BUILD_VERSION}/scripts/deploy/production-api-deploy.sh" \ --source-dir "build/${BUILD_VERSION}" \ --version "${BUILD_VERSION}" \ diff --git a/scripts/check-production-ops-guardrails.mjs b/scripts/check-production-ops-guardrails.mjs index 4ab1e3ff1..948de009a 100644 --- a/scripts/check-production-ops-guardrails.mjs +++ b/scripts/check-production-ops-guardrails.mjs @@ -6098,6 +6098,12 @@ const checks = [ 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', reason: 'API Build 归档必须包含 Pingora gateway env shadow 切换脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-build', + includes: + 'build/${env.EFFECTIVE_BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs', + reason: 'API Build 归档必须包含 Pingora TLS 证书同步脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-build', includes: @@ -6246,6 +6252,12 @@ const checks = [ 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-gateway-env-shadow-switch.mjs', reason: 'API Deploy 必须复制 build 产物内的 Pingora gateway env shadow 切换脚本。', }, + { + file: 'jenkins/Jenkinsfile.production-api-deploy', + includes: + 'build/${params.BUILD_VERSION}/scripts/deploy/pingora-tls-cert-sync.mjs', + reason: 'API Deploy 必须复制 build 产物内的 Pingora TLS 证书同步脚本。', + }, { file: 'jenkins/Jenkinsfile.production-api-deploy', includes: 'build/${params.BUILD_VERSION}/scripts/deploy/maintenance-off.sh',