#!/usr/bin/env node import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; const failures = []; const tmpRoot = mkdtempSync( path.join(tmpdir(), 'genarrative-production-api-deploy-'), ); try { main(); } finally { rmSync(tmpRoot, { recursive: true, force: true }); } if (failures.length > 0) { console.error('[check:production-api-deploy] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } console.log('[check:production-api-deploy] OK'); function main() { assertDeployCopiesPingoraDirectReleaseDependencies(); assertDeployRestartsActivePingoraWhenArtifactIncluded(); assertDeploySkipsInactivePingoraWhenArtifactIncluded(); assertDeployRejectsPingoraArtifactMissingManifestEntry(); assertMissingReleaseManifestFails(); assertReleaseManifestMissingApiArtifactFails(); assertDeployRejectsDotVersion(); assertDeployRejectsDotDotVersion(); assertDeployRejectsVersionStartingWithDot(); assertDeployRejectsExistingReleaseDirectory(); assertDeployRejectsDirectoryCurrentLink(); assertDeployRejectsRelativeReleaseRoot(); assertDeployRejectsRelativeCurrentLink(); assertDeployRejectsRelativeApiEnvFile(); assertDeployCleansStagingReleaseOnFailure(); assertDeployRejectsFinalReleaseRaceAndCleansStaging(); assertMissingBackupScriptFails(); assertMissingHealthPatrolScriptFails(); assertMissingPingoraCurrentReleaseAuditFails(); assertMissingPingoraCutoverStatusSnapshotFails(); assertMissingPingoraCutoverEvidenceBundleFails(); assertMissingPingoraCutoverCommandEvidenceFails(); assertMissingPingoraCutoverEvidenceVerifyFails(); assertMissingPingoraCutoverEvidenceAuditFails(); assertMissingHealthPatrolEnvCheckFails(); assertMissingPingoraReleaseReadinessFails(); assertMissingPingoraHealthPatrolEnvSwitchFails(); assertMissingEnvExamplesFails(); assertMissingPingoraDirectCheckFails(); assertMissingPingoraCanaryLiveFails(); assertMissingPingoraCanaryAccessLogParityFails(); } function assertDeployCopiesPingoraDirectReleaseDependencies() { const fixture = prepareFixture('with-direct-checks'); const result = runDeploy(fixture); assertStatus(result, 0, '完整 fixture 应部署成功。'); if (result.status !== 0) { return; } assertIncludes( result.stdout, '[production-api-deploy] 完成:', '部署成功时必须输出完成信息。', ); const releaseDir = path.join(fixture.releaseRoot, fixture.version); const currentTarget = readlinkSync(fixture.currentLink); if (currentTarget !== releaseDir) { failures.push( `current link 应指向新 release。实际 ${currentTarget},预期 ${releaseDir}`, ); } assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), 'current release 必须包含 Pingora 直连启用脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), 'current release 必须包含 Pingora 直连回退脚本。', ); assertFileExists( path.join( releaseDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs', ), 'current release 必须包含 Pingora health patrol env 切换脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), 'current release 必须包含 Pingora 直连预检脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), 'current release 必须包含健康巡检 env 复核脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), 'current release 必须包含 Pingora release readiness 聚合门禁脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), 'current release 必须包含 Pingora current release 自审脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), 'current release 必须包含 Pingora 直连切换状态快照脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), 'current release 必须包含 Pingora 直连切换证据包脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), 'current release 必须包含 Pingora 直连切换命令证据脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), 'current release 必须包含 Pingora 直连切换证据验真脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), 'current release 必须包含 Pingora 直连切换证据根目录审计脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), 'current release 必须包含 Pingora 直连 live smoke 脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), 'current release 必须包含 Pingora canary live smoke 脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), 'current release 必须包含 Pingora canary access log 对账脚本。', ); assertFileExists( path.join(releaseDir, 'release-manifest.api-server.json'), 'current release 必须包含 API release manifest 副本。', ); assertFileExists( path.join(releaseDir, 'deploy/systemd/genarrative-pingora-gateway.service'), 'current release 必须包含 systemd 主 service 模板。', ); assertFileExists( path.join( releaseDir, 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', ), 'current release 必须包含 Pingora 直连 drop-in 模板。', ); assertFileExists( path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), 'current release 必须包含 Pingora env 示例。', ); assertFileExists( path.join( releaseDir, 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', ), 'current release 必须包含 Pingora 真实路径 canary Nginx snippet。', ); assertFileExists( path.join(releaseDir, 'deploy/env/health-patrol.env.example'), 'current release 必须包含健康巡检 env 示例。', ); assertFileExists( path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), 'current release 必须包含 Pingora direct live env 示例。', ); assertFileExists( path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), 'current release 必须包含 Pingora canary live env 示例。', ); assertPingoraEnvProductionDefaults( path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), 'current release 内 Pingora env 示例', ); const copiedPreflight = readFileSync( path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), 'utf8', ); assertIncludes( copiedPreflight, 'deploy/pingora/pingora-gateway.env.example', '复制后的 preflight 脚本必须仍能从 release root 推导 Pingora 配置。', ); const apiEnv = readFileSync(fixture.apiEnvFile, 'utf8'); assertIncludes( apiEnv, 'GENARRATIVE_SPACETIME_DATABASE=genarrative-prod', '部署脚本必须写入 SpacetimeDB database。', ); assertIncludes( apiEnv, 'GENARRATIVE_SPACETIME_SERVER_URL=http://127.0.0.1:3101', '部署脚本必须写入 SpacetimeDB server URL。', ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl restart genarrative-api.service', '部署脚本必须重启 API service。', ); assertIncludes( commandsLog, 'curl -fsS http://127.0.0.1:18082/readyz', '部署脚本必须执行 readiness curl。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('部署成功后应退出维护模式。'); } } function assertDeployRestartsActivePingoraWhenArtifactIncluded() { const fixture = prepareFixture('with-active-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture); assertStatus( result, 0, '包含 Pingora 且 shadow service active 时应部署成功。', ); if (result.status !== 0) { return; } const releaseDir = path.join(fixture.releaseRoot, fixture.version); assertFileExists( path.join(releaseDir, 'pingora-gateway'), '包含 Pingora 的发布包部署后 current release 必须包含网关二进制。', ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl is-active --quiet genarrative-pingora-gateway.service', '部署脚本看到 Pingora 产物后必须先检查 shadow service 是否 active。', ); assertIncludes( commandsLog, 'systemctl try-restart genarrative-pingora-gateway.service', 'Pingora shadow service 已 active 时必须随 current release 切换 try-restart。', ); assertIncludes( result.stdout, '发布包包含 Pingora,重启已运行的影子服务', 'Pingora shadow service 自动重启时必须输出明确提示。', ); } function assertDeploySkipsInactivePingoraWhenArtifactIncluded() { const fixture = prepareFixture('with-inactive-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture, { pingoraActive: false }); assertStatus( result, 0, '包含 Pingora 但 shadow service inactive 时应部署成功。', ); if (result.status !== 0) { return; } const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl is-active --quiet genarrative-pingora-gateway.service', '部署脚本看到 Pingora 产物后必须检查 shadow service 是否 active。', ); if ( commandsLog.includes( 'systemctl try-restart genarrative-pingora-gateway.service', ) ) { failures.push('Pingora shadow service inactive 时不能主动 try-restart。'); } assertIncludes( result.stdout, 'Pingora 影子服务未处于 active,跳过自动重启', 'Pingora shadow service inactive 时必须明确说明不会主动拉起。', ); } function assertDeployRejectsPingoraArtifactMissingManifestEntry() { const fixture = prepareFixture('pingora-artifact-missing-manifest-entry'); addPingoraGatewayArtifact(fixture, { registerInManifest: false }); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布包包含 Pingora 但 manifest 未登记时部署必须失败。'); } assertIncludes( result.stderr, 'release-manifest.json 缺少 pingora-gateway artifact', 'manifest 未登记 Pingora 时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('manifest 未登记 Pingora 导致部署失败时必须保持维护模式。'); } } function assertMissingReleaseManifestFails() { const fixture = prepareFixture('missing-release-manifest'); rmSync(path.join(fixture.sourceDir, 'release-manifest.json')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 release-manifest.json 时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 release-manifest.json', '缺少 release-manifest.json 时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('缺少 release-manifest.json 导致部署失败时必须保持维护模式。'); } } function assertReleaseManifestMissingApiArtifactFails() { const fixture = prepareFixture('release-manifest-missing-api-artifact'); writeFileSync( path.join(fixture.sourceDir, 'release-manifest.json'), `${JSON.stringify({ version: fixture.version, artifacts: [] }, null, 2)}\n`, 'utf8', ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('release manifest 缺少 api-server artifact 时部署必须失败。'); } assertIncludes( result.stderr, 'release-manifest.json 缺少 api-server artifact', 'manifest 缺少 api-server artifact 时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('manifest 缺少 api-server artifact 导致部署失败时必须保持维护模式。'); } } function assertDeployRejectsDotVersion() { const fixture = prepareFixture('dot-version'); const result = runDeploy(fixture, { version: '.' }); if (result.status === 0) { failures.push('--version=. 时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version=. 时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version=. 且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsDotDotVersion() { const fixture = prepareFixture('dot-dot-version'); const result = runDeploy(fixture, { version: '..' }); if (result.status === 0) { failures.push('--version=.. 时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version=.. 时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version=.. 且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsVersionStartingWithDot() { const fixture = prepareFixture('dot-prefix-version'); const result = runDeploy(fixture, { version: '.hidden-release' }); if (result.status === 0) { failures.push('--version 以点开头时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version 以点开头时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version 以点开头且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsExistingReleaseDirectory() { const fixture = prepareFixture('existing-release-directory'); const releaseDir = path.join(fixture.releaseRoot, fixture.version); mkdirSync(releaseDir, { recursive: true }); writeFileSync(path.join(releaseDir, 'old-file'), 'old\n', 'utf8'); const result = runDeploy(fixture); if (result.status === 0) { failures.push('目标 release 目录已存在时部署必须失败。'); } assertIncludes( result.stderr, '目标 release 已存在,拒绝覆盖或合并旧文件', '目标 release 目录已存在时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('目标 release 目录已存在且未进入部署阶段时不应开启维护模式。'); } assertFileExists( path.join(releaseDir, 'old-file'), '拒绝覆盖既有 release 时必须保留原目录。', ); } function assertDeployRejectsDirectoryCurrentLink() { const fixture = prepareFixture('directory-current-link'); mkdirSync(fixture.currentLink, { recursive: true }); writeFileSync(path.join(fixture.currentLink, 'old-file'), 'old\n', 'utf8'); const result = runDeploy(fixture); if (result.status === 0) { failures.push('current 路径已存在但不是符号链接时部署必须失败。'); } assertIncludes( result.stderr, 'current 链接路径已存在但不是符号链接,拒绝覆盖', 'current 路径不是符号链接时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('current 路径不是符号链接且未进入部署阶段时不应开启维护模式。'); } assertFileExists( path.join(fixture.currentLink, 'old-file'), '拒绝覆盖目录型 current 时必须保留原目录内容。', ); } function assertDeployRejectsRelativeReleaseRoot() { const fixture = prepareFixture('relative-release-root'); const result = runDeploy(fixture, { releaseRoot: 'relative/releases' }); if (result.status === 0) { failures.push('--release-root 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--release-root 必须使用绝对路径', '--release-root 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--release-root 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsRelativeCurrentLink() { const fixture = prepareFixture('relative-current-link'); const result = runDeploy(fixture, { currentLink: 'relative/current' }); if (result.status === 0) { failures.push('--current-link 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--current-link 必须使用绝对路径', '--current-link 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--current-link 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsRelativeApiEnvFile() { const fixture = prepareFixture('relative-api-env-file'); const result = runDeploy(fixture, { apiEnvFile: 'relative/api-server.env' }); if (result.status === 0) { failures.push('--api-env-file 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--api-env-file 必须使用绝对路径', '--api-env-file 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--api-env-file 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployCleansStagingReleaseOnFailure() { const fixture = prepareFixture('cleans-staging-on-failure'); rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('缺少数据库备份脚本时部署必须失败。'); } const releaseDir = path.join(fixture.releaseRoot, fixture.version); if (existsSync(releaseDir)) { failures.push('部署失败时不应留下正式 release 目录。'); } const entries = existsSync(fixture.releaseRoot) ? readDirNames(fixture.releaseRoot) : []; const stagingEntries = entries.filter((entry) => entry.includes(`${fixture.version}.staging`), ); if (stagingEntries.length > 0) { failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`); } if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { const fixture = prepareFixture('final-release-race'); const result = runDeploy(fixture, { createReleaseDuringCopy: true }); if (result.status === 0) { failures.push('最终提升前目标 release 目录被外部创建时部署必须失败。'); } assertIncludes( result.stderr, '目标 release 在发布过程中出现,拒绝合并 staging', '最终提升前目标 release 目录被外部创建时必须给出明确错误。', ); const releaseDir = path.join(fixture.releaseRoot, fixture.version); assertFileExists( path.join(releaseDir, 'raced-file'), '部署脚本拒绝竞态目标 release 时必须保留外部创建的目录。', ); if (existsSync(path.join(releaseDir, 'api-server'))) { failures.push('目标 release 竞态出现后不应把 staging 内容合并进去。'); } const entries = existsSync(fixture.releaseRoot) ? readDirNames(fixture.releaseRoot) : []; const stagingEntries = entries.filter((entry) => entry.includes(`${fixture.version}.staging`), ); if (stagingEntries.length > 0) { failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`); } if (!existsSync(fixture.maintenanceFile)) { failures.push('目标 release 竞态失败时必须保持维护模式。'); } } function assertMissingPingoraDirectCheckFails() { const fixture = prepareFixture('missing-direct-live'); rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 direct live smoke 脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连 live smoke 脚本', '缺少 direct live smoke 脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCanaryLiveFails() { const fixture = prepareFixture('missing-canary-live'); rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-canary-live.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 canary live smoke 脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora canary live smoke 脚本', '缺少 canary live smoke 脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCanaryAccessLogParityFails() { const fixture = prepareFixture('missing-canary-log-parity'); rmSync( path.join( fixture.sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 canary access log 对账脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora canary access log 对账脚本', '缺少 canary access log 对账脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingBackupScriptFails() { const fixture = prepareFixture('missing-backup-script'); rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少数据库备份脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少数据库备份脚本', '缺少数据库备份脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingHealthPatrolScriptFails() { const fixture = prepareFixture('missing-health-patrol-script'); rmSync( path.join(fixture.sourceDir, 'scripts/ops/production-health-patrol.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少生产健康巡检脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少生产健康巡检脚本', '缺少生产健康巡检脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCurrentReleaseAuditFails() { const fixture = prepareFixture('missing-pingora-current-release-audit'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-current-release-audit.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora current release 自审脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora current release 自审脚本', '缺少 Pingora current release 自审脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCutoverStatusSnapshotFails() { const fixture = prepareFixture('missing-pingora-cutover-status-snapshot'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换状态快照脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换状态快照脚本', '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCutoverEvidenceBundleFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-bundle'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换证据包脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据包脚本', '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCutoverCommandEvidenceFails() { const fixture = prepareFixture('missing-pingora-cutover-command-evidence'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换命令证据脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换命令证据脚本', '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCutoverEvidenceVerifyFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-verify'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换证据验真脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据验真脚本', '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraCutoverEvidenceAuditFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-audit'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '发布产物缺少 Pingora 直连切换证据根目录审计脚本时部署必须失败。', ); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据根目录审计脚本', '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingHealthPatrolEnvCheckFails() { const fixture = prepareFixture('missing-health-patrol-env-check'); rmSync( path.join(fixture.sourceDir, 'scripts/check-production-health-patrol-env.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少生产健康巡检 env 复核脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少生产健康巡检 env 复核脚本', '缺少生产健康巡检 env 复核脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraReleaseReadinessFails() { const fixture = prepareFixture('missing-pingora-release-readiness'); rmSync( path.join(fixture.sourceDir, 'scripts/check-pingora-release-readiness.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '发布产物缺少 Pingora release readiness 聚合门禁脚本时部署必须失败。', ); } assertIncludes( result.stderr, '发布产物缺少 Pingora release readiness 聚合门禁脚本', '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingPingoraHealthPatrolEnvSwitchFails() { const fixture = prepareFixture('missing-pingora-health-patrol-env-switch'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora health patrol env 切换脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora health patrol env 切换脚本', '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function assertMissingEnvExamplesFails() { const fixture = prepareFixture('missing-env-examples'); rmSync(path.join(fixture.sourceDir, 'deploy/env'), { recursive: true, force: true, }); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少环境变量示例目录时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少环境变量示例目录', '缺少环境变量示例目录时必须给出明确错误。', ); if (!existsSync(fixture.maintenanceFile)) { failures.push('部署失败时必须保持维护模式。'); } } function prepareFixture(name) { const root = path.join(tmpRoot, name); const sourceDir = path.join(root, 'source'); const releaseRoot = path.join(root, 'releases'); const currentLink = path.join(root, 'current'); const apiEnvFile = path.join(root, 'etc', 'api-server.env'); const maintenanceFile = path.join(root, 'maintenance', 'enabled'); const fakeBin = path.join(root, 'bin'); const commandsLog = path.join(root, 'commands.log'); const version = `20260614-${name}`; mkdirSync(sourceDir, { recursive: true }); mkdirSync(fakeBin, { recursive: true }); mkdirSync(path.dirname(apiEnvFile), { recursive: true }); mkdirSync(path.join(sourceDir, 'scripts/deploy'), { recursive: true }); mkdirSync(path.join(sourceDir, 'scripts/ops'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/systemd'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/pingora'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/env'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/nginx/snippets'), { recursive: true }); writeFileSync( path.join(sourceDir, 'api-server'), '#!/usr/bin/env bash\n', 'utf8', ); writeFileSync( apiEnvFile, [ 'GENARRATIVE_TRACKING_OUTBOX_ENABLED=false', 'GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS=5000', '', ].join('\n'), 'utf8', ); chmodExecutable(path.join(sourceDir, 'api-server')); writeSha256(sourceDir, 'api-server'); writeFileSync( path.join(sourceDir, 'release-manifest.json'), `${JSON.stringify( { version, artifacts: [ { component: 'api-server', path: 'api-server', checksum_path: 'api-server.sha256', }, ], }, null, 2, )}\n`, 'utf8', ); writeFileSync( path.join(sourceDir, 'scripts/database-backup-to-oss.mjs'), 'console.log("backup");\n', 'utf8', ); writeFileSync( path.join(sourceDir, 'scripts/ops/production-health-patrol.mjs'), 'console.log("patrol");\n', 'utf8', ); copyFile( 'scripts/ops/pingora-current-release-audit.mjs', path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-status-snapshot.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-bundle.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-command-evidence.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-verify.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-audit.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), ); copyFile( 'scripts/deploy/production-api-deploy.sh', path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), ); copyFile( 'scripts/deploy/maintenance-on.sh', path.join(sourceDir, 'scripts/deploy/maintenance-on.sh'), ); copyFile( 'scripts/deploy/maintenance-off.sh', path.join(sourceDir, 'scripts/deploy/maintenance-off.sh'), ); copyFile( 'scripts/deploy/pingora-direct-enable.sh', path.join(sourceDir, 'scripts/deploy/pingora-direct-enable.sh'), ); copyFile( 'scripts/deploy/pingora-direct-rollback.sh', path.join(sourceDir, 'scripts/deploy/pingora-direct-rollback.sh'), ); copyFile( 'scripts/deploy/pingora-health-patrol-env-switch.mjs', path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), ); copyFile( 'scripts/check-production-health-patrol-env.mjs', path.join(sourceDir, 'scripts/check-production-health-patrol-env.mjs'), ); copyFile( 'scripts/check-pingora-release-readiness.mjs', path.join(sourceDir, 'scripts/check-pingora-release-readiness.mjs'), ); copyFile( 'scripts/check-pingora-direct-preflight.mjs', path.join(sourceDir, 'scripts/check-pingora-direct-preflight.mjs'), ); copyFile( 'scripts/check-pingora-direct-live.mjs', path.join(sourceDir, 'scripts/check-pingora-direct-live.mjs'), ); copyFile( 'scripts/check-pingora-canary-live.mjs', path.join(sourceDir, 'scripts/check-pingora-canary-live.mjs'), ); copyFile( 'scripts/check-pingora-canary-access-log-parity.mjs', path.join(sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), ); copyFile( 'deploy/systemd/genarrative-pingora-gateway.service', path.join(sourceDir, 'deploy/systemd/genarrative-pingora-gateway.service'), ); copyFile( 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', path.join( sourceDir, 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', ), ); copyFile( 'deploy/pingora/pingora-gateway.env.example', path.join(sourceDir, 'deploy/pingora/pingora-gateway.env.example'), ); copyFile( 'deploy/env/health-patrol.env.example', path.join(sourceDir, 'deploy/env/health-patrol.env.example'), ); copyFile( 'deploy/env/pingora-direct-live.env.example', path.join(sourceDir, 'deploy/env/pingora-direct-live.env.example'), ); copyFile( 'deploy/env/pingora-canary-live.env.example', path.join(sourceDir, 'deploy/env/pingora-canary-live.env.example'), ); copyFile( 'deploy/nginx/snippets/genarrative-pingora-canary.conf', path.join( sourceDir, 'deploy/nginx/snippets/genarrative-pingora-canary.conf', ), ); copyFile( 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', path.join( sourceDir, 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', ), ); chmodExecutable( path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), ); chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-on.sh')); chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-off.sh')); writeFileSync( path.join(fakeBin, 'systemctl'), [ '#!/usr/bin/env bash', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'if [[ "$1 $2 $3" == "is-active --quiet genarrative-pingora-gateway.service" && "${FAKE_PINGORA_ACTIVE:-true}" == "false" ]]; then', ' exit 3', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'curl'), [ '#!/usr/bin/env bash', `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'cp'), [ '#!/usr/bin/env bash', 'set -euo pipefail', '/usr/bin/cp "$@"', 'if [[ "${FAKE_CREATE_RELEASE_DURING_COPY:-false}" == "true" ]]; then', ' marker="${FAKE_RELEASE_ROOT}/.${FAKE_RELEASE_VERSION}.race-created"', ' if [[ ! -e "${marker}" ]]; then', ' mkdir -p "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}"', ' printf "race\\n" > "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}/raced-file"', ' printf "created\\n" > "${marker}"', ' fi', 'fi', '', ].join('\n'), 'utf8', ); chmodExecutable(path.join(fakeBin, 'systemctl')); chmodExecutable(path.join(fakeBin, 'curl')); chmodExecutable(path.join(fakeBin, 'cp')); return { root, sourceDir, releaseRoot, currentLink, apiEnvFile, maintenanceFile, fakeBin, commandsLog, version, }; } function addPingoraGatewayArtifact(fixture, options = {}) { writeFileSync( path.join(fixture.sourceDir, 'pingora-gateway'), '#!/usr/bin/env bash\n', 'utf8', ); chmodExecutable(path.join(fixture.sourceDir, 'pingora-gateway')); writeSha256(fixture.sourceDir, 'pingora-gateway'); if (options.registerInManifest === false) { return; } const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); manifest.artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; manifest.artifacts.push({ component: 'pingora-gateway', path: 'pingora-gateway', checksum_path: 'pingora-gateway.sha256', }); writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); } function runDeploy(fixture, options = {}) { const deployScript = path.join( fixture.sourceDir, 'scripts/deploy/production-api-deploy.sh', ); return spawnSync( 'bash', [ deployScript, '--source-dir', fixture.sourceDir, '--version', options.version ?? fixture.version, '--release-root', options.releaseRoot ?? fixture.releaseRoot, '--current-link', options.currentLink ?? fixture.currentLink, '--service', 'genarrative-api.service', '--health-url', 'http://127.0.0.1:18082/readyz', '--api-env-file', options.apiEnvFile ?? fixture.apiEnvFile, '--database', 'genarrative-prod', '--spacetime-server-url', 'http://127.0.0.1:3101', ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile, FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true', FAKE_CREATE_RELEASE_DURING_COPY: options.createReleaseDuringCopy === true ? 'true' : 'false', FAKE_RELEASE_ROOT: fixture.releaseRoot, FAKE_RELEASE_VERSION: fixture.version, }, }, ); } function copyFile(source, target) { writeFileSync(target, readFileSync(source, 'utf8'), 'utf8'); } function writeSha256(directory, fileName) { const result = spawnSync('sha256sum', [fileName], { cwd: directory, encoding: 'utf8', }); if (result.status !== 0) { throw new Error(result.stderr || `sha256sum failed for ${fileName}`); } writeFileSync( path.join(directory, `${fileName}.sha256`), result.stdout, 'utf8', ); } function chmodExecutable(filePath) { spawnSync('chmod', ['0755', filePath], { cwd: process.cwd(), encoding: 'utf8', }); } function shellQuote(value) { return `'${String(value).replace(/'/g, "'\\''")}'`; } function assertFileExists(filePath, reason) { if (!existsSync(filePath)) { failures.push(`${reason} 缺少: ${filePath}`); } } function readDirNames(directory) { return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], { cwd: process.cwd(), encoding: 'utf8', }) .stdout.split('\n') .filter(Boolean); } function assertStatus(result, expected, reason) { const actual = result.status ?? 0; if (actual !== expected) { failures.push( `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } } function assertIncludes(content, needle, reason) { if (!content.includes(needle)) { failures.push(`${reason} 缺少: ${needle}`); } } function assertPingoraEnvProductionDefaults(filePath, label) { if (!existsSync(filePath)) { return; } const content = readFileSync(filePath, 'utf8'); const requiredLines = [ [ 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', '压缩算法必须保持 gzip-only,避免 Brotli 在未验证前进入 current release。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', '前置代理信任确认开关必须默认关闭。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', '接流保护必须默认开启。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', '内部探针 token 示例必须保持空值,避免 current release 夹带真实 token。', ], ]; for (const [line, reason] of requiredLines) { assertIncludes(content, line, `${label} ${reason}`); } }