#!/usr/bin/env node import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; const failures = []; const tmpRoot = mkdtempSync( path.join(tmpdir(), 'genarrative-production-api-deploy-'), ); try { main(); } finally { rmSync(tmpRoot, { recursive: true, force: true }); } if (failures.length > 0) { console.error('[check:production-api-deploy] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } console.log('[check:production-api-deploy] OK'); function main() { assertDeployCopiesPingoraDirectReleaseDependencies(); assertSuccessfulDeployCanKeepMaintenance(); assertDeployRestartsActivePingoraWhenArtifactIncluded(); assertDeployStartsInactivePingoraWhenArtifactIncluded(); assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded(); assertDeployStopsAfterPingoraSystemdReadFailure(); assertDeployRejectsPingoraPublicListenWhenArtifactIncluded(); assertDeployRejectsPingoraArtifactMissingManifestEntry(); assertDeployRejectsPingoraManifestEntryMissingArtifact(); assertDeployRequiresPingoraWhenRequested(); assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch(); assertMissingReleaseManifestFails(); assertReleaseManifestMissingApiArtifactFails(); assertDeployRejectsDotVersion(); assertDeployRejectsDotDotVersion(); assertDeployRejectsVersionStartingWithDot(); assertDeployRejectsExistingReleaseDirectory(); assertDeployRejectsDirectoryCurrentLink(); assertDeployRejectsRelativeReleaseRoot(); assertDeployRejectsRelativeCurrentLink(); assertDeployRejectsRelativeApiEnvFile(); assertRealWechatPayRequiresRefundReconciliation(); assertRealWechatPayUsesLastRefundReconciliationAssignment(); assertDeployCleansStagingReleaseOnFailure(); assertDeployRejectsFinalReleaseRaceAndCleansStaging(); assertMissingBackupScriptFails(); assertMissingHealthPatrolScriptFails(); assertMissingPingoraCurrentReleaseAuditFails(); assertMissingPingoraDirectRehearsalStatusFails(); assertMissingPingoraCutoverStatusSnapshotFails(); assertMissingPingoraCutoverEvidenceBundleFails(); assertMissingPingoraCutoverCommandEvidenceFails(); assertMissingPingoraCutoverEvidenceVerifyFails(); assertMissingPingoraCutoverEvidenceAuditFails(); assertMissingHealthPatrolEnvCheckFails(); assertMissingPingoraReleaseReadinessFails(); assertMissingPingoraRealpathCanaryEnableFails(); assertMissingPingoraRealpathCanaryDisableFails(); assertMissingPingoraHealthPatrolEnvSwitchFails(); assertMissingPingoraGatewayEnvShadowSwitchFails(); assertMissingPingoraTlsCertSyncFails(); assertMissingEnvExamplesFails(); assertMissingPingoraDirectCheckFails(); assertMissingPingoraCanaryLiveFails(); assertMissingPingoraCanaryAccessLogParityFails(); } function assertSuccessfulDeployCanKeepMaintenance() { const fixture = prepareFixture('keep-maintenance-after-success'); const result = runDeploy(fixture, { keepMaintenance: true }); assertStatus(result, 0, '显式保持维护时完整 fixture 应部署成功。'); if (result.status !== 0) { return; } assertMaintenanceKept(fixture, '显式要求成功部署后保持维护'); assertIncludes( result.stdout, '按参数保持维护模式', '成功部署并保持维护时必须输出明确状态。', ); } function readOptionalCommandsLog(fixture) { if (!existsSync(fixture.commandsLog)) { return ''; } return readFileSync(fixture.commandsLog, 'utf8'); } function assertMaintenanceCleared(fixture, reason) { if (existsSync(fixture.maintenanceFile)) { failures.push(`${reason} 时应退出本次打开的维护模式。`); } } function assertMaintenanceKept(fixture, reason) { if (!existsSync(fixture.maintenanceFile)) { failures.push(`${reason} 时必须保持维护模式。`); } } function assertDeployCopiesPingoraDirectReleaseDependencies() { const fixture = prepareFixture('with-direct-checks'); const result = runDeploy(fixture); assertStatus(result, 0, '完整 fixture 应部署成功。'); if (result.status !== 0) { return; } assertIncludes( result.stdout, '[production-api-deploy] 完成:', '部署成功时必须输出完成信息。', ); assertIncludes( readFileSync(fixture.apiEnvFile, 'utf8'), 'WECHAT_PAY_REFUND_RECONCILIATION_ENABLED=true', '生产部署必须为存量 API env 补齐退款 reconciliation 开关。', ); const releaseDir = path.join(fixture.releaseRoot, fixture.version); const currentTarget = readlinkSync(fixture.currentLink); if (currentTarget !== releaseDir) { failures.push( `current link 应指向新 release。实际 ${currentTarget},预期 ${releaseDir}`, ); } assertFileExists( path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'), 'current release 必须包含 API 部署脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'), 'current release 必须包含进入维护脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'), 'current release 必须包含退出维护脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'), 'current release 必须包含 Pingora 直连启用脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'), 'current release 必须包含 Pingora 直连回退脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), 'current release 必须包含 Pingora realpath canary 启用脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), 'current release 必须包含 Pingora realpath canary 关闭脚本。', ); assertFileExists( path.join( releaseDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs', ), 'current release 必须包含 Pingora health patrol env 切换脚本。', ); assertFileExists( path.join( releaseDir, 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', ), 'current release 必须包含 Pingora gateway env shadow 切换脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), 'current release 必须包含 Pingora TLS 证书同步脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), 'current release 必须包含 Pingora 直连预检脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'), 'current release 必须包含健康巡检 env 复核脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'), 'current release 必须包含 Pingora release readiness 聚合门禁脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-current-release-audit.mjs'), 'current release 必须包含 Pingora current release 自审脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'), 'current release 必须包含 Pingora 直连彩排状态脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), 'current release 必须包含 Pingora 直连切换状态快照脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), 'current release 必须包含 Pingora 直连切换证据包脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), 'current release 必须包含 Pingora 直连切换命令证据脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), 'current release 必须包含 Pingora 直连切换证据验真脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), 'current release 必须包含 Pingora 直连切换证据根目录审计脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'), 'current release 必须包含 Pingora 直连 live smoke 脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'), 'current release 必须包含 Pingora canary live smoke 脚本。', ); assertFileExists( path.join(releaseDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), 'current release 必须包含 Pingora canary access log 对账脚本。', ); assertFileExists( path.join(releaseDir, 'release-manifest.api-server.json'), 'current release 必须包含 API release manifest 副本。', ); assertFileExists( path.join(releaseDir, 'api-server.sha256'), 'current release 必须包含 api-server checksum,供随包 current release 自审校验。', ); assertFileExists( path.join(releaseDir, 'deploy/systemd/genarrative-pingora-gateway.service'), 'current release 必须包含 systemd 主 service 模板。', ); assertFileExists( path.join( releaseDir, 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', ), 'current release 必须包含 Pingora 直连 drop-in 模板。', ); assertFileExists( path.join( releaseDir, 'deploy/systemd/genarrative-external-generation-worker@.service', ), 'current release 必须包含外部生成 worker systemd 模板。', ); assertFileExists( path.join( releaseDir, 'deploy/systemd/genarrative-external-generation-controller.service', ), 'current release 必须包含外部生成 worker controller systemd 单元。', ); assertFileExists( path.join( fixture.systemdUnitDir, 'genarrative-external-generation-worker@.service', ), 'API deploy 必须把随包外部生成 worker 模板安装到 systemd unit 目录。', ); assertFileExists( path.join( fixture.systemdUnitDir, 'genarrative-external-generation-controller.service', ), 'API deploy 必须把随包外部生成 worker controller 单元安装到 systemd unit 目录。', ); assertFileExists( path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), 'current release 必须包含 Pingora env 示例。', ); assertFileExists( path.join( releaseDir, 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', ), 'current release 必须包含 Pingora 真实路径 canary Nginx snippet。', ); assertFileExists( path.join(releaseDir, 'deploy/env/health-patrol.env.example'), 'current release 必须包含健康巡检 env 示例。', ); assertFileExists( path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'), 'current release 必须包含 Pingora direct live env 示例。', ); assertFileExists( path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'), 'current release 必须包含 Pingora canary live env 示例。', ); assertPingoraEnvProductionDefaults( path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'), 'current release 内 Pingora env 示例', ); const copiedPreflight = readFileSync( path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'), 'utf8', ); assertIncludes( copiedPreflight, 'deploy/pingora/pingora-gateway.env.example', '复制后的 preflight 脚本必须仍能从 release root 推导 Pingora 配置。', ); const apiEnv = readFileSync(fixture.apiEnvFile, 'utf8'); assertIncludes( apiEnv, 'GENARRATIVE_SPACETIME_DATABASE=genarrative-prod', '部署脚本必须写入 SpacetimeDB database。', ); assertIncludes( apiEnv, 'GENARRATIVE_SPACETIME_SERVER_URL=http://127.0.0.1:3101', '部署脚本必须写入 SpacetimeDB server URL。', ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl daemon-reload', '安装 worker systemd 单元后必须 daemon-reload。', ); assertIncludes( commandsLog, 'systemctl restart genarrative-api.service', '部署脚本必须重启 API service。', ); assertIncludes( commandsLog, 'curl -fsS --max-time 2 http://127.0.0.1:18082/readyz', '部署脚本必须为 readiness curl 设置单次超时,避免端口已建立但服务未响应时无限等待。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('部署成功后应退出维护模式。'); } } function assertRealWechatPayRequiresRefundReconciliation() { const fixture = prepareFixture('wechat-refund-reconciliation-disabled'); writeFileSync( fixture.apiEnvFile, `${readFileSync(fixture.apiEnvFile, 'utf8')}WECHAT_PAY_ENABLED=true\nWECHAT_PAY_PROVIDER=real\nWECHAT_PAY_REFUND_RECONCILIATION_ENABLED=false\n`, 'utf8', ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('真实微信支付关闭退款 reconciliation 时部署必须失败。'); } assertIncludes( result.stderr, '真实微信支付必须设置 WECHAT_PAY_REFUND_RECONCILIATION_ENABLED=true', '退款 reconciliation 门禁失败时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '退款 reconciliation 门禁失败'); } function assertRealWechatPayUsesLastRefundReconciliationAssignment() { const fixture = prepareFixture('wechat-refund-reconciliation-duplicate-disabled-last'); writeFileSync( fixture.apiEnvFile, `${readFileSync(fixture.apiEnvFile, 'utf8')}WECHAT_PAY_ENABLED=true\nWECHAT_PAY_PROVIDER=real\nWECHAT_PAY_REFUND_RECONCILIATION_ENABLED=true\nWECHAT_PAY_REFUND_RECONCILIATION_ENABLED=false\n`, 'utf8', ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '退款 reconciliation 重复配置时必须按 systemd EnvironmentFile 的最后一次赋值拦截部署。', ); } assertIncludes( result.stderr, '真实微信支付必须设置 WECHAT_PAY_REFUND_RECONCILIATION_ENABLED=true', '退款 reconciliation 重复配置的最后一项为 false 时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '退款 reconciliation 重复配置门禁失败'); } function assertDeployRestartsActivePingoraWhenArtifactIncluded() { const fixture = prepareFixture('with-active-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture); assertStatus( result, 0, '包含 Pingora 且 shadow service active 时应部署成功。', ); if (result.status !== 0) { return; } const releaseDir = path.join(fixture.releaseRoot, fixture.version); assertFileExists( path.join(releaseDir, 'pingora-gateway'), '包含 Pingora 的发布包部署后 current release 必须包含网关二进制。', ); assertFileExists( path.join(releaseDir, 'pingora-gateway.sha256'), '包含 Pingora 的发布包部署后 current release 必须包含网关 checksum。', ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl cat genarrative-pingora-gateway.service', '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', ); assertIncludes( commandsLog, 'systemctl restart genarrative-pingora-gateway.service', 'Pingora shadow service 已 active 时必须随 current release 切换 restart。', ); assertIncludes( result.stdout, '发布包包含 Pingora,启动或重启 shadow 影子服务', 'Pingora shadow service 自动重启时必须输出明确提示。', ); } function assertDeployStartsInactivePingoraWhenArtifactIncluded() { const fixture = prepareFixture('with-inactive-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture, { pingoraActive: false }); assertStatus( result, 0, '包含 Pingora 且 shadow service inactive 但仍是安全 shadow 配置时应部署成功。', ); if (result.status !== 0) { return; } const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl cat genarrative-pingora-gateway.service', '部署脚本看到 Pingora 产物后必须先读取 systemd 最终配置。', ); assertIncludes( commandsLog, 'systemctl restart genarrative-pingora-gateway.service', 'Pingora shadow service inactive 且 shadow 配置安全时必须随 current release 启动。', ); assertIncludes( commandsLog, 'systemctl is-active --quiet genarrative-pingora-gateway.service', '启动或重启 Pingora shadow 后必须复核 active。', ); assertIncludes( result.stdout, '发布包包含 Pingora,启动或重启 shadow 影子服务', 'Pingora shadow service 自动启动时必须输出明确提示。', ); } function assertDeployRejectsPingoraDirectEntryWhenArtifactIncluded() { const fixture = prepareFixture('with-direct-entry-pingora-artifact'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture, { pingoraDirectEntry: true }); if (result.status === 0) { failures.push('包含 Pingora 但 systemd 已启用 direct-entry capability 时部署必须失败。'); } assertIncludes( result.stderr, 'Pingora systemd 已包含 CAP_NET_BIND_SERVICE', 'direct-entry capability 存在时必须给出明确错误。', ); if ( result.stderr.includes( 'Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile', ) ) { failures.push( 'direct-entry capability 已被识别后不应继续误报 Pingora EnvironmentFile 缺失。', ); } if ( result.stderr.includes( 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', ) ) { failures.push( 'direct-entry capability 已被识别后不应继续用空 env 误报 shadow listen。', ); } const commandsLog = readOptionalCommandsLog(fixture); if ( commandsLog.includes( 'systemctl restart genarrative-pingora-gateway.service', ) ) { failures.push('direct-entry capability 存在时不能自动 restart Pingora。'); } assertMaintenanceCleared(fixture, 'direct-entry capability 导致部署失败'); assertNoReleasePromoted( fixture, 'direct-entry capability 导致部署失败时不能提升正式 release。', ); } function assertDeployStopsAfterPingoraSystemdReadFailure() { const fixture = prepareFixture('with-unreadable-pingora-systemd'); addPingoraGatewayArtifact(fixture); const result = runDeploy(fixture, { pingoraSystemctlCatFails: true }); if (result.status === 0) { failures.push('包含 Pingora 但无法读取 systemd 最终配置时部署必须失败。'); } assertIncludes( result.stderr, '无法读取 Pingora systemd 最终配置', 'systemctl cat 失败时必须保留权威错误。', ); if ( result.stderr.includes( 'Pingora systemd 配置缺少包含 GENARRATIVE_PINGORA_GATEWAY_LISTEN 的 EnvironmentFile', ) ) { failures.push( 'systemctl cat 失败后不应继续误报 Pingora EnvironmentFile 缺失。', ); } if ( result.stderr.includes( 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', ) ) { failures.push('systemctl cat 失败后不应继续用空 env 误报 shadow listen。'); } const commandsLog = readOptionalCommandsLog(fixture); if ( commandsLog.includes( 'systemctl restart genarrative-pingora-gateway.service', ) ) { failures.push('systemctl cat 失败后不能自动 restart Pingora。'); } assertMaintenanceCleared(fixture, '读取 Pingora systemd 配置失败'); assertNoReleasePromoted(fixture, '读取 Pingora systemd 配置失败时'); } function assertDeployRejectsPingoraPublicListenWhenArtifactIncluded() { const fixture = prepareFixture('with-public-listen-pingora-artifact'); addPingoraGatewayArtifact(fixture); writePingoraEnv(fixture, { listen: '0.0.0.0:443', }); const result = runDeploy(fixture); if (result.status === 0) { failures.push('包含 Pingora 但 env 已配置公网监听时部署必须失败。'); } assertIncludes( result.stderr, 'Pingora 自动启动只允许 shadow 监听 127.0.0.1:18081', '公网监听 env 存在时必须给出明确错误。', ); const commandsLog = readOptionalCommandsLog(fixture); if ( commandsLog.includes( 'systemctl restart genarrative-pingora-gateway.service', ) ) { failures.push('公网监听 env 存在时不能自动 restart Pingora。'); } assertMaintenanceCleared(fixture, '公网监听 env 导致部署失败'); assertNoReleasePromoted( fixture, '公网监听 env 导致部署失败时不能提升正式 release。', ); } function assertDeployRejectsPingoraArtifactMissingManifestEntry() { const fixture = prepareFixture('pingora-artifact-missing-manifest-entry'); addPingoraGatewayArtifact(fixture, { registerInManifest: false }); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布包包含 Pingora 但 manifest 未登记时部署必须失败。'); } assertIncludes( result.stderr, 'release-manifest.json 缺少 pingora-gateway artifact', 'manifest 未登记 Pingora 时必须给出明确错误。', ); assertMaintenanceCleared(fixture, 'manifest 未登记 Pingora 导致部署失败'); } function assertDeployRejectsPingoraManifestEntryMissingArtifact() { const fixture = prepareFixture('pingora-manifest-entry-missing-artifact'); const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); manifest.artifacts.push({ component: 'pingora-gateway', path: 'pingora-gateway', checksum_path: 'pingora-gateway.sha256', }); writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); const result = runDeploy(fixture); if (result.status === 0) { failures.push('manifest 登记 Pingora 但发布目录缺少二进制时部署必须失败。'); } assertIncludes( result.stderr, 'release-manifest.json 登记了 pingora-gateway artifact', 'manifest 登记 Pingora 但文件缺失时必须给出明确错误。', ); assertMaintenanceCleared(fixture, 'manifest 登记 Pingora 但文件缺失导致部署失败'); } function assertDeployRequiresPingoraWhenRequested() { const fixture = prepareFixture('require-pingora-missing-artifact'); const result = runDeploy(fixture, { requirePingoraGateway: true }); if (result.status === 0) { failures.push('--require-pingora-gateway 但发布目录缺少 Pingora 时部署必须失败。'); } assertIncludes( result.stderr, '本次部署要求 Pingora', '--require-pingora-gateway 缺少 Pingora 文件时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '--require-pingora-gateway 缺少 Pingora 文件导致部署失败'); } function assertReadinessFailureKeepsMaintenanceAfterCurrentSwitch() { const fixture = prepareFixture('readiness-failure'); const result = runDeploy(fixture, { curlFails: true }); if (result.status === 0) { failures.push('current 切换后的 readiness 失败必须让部署失败。'); } assertIncludes( result.stderr, 'readiness 检查超时', 'readiness 失败时必须给出明确错误。', ); assertMaintenanceKept(fixture, 'current 切换后的 readiness 失败'); const releaseDir = path.join(fixture.releaseRoot, fixture.version); const currentTarget = readlinkSync(fixture.currentLink); if (currentTarget !== releaseDir) { failures.push( `readiness 失败发生在 current 切换后,current link 应指向新 release。实际 ${currentTarget},预期 ${releaseDir}`, ); } } function assertMissingReleaseManifestFails() { const fixture = prepareFixture('missing-release-manifest'); rmSync(path.join(fixture.sourceDir, 'release-manifest.json')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 release-manifest.json 时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 release-manifest.json', '缺少 release-manifest.json 时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 release-manifest.json 导致部署失败'); } function assertReleaseManifestMissingApiArtifactFails() { const fixture = prepareFixture('release-manifest-missing-api-artifact'); writeFileSync( path.join(fixture.sourceDir, 'release-manifest.json'), `${JSON.stringify({ version: fixture.version, artifacts: [] }, null, 2)}\n`, 'utf8', ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('release manifest 缺少 api-server artifact 时部署必须失败。'); } assertIncludes( result.stderr, 'release-manifest.json 缺少 api-server artifact', 'manifest 缺少 api-server artifact 时必须给出明确错误。', ); assertMaintenanceCleared(fixture, 'manifest 缺少 api-server artifact 导致部署失败'); } function assertDeployRejectsDotVersion() { const fixture = prepareFixture('dot-version'); const result = runDeploy(fixture, { version: '.' }); if (result.status === 0) { failures.push('--version=. 时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version=. 时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version=. 且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsDotDotVersion() { const fixture = prepareFixture('dot-dot-version'); const result = runDeploy(fixture, { version: '..' }); if (result.status === 0) { failures.push('--version=.. 时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version=.. 时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version=.. 且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsVersionStartingWithDot() { const fixture = prepareFixture('dot-prefix-version'); const result = runDeploy(fixture, { version: '.hidden-release' }); if (result.status === 0) { failures.push('--version 以点开头时部署必须失败。'); } assertIncludes( result.stderr, '--version 必须以数字或字母开头', '--version 以点开头时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--version 以点开头且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsExistingReleaseDirectory() { const fixture = prepareFixture('existing-release-directory'); const releaseDir = path.join(fixture.releaseRoot, fixture.version); mkdirSync(releaseDir, { recursive: true }); writeFileSync(path.join(releaseDir, 'old-file'), 'old\n', 'utf8'); const result = runDeploy(fixture); if (result.status === 0) { failures.push('目标 release 目录已存在时部署必须失败。'); } assertIncludes( result.stderr, '目标 release 已存在,拒绝覆盖或合并旧文件', '目标 release 目录已存在时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('目标 release 目录已存在且未进入部署阶段时不应开启维护模式。'); } assertFileExists( path.join(releaseDir, 'old-file'), '拒绝覆盖既有 release 时必须保留原目录。', ); } function assertDeployRejectsDirectoryCurrentLink() { const fixture = prepareFixture('directory-current-link'); mkdirSync(fixture.currentLink, { recursive: true }); writeFileSync(path.join(fixture.currentLink, 'old-file'), 'old\n', 'utf8'); const result = runDeploy(fixture); if (result.status === 0) { failures.push('current 路径已存在但不是符号链接时部署必须失败。'); } assertIncludes( result.stderr, 'current 链接路径已存在但不是符号链接,拒绝覆盖', 'current 路径不是符号链接时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('current 路径不是符号链接且未进入部署阶段时不应开启维护模式。'); } assertFileExists( path.join(fixture.currentLink, 'old-file'), '拒绝覆盖目录型 current 时必须保留原目录内容。', ); } function assertDeployRejectsRelativeReleaseRoot() { const fixture = prepareFixture('relative-release-root'); const result = runDeploy(fixture, { releaseRoot: 'relative/releases' }); if (result.status === 0) { failures.push('--release-root 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--release-root 必须使用绝对路径', '--release-root 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--release-root 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsRelativeCurrentLink() { const fixture = prepareFixture('relative-current-link'); const result = runDeploy(fixture, { currentLink: 'relative/current' }); if (result.status === 0) { failures.push('--current-link 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--current-link 必须使用绝对路径', '--current-link 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--current-link 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployRejectsRelativeApiEnvFile() { const fixture = prepareFixture('relative-api-env-file'); const result = runDeploy(fixture, { apiEnvFile: 'relative/api-server.env' }); if (result.status === 0) { failures.push('--api-env-file 使用相对路径时部署必须失败。'); } assertIncludes( result.stderr, '--api-env-file 必须使用绝对路径', '--api-env-file 使用相对路径时必须给出明确错误。', ); if (existsSync(fixture.maintenanceFile)) { failures.push('--api-env-file 相对路径且未进入部署阶段时不应开启维护模式。'); } } function assertDeployCleansStagingReleaseOnFailure() { const fixture = prepareFixture('cleans-staging-on-failure'); rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('缺少数据库备份脚本时部署必须失败。'); } const releaseDir = path.join(fixture.releaseRoot, fixture.version); if (existsSync(releaseDir)) { failures.push('部署失败时不应留下正式 release 目录。'); } const entries = existsSync(fixture.releaseRoot) ? readDirNames(fixture.releaseRoot) : []; const stagingEntries = entries.filter((entry) => entry.includes(`${fixture.version}.staging`), ); if (stagingEntries.length > 0) { failures.push(`部署失败时不应留下 staging release: ${stagingEntries.join(', ')}`); } assertMaintenanceCleared(fixture, 'staging 构建中失败'); } function assertDeployRejectsFinalReleaseRaceAndCleansStaging() { const fixture = prepareFixture('final-release-race'); const result = runDeploy(fixture, { createReleaseDuringCopy: true }); if (result.status === 0) { failures.push('最终提升前目标 release 目录被外部创建时部署必须失败。'); } assertIncludes( result.stderr, '目标 release 在发布过程中出现,拒绝合并 staging', '最终提升前目标 release 目录被外部创建时必须给出明确错误。', ); const releaseDir = path.join(fixture.releaseRoot, fixture.version); assertFileExists( path.join(releaseDir, 'raced-file'), '部署脚本拒绝竞态目标 release 时必须保留外部创建的目录。', ); if (existsSync(path.join(releaseDir, 'api-server'))) { failures.push('目标 release 竞态出现后不应把 staging 内容合并进去。'); } const entries = existsSync(fixture.releaseRoot) ? readDirNames(fixture.releaseRoot) : []; const stagingEntries = entries.filter((entry) => entry.includes(`${fixture.version}.staging`), ); if (stagingEntries.length > 0) { failures.push(`目标 release 竞态失败后不应留下 staging release: ${stagingEntries.join(', ')}`); } assertMaintenanceCleared(fixture, '目标 release 竞态失败'); } function assertMissingPingoraDirectCheckFails() { const fixture = prepareFixture('missing-direct-live'); rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-direct-live.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 direct live smoke 脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连 live smoke 脚本', '缺少 direct live smoke 脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 direct live smoke 脚本导致部署失败'); } function assertMissingPingoraCanaryLiveFails() { const fixture = prepareFixture('missing-canary-live'); rmSync(path.join(fixture.sourceDir, 'scripts/check-pingora-canary-live.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 canary live smoke 脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora canary live smoke 脚本', '缺少 canary live smoke 脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 canary live smoke 脚本导致部署失败'); } function assertMissingPingoraCanaryAccessLogParityFails() { const fixture = prepareFixture('missing-canary-log-parity'); rmSync( path.join( fixture.sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 canary access log 对账脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora canary access log 对账脚本', '缺少 canary access log 对账脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 canary access log 对账脚本导致部署失败'); } function assertMissingBackupScriptFails() { const fixture = prepareFixture('missing-backup-script'); rmSync(path.join(fixture.sourceDir, 'scripts/database-backup-to-oss.mjs')); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少数据库备份脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少数据库备份脚本', '缺少数据库备份脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少数据库备份脚本导致部署失败'); } function assertMissingHealthPatrolScriptFails() { const fixture = prepareFixture('missing-health-patrol-script'); rmSync( path.join(fixture.sourceDir, 'scripts/ops/production-health-patrol.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少生产健康巡检脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少生产健康巡检脚本', '缺少生产健康巡检脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少生产健康巡检脚本导致部署失败'); } function assertMissingPingoraCurrentReleaseAuditFails() { const fixture = prepareFixture('missing-pingora-current-release-audit'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-current-release-audit.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora current release 自审脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora current release 自审脚本', '缺少 Pingora current release 自审脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora current release 自审脚本导致部署失败'); } function assertMissingPingoraDirectRehearsalStatusFails() { const fixture = prepareFixture('missing-pingora-direct-rehearsal-status'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连彩排状态脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连彩排状态脚本', '缺少 Pingora 直连彩排状态脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连彩排状态脚本导致部署失败'); } function assertMissingPingoraCutoverStatusSnapshotFails() { const fixture = prepareFixture('missing-pingora-cutover-status-snapshot'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换状态快照脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换状态快照脚本', '缺少 Pingora 直连切换状态快照脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换状态快照脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceBundleFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-bundle'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换证据包脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据包脚本', '缺少 Pingora 直连切换证据包脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据包脚本导致部署失败'); } function assertMissingPingoraCutoverCommandEvidenceFails() { const fixture = prepareFixture('missing-pingora-cutover-command-evidence'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换命令证据脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换命令证据脚本', '缺少 Pingora 直连切换命令证据脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换命令证据脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceVerifyFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-verify'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora 直连切换证据验真脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据验真脚本', '缺少 Pingora 直连切换证据验真脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据验真脚本导致部署失败'); } function assertMissingPingoraCutoverEvidenceAuditFails() { const fixture = prepareFixture('missing-pingora-cutover-evidence-audit'); rmSync( path.join( fixture.sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '发布产物缺少 Pingora 直连切换证据根目录审计脚本时部署必须失败。', ); } assertIncludes( result.stderr, '发布产物缺少 Pingora 直连切换证据根目录审计脚本', '缺少 Pingora 直连切换证据根目录审计脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora 直连切换证据根目录审计脚本导致部署失败'); } function assertMissingHealthPatrolEnvCheckFails() { const fixture = prepareFixture('missing-health-patrol-env-check'); rmSync( path.join(fixture.sourceDir, 'scripts/check-production-health-patrol-env.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少生产健康巡检 env 复核脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少生产健康巡检 env 复核脚本', '缺少生产健康巡检 env 复核脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少生产健康巡检 env 复核脚本导致部署失败'); } function assertMissingPingoraReleaseReadinessFails() { const fixture = prepareFixture('missing-pingora-release-readiness'); rmSync( path.join(fixture.sourceDir, 'scripts/check-pingora-release-readiness.mjs'), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '发布产物缺少 Pingora release readiness 聚合门禁脚本时部署必须失败。', ); } assertIncludes( result.stderr, '发布产物缺少 Pingora release readiness 聚合门禁脚本', '缺少 Pingora release readiness 聚合门禁脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora release readiness 聚合门禁脚本导致部署失败'); } function assertMissingPingoraHealthPatrolEnvSwitchFails() { const fixture = prepareFixture('missing-pingora-health-patrol-env-switch'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora health patrol env 切换脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora health patrol env 切换脚本', '缺少 Pingora health patrol env 切换脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora health patrol env 切换脚本导致部署失败'); } function assertMissingPingoraGatewayEnvShadowSwitchFails() { const fixture = prepareFixture('missing-pingora-gateway-env-shadow-switch'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push( '发布产物缺少 Pingora gateway env shadow 切换脚本时部署必须失败。', ); } assertIncludes( result.stderr, '发布产物缺少 Pingora gateway env shadow 切换脚本', '缺少 Pingora gateway env shadow 切换脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora gateway env shadow 切换脚本导致部署失败'); } function assertMissingPingoraRealpathCanaryEnableFails() { const fixture = prepareFixture('missing-pingora-realpath-canary-enable'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-realpath-canary-enable.sh', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora realpath canary 启用脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora realpath canary 启用脚本', '缺少 Pingora realpath canary 启用脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 启用脚本导致部署失败'); } function assertMissingPingoraRealpathCanaryDisableFails() { const fixture = prepareFixture('missing-pingora-realpath-canary-disable'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-realpath-canary-disable.sh', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora realpath canary 关闭脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora realpath canary 关闭脚本', '缺少 Pingora realpath canary 关闭脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora realpath canary 关闭脚本导致部署失败'); } function assertMissingPingoraTlsCertSyncFails() { const fixture = prepareFixture('missing-pingora-tls-cert-sync'); rmSync( path.join( fixture.sourceDir, 'scripts/deploy/pingora-tls-cert-sync.mjs', ), ); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少 Pingora TLS 证书同步脚本时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少 Pingora TLS 证书同步脚本', '缺少 Pingora TLS 证书同步脚本时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少 Pingora TLS 证书同步脚本导致部署失败'); } function assertMissingEnvExamplesFails() { const fixture = prepareFixture('missing-env-examples'); rmSync(path.join(fixture.sourceDir, 'deploy/env'), { recursive: true, force: true, }); const result = runDeploy(fixture); if (result.status === 0) { failures.push('发布产物缺少环境变量示例目录时部署必须失败。'); } assertIncludes( result.stderr, '发布产物缺少环境变量示例目录', '缺少环境变量示例目录时必须给出明确错误。', ); assertMaintenanceCleared(fixture, '缺少环境变量示例目录导致部署失败'); } function prepareFixture(name) { const root = path.join(tmpRoot, name); const sourceDir = path.join(root, 'source'); const releaseRoot = path.join(root, 'releases'); const currentLink = path.join(root, 'current'); const apiEnvFile = path.join(root, 'etc', 'api-server.env'); const pingoraEnvFile = path.join(root, 'etc', 'pingora-gateway.env'); const maintenanceFile = path.join(root, 'maintenance', 'enabled'); const fakeBin = path.join(root, 'bin'); const commandsLog = path.join(root, 'commands.log'); const systemdUnitDir = path.join(root, 'etc', 'systemd', 'system'); const workerStateFile = path.join(root, 'worker-service-enabled'); const pingoraStateFile = path.join(root, 'pingora-service-active'); const version = `20260614-${name}`; mkdirSync(sourceDir, { recursive: true }); mkdirSync(fakeBin, { recursive: true }); mkdirSync(path.dirname(apiEnvFile), { recursive: true }); mkdirSync(path.join(sourceDir, 'scripts/deploy'), { recursive: true }); mkdirSync(path.join(sourceDir, 'scripts/ops'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/systemd'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/pingora'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/env'), { recursive: true }); mkdirSync(path.join(sourceDir, 'deploy/nginx/snippets'), { recursive: true }); writeFileSync( path.join(sourceDir, 'api-server'), '#!/usr/bin/env bash\n', 'utf8', ); writeFileSync( apiEnvFile, [ 'GENARRATIVE_TRACKING_OUTBOX_ENABLED=false', `GENARRATIVE_WALLET_REFUND_OUTBOX_DIR=${path.join(root, 'wallet-refund-outbox')}`, 'GENARRATIVE_API_SHUTDOWN_OUTBOX_FLUSH_TIMEOUT_MS=5000', '', ].join('\n'), 'utf8', ); writePingoraEnv({ pingoraEnvFile }); chmodExecutable(path.join(sourceDir, 'api-server')); writeSha256(sourceDir, 'api-server'); writeFileSync( path.join(sourceDir, 'release-manifest.json'), `${JSON.stringify( { version, artifacts: [ { component: 'api-server', path: 'api-server', checksum_path: 'api-server.sha256', }, ], }, null, 2, )}\n`, 'utf8', ); writeFileSync( path.join(sourceDir, 'scripts/database-backup-to-oss.mjs'), 'console.log("backup");\n', 'utf8', ); writeFileSync( path.join(sourceDir, 'scripts/ops/production-health-patrol.mjs'), 'console.log("patrol");\n', 'utf8', ); copyFile( 'scripts/ops/pingora-current-release-audit.mjs', path.join(sourceDir, 'scripts/ops/pingora-current-release-audit.mjs'), ); copyFile( 'scripts/ops/pingora-direct-rehearsal-status.mjs', path.join(sourceDir, 'scripts/ops/pingora-direct-rehearsal-status.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-status-snapshot.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-status-snapshot.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-bundle.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-bundle.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-command-evidence.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-command-evidence.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-verify.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-verify.mjs'), ); copyFile( 'scripts/ops/pingora-cutover-evidence-audit.mjs', path.join(sourceDir, 'scripts/ops/pingora-cutover-evidence-audit.mjs'), ); copyFile( 'scripts/deploy/production-api-deploy.sh', path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), ); copyFile( 'scripts/deploy/maintenance-on.sh', path.join(sourceDir, 'scripts/deploy/maintenance-on.sh'), ); copyFile( 'scripts/deploy/maintenance-off.sh', path.join(sourceDir, 'scripts/deploy/maintenance-off.sh'), ); copyFile( 'scripts/deploy/pingora-direct-enable.sh', path.join(sourceDir, 'scripts/deploy/pingora-direct-enable.sh'), ); copyFile( 'scripts/deploy/pingora-direct-rollback.sh', path.join(sourceDir, 'scripts/deploy/pingora-direct-rollback.sh'), ); copyFile( 'scripts/deploy/pingora-realpath-canary-enable.sh', path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'), ); copyFile( 'scripts/deploy/pingora-realpath-canary-disable.sh', path.join(sourceDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'), ); copyFile( 'scripts/deploy/pingora-health-patrol-env-switch.mjs', path.join(sourceDir, 'scripts/deploy/pingora-health-patrol-env-switch.mjs'), ); copyFile( 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', path.join(sourceDir, 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs'), ); copyFile( 'scripts/deploy/pingora-tls-cert-sync.mjs', path.join(sourceDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'), ); copyFile( 'scripts/check-production-health-patrol-env.mjs', path.join(sourceDir, 'scripts/check-production-health-patrol-env.mjs'), ); copyFile( 'scripts/check-pingora-release-readiness.mjs', path.join(sourceDir, 'scripts/check-pingora-release-readiness.mjs'), ); copyFile( 'scripts/check-pingora-direct-preflight.mjs', path.join(sourceDir, 'scripts/check-pingora-direct-preflight.mjs'), ); copyFile( 'scripts/check-pingora-direct-live.mjs', path.join(sourceDir, 'scripts/check-pingora-direct-live.mjs'), ); copyFile( 'scripts/check-pingora-canary-live.mjs', path.join(sourceDir, 'scripts/check-pingora-canary-live.mjs'), ); copyFile( 'scripts/check-pingora-canary-access-log-parity.mjs', path.join(sourceDir, 'scripts/check-pingora-canary-access-log-parity.mjs'), ); copyFile( 'deploy/systemd/genarrative-pingora-gateway.service', path.join(sourceDir, 'deploy/systemd/genarrative-pingora-gateway.service'), ); copyFile( 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', path.join( sourceDir, 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', ), ); copyFile( 'deploy/systemd/genarrative-external-generation-worker@.service', path.join( sourceDir, 'deploy/systemd/genarrative-external-generation-worker@.service', ), ); copyFile( 'deploy/systemd/genarrative-external-generation-controller.service', path.join( sourceDir, 'deploy/systemd/genarrative-external-generation-controller.service', ), ); copyFile( 'deploy/pingora/pingora-gateway.env.example', path.join(sourceDir, 'deploy/pingora/pingora-gateway.env.example'), ); copyFile( 'deploy/env/health-patrol.env.example', path.join(sourceDir, 'deploy/env/health-patrol.env.example'), ); copyFile( 'deploy/env/pingora-direct-live.env.example', path.join(sourceDir, 'deploy/env/pingora-direct-live.env.example'), ); copyFile( 'deploy/env/pingora-canary-live.env.example', path.join(sourceDir, 'deploy/env/pingora-canary-live.env.example'), ); copyFile( 'deploy/nginx/snippets/genarrative-pingora-canary.conf', path.join( sourceDir, 'deploy/nginx/snippets/genarrative-pingora-canary.conf', ), ); copyFile( 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', path.join( sourceDir, 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', ), ); chmodExecutable( path.join(sourceDir, 'scripts/deploy/production-api-deploy.sh'), ); chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-on.sh')); chmodExecutable(path.join(sourceDir, 'scripts/deploy/maintenance-off.sh')); writeFileSync( path.join(fakeBin, 'systemctl'), [ '#!/usr/bin/env bash', 'set -euo pipefail', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'worker_state_file="${FAKE_WORKER_STATE_FILE}"', 'pingora_state_file="${FAKE_PINGORA_STATE_FILE}"', 'if [[ "$1" == "cat" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', ' if [[ "${FAKE_PINGORA_SYSTEMCTL_CAT_FAIL:-false}" == "true" ]]; then', ' exit 1', ' fi', ' printf "[Service]\\n"', ' printf "EnvironmentFile=%s\\n" "${FAKE_PINGORA_ENV_FILE}"', ' printf "ExecStart=/opt/genarrative/current/pingora-gateway\\n"', ' if [[ "${FAKE_PINGORA_DIRECT_ENTRY:-false}" == "true" ]]; then', ' printf "AmbientCapabilities=CAP_NET_BIND_SERVICE\\n"', ' printf "CapabilityBoundingSet=CAP_NET_BIND_SERVICE\\n"', ' fi', ' exit 0', 'fi', 'if [[ "$1" == "list-units" ]]; then', ' pattern="${@: -1}"', ' if [[ "${pattern}" == "genarrative-external-generation-worker@*.service" && -f "${worker_state_file}" ]]; then', ' printf "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker\\n"', ' fi', ' exit 0', 'fi', 'if [[ "$1 ${2:-}" == "enable --now" && "${3:-}" == "genarrative-external-generation-worker@1.service" ]]; then', ' printf "enabled\\n" > "${worker_state_file}"', ' exit 0', 'fi', 'if [[ "$1 ${2:-} ${3:-}" == "is-active --quiet genarrative-pingora-gateway.service" ]]; then', ' if [[ "${FAKE_PINGORA_ACTIVE:-true}" == "true" || -f "${pingora_state_file}" ]]; then', ' exit 0', ' fi', ' exit 3', 'fi', 'if [[ "$1" == "restart" && "${2:-}" == "genarrative-pingora-gateway.service" ]]; then', ' printf "active\\n" > "${pingora_state_file}"', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'curl'), [ '#!/usr/bin/env bash', `printf 'curl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'if [[ "${FAKE_CURL_FAIL:-false}" == "true" ]]; then', ' exit 22', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'sleep'), [ '#!/usr/bin/env bash', `printf 'sleep %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'cp'), [ '#!/usr/bin/env bash', 'set -euo pipefail', '/usr/bin/cp "$@"', 'if [[ "${FAKE_CREATE_RELEASE_DURING_COPY:-false}" == "true" ]]; then', ' marker="${FAKE_RELEASE_ROOT}/.${FAKE_RELEASE_VERSION}.race-created"', ' if [[ ! -e "${marker}" ]]; then', ' mkdir -p "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}"', ' printf "race\\n" > "${FAKE_RELEASE_ROOT}/${FAKE_RELEASE_VERSION}/raced-file"', ' printf "created\\n" > "${marker}"', ' fi', 'fi', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'sudo'), [ '#!/usr/bin/env bash', 'set -euo pipefail', 'if [[ "${1:-}" == "-n" ]]; then', ' shift', 'fi', 'if [[ "${1:-}" == "true" ]]; then', ' exit 0', 'fi', 'if [[ "${1:-}" == "install" && "${2:-}" == "-d" ]]; then', ' path="${@: -1}"', ' mkdir -p "${path}"', ' exit 0', 'fi', 'exec "$@"', '', ].join('\n'), 'utf8', ); chmodExecutable(path.join(fakeBin, 'systemctl')); chmodExecutable(path.join(fakeBin, 'curl')); chmodExecutable(path.join(fakeBin, 'sleep')); chmodExecutable(path.join(fakeBin, 'cp')); chmodExecutable(path.join(fakeBin, 'sudo')); return { root, sourceDir, releaseRoot, currentLink, apiEnvFile, pingoraEnvFile, maintenanceFile, fakeBin, commandsLog, systemdUnitDir, workerStateFile, pingoraStateFile, version, }; } function writePingoraEnv(fixture, options = {}) { const filePath = fixture.pingoraEnvFile; const lines = [ `GENARRATIVE_PINGORA_GATEWAY_LISTEN=${options.listen ?? '127.0.0.1:18081'}`, ]; if (options.tlsListen) { lines.push(`GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN=${options.tlsListen}`); } if (options.redirectListen) { lines.push( `GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN=${options.redirectListen}`, ); } writeFileSync(filePath, `${lines.join('\n')}\n`, 'utf8'); } function addPingoraGatewayArtifact(fixture, options = {}) { writeFileSync( path.join(fixture.sourceDir, 'pingora-gateway'), '#!/usr/bin/env bash\n', 'utf8', ); chmodExecutable(path.join(fixture.sourceDir, 'pingora-gateway')); writeSha256(fixture.sourceDir, 'pingora-gateway'); if (options.registerInManifest === false) { return; } const manifestPath = path.join(fixture.sourceDir, 'release-manifest.json'); const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); manifest.artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; manifest.artifacts.push({ component: 'pingora-gateway', path: 'pingora-gateway', checksum_path: 'pingora-gateway.sha256', }); writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, 'utf8'); } function runDeploy(fixture, options = {}) { const deployScript = path.join( fixture.sourceDir, 'scripts/deploy/production-api-deploy.sh', ); const args = [ deployScript, '--source-dir', fixture.sourceDir, '--version', options.version ?? fixture.version, '--release-root', options.releaseRoot ?? fixture.releaseRoot, '--current-link', options.currentLink ?? fixture.currentLink, '--service', 'genarrative-api.service', '--health-url', 'http://127.0.0.1:18082/readyz', '--api-env-file', options.apiEnvFile ?? fixture.apiEnvFile, '--database', 'genarrative-prod', '--spacetime-server-url', 'http://127.0.0.1:3101', ]; if (options.requirePingoraGateway) { args.push('--require-pingora-gateway'); } if (options.keepMaintenance) { args.push('--keep-maintenance-mode'); } return spawnSync( 'bash', args, { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, GENARRATIVE_MAINTENANCE_FILE: fixture.maintenanceFile, FAKE_PINGORA_ACTIVE: options.pingoraActive === false ? 'false' : 'true', FAKE_PINGORA_DIRECT_ENTRY: options.pingoraDirectEntry === true ? 'true' : 'false', FAKE_PINGORA_SYSTEMCTL_CAT_FAIL: options.pingoraSystemctlCatFails === true ? 'true' : 'false', FAKE_PINGORA_ENV_FILE: fixture.pingoraEnvFile, FAKE_PINGORA_STATE_FILE: fixture.pingoraStateFile, FAKE_CURL_FAIL: options.curlFails === true ? 'true' : 'false', FAKE_CREATE_RELEASE_DURING_COPY: options.createReleaseDuringCopy === true ? 'true' : 'false', FAKE_RELEASE_ROOT: fixture.releaseRoot, FAKE_RELEASE_VERSION: fixture.version, FAKE_WORKER_STATE_FILE: fixture.workerStateFile, GENARRATIVE_SYSTEMD_UNIT_DIR: fixture.systemdUnitDir, }, }, ); } function copyFile(source, target) { writeFileSync(target, readFileSync(source, 'utf8'), 'utf8'); } function writeSha256(directory, fileName) { const result = spawnSync('sha256sum', [fileName], { cwd: directory, encoding: 'utf8', }); if (result.status !== 0) { throw new Error(result.stderr || `sha256sum failed for ${fileName}`); } writeFileSync( path.join(directory, `${fileName}.sha256`), result.stdout, 'utf8', ); } function chmodExecutable(filePath) { spawnSync('chmod', ['0755', filePath], { cwd: process.cwd(), encoding: 'utf8', }); } function shellQuote(value) { return `'${String(value).replace(/'/g, "'\\''")}'`; } function assertFileExists(filePath, reason) { if (!existsSync(filePath)) { failures.push(`${reason} 缺少: ${filePath}`); } } function assertNoReleasePromoted(fixture, reason) { const releaseDir = path.join(fixture.releaseRoot, fixture.version); if (existsSync(releaseDir)) { failures.push(`${reason} 已存在: ${releaseDir}`); } if (existsSync(fixture.currentLink)) { failures.push(`${reason} 不应切换 current: ${fixture.currentLink}`); } } function readDirNames(directory) { return spawnSync('find', [directory, '-maxdepth', '1', '-mindepth', '1', '-printf', '%f\n'], { cwd: process.cwd(), encoding: 'utf8', }) .stdout.split('\n') .filter(Boolean); } function assertStatus(result, expected, reason) { const actual = result.status ?? 0; if (actual !== expected) { failures.push( `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } } function assertIncludes(content, needle, reason) { if (!content.includes(needle)) { failures.push(`${reason} 缺少: ${needle}`); } } function assertPingoraEnvProductionDefaults(filePath, label) { if (!existsSync(filePath)) { return; } const content = readFileSync(filePath, 'utf8'); const requiredLines = [ [ 'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip', '压缩算法必须保持 gzip-only,避免 Brotli 在未验证前进入 current release。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false', '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false', '前置代理信任确认开关必须默认关闭。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true', '接流保护必须默认开启。', ], [ 'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=', '内部探针 token 示例必须保持空值,避免 current release 夹带真实 token。', ], ]; for (const [line, reason] of requiredLines) { assertIncludes(content, line, `${label} ${reason}`); } }