#!/usr/bin/env node import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync, } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; const ROLLBACK_SCRIPT = 'scripts/deploy/pingora-direct-rollback.sh'; const failures = []; const tmpRoot = mkdtempSync( path.join(tmpdir(), 'genarrative-pingora-rollback-'), ); try { main(); } finally { rmSync(tmpRoot, { recursive: true, force: true }); } if (failures.length > 0) { console.error('[check:pingora-direct-rollback] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } console.log('[check:pingora-direct-rollback] OK'); function main() { assertScriptShape(); assertDryRunKeepsDropin(); assertDryRunMissingDropin(); assertRelativeDropinRejected(); assertRelativeServiceUnitPathRejected(); assertRejectsRelativeNginxBinaryPath(); assertRejectsRelativeCurlBinaryPath(); assertRejectsFilesystemRootPathsBeforeApply(); assertApplyRequiresReloadNginx(); assertApplyRequiresNginxSmokeUrl(); assertRejectsInvalidNginxSmokeUrl(); assertApplyRequiresHostForLoopbackSmokeUrl(); assertRejectsInvalidNginxSmokeHost(); assertRejectsRelativeHealthPatrolEnvFile(); assertRejectsRelativeHealthPatrolEnvCheckScript(); assertRejectsConflictingHealthPatrolPublicHostOptions(); assertRejectsInvalidHealthPatrolExpectedPublicBaseUrl(); assertRejectsInvalidHealthPatrolExpectedPublicHost(); assertRejectsIncompletePingoraShadowProbe(); assertRejectsInvalidPingoraShadowProbeUrl(); assertRejectsControlCharacterInputsBeforeApply(); assertApplyRejectsSymlinkDropinDirectoryBeforeNginxTest(); assertApplyRejectsSymlinkDropinFileBeforeRemoval(); assertApplyRejectsDirectoryDropinTargetBeforeNginxTest(); assertApplyStopsBeforeRemovingDropinWhenNginxTestFails(); assertApplyFailsWhenNginxSmokeFails(); assertApplyFailsWhenNginxSmokeBodyMismatches(); assertApplyFailsWhenHealthPatrolEnvStillDirect(); assertApplyFailsWhenHealthPatrolPublicBaseUrlDrifts(); assertApplySupportsExpectedHealthPatrolPublicHost(); assertApplyFailsWhenSystemdExecStartDiffers(); assertApplyFailsWhenPingoraShadowProbeBodyIsWrong(); } function assertScriptShape() { const content = readFileSync(ROLLBACK_SCRIPT, 'utf8'); assertIncludes(content, 'APPLY="false"', '回退脚本必须默认 dry-run。'); assertIncludes( content, '--apply', '回退脚本必须显式要求 --apply 才修改系统。', ); assertIncludes( content, 'systemctl daemon-reload', '移除 drop-in 后必须 reload systemd。', ); assertIncludes( content, 'restart "${SERVICE_NAME}"', '回退脚本必须重启 Pingora 服务。', ); assertIncludes( content, '--reload-nginx', '回退脚本必须保留显式 Nginx reload。', ); assertIncludes( content, '--nginx-binary', '回退脚本必须支持覆盖 Nginx 可执行文件,便于测试和特殊环境。', ); assertIncludes( content, '--nginx-smoke-url', '回退脚本必须支持 Nginx 入口 smoke URL。', ); assertIncludes( content, '--nginx-smoke-host', '回退脚本必须支持 Nginx 入口 smoke Host。', ); assertIncludes( content, '--nginx-smoke-expect-body', '回退脚本必须支持 Nginx smoke 响应体证据片段。', ); assertIncludes( content, '--nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host', '回退脚本 apply 用本机 smoke URL 时必须强制正式 Host。', ); assertIncludes( content, '只能是 host 或 host:port', '回退脚本必须拒绝非法 Nginx smoke Host。', ); assertIncludes( content, '--curl-binary', '回退脚本必须支持覆盖 curl 可执行文件,便于测试和特殊环境。', ); assertIncludes( content, '--health-patrol-env-file', '回退脚本必须支持回退后 health patrol env 复核。', ); assertIncludes( content, '--health-patrol-env-check-script', '回退脚本必须支持覆盖 health patrol env 复核脚本,便于 release layout 与测试。', ); assertIncludes( content, '--pingora-shadow-probe-url', '回退脚本必须支持回退后 Pingora shadow probe URL。', ); assertIncludes( content, '--pingora-shadow-probe-token', '回退脚本必须支持回退后 Pingora shadow probe token。', ); assertIncludes( content, '--expected-gateway-mode', '回退脚本必须调用 health patrol env 复核脚本并要求 nginx 模式。', ); assertIncludes( content, '--require-empty-public-host', '回退脚本必须要求回退后 health patrol public Host 为空。', ); assertIncludes( content, '--health-patrol-expected-public-base-url', '回退脚本必须支持校验回退后 health patrol public base URL。', ); assertIncludes( content, '--health-patrol-expected-public-host', '回退脚本必须支持校验回退后 health patrol public Host。', ); assertIncludes( content, '--health-patrol-require-empty-public-host', '回退脚本必须支持显式要求回退后 health patrol public Host 为空。', ); assertIncludes( content, 'dry-run:--apply 后会复核 health patrol env 已切回 nginx,且 public base URL / Host 已恢复为预期值。', '回退脚本 dry-run 必须说明 health patrol env 回退后复核。', ); assertIncludes( content, 'dry-run:--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。', '回退脚本 dry-run 必须说明 Pingora shadow probe 回退后复核。', ); assertIncludes( content, '--apply 必须同时提供 --reload-nginx', '回退脚本 apply 时必须强制 reload Nginx。', ); assertIncludes( content, '--apply 必须同时提供 --nginx-smoke-url', '回退脚本 apply 时必须强制 Nginx 入口 smoke。', ); assertIncludes( content, '${NGINX_BINARY} -t', '回退脚本 reload Nginx 前必须先跑 nginx -t。', ); assertIncludes( content, 'drop-in 目录不能是符号链接', '回退脚本 apply 前必须拒绝符号链接 drop-in 目录。', ); assertIncludes( content, 'drop-in 目标不能是符号链接', '回退脚本 apply 前必须拒绝符号链接 drop-in 目标。', ); assertIncludes( content, 'drop-in 目标已存在但不是普通文件', '回退脚本 apply 前必须拒绝非普通文件 drop-in 目标。', ); assertIncludes( content, 'direct-entry.conf', '回退脚本必须默认指向 direct-entry drop-in。', ); assertIncludes( content, '--no-postcheck', '回退脚本必须支持少数异常场景跳过 post-check。', ); assertIncludes( content, '--service-unit-path', '回退脚本必须支持覆盖主 service 模板路径,便于 current release 随包执行。', ); assertIncludes( content, 'systemctl cat ${SERVICE_NAME}', '回退脚本必须在 apply 后核验 systemd 最终配置。', ); assertIncludes( content, 'systemctl show ${SERVICE_NAME} --property=ExecStart --value --no-pager', '回退脚本必须在 apply 后核验 Pingora ExecStart 指向 current release。', ); assertIncludes( content, '主 service 模板中的', '回退脚本 ExecStart 核验必须以随包主 service 模板为真相源。', ); assertIncludes( content, '仍显示 AmbientCapabilities=CAP_NET_BIND_SERVICE', '回退脚本必须拒绝 AmbientCapabilities 残留。', ); assertIncludes( content, '仍显示 CapabilityBoundingSet=CAP_NET_BIND_SERVICE', '回退脚本必须拒绝 CapabilityBoundingSet 残留。', ); assertIncludes( content, 'systemctl is-active ${NGINX_SERVICE}', '回退脚本必须在 reload Nginx 后核验 Nginx 状态。', ); assertIncludes( content, '${CURL_BINARY} --fail', '回退脚本必须在 reload Nginx 后执行 Nginx smoke。', ); assertIncludes( content, 'nginx smoke evidence matched expected body fragment', '回退脚本必须在 Nginx smoke 响应体匹配时打印证据摘要。', ); assertIncludes( content, 'Nginx smoke 响应缺少预期片段', '回退脚本必须拒绝缺少预期响应体片段的 Nginx smoke。', ); assertIncludes( content, 'Nginx reload 后状态不是 active', '回退脚本必须拒绝 Nginx reload 后非 active 状态。', ); assertIncludes( content, 'reject_control_characters', '回退脚本必须统一拒绝换行或 NUL 参数。', ); assertIncludes( content, '不能包含换行或 NUL 字符', '回退脚本拒绝控制字符时必须给出明确错误。', ); assertIncludes( content, 'reject_filesystem_root_path', '回退脚本必须统一拒绝文件系统根目录路径参数。', ); assertIncludes( content, '不能是文件系统根目录', '回退脚本拒绝文件系统根目录路径时必须给出明确错误。', ); } function assertDryRunKeepsDropin() { const dropinPath = path.join(tmpRoot, 'direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--dropin-path', dropinPath, '--service', 'genarrative-pingora-gateway.service', '--reload-nginx', '--nginx-service', 'nginx.service', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--nginx-smoke-expect-body', '"ok":true', '--no-status', ]); assertStatus(result, 0, 'dry-run 应成功退出。'); assertIncludes( result.stdout, 'apply=false', 'dry-run 输出必须明确 apply=false。', ); assertIncludes( result.stdout, `+ rm -f ${dropinPath}`, 'dry-run 必须展示将移除 drop-in。', ); assertIncludes( result.stdout, '+ systemctl daemon-reload', 'dry-run 必须展示 daemon-reload。', ); assertIncludes( result.stdout, '+ systemctl restart genarrative-pingora-gateway.service', 'dry-run 必须展示重启 Pingora。', ); assertIncludes( result.stdout, '+ nginx -t', 'dry-run 必须展示 reload Nginx 前的 nginx -t。', ); assertIncludes( result.stdout, 'dry-run:--apply --reload-nginx 后会先执行 nginx -t,通过后再 reload。', 'dry-run 必须说明 apply reload 前会先跑 nginx -t。', ); assertIncludes( result.stdout, '+ systemctl reload nginx.service', 'dry-run 必须展示可选 reload Nginx。', ); assertIncludes( result.stdout, '+ systemctl is-active nginx.service', 'dry-run 必须展示 reload Nginx 后 active 核验。', ); assertIncludes( result.stdout, '+ curl --fail --silent --show-error --max-time 5 -H Host: example.com http://127.0.0.1/healthz', 'dry-run 必须展示 Nginx 入口 smoke。', ); assertIncludes( result.stdout, 'dry-run:--apply --reload-nginx 后会验证 Nginx smoke URL 可访问,且响应体包含预期片段。', 'dry-run 带响应体片段时必须说明会校验 Nginx smoke body。', ); const healthPatrolEnvFile = path.join(tmpRoot, 'health-patrol.env'); const healthPatrolEnvCheckScript = path.join( process.cwd(), 'scripts/check-production-health-patrol-env.mjs', ); const healthPatrolResult = runRollback([ '--dropin-path', dropinPath, '--service', 'genarrative-pingora-gateway.service', '--reload-nginx', '--nginx-service', 'nginx.service', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', healthPatrolEnvFile, '--health-patrol-env-check-script', healthPatrolEnvCheckScript, '--health-patrol-expected-public-base-url', 'https://nginx.example.com', '--no-status', ]); assertStatus( healthPatrolResult, 0, '带 health patrol env 的 dry-run 应成功退出。', ); assertIncludes( healthPatrolResult.stdout, `+ node -- ${healthPatrolEnvCheckScript} --env-file ${healthPatrolEnvFile} --expected-gateway-mode nginx --expected-public-base-url https://nginx.example.com --require-empty-public-host`, 'dry-run 必须展示 health patrol env 回退后复核命令。', ); assertIncludes( healthPatrolResult.stdout, 'dry-run:--apply 后会复核 health patrol env 已切回 nginx,且 public base URL / Host 已恢复为预期值。', 'dry-run 必须说明 apply 后会复核 health patrol env。', ); const healthPatrolHostResult = runRollback([ '--dropin-path', dropinPath, '--service', 'genarrative-pingora-gateway.service', '--reload-nginx', '--nginx-service', 'nginx.service', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', healthPatrolEnvFile, '--health-patrol-env-check-script', healthPatrolEnvCheckScript, '--health-patrol-expected-public-base-url', 'https://nginx.example.com', '--health-patrol-expected-public-host', 'nginx.example.com', '--no-status', ]); assertStatus( healthPatrolHostResult, 0, '带显式 health patrol public Host 的 dry-run 应成功退出。', ); assertIncludes( healthPatrolHostResult.stdout, `+ node -- ${healthPatrolEnvCheckScript} --env-file ${healthPatrolEnvFile} --expected-gateway-mode nginx --expected-public-base-url https://nginx.example.com --expected-public-host nginx.example.com`, 'dry-run 必须展示显式 public Host 的 health patrol env 回退后复核命令。', ); const shadowProbeResult = runRollback([ '--dropin-path', dropinPath, '--service', 'genarrative-pingora-gateway.service', '--reload-nginx', '--nginx-service', 'nginx.service', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--pingora-shadow-probe-url', 'http://127.0.0.1:18081/__genarrative_pingora/healthz', '--pingora-shadow-probe-token', 'test-shadow-probe-token', '--no-status', ]); assertStatus( shadowProbeResult, 0, '带 Pingora shadow probe 的 dry-run 应成功退出。', ); assertIncludes( shadowProbeResult.stdout, '+ curl --fail --silent --show-error --max-time 5 -H X-Genarrative-Pingora-Probe: http://127.0.0.1:18081/__genarrative_pingora/healthz', 'dry-run 必须展示 Pingora shadow probe 命令且隐藏 token。', ); if (shadowProbeResult.stdout.includes('test-shadow-probe-token')) { failures.push('dry-run Pingora shadow probe 命令不应输出 probe token 原文。'); } assertIncludes( shadowProbeResult.stdout, 'dry-run:--apply 后会验证 Pingora shadow 探针仍为 gateway=pingora-shadow。', 'dry-run 必须说明 apply 后会复核 Pingora shadow probe。', ); assertIncludes( result.stdout, '+ systemctl cat genarrative-pingora-gateway.service', 'dry-run 必须展示回退后 systemd drop-in 移除核验。', ); assertIncludes( result.stdout, '+ systemctl show genarrative-pingora-gateway.service --property=ExecStart --value --no-pager', 'dry-run 必须展示回退后 ExecStart 指向核验。', ); assertIncludes( result.stdout, 'dry-run:--apply 后会校验 systemd 最终配置不再包含 CAP_NET_BIND_SERVICE。', 'dry-run 必须说明 apply 后会核验 capability 已移除。', ); assertIncludes( result.stdout, 'dry-run:--apply 后会核验 systemd ExecStart 指向主 service 模板中的 /opt/genarrative/current/pingora-gateway。', 'dry-run 必须说明 apply 后会核验 Pingora ExecStart 指向 current release。', ); assertIncludes( result.stdout, 'dry-run:--apply --reload-nginx 后会 reload Nginx 并校验 service 仍为 active。', 'dry-run 必须说明 apply reload 后会核验 Nginx active。', ); if (!existsSync(dropinPath)) { failures.push('dry-run 不应删除临时 drop-in 文件。'); } } function assertDryRunMissingDropin() { const dropinPath = path.join(tmpRoot, 'missing-direct-entry.conf'); const result = runRollback(['--dropin-path', dropinPath, '--no-status']); assertStatus(result, 0, 'drop-in 不存在时 dry-run 应视为已回退。'); assertIncludes( result.stdout, 'direct-entry drop-in 不存在,视为已移除', 'drop-in 不存在时必须给出清晰提示。', ); assertIncludes( result.stdout, '+ systemctl daemon-reload', 'drop-in 不存在时仍应 reload systemd。', ); assertIncludes( result.stdout, '+ systemctl restart genarrative-pingora-gateway.service', 'drop-in 不存在时仍应重启 Pingora 进入 shadow 口径。', ); } function assertRelativeDropinRejected() { const result = runRollback([ '--dropin-path', 'relative/direct-entry.conf', '--no-status', ]); if (result.status === 0) { failures.push('相对路径 drop-in 必须被拒绝。'); } assertIncludes( result.stderr, '--dropin-path 必须是绝对路径', '相对路径负例必须说明需要绝对路径。', ); } function assertRelativeServiceUnitPathRejected() { const dropinPath = path.join(tmpRoot, 'relative-service-unit-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--service-unit-path', 'relative/genarrative-pingora-gateway.service', '--dropin-path', dropinPath, '--no-status', ]); if (result.status === 0) { failures.push('相对路径 --service-unit-path 必须被拒绝。'); } assertIncludes( result.stderr, '--service-unit-path 必须是绝对路径', 'service unit 相对路径负例必须说明需要绝对路径。', ); if (!existsSync(dropinPath)) { failures.push('service unit 相对路径负例不应删除 drop-in。'); } } function assertRejectsRelativeNginxBinaryPath() { const dropinPath = path.join(tmpRoot, 'relative-nginx-binary-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', './nginx', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('相对路径 --nginx-binary 必须被拒绝。'); } assertIncludes( result.stderr, '--nginx-binary 包含路径分隔符时必须是绝对路径', 'Nginx 二进制相对路径负例必须说明需要绝对路径。', ); if (!existsSync(dropinPath)) { failures.push('Nginx 二进制相对路径负例不应删除 drop-in。'); } } function assertRejectsRelativeCurlBinaryPath() { const dropinPath = path.join(tmpRoot, 'relative-curl-binary-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--curl-binary', 'tools/curl', '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('相对路径 --curl-binary 必须被拒绝。'); } assertIncludes( result.stderr, '--curl-binary 包含路径分隔符时必须是绝对路径', 'curl 二进制相对路径负例必须说明需要绝对路径。', ); if (!existsSync(dropinPath)) { failures.push('curl 二进制相对路径负例不应删除 drop-in。'); } } function assertRejectsFilesystemRootPathsBeforeApply() { const nginxBinary = path.join(tmpRoot, 'fake-nginx-should-not-run-root.sh'); const dropinPath = path.join(tmpRoot, 'root-path-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "nginx should not run for root path" >&2\nexit 99\n', 'utf8', ); chmodExecutable(nginxBinary); const rootDropinResult = runRollback([ '--apply', '--reload-nginx', '--dropin-path', '/', '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (rootDropinResult.status === 0) { failures.push('文件系统根目录 --dropin-path 必须被拒绝。'); } assertIncludes( rootDropinResult.stderr, '--dropin-path 不能是文件系统根目录', '文件系统根目录 drop-in 路径负例必须给出明确错误。', ); if (rootDropinResult.stderr.includes('nginx should not run for root path')) { failures.push('文件系统根目录 drop-in 被拒绝后不应继续执行 nginx -t。'); } const rootHealthPatrolEnvResult = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', '/', '--no-status', ]); if (rootHealthPatrolEnvResult.status === 0) { failures.push('文件系统根目录 --health-patrol-env-file 必须被拒绝。'); } assertIncludes( rootHealthPatrolEnvResult.stderr, '--health-patrol-env-file 不能是文件系统根目录', '文件系统根目录 health patrol env 路径负例必须给出明确错误。', ); if (rootHealthPatrolEnvResult.stderr.includes('nginx should not run for root path')) { failures.push('文件系统根目录 health patrol env 被拒绝后不应继续执行 nginx -t。'); } if (!existsSync(dropinPath)) { failures.push('文件系统根目录 health patrol env 被拒绝后不应删除 drop-in。'); } } function assertApplyRequiresReloadNginx() { const dropinPath = path.join(tmpRoot, 'apply-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--dropin-path', dropinPath, '--no-status', ]); if (result.status === 0) { failures.push('--apply 缺少 --reload-nginx 必须被拒绝。'); } assertIncludes( result.stderr, '--apply 必须同时提供 --reload-nginx', 'apply 负例必须说明不能跳过 Nginx reload。', ); if (!existsSync(dropinPath)) { failures.push('--apply 缺少 --reload-nginx 时不应删除 drop-in。'); } } function assertApplyRequiresNginxSmokeUrl() { const dropinPath = path.join(tmpRoot, 'apply-smoke-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--no-status', ]); if (result.status === 0) { failures.push('--apply 缺少 --nginx-smoke-url 必须被拒绝。'); } assertIncludes( result.stderr, '--apply 必须同时提供 --nginx-smoke-url', 'apply 负例必须说明不能跳过 Nginx smoke。', ); if (!existsSync(dropinPath)) { failures.push('--apply 缺少 --nginx-smoke-url 时不应删除 drop-in。'); } } function assertRejectsInvalidNginxSmokeUrl() { const dropinPath = path.join(tmpRoot, 'invalid-smoke-url-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', '127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('非法 --nginx-smoke-url 必须被拒绝。'); } assertIncludes( result.stderr, '--nginx-smoke-url 必须是 http(s) URL', '非法 Nginx smoke URL 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push('非法 Nginx smoke URL 时不应删除 drop-in。'); } } function assertApplyRequiresHostForLoopbackSmokeUrl() { const dropinPath = path.join(tmpRoot, 'apply-smoke-host-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--no-status', ]); if (result.status === 0) { failures.push( '--apply 使用 loopback --nginx-smoke-url 时缺少 --nginx-smoke-host 必须被拒绝。', ); } assertIncludes( result.stderr, '--nginx-smoke-url 指向本机地址时必须同时提供 --nginx-smoke-host', 'loopback Nginx smoke 负例必须说明需要正式 Host。', ); if (!existsSync(dropinPath)) { failures.push('缺少 loopback smoke Host 时不应删除 drop-in。'); } } function assertRejectsInvalidNginxSmokeHost() { const dropinPath = path.join(tmpRoot, 'invalid-smoke-host-direct-entry.conf'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'https://example.com/path', '--no-status', ]); if (result.status === 0) { failures.push('非法 --nginx-smoke-host 必须被拒绝。'); } assertIncludes( result.stderr, '--nginx-smoke-host 只能是 host 或 host:port', '非法 Nginx smoke Host 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push('非法 smoke Host 时不应删除 drop-in。'); } } function assertRejectsRelativeHealthPatrolEnvFile() { const dropinPath = path.join( tmpRoot, 'relative-health-patrol-env-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', 'relative/health-patrol.env', '--no-status', ]); if (result.status === 0) { failures.push('相对路径 --health-patrol-env-file 必须被拒绝。'); } assertIncludes( result.stderr, '--health-patrol-env-file 必须是绝对路径', 'health patrol env 相对路径负例必须说明需要绝对路径。', ); if (!existsSync(dropinPath)) { failures.push('health patrol env 相对路径负例不应删除 drop-in。'); } } function assertRejectsRelativeHealthPatrolEnvCheckScript() { const dropinPath = path.join( tmpRoot, 'relative-health-patrol-check-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', path.join(tmpRoot, 'health-patrol.env'), '--health-patrol-env-check-script', 'relative/check-production-health-patrol-env.mjs', '--no-status', ]); if (result.status === 0) { failures.push('相对路径 --health-patrol-env-check-script 必须被拒绝。'); } assertIncludes( result.stderr, '--health-patrol-env-check-script 必须是绝对路径', 'health patrol env check script 相对路径负例必须说明需要绝对路径。', ); if (!existsSync(dropinPath)) { failures.push('health patrol check script 相对路径负例不应删除 drop-in。'); } } function assertRejectsConflictingHealthPatrolPublicHostOptions() { const dropinPath = path.join( tmpRoot, 'conflicting-health-patrol-host-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', path.join(tmpRoot, 'health-patrol.env'), '--health-patrol-expected-public-host', 'nginx.example.com', '--health-patrol-require-empty-public-host', '--no-status', ]); if (result.status === 0) { failures.push( '显式 public Host 与 require empty public Host 同时提供时必须被拒绝。', ); } assertIncludes( result.stderr, '--health-patrol-expected-public-host 和 --health-patrol-require-empty-public-host 不能同时使用', 'health patrol public Host 冲突参数必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push('health patrol public Host 冲突参数时不应删除 drop-in。'); } } function assertRejectsInvalidHealthPatrolExpectedPublicBaseUrl() { const dropinPath = path.join( tmpRoot, 'invalid-health-patrol-public-base-url-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', path.join(tmpRoot, 'health-patrol.env'), '--health-patrol-expected-public-base-url', '127.0.0.1', '--no-status', ]); if (result.status === 0) { failures.push('非法 health patrol expected public base URL 必须被拒绝。'); } assertIncludes( result.stderr, '--health-patrol-expected-public-base-url 必须是 http(s) URL', '非法 health patrol expected public base URL 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push( '非法 health patrol expected public base URL 时不应删除 drop-in。', ); } } function assertRejectsInvalidHealthPatrolExpectedPublicHost() { const dropinPath = path.join( tmpRoot, 'invalid-health-patrol-public-host-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', path.join(tmpRoot, 'health-patrol.env'), '--health-patrol-expected-public-host', 'https://nginx.example.com/path', '--no-status', ]); if (result.status === 0) { failures.push('非法 health patrol expected public Host 必须被拒绝。'); } assertIncludes( result.stderr, '--health-patrol-expected-public-host 只能是 host 或 host:port', '非法 health patrol expected public Host 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push( '非法 health patrol expected public Host 时不应删除 drop-in。', ); } } function assertRejectsIncompletePingoraShadowProbe() { const dropinPath = path.join( tmpRoot, 'incomplete-shadow-probe-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const missingToken = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--pingora-shadow-probe-url', 'http://127.0.0.1:18081/__genarrative_pingora/healthz', '--no-status', ]); if (missingToken.status === 0) { failures.push('shadow probe URL 缺少 token 必须被拒绝。'); } assertIncludes( missingToken.stderr, '--pingora-shadow-probe-url 必须同时提供 --pingora-shadow-probe-token', 'shadow probe URL 缺少 token 负例必须给出明确错误。', ); const missingUrl = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--pingora-shadow-probe-token', 'test-shadow-probe-token', '--no-status', ]); if (missingUrl.status === 0) { failures.push('shadow probe token 缺少 URL 必须被拒绝。'); } assertIncludes( missingUrl.stderr, '--pingora-shadow-probe-token 必须同时提供 --pingora-shadow-probe-url', 'shadow probe token 缺少 URL 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push('shadow probe 参数不完整时不应删除 drop-in。'); } } function assertRejectsInvalidPingoraShadowProbeUrl() { const dropinPath = path.join( tmpRoot, 'invalid-shadow-probe-url-direct-entry.conf', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--pingora-shadow-probe-url', '127.0.0.1:18081/__genarrative_pingora/healthz', '--pingora-shadow-probe-token', 'test-shadow-probe-token', '--no-status', ]); if (result.status === 0) { failures.push('非法 --pingora-shadow-probe-url 必须被拒绝。'); } assertIncludes( result.stderr, '--pingora-shadow-probe-url 必须是 http(s) URL', '非法 shadow probe URL 负例必须给出明确错误。', ); if (!existsSync(dropinPath)) { failures.push('非法 shadow probe URL 时不应删除 drop-in。'); } } function assertRejectsControlCharacterInputsBeforeApply() { const dropinPath = path.join(tmpRoot, 'control-character-direct-entry.conf'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-should-not-run-control.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "nginx should not run" >&2\nexit 99\n', 'utf8', ); chmodExecutable(nginxBinary); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com\ninjected.example.com', '--no-status', ]); if (result.status === 0) { failures.push('带换行的 Nginx smoke Host 必须在 rollback apply 修改系统前被拒绝。'); } assertIncludes( result.stderr, '--nginx-smoke-host 不能包含换行或 NUL 字符', 'Nginx smoke Host 控制字符负例必须给出明确错误。', ); if (result.stderr.includes('nginx should not run')) { failures.push('带换行的 Nginx smoke Host 被拒绝后不应继续执行 nginx -t。'); } if (!existsSync(dropinPath)) { failures.push('带换行的 Nginx smoke Host 被拒绝后不应删除 drop-in。'); } } function assertApplyRejectsSymlinkDropinDirectoryBeforeNginxTest() { const realDropinDir = path.join(tmpRoot, 'rollback-real-service.d'); const symlinkDropinDir = path.join(tmpRoot, 'rollback-linked-service.d'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-should-not-run-dir.sh'); mkdirSync(realDropinDir, { recursive: true }); symlinkSync(realDropinDir, symlinkDropinDir); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "nginx should not run" >&2\nexit 99\n', 'utf8', ); chmodExecutable(nginxBinary); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', path.join(symlinkDropinDir, 'direct-entry.conf'), '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('drop-in 目录是符号链接时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'drop-in 目录不能是符号链接', 'rollback drop-in 符号链接目录负例必须给出明确错误。', ); if (result.stderr.includes('nginx should not run')) { failures.push('drop-in 目录是符号链接时不应继续执行 nginx -t。'); } if (existsSync(path.join(realDropinDir, 'direct-entry.conf'))) { failures.push('drop-in 目录是符号链接时不应删除或写入真实目标目录。'); } } function assertApplyRejectsSymlinkDropinFileBeforeRemoval() { const dropinDir = path.join(tmpRoot, 'rollback-symlink-file-service.d'); const realDropinFile = path.join(tmpRoot, 'rollback-real-direct-entry.conf'); const symlinkDropinFile = path.join(dropinDir, 'direct-entry.conf'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-should-not-run-file.sh'); mkdirSync(dropinDir, { recursive: true }); writeFileSync(realDropinFile, 'original rollback dropin\n', 'utf8'); symlinkSync(realDropinFile, symlinkDropinFile); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "nginx should not run" >&2\nexit 99\n', 'utf8', ); chmodExecutable(nginxBinary); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', symlinkDropinFile, '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('drop-in 目标是符号链接时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'drop-in 目标不能是符号链接', 'rollback drop-in 符号链接目标负例必须给出明确错误。', ); if (result.stderr.includes('nginx should not run')) { failures.push('drop-in 目标是符号链接时不应继续执行 nginx -t。'); } const realContent = readFileSync(realDropinFile, 'utf8'); if (realContent !== 'original rollback dropin\n') { failures.push('drop-in 目标是符号链接时不应改写真实目标文件。'); } } function assertApplyRejectsDirectoryDropinTargetBeforeNginxTest() { const dropinDir = path.join(tmpRoot, 'rollback-directory-target-service.d'); const dropinPath = path.join(dropinDir, 'direct-entry.conf'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-should-not-run-target.sh'); mkdirSync(dropinPath, { recursive: true }); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "nginx should not run" >&2\nexit 99\n', 'utf8', ); chmodExecutable(nginxBinary); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('drop-in 目标是目录时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'drop-in 目标已存在但不是普通文件', 'rollback drop-in 目录目标负例必须给出明确错误。', ); if (result.stderr.includes('nginx should not run')) { failures.push('drop-in 目标是目录时不应继续执行 nginx -t。'); } if (!existsSync(dropinPath)) { failures.push('drop-in 目标是目录时不应删除该目录。'); } } function assertApplyStopsBeforeRemovingDropinWhenNginxTestFails() { const dropinPath = path.join(tmpRoot, 'nginx-test-fail-direct-entry.conf'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-fail.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync( nginxBinary, '#!/usr/bin/env bash\necho "fake nginx -t failed" >&2\nexit 1\n', 'utf8', ); chmodExecutable(nginxBinary); const result = runRollback([ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ]); if (result.status === 0) { failures.push('nginx -t 失败时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'fake nginx -t failed', 'nginx -t 失败时必须暴露 Nginx 检查错误。', ); if (!existsSync(dropinPath)) { failures.push('nginx -t 失败时不应删除 drop-in。'); } } function assertApplyFailsWhenNginxSmokeFails() { const dropinPath = path.join(tmpRoot, 'nginx-smoke-fail-direct-entry.conf'); const nginxBinary = path.join(tmpRoot, 'fake-nginx-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-fail.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, '#!/usr/bin/env bash\necho "fake nginx smoke failed" >&2\nexit 22\n', 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push('Nginx smoke 失败时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'fake nginx smoke failed', 'Nginx smoke 失败时必须暴露 curl 错误。', ); } function assertApplyFailsWhenNginxSmokeBodyMismatches() { const dropinPath = path.join( tmpRoot, 'nginx-smoke-body-mismatch-direct-entry.conf', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-body-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-body-mismatch.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, '#!/usr/bin/env bash\necho \'{"ok":false,"service":"unexpected"}\'\n', 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--nginx-smoke-expect-body', '"ok":true', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push('Nginx smoke 响应体不匹配时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'Nginx smoke 响应缺少预期片段: "ok":true', 'Nginx smoke 响应体不匹配时必须暴露明确错误。', ); assertIncludes( result.stderr, 'Nginx smoke 响应前 500 字符', 'Nginx smoke 响应体不匹配时必须输出截断响应便于排障。', ); } function assertApplyFailsWhenHealthPatrolEnvStillDirect() { const dropinPath = path.join( tmpRoot, 'health-patrol-direct-leftover-direct-entry.conf', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-health-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-health-ok.sh'); const healthPatrolEnvFile = path.join( tmpRoot, 'health-patrol-direct-leftover.env', ); const healthPatrolEnvCheckScript = path.join( process.cwd(), 'scripts/check-production-health-patrol-env.mjs', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, '#!/usr/bin/env bash\necho \'{"ok":true,"service":"genarrative-api-server"}\'\n', 'utf8', ); writeFileSync( healthPatrolEnvFile, [ 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=pingora-direct', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://127.0.0.1', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=example.com', '', ].join('\n'), 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', healthPatrolEnvFile, '--health-patrol-env-check-script', healthPatrolEnvCheckScript, '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push( 'health patrol env 仍停留在 pingora-direct 时 rollback apply 必须失败。', ); } assertIncludes( result.stderr, 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 nginx,实际 pingora-direct', 'health patrol env 模式错误时必须暴露明确错误。', ); assertIncludes( result.stderr, 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空,实际 example.com', 'health patrol env Host 残留时必须暴露明确错误。', ); } function assertApplyFailsWhenHealthPatrolPublicBaseUrlDrifts() { const dropinPath = path.join( tmpRoot, 'health-patrol-base-url-drift-direct-entry.conf', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-base-url-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-base-url-ok.sh'); const healthPatrolEnvFile = path.join( tmpRoot, 'health-patrol-base-url-drift.env', ); const healthPatrolEnvCheckScript = path.join( process.cwd(), 'scripts/check-production-health-patrol-env.mjs', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, '#!/usr/bin/env bash\necho \'{"ok":true,"service":"genarrative-api-server"}\'\n', 'utf8', ); writeFileSync( healthPatrolEnvFile, [ 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://wrong.example.com', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=', '', ].join('\n'), 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', healthPatrolEnvFile, '--health-patrol-env-check-script', healthPatrolEnvCheckScript, '--health-patrol-expected-public-base-url', 'https://nginx.example.com', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push( 'health patrol public base URL 漂移时 rollback apply 必须失败。', ); } assertIncludes( result.stderr, 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 https://nginx.example.com,实际 https://wrong.example.com', 'health patrol public base URL 漂移时必须暴露明确错误。', ); } function assertApplySupportsExpectedHealthPatrolPublicHost() { const dropinPath = path.join( tmpRoot, 'health-patrol-expected-host-direct-entry.conf', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-expected-host-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-expected-host-ok.sh'); const healthPatrolEnvFile = path.join( tmpRoot, 'health-patrol-expected-host.env', ); const healthPatrolEnvCheckScript = path.join( process.cwd(), 'scripts/check-production-health-patrol-env.mjs', ); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, '#!/usr/bin/env bash\necho \'{"ok":true,"service":"genarrative-api-server"}\'\n', 'utf8', ); writeFileSync( healthPatrolEnvFile, [ 'GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE=nginx', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL=https://nginx.example.com', 'GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST=nginx.example.com', '', ].join('\n'), 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--health-patrol-env-file', healthPatrolEnvFile, '--health-patrol-env-check-script', healthPatrolEnvCheckScript, '--health-patrol-expected-public-base-url', 'https://nginx.example.com', '--health-patrol-expected-public-host', 'nginx.example.com', '--nginx-smoke-expect-body', '"ok":true', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); assertStatus( result, 0, 'health patrol public Host 与切换前记录一致时 rollback apply 应成功。', ); assertIncludes( result.stdout, '--expected-public-host nginx.example.com', 'rollback apply 必须把显式 health patrol public Host 传给复核脚本。', ); assertIncludes( result.stdout, '[check:production-health-patrol-env] OK', 'rollback apply 应执行 health patrol env 复核并通过。', ); assertIncludes( result.stdout, 'nginx smoke evidence matched expected body fragment.', 'rollback apply 应在 Nginx smoke body 匹配后输出证据摘要。', ); } function assertApplyFailsWhenSystemdExecStartDiffers() { const dropinPath = path.join( tmpRoot, 'systemd-exec-mismatch-direct-entry.conf', ); const serviceUnitPath = path.join( tmpRoot, 'systemd-exec-mismatch.service', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-exec-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-exec-ok.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync( serviceUnitPath, [ '[Service]', 'ExecStart=/opt/genarrative/current/pingora-gateway', '', ].join('\n'), 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/old-release/pingora-gateway ; argv[]=/opt/genarrative/old-release/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync(curlBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--service-unit-path', serviceUnitPath, '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push( 'systemctl show ExecStart 指向旧 release 时 rollback apply 必须失败。', ); } assertIncludes( result.stderr, 'systemctl show ExecStart 未指向主 service 模板中的 /opt/genarrative/current/pingora-gateway', 'systemd 最终 ExecStart 指向旧 release 时必须给出明确错误。', ); } function assertApplyFailsWhenPingoraShadowProbeBodyIsWrong() { const dropinPath = path.join( tmpRoot, 'shadow-probe-body-wrong-direct-entry.conf', ); const nginxBinary = path.join(tmpRoot, 'fake-nginx-shadow-ok.sh'); const systemctlBinary = path.join(tmpRoot, 'systemctl'); const curlBinary = path.join(tmpRoot, 'fake-curl-shadow-body-wrong.sh'); writeFileSync( dropinPath, '[Service]\nAmbientCapabilities=CAP_NET_BIND_SERVICE\n', 'utf8', ); writeFileSync(nginxBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8'); writeFileSync( systemctlBinary, [ '#!/usr/bin/env bash', 'if [[ "$1" == "cat" ]]; then', ' echo "[Service]"', ' exit 0', 'fi', 'if [[ "$1" == "show" ]]; then', ' echo "{ path=/opt/genarrative/current/pingora-gateway ; argv[]=/opt/genarrative/current/pingora-gateway ; }"', ' exit 0', 'fi', 'if [[ "$1" == "is-active" ]]; then', ' echo active', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( curlBinary, [ '#!/usr/bin/env bash', 'last_arg="${@: -1}"', 'if [[ "${last_arg}" == *"__genarrative_pingora/healthz"* ]]; then', ' echo \'{"ok":true,"gateway":"not-pingora"}\'', ' exit 0', 'fi', 'exit 0', '', ].join('\n'), 'utf8', ); chmodExecutable(nginxBinary); chmodExecutable(systemctlBinary); chmodExecutable(curlBinary); const result = runRollback( [ '--apply', '--reload-nginx', '--dropin-path', dropinPath, '--nginx-binary', nginxBinary, '--curl-binary', curlBinary, '--nginx-smoke-url', 'http://127.0.0.1/healthz', '--nginx-smoke-host', 'example.com', '--pingora-shadow-probe-url', 'http://127.0.0.1:18081/__genarrative_pingora/healthz', '--pingora-shadow-probe-token', 'test-shadow-probe-token', '--no-status', ], { PATH: `${tmpRoot}:${process.env.PATH || ''}` }, ); if (result.status === 0) { failures.push('Pingora shadow probe body 错误时 rollback apply 必须失败。'); } assertIncludes( result.stderr, 'Pingora shadow 探针响应不是预期 JSON', 'Pingora shadow probe body 错误时必须暴露明确错误。', ); } function runRollback(args, env = {}) { return spawnSync('bash', [ROLLBACK_SCRIPT, ...args], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, ...env, }, }); } function chmodExecutable(filePath) { spawnSync('chmod', ['0755', filePath], { cwd: process.cwd(), encoding: 'utf8', }); } function assertStatus(result, expected, reason) { const actual = result.status ?? 0; if (actual !== expected) { failures.push( `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } } function assertIncludes(content, needle, reason) { if (!content.includes(needle)) { failures.push(`${reason} 缺少: ${needle}`); } }