Genarrative/scripts/check-production-api-release.mjs

#!/usr/bin/env node

import { spawnSync } from 'node:child_process';
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  rmSync,
  statSync,
  writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';

const BUILD_SCRIPT = 'scripts/build-production-release.sh';
const failures = [];
const tmpRoot = mkdtempSync(
  path.join(tmpdir(), 'genarrative-production-api-release-'),
);

try {
  main();
} finally {
  rmSync(tmpRoot, { recursive: true, force: true });
}

if (failures.length > 0) {
  console.error('[check:production-api-release] FAILED');
  for (const failure of failures) {
    console.error(`- ${failure}`);
  }
  process.exit(1);
}

console.log('[check:production-api-release] OK');

function main() {
  assertApiReleaseContainsPingoraDirectDependencies();
  assertApiReleaseCanIncludePingoraGatewayArtifact();
}

function assertApiReleaseContainsPingoraDirectDependencies() {
  const fixture = prepareFixture('api-direct-dependencies');
  const result = runBuild(fixture);
  assertStatus(result, 0, 'API release build 烟测应成功。');
  if (result.status !== 0) {
    return;
  }

  const releaseDir = path.join(process.cwd(), 'build', fixture.version);
  try {
    assertFileExists(
      path.join(releaseDir, 'api-server'),
      'API release 必须包含 api-server。',
    );
    assertExecutable(
      path.join(releaseDir, 'api-server'),
      'api-server 必须保留可执行权限。',
    );
    assertFileExists(
      path.join(releaseDir, 'api-server.sha256'),
      'API release 必须包含 api-server checksum。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'),
      'API release 必须包含随产物执行的 API deploy 脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/production-api-deploy.sh'),
      '随产物执行的 API deploy 脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'),
      'API release 必须在 deploy 脚本同目录包含 maintenance-on.sh。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/maintenance-on.sh'),
      'deploy 同目录 maintenance-on.sh 必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'),
      'API release 必须在 deploy 脚本同目录包含 maintenance-off.sh。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/maintenance-off.sh'),
      'deploy 同目录 maintenance-off.sh 必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'),
      'API release 必须包含 Pingora 直连启用脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/pingora-direct-enable.sh'),
      'Pingora 直连启用脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'),
      'API release 必须包含 Pingora 直连回退脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/pingora-direct-rollback.sh'),
      'Pingora 直连回退脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'),
      'API release 必须包含 Pingora realpath canary 启用脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-enable.sh'),
      'Pingora realpath canary 启用脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'),
      'API release 必须包含 Pingora realpath canary 关闭脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/pingora-realpath-canary-disable.sh'),
      'Pingora realpath canary 关闭脚本必须可执行。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/deploy/pingora-health-patrol-env-switch.mjs',
      ),
      'API release 必须包含 Pingora health patrol env 切换脚本。',
    );
    assertExecutable(
      path.join(
        releaseDir,
        'scripts/deploy/pingora-health-patrol-env-switch.mjs',
      ),
      'Pingora health patrol env 切换脚本必须可执行。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
      ),
      'API release 必须包含 Pingora gateway env shadow 切换脚本。',
    );
    assertExecutable(
      path.join(
        releaseDir,
        'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
      ),
      'Pingora gateway env shadow 切换脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'),
      'API release 必须包含 Pingora TLS 证书同步脚本。',
    );
    assertExecutable(
      path.join(releaseDir, 'scripts/deploy/pingora-tls-cert-sync.mjs'),
      'Pingora TLS 证书同步脚本必须可执行。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/check-production-health-patrol-env.mjs'),
      'API release 必须包含健康巡检 env 复核脚本。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/check-pingora-release-readiness.mjs'),
      'API release 必须包含 Pingora release readiness 聚合门禁脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-current-release-audit.mjs',
      ),
      'API release 必须包含 Pingora current release 自审脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-direct-rehearsal-status.mjs',
      ),
      'API release 必须包含 Pingora 直连彩排状态脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-cutover-status-snapshot.mjs',
      ),
      'API release 必须包含 Pingora 直连切换状态快照脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-cutover-evidence-bundle.mjs',
      ),
      'API release 必须包含 Pingora 直连切换证据包脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-cutover-command-evidence.mjs',
      ),
      'API release 必须包含 Pingora 直连切换命令证据脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-cutover-evidence-verify.mjs',
      ),
      'API release 必须包含 Pingora 直连切换证据验真脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/ops/pingora-cutover-evidence-audit.mjs',
      ),
      'API release 必须包含 Pingora 直连切换证据根目录审计脚本。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/check-pingora-direct-preflight.mjs'),
      'API release 必须包含 Pingora 直连预检脚本。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/check-pingora-direct-live.mjs'),
      'API release 必须包含 Pingora 直连 live smoke 脚本。',
    );
    assertFileExists(
      path.join(releaseDir, 'scripts/check-pingora-canary-live.mjs'),
      'API release 必须包含 Pingora canary live smoke 脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'scripts/check-pingora-canary-access-log-parity.mjs',
      ),
      'API release 必须包含 Pingora canary access log 对账脚本。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-pingora-gateway.service',
      ),
      'API release 必须包含 Pingora systemd 主 service 模板。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf',
      ),
      'API release 必须包含 Pingora 直连 drop-in 模板。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-external-generation-worker@.service',
      ),
      'API release 必须包含外部生成 worker systemd 模板。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-external-generation-controller.service',
      ),
      'API release 必须包含外部生成 worker controller systemd 单元。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-bgfilter-worker.service',
      ),
      'API release 必须包含唯一 BgFilter worker systemd 单元。',
    );
    assertFileExists(
      path.join(releaseDir, 'deploy/env/bgfilter-worker.env.example'),
      'API release 必须包含 BgFilter worker env 示例。',
    );
    const bgfilterUnit = readFileSync(
      path.join(
        releaseDir,
        'deploy/systemd/genarrative-bgfilter-worker.service',
      ),
      'utf8',
    );
    const sharedEnvIndex = bgfilterUnit.indexOf(
      'EnvironmentFile=/etc/genarrative/api-server.env',
    );
    const dedicatedEnvIndex = bgfilterUnit.indexOf(
      'EnvironmentFile=/etc/genarrative/bgfilter-worker.env',
    );
    if (
      sharedEnvIndex < 0 ||
      dedicatedEnvIndex < 0 ||
      sharedEnvIndex > dedicatedEnvIndex
    ) {
      failures.push('API release 的 BgFilter unit 必须按共享 env → 专属 env 加载。');
    }
    assertIncludes(
      bgfilterUnit,
      'TimeoutStopSec=900',
      'API release 的 BgFilter unit 必须给取得 permit 后的公式化 callBudget 留足优雅排空时间。',
    );
    assertFileExists(
      path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'),
      'API release 必须包含 Pingora env 示例。',
    );
    assertFileExists(
      path.join(
        releaseDir,
        'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf',
      ),
      'API release 必须包含 Pingora 真实路径 canary Nginx snippet。',
    );
    assertFileExists(
      path.join(releaseDir, 'deploy/env/health-patrol.env.example'),
      'API release 必须包含健康巡检 env 示例。',
    );
    assertFileExists(
      path.join(releaseDir, 'deploy/env/pingora-direct-live.env.example'),
      'API release 必须包含 Pingora direct live env 示例。',
    );
    assertFileExists(
      path.join(releaseDir, 'deploy/env/pingora-canary-live.env.example'),
      'API release 必须包含 Pingora canary live env 示例。',
    );
    assertPingoraEnvProductionDefaults(
      path.join(releaseDir, 'deploy/pingora/pingora-gateway.env.example'),
      'API release 内 Pingora env 示例',
    );

    const manifest = readJson(path.join(releaseDir, 'release-manifest.json'));
    if (manifest.component_type !== 'api-server') {
      failures.push(
        `release manifest component_type 应为 api-server，实际 ${manifest.component_type}`,
      );
    }
    if (!manifest.artifacts?.some((item) => item.path === 'api-server')) {
      failures.push('release manifest 必须登记 api-server artifact。');
    }
    if (manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) {
      failures.push('默认 API release 不应登记 pingora-gateway artifact。');
    }

    const releaseReadme = readFileSync(
      path.join(releaseDir, 'README.md'),
      'utf8',
    );
    assertIncludes(
      releaseReadme,
      'Pingora release readiness 聚合门禁、直连启用 / 回退',
      'API release README 必须说明随包携带 Pingora 直连脚本。',
    );
    assertIncludes(
      releaseReadme,
      'realpath canary 启用 / 关闭',
      'API release README 必须说明随包携带 Pingora realpath canary 启停脚本。',
    );
    assertIncludes(
      releaseReadme,
      'health patrol env 切换 / TLS 证书同步 / 预检 / direct live smoke / canary live smoke / canary access log 对账 / current release 自审 / 直连彩排状态',
      'API release README 必须说明随包携带 Pingora 运行态复核脚本。',
    );
    assertIncludes(
      releaseReadme,
      '状态快照 / 证据包 / 命令证据 / 证据验真 / 证据根目录审计脚本',
      'API release README 必须说明随包携带 Pingora 切换证据脚本。',
    );
    for (const executableArg of requiredCutoverAuditExecutableArgs()) {
      assertIncludes(
        releaseReadme,
        `--require-command-executable ${executableArg}`,
        'API release README 必须记录最终证据根目录总审计绑定 current release 切换脚本。',
      );
    }
    assertIncludes(
      releaseReadme,
      'scripts/deploy/production-api-deploy.sh',
      'API release README 必须说明随包携带 API Deploy 执行入口。',
    );
    assertIncludes(
      releaseReadme,
      '同目录的 `maintenance-on.sh` / `maintenance-off.sh` 必须来自同一发布包',
      'API release README 必须说明 deploy 同目录维护脚本来自同一发布包。',
    );
    assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir);
  } finally {
    rmSync(releaseDir, { recursive: true, force: true });
  }
}

function assertApiReleaseCanIncludePingoraGatewayArtifact() {
  const fixture = prepareFixture('api-with-pingora-artifact', {
    includePingoraGateway: true,
  });
  const result = runBuild(fixture, {
    includePingoraGateway: true,
    skipPingoraGatewayBuild: true,
  });
  assertStatus(result, 0, '包含 Pingora 的 API release build 烟测应成功。');
  if (result.status !== 0) {
    return;
  }

  const releaseDir = path.join(process.cwd(), 'build', fixture.version);
  try {
    assertFileExists(
      path.join(releaseDir, 'pingora-gateway'),
      '显式 include Pingora 时 API release 必须包含 pingora-gateway。',
    );
    assertExecutable(
      path.join(releaseDir, 'pingora-gateway'),
      'pingora-gateway 必须保留可执行权限。',
    );
    assertFileExists(
      path.join(releaseDir, 'pingora-gateway.sha256'),
      '显式 include Pingora 时 API release 必须包含 pingora-gateway checksum。',
    );

    const manifest = readJson(path.join(releaseDir, 'release-manifest.json'));
    if (!manifest.artifacts?.some((item) => item.path === 'pingora-gateway')) {
      failures.push(
        '显式 include Pingora 时 release manifest 必须登记 pingora-gateway artifact。',
      );
    }
  } finally {
    rmSync(releaseDir, { recursive: true, force: true });
  }
}

function prepareFixture(name, options = {}) {
  const cargoTargetDir = path.join(tmpRoot, name, 'cargo-target');
  const binaryDir = path.join(
    cargoTargetDir,
    'x86_64-unknown-linux-gnu/release',
  );
  const version = `check-production-api-release-${process.pid}-${Date.now()}`;
  mkdirSync(binaryDir, { recursive: true });
  const apiBinary = path.join(binaryDir, 'api-server');
  writeFileSync(apiBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8');
  spawnSync('chmod', ['0755', apiBinary], { encoding: 'utf8' });
  if (options.includePingoraGateway) {
    const pingoraBinary = path.join(binaryDir, 'pingora-gateway');
    writeFileSync(pingoraBinary, '#!/usr/bin/env bash\nexit 0\n', 'utf8');
    spawnSync('chmod', ['0755', pingoraBinary], { encoding: 'utf8' });
  }
  return { cargoTargetDir, version };
}

function runBuild(fixture, options = {}) {
  const args = [
    BUILD_SCRIPT,
    '--component',
    'api-server',
    '--name',
    fixture.version,
    '--skip-api-build',
  ];
  if (options.includePingoraGateway) {
    args.push('--include-pingora-gateway');
  }
  if (options.skipPingoraGatewayBuild) {
    args.push('--skip-pingora-gateway-build');
  }

  return spawnSync('bash', args, {
    cwd: process.cwd(),
    encoding: 'utf8',
    env: {
      ...process.env,
      CARGO_TARGET_DIR: fixture.cargoTargetDir,
      SOURCE_BRANCH: 'test-branch',
      SOURCE_COMMIT: 'test-commit',
    },
  });
}

function readJson(filePath) {
  try {
    return JSON.parse(readFileSync(filePath, 'utf8'));
  } catch (error) {
    failures.push(`${filePath} 不是合法 JSON: ${error.message}`);
    return {};
  }
}

function assertFileExists(filePath, reason) {
  if (!existsSync(filePath)) {
    failures.push(`${reason} 缺少: ${filePath}`);
  }
}

function assertExecutable(filePath, reason) {
  if (!existsSync(filePath)) {
    return;
  }
  if ((statSync(filePath).mode & 0o111) === 0) {
    failures.push(`${reason} 文件不可执行: ${filePath}`);
  }
}

function assertStatus(result, expected, reason) {
  const actual = result.status ?? 0;
  if (actual !== expected) {
    failures.push(
      `${reason} 预期退出码 ${expected}，实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
    );
  }
}

function assertIncludes(content, needle, reason) {
  if (!content.includes(needle)) {
    failures.push(`${reason} 缺少: ${needle}`);
  }
}

function assertReleaseReadinessCutoverPlanBindsCurrentExecutables(releaseDir) {
  const readinessScript = path.join(
    releaseDir,
    'scripts/check-pingora-release-readiness.mjs',
  );
  const result = spawnSync(
    'node',
    [
      readinessScript,
      '--dry-run-cutover',
      '--require-direct',
      '--direct-https-base-url',
      'https://127.0.0.1',
      '--direct-http-base-url',
      'http://127.0.0.1',
      '--direct-host',
      'example.com',
      '--direct-redirect-host',
      'example.com',
      '--direct-spacetime-database',
      'genarrative-prod',
      '--direct-pingora-access-log',
      '/var/log/genarrative/pingora-gateway.access.log',
      '--direct-health-patrol-env-file',
      '/etc/genarrative/health-patrol.env',
      '--direct-preflight-env-file',
      '/etc/genarrative/pingora-gateway.env',
      '--direct-preflight-systemd',
      '--direct-preflight-check-cert-readable',
      '--direct-preflight-check-service-env-file',
      '--direct-preflight-check-service-user-cert-readable',
      '--direct-preflight-check-service-binary-executable',
      '--direct-preflight-check-ports-free',
      '--cutover-run-id',
      'check-production-api-release',
      '--rollback-nginx-smoke-url',
      'https://example.com/',
      '--rollback-nginx-smoke-expect-body',
      '<!doctype html>',
      '--rollback-health-patrol-public-base-url',
      'http://127.0.0.1',
    ],
    {
      cwd: releaseDir,
      encoding: 'utf8',
      env: {
        ...process.env,
        GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT: '',
      },
    },
  );
  assertStatus(
    result,
    0,
    'API release 随包 readiness 脚本必须能生成直连切换 dry-run runbook。',
  );
  if (result.status !== 0) {
    return;
  }

  let plan;
  try {
    plan = JSON.parse(result.stdout);
  } catch (error) {
    failures.push(
      `API release 随包 readiness dry-run cutover 输出不是合法 JSON: ${error.message}`,
    );
    return;
  }

  const auditStep = Array.isArray(plan)
    ? plan.find((step) => step?.name === '切换证据根目录三阶段总审计')
    : null;
  if (!auditStep) {
    failures.push(
      'API release 随包 readiness dry-run cutover 必须包含切换证据根目录三阶段总审计步骤。',
    );
    return;
  }

  if (!auditStep.args?.includes('--require-command-executable')) {
    failures.push(
      'API release 随包 readiness 总审计步骤必须携带 --require-command-executable。',
    );
  }
  for (const executableArg of requiredCutoverAuditExecutableArgs()) {
    if (!auditStep.args?.includes(executableArg)) {
      failures.push(
        `API release 随包 readiness 总审计步骤必须绑定 current release 切换脚本，缺少: ${executableArg}`,
      );
    }
  }
}

function requiredCutoverAuditExecutableArgs() {
  return [
    'enable-apply:pingora-direct-enable-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-enable.sh',
    'rollback-apply:pingora-direct-rollback-apply:/opt/genarrative/current/scripts/deploy/pingora-direct-rollback.sh',
  ];
}

function assertPingoraEnvProductionDefaults(filePath, label) {
  if (!existsSync(filePath)) {
    return;
  }
  const content = readFileSync(filePath, 'utf8');
  const requiredLines = [
    [
      'GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS=gzip',
      '压缩算法必须保持 gzip-only，避免 Brotli 在未验证前进入直连发布包。',
    ],
    [
      'GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR=false',
      '公网直连默认不能信任客户端可伪造的 X-Forwarded-For。',
    ],
    [
      'GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED=false',
      '前置代理信任确认开关必须默认关闭。',
    ],
    [
      'GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED=true',
      '接流保护必须默认开启。',
    ],
    [
      'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN=',
      '内部探针 token 示例必须保持空值，避免发布包夹带真实 token。',
    ],
  ];
  for (const [line, reason] of requiredLines) {
    assertIncludes(content, line, `${label} ${reason}`);
  }
}