Genarrative/scripts/ops/pingora-cutover-evidence-bundle.mjs

#!/usr/bin/env node

import { execFile } from 'node:child_process';
import { createHash } from 'node:crypto';
import { constants as fsConstants } from 'node:fs';
import {
  access,
  chmod,
  lstat,
  mkdir,
  readFile,
  stat,
  writeFile,
} from 'node:fs/promises';
import path from 'node:path';

const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']);
const PINGORA_ENV_MODES = new Set(['shadow', 'direct']);
const BUNDLE_DIR_MODE = 0o750;
const EVIDENCE_FILE_MODE = 0o640;
const SECRET_VALUE_FLAGS = new Set([
  '--direct-probe-token',
  '--probe-token',
  '--pingora-shadow-probe-token',
  '--rollback-pingora-shadow-probe-token',
]);
const DIRECT_LIVE_JSON_PATTERN = /\{\s*"ok"\s*:/u;

const config = parseArgs(process.argv.slice(2));
const startedAt = new Date();
const bundleDir = await createBundleDir(
  config.outputRoot,
  config.phase,
  startedAt,
);
const snapshotArgs = buildSnapshotArgs(config);
const snapshotRun = await runEvidenceCommand({
  name: 'pingora-cutover-status-snapshot',
  executable: 'node',
  args: [config.snapshotScript, ...snapshotArgs],
  stdoutFile: 'snapshot.stdout.txt',
  stderrFile: 'snapshot.stderr.txt',
  jsonFile: 'snapshot.json',
  parseErrorFile: 'snapshot-parse-error.txt',
  commandFile: 'snapshot-command.json',
  parseJson: parseSnapshotJson,
});
const manifestPath = path.join(bundleDir, 'manifest.json');
const snapshot = snapshotRun.parsed;
const snapshotStatus = snapshot.ok
  ? snapshot.value?.summary?.status ||
    (snapshotRun.result.code === 0 ? 'OK' : 'CRITICAL')
  : 'CRITICAL';

let directLiveRun = null;
if (
  config.runDirectLive &&
  snapshotRun.result.code === 0 &&
  snapshot.ok &&
  (!config.failOnCritical || snapshotStatus !== 'CRITICAL')
) {
  directLiveRun = await runEvidenceCommand({
    name: 'pingora-direct-live',
    executable: 'node',
    args: [config.directLiveScript, ...buildDirectLiveArgs(config)],
    stdoutFile: 'direct-live.stdout.txt',
    stderrFile: 'direct-live.stderr.txt',
    jsonFile: 'direct-live.json',
    parseErrorFile: 'direct-live-parse-error.txt',
    commandFile: 'direct-live-command.json',
    parseJson: parseDirectLiveJson,
  });
}

const directLiveAccessLog = summarizeDirectLiveAccessLog(directLiveRun);
const directLiveStaticHeaders = summarizeDirectLiveStaticHeaders(directLiveRun);
const pingoraEnvShadow = summarizePingoraEnvShadow(snapshotRun);
const directLiveStatus = directLiveRun
  ? isDirectLiveEvidenceOk(
      directLiveRun,
      directLiveAccessLog,
      directLiveStaticHeaders,
    )
    ? 'OK'
    : 'CRITICAL'
  : 'SKIPPED';
const snapshotCriticalCount = snapshot.ok
  ? Number(snapshot.value?.summary?.criticalCount || 0)
  : 1;
const directLiveCriticalCount = directLiveStatus === 'CRITICAL' ? 1 : 0;
const bundleStatus =
  snapshotStatus === 'CRITICAL' || directLiveStatus === 'CRITICAL'
    ? 'CRITICAL'
    : snapshotStatus;
const manifest = {
  schemaVersion: 1,
  generatedAt: new Date().toISOString(),
  phase: config.phase,
  ...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}),
  releaseRoot: config.releaseRoot,
  outputRoot: config.outputRoot,
  bundleDir,
  summary: {
    status: bundleStatus,
    snapshotStatus,
    snapshotExitCode: snapshotRun.result.code,
    directLiveStatus,
    directLiveExitCode: directLiveRun?.result.code ?? null,
    directLiveAccessLog: directLiveAccessLog?.summary ?? null,
    directLiveStaticHeaders: directLiveStaticHeaders?.summary ?? null,
    pingoraEnvShadow: pingoraEnvShadow?.summary ?? null,
    criticalCount: snapshotCriticalCount + directLiveCriticalCount,
    warningCount: snapshot.ok
      ? Number(snapshot.value?.summary?.warningCount || 0)
      : 0,
  },
  files: {
    manifest: path.basename(manifestPath),
    snapshot: snapshot.ok ? snapshotRun.files.json : null,
    snapshotParseError: snapshot.ok ? null : snapshotRun.files.parseError,
    snapshotStdout: snapshotRun.files.stdout,
    snapshotStderr: snapshotRun.files.stderr,
    snapshotCommand: snapshotRun.files.command,
    directLive: directLiveRun?.parsed.ok ? directLiveRun.files.json : null,
    directLiveParseError:
      directLiveRun && !directLiveRun.parsed.ok
        ? directLiveRun.files.parseError
        : null,
    directLiveStdout: directLiveRun ? directLiveRun.files.stdout : null,
    directLiveStderr: directLiveRun ? directLiveRun.files.stderr : null,
    directLiveCommand: directLiveRun ? directLiveRun.files.command : null,
  },
  commands: [
    snapshotRun.commandRecord,
    ...(directLiveRun ? [directLiveRun.commandRecord] : []),
  ],
};
await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);

const directLiveOk =
  !config.runDirectLive ||
  isDirectLiveEvidenceOk(
    directLiveRun,
    directLiveAccessLog,
    directLiveStaticHeaders,
  );
const result = {
  ok:
    snapshotRun.result.code === 0 &&
    snapshot.ok &&
    (!config.failOnCritical || snapshotStatus !== 'CRITICAL') &&
    directLiveOk,
  phase: config.phase,
  cutoverRunId: config.cutoverRunId || null,
  status: bundleStatus,
  bundleDir,
  manifestPath,
  snapshotPath: snapshot.ok ? snapshotRun.jsonPath : null,
  snapshotParseErrorPath: snapshot.ok ? null : snapshotRun.parseErrorPath,
  directLivePath: directLiveRun?.parsed.ok ? directLiveRun.jsonPath : null,
  directLiveParseErrorPath:
    directLiveRun && !directLiveRun.parsed.ok
      ? directLiveRun.parseErrorPath
      : null,
};
console.log(`${JSON.stringify(result, null, 2)}\n`);

if (!result.ok) {
  process.exit(1);
}

function summarizePingoraEnvShadow(snapshotRun) {
  if (!snapshotRun.parsed.ok) {
    return {
      ok: false,
      summary: {
        present: false,
        listen: '',
        tlsListen: '',
        httpRedirectListen: '',
        ok: false,
        diagnostics: ['snapshot JSON parse failed'],
      },
    };
  }

  const pingoraEnv = snapshotRun.parsed.value?.pingoraEnv;
  const values = pingoraEnv?.values;
  if (!values || typeof values !== 'object' || Array.isArray(values)) {
    return {
      ok: false,
      summary: {
        present: false,
        listen: '',
        tlsListen: '',
        httpRedirectListen: '',
        ok: false,
        diagnostics: ['snapshot JSON missing pingoraEnv.values'],
      },
    };
  }

  const summary = {
    present: true,
    mode: String(pingoraEnv?.posture?.mode || ''),
    shadowReady: pingoraEnv?.posture?.shadowReady === true,
    directReady: pingoraEnv?.posture?.directReady === true,
    listen: String(values.listen || ''),
    tlsListen: String(values.tlsListen || ''),
    httpRedirectListen: String(values.httpRedirectListen || ''),
    tlsCertFile: String(values.tlsCertFile || ''),
    tlsKeyFile: String(values.tlsKeyFile || ''),
    ok: false,
    diagnostics: [],
  };
  if (summary.listen !== '127.0.0.1:18081') {
    summary.diagnostics.push(
      `listen 应为 127.0.0.1:18081，实际 ${summary.listen || '-'}`,
    );
  }
  if (summary.tlsListen !== '') {
    summary.diagnostics.push(
      `tlsListen 应为空，实际 ${summary.tlsListen}`,
    );
  }
  if (summary.httpRedirectListen !== '') {
    summary.diagnostics.push(
      `httpRedirectListen 应为空，实际 ${summary.httpRedirectListen}`,
    );
  }
  if (summary.tlsCertFile !== '') {
    summary.diagnostics.push(
      `tlsCertFile 应为空，实际 ${summary.tlsCertFile}`,
    );
  }
  if (summary.tlsKeyFile !== '') {
    summary.diagnostics.push(
      `tlsKeyFile 应为空，实际 ${summary.tlsKeyFile}`,
    );
  }
  if (summary.mode && summary.mode !== 'shadow') {
    summary.diagnostics.push(`mode 应为 shadow，实际 ${summary.mode}`);
  }
  if (summary.shadowReady === false) {
    summary.diagnostics.push('shadowReady 应为 true');
  }
  summary.ok = summary.diagnostics.length === 0;
  return { ok: summary.ok, summary };
}

function summarizeDirectLiveAccessLog(directLiveRun) {
  if (!directLiveRun) {
    return null;
  }
  if (!directLiveRun.parsed.ok) {
    return {
      ok: false,
      summary: {
        present: false,
        reason: 'direct live JSON parse failed',
      },
    };
  }
  const accessLogCheck = directLiveRun.parsed.value?.results?.find(
    (item) => item?.name === 'direct-access-log',
  );
  if (!accessLogCheck) {
    return {
      ok: false,
      summary: {
        present: false,
        reason: 'direct live JSON missing direct-access-log result',
      },
    };
  }

  const summary = {
    present: true,
    logFile: accessLogCheck.logFile || '',
    sinceLines: coerceNonNegativeInteger(accessLogCheck.sinceLines),
    scannedLineCount: coerceNonNegativeInteger(
      accessLogCheck.scannedLineCount,
    ),
    checked: coerceNonNegativeInteger(accessLogCheck.checked),
    matchedCount: coerceNonNegativeInteger(accessLogCheck.matchedCount),
    missingCount: coerceNonNegativeInteger(accessLogCheck.missingCount),
    mismatchCount: coerceNonNegativeInteger(accessLogCheck.mismatchCount),
    missingDetailsCount: Array.isArray(accessLogCheck.missing)
      ? accessLogCheck.missing.length
      : null,
    mismatchDetailsCount: Array.isArray(accessLogCheck.mismatches)
      ? accessLogCheck.mismatches.length
      : null,
  };
  const ok =
    summary.checked !== null &&
    summary.matchedCount === summary.checked &&
    summary.missingCount === 0 &&
    summary.mismatchCount === 0 &&
    summary.missingDetailsCount === 0 &&
    summary.mismatchDetailsCount === 0;

  return { ok, summary };
}

function summarizeDirectLiveStaticHeaders(directLiveRun) {
  if (!directLiveRun) {
    return null;
  }
  if (!directLiveRun.parsed.ok) {
    return {
      ok: false,
      summary: {
        present: false,
        reason: 'direct live JSON parse failed',
      },
    };
  }

  const staticAssetCheck = directLiveRun.parsed.value?.results?.find(
    (item) => item?.name === 'https-static-asset',
  );
  if (!staticAssetCheck) {
    return {
      ok: false,
      summary: {
        present: false,
        skipped: false,
        reason: 'direct live JSON missing https-static-asset result',
      },
    };
  }
  if (staticAssetCheck.skipped) {
    return {
      ok: true,
      summary: {
        present: false,
        skipped: true,
        reason: staticAssetCheck.reason || '',
      },
    };
  }

  const normal = summarizeStaticHeaderEntry(staticAssetCheck);
  const fingerprinted = summarizeStaticHeaderEntry(
    staticAssetCheck.fingerprinted,
    {
      expectedCacheControl: 'public, max-age=31536000, immutable',
    },
  );
  const diagnostics = [
    ...normal.diagnostics.map((item) => `normal: ${item}`),
    ...(fingerprinted.present
      ? fingerprinted.diagnostics.map((item) => `fingerprinted: ${item}`)
      : []),
  ];
  const summary = {
    present: true,
    normal,
    fingerprinted,
    diagnostics,
  };
  return {
    ok: normal.ok && (!fingerprinted.present || fingerprinted.ok),
    summary,
  };
}

function summarizeStaticHeaderEntry(entry, options = {}) {
  if (!entry || entry.skipped) {
    return {
      ok: true,
      present: false,
      skipped: Boolean(entry?.skipped),
      reason: entry?.reason || '',
      diagnostics: [],
    };
  }

  const summary = {
    present: true,
    path: safePathFromUrl(entry.url),
    statusCode: coerceNonNegativeInteger(entry.statusCode),
    cacheControl: entry.headers?.['cache-control'] || '',
    etag: entry.headers?.etag || '',
    lastModified: entry.headers?.['last-modified'] || '',
    acceptRanges: entry.headers?.['accept-ranges'] || '',
    contentLength: entry.headers?.['content-length'] || '',
    rangeStatusCode: coerceNonNegativeInteger(entry.range?.statusCode),
    rangeContentRange: entry.range?.headers?.['content-range'] || '',
    rangeContentLength: entry.range?.headers?.['content-length'] || '',
    etag304StatusCode: coerceNonNegativeInteger(
      entry.notModified?.etag?.statusCode,
    ),
    lastModified304StatusCode: coerceNonNegativeInteger(
      entry.notModified?.lastModified?.statusCode,
    ),
  };
  const diagnostics = [];
  if (summary.statusCode !== 200) {
    diagnostics.push(`GET statusCode 应为 200，实际 ${formatValue(summary.statusCode)}`);
  }
  if (!summary.cacheControl) {
    diagnostics.push('缺少 Cache-Control');
  }
  if (
    options.expectedCacheControl &&
    summary.cacheControl !== options.expectedCacheControl
  ) {
    diagnostics.push(
      `Cache-Control 应为 ${options.expectedCacheControl}，实际 ${summary.cacheControl || '-'}`,
    );
  }
  if (!summary.etag) {
    diagnostics.push('缺少 ETag');
  }
  if (!summary.lastModified) {
    diagnostics.push('缺少 Last-Modified');
  }
  if (summary.acceptRanges !== 'bytes') {
    diagnostics.push(
      `Accept-Ranges 应为 bytes，实际 ${summary.acceptRanges || '-'}`,
    );
  }
  if (!summary.contentLength) {
    diagnostics.push('缺少 Content-Length');
  }
  if (summary.rangeStatusCode !== 206) {
    diagnostics.push(
      `Range statusCode 应为 206，实际 ${formatValue(summary.rangeStatusCode)}`,
    );
  }
  if (!/^bytes 0-0\/\d+$/u.test(summary.rangeContentRange)) {
    diagnostics.push(
      `Range Content-Range 应匹配 bytes 0-0/<len>，实际 ${summary.rangeContentRange || '-'}`,
    );
  }
  if (summary.rangeContentLength !== '1') {
    diagnostics.push(
      `Range Content-Length 应为 1，实际 ${summary.rangeContentLength || '-'}`,
    );
  }
  if (summary.etag304StatusCode !== 304) {
    diagnostics.push(
      `ETag 304 statusCode 应为 304，实际 ${formatValue(summary.etag304StatusCode)}`,
    );
  }
  if (summary.lastModified304StatusCode !== 304) {
    diagnostics.push(
      `Last-Modified 304 statusCode 应为 304，实际 ${formatValue(summary.lastModified304StatusCode)}`,
    );
  }

  return {
    ok: diagnostics.length === 0,
    ...summary,
    diagnostics,
  };
}

function safePathFromUrl(value) {
  if (!value) {
    return '';
  }
  try {
    return new URL(value).pathname;
  } catch {
    return '';
  }
}

function coerceNonNegativeInteger(value) {
  const number = Number(value);
  if (!Number.isInteger(number) || number < 0) {
    return null;
  }
  return number;
}

function isDirectLiveEvidenceOk(
  directLiveRun,
  directLiveAccessLog,
  directLiveStaticHeaders,
) {
  return Boolean(
    directLiveRun &&
      directLiveRun.result.code === 0 &&
      directLiveRun.parsed.ok &&
      directLiveRun.parsed.value?.ok !== false &&
      directLiveAccessLog?.ok &&
      directLiveStaticHeaders?.ok,
  );
}

function formatValue(value) {
  return value === null || value === undefined ? '-' : String(value);
}

function parseArgs(argv) {
  const result = {
    phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual',
    cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '',
    releaseRoot:
      process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT ||
      '/opt/genarrative/current',
    outputRoot:
      process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT ||
      '/var/log/genarrative/pingora-cutover-evidence',
    healthPatrolEnvFile:
      process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE ||
      '/etc/genarrative/health-patrol.env',
    pingoraEnvFile:
      process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE ||
      '/etc/genarrative/pingora-gateway.env',
    snapshotScript: '',
    directLiveScript: '',
    runDirectLive: readBoolEnv(
      'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_RUN_DIRECT_LIVE',
    ),
    directHttpsBaseUrl:
      process.env.GENARRATIVE_PINGORA_DIRECT_HTTPS_BASE_URL || '',
    directHttpBaseUrl:
      process.env.GENARRATIVE_PINGORA_DIRECT_HTTP_BASE_URL || '',
    directHost: process.env.GENARRATIVE_PINGORA_DIRECT_HOST || '',
    directRedirectHost:
      process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_HOST || '',
    directRedirectBaseUrl:
      process.env.GENARRATIVE_PINGORA_DIRECT_REDIRECT_BASE_URL || '',
    directProbeToken: process.env.GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN || '',
    directSpacetimeDatabase:
      process.env.GENARRATIVE_PINGORA_DIRECT_SPACETIME_DATABASE || '',
    directPingoraAccessLog:
      process.env.GENARRATIVE_PINGORA_DIRECT_PINGORA_ACCESS_LOG || '',
    directAccessLogSinceLines:
      process.env.GENARRATIVE_PINGORA_DIRECT_ACCESS_LOG_SINCE_LINES || '2000',
    expectedGatewayMode:
      process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '',
    expectedPingoraEnvMode:
      process.env.GENARRATIVE_PINGORA_EXPECTED_ENV_MODE || '',
    expectedPublicBaseUrl:
      process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '',
    expectedPublicHost:
      process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null,
    requireEmptyPublicHost: readBoolEnv(
      'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST',
    ),
    runHealthPatrol: readBoolEnv(
      'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL',
    ),
    requirePingoraGateway: readBoolEnv(
      'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY',
    ),
    timeoutMs: parseOptionalPositiveInt(
      process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS,
      5000,
      'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS',
    ),
    failOnCritical: readBoolEnv(
      'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL',
    ),
  };

  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    switch (arg) {
      case '-h':
      case '--help':
        printUsage();
        process.exit(0);
        break;
      case '--phase':
        result.phase = requireValue(argv, ++index, arg);
        break;
      case '--cutover-run-id':
        result.cutoverRunId = requireValue(argv, ++index, arg);
        break;
      case '--release-root':
        result.releaseRoot = requireValue(argv, ++index, arg);
        break;
      case '--output-root':
        result.outputRoot = requireValue(argv, ++index, arg);
        break;
      case '--health-patrol-env-file':
        result.healthPatrolEnvFile = requireValue(argv, ++index, arg);
        break;
      case '--pingora-env-file':
        result.pingoraEnvFile = requireValue(argv, ++index, arg);
        break;
      case '--snapshot-script':
        result.snapshotScript = requireValue(argv, ++index, arg);
        break;
      case '--direct-live-script':
        result.directLiveScript = requireValue(argv, ++index, arg);
        break;
      case '--run-direct-live':
        result.runDirectLive = true;
        break;
      case '--direct-https-base-url':
        result.directHttpsBaseUrl = requireValue(argv, ++index, arg);
        break;
      case '--direct-http-base-url':
        result.directHttpBaseUrl = requireValue(argv, ++index, arg);
        break;
      case '--direct-host':
        result.directHost = requireValue(argv, ++index, arg);
        break;
      case '--direct-redirect-host':
        result.directRedirectHost = requireValue(argv, ++index, arg);
        break;
      case '--direct-redirect-base-url':
        result.directRedirectBaseUrl = requireValue(argv, ++index, arg);
        break;
      case '--direct-probe-token':
        result.directProbeToken = requireValue(argv, ++index, arg);
        break;
      case '--direct-spacetime-database':
        result.directSpacetimeDatabase = requireValue(argv, ++index, arg);
        break;
      case '--direct-pingora-access-log':
        result.directPingoraAccessLog = requireValue(argv, ++index, arg);
        break;
      case '--direct-access-log-since-lines':
        result.directAccessLogSinceLines = requireValue(argv, ++index, arg);
        break;
      case '--expected-gateway-mode':
        result.expectedGatewayMode = requireValue(argv, ++index, arg);
        break;
      case '--expected-pingora-env-mode':
        result.expectedPingoraEnvMode = requireValue(argv, ++index, arg);
        break;
      case '--expected-public-base-url':
        result.expectedPublicBaseUrl = requireValue(argv, ++index, arg);
        break;
      case '--expected-public-host':
        result.expectedPublicHost = requireValue(argv, ++index, arg);
        break;
      case '--require-empty-public-host':
        result.requireEmptyPublicHost = true;
        break;
      case '--run-health-patrol':
        result.runHealthPatrol = true;
        break;
      case '--require-pingora-gateway':
        result.requirePingoraGateway = true;
        break;
      case '--timeout-ms':
        result.timeoutMs = parseRequiredPositiveInt(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--fail-on-critical':
        result.failOnCritical = true;
        break;
      default:
        throw new Error(`未知参数: ${arg}`);
    }
  }

  result.snapshotScript =
    result.snapshotScript ||
    path.join(
      result.releaseRoot,
      'scripts/ops/pingora-cutover-status-snapshot.mjs',
    );
  result.directLiveScript =
    result.directLiveScript ||
    path.join(result.releaseRoot, 'scripts/check-pingora-direct-live.mjs');

  validateConfig(result);
  return result;
}

function printUsage() {
  console.log(`Usage:
  node scripts/ops/pingora-cutover-evidence-bundle.mjs [options]

Options:
  --phase <name>                    写入 manifest 的阶段标签，例如 pre-cutover / post-enable / post-rollback。
  --cutover-run-id <id>             可选，写入 manifest 的本次切换批次 ID；正式 runbook 会用同一 ID 串联所有阶段和命令证据。
  --release-root <path>             current release 根目录，默认 /opt/genarrative/current。
  --output-root <path>              证据包输出根目录，默认 /var/log/genarrative/pingora-cutover-evidence。
  --health-patrol-env-file <path>   health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。
  --pingora-env-file <path>         pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。
  --snapshot-script <path>          状态快照脚本，默认 current release 随包脚本。
  --direct-live-script <path>       direct live smoke 脚本，默认 current release 随包脚本。
  --run-direct-live                 额外执行 direct live smoke，并把 request_id / access log 结果归档。
  --direct-https-base-url <url>     透传给 direct live smoke 的 HTTPS 入口。
  --direct-http-base-url <url>      透传给 direct live smoke 的 HTTP redirect / ACME 入口。
  --direct-host <host>              透传给 direct live smoke 的正式 Host/SNI。
  --direct-redirect-host <host>     透传给 direct live smoke 的 redirect Location host。
  --direct-redirect-base-url <url>  可选，透传给 direct live smoke 的 redirect Location HTTPS base URL。
  --direct-probe-token <token>      可选，透传给 direct live smoke 的内部探针 token；证据中会脱敏。
  --direct-spacetime-database <db>  透传给 direct live smoke 的 SpacetimeDB 数据库名。
  --direct-pingora-access-log <path>
                                    透传给 direct live smoke 的 Pingora access log 路径。
  --direct-access-log-since-lines <n>
                                    透传给 direct live smoke 的 access log tail 行数，默认 2000。
  --expected-gateway-mode <mode>    可选，nginx 或 pingora-direct；会透传给状态快照。
  --expected-pingora-env-mode <mode>
                                    可选，shadow 或 direct；会透传给状态快照。
  --expected-public-base-url <url>  可选，要求 health-patrol public base URL 与该值一致。
  --expected-public-host <host>     可选，要求 health-patrol public Host 与该值一致。
  --require-empty-public-host       可选，要求 health-patrol public Host 为空。
  --run-health-patrol               让状态快照执行 current release 随包生产巡检脚本并收录结果。
  --require-pingora-gateway         让状态快照自审 current release 的 Pingora 二进制、checksum 和 manifest。
  --timeout-ms <ms>                 透传给状态快照脚本的 systemctl / 子检查超时，默认 5000。
  --fail-on-critical                如果快照中出现 CRITICAL，则证据包脚本也以退出码 1 结束。

该脚本只写 --output-root 下的新证据目录，且 --output-root 不能是符号链接或非目录路径；证据目录权限固定 0750，证据文件权限固定 0640；不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。
`);
}

function validateConfig(config) {
  validatePhase(config.phase);
  if (config.cutoverRunId) {
    validateSafeName(config.cutoverRunId, '--cutover-run-id');
  }
  for (const [label, value] of [
    ['--release-root', config.releaseRoot],
    ['--output-root', config.outputRoot],
    ['--health-patrol-env-file', config.healthPatrolEnvFile],
    ['--pingora-env-file', config.pingoraEnvFile],
    ['--snapshot-script', config.snapshotScript],
    ['--direct-live-script', config.directLiveScript],
  ]) {
    if (!path.isAbsolute(value)) {
      throw new Error(`${label} 必须是绝对路径。`);
    }
    if (path.resolve(value) === path.parse(path.resolve(value)).root) {
      throw new Error(`${label} 不能是文件系统根目录。`);
    }
  }
  if (
    config.expectedGatewayMode &&
    !GATEWAY_MODES.has(config.expectedGatewayMode)
  ) {
    throw new Error(
      `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${config.expectedGatewayMode}`,
    );
  }
  if (
    config.expectedPingoraEnvMode &&
    !PINGORA_ENV_MODES.has(config.expectedPingoraEnvMode)
  ) {
    throw new Error(
      `--expected-pingora-env-mode 只支持 shadow 或 direct: ${config.expectedPingoraEnvMode}`,
    );
  }
  if (config.expectedPublicHost !== null) {
    validateHostOption(config.expectedPublicHost, '--expected-public-host');
  }
  if (config.expectedPublicHost !== null && config.requireEmptyPublicHost) {
    throw new Error(
      '--expected-public-host 和 --require-empty-public-host 不能同时使用。',
    );
  }
  if (config.expectedPublicBaseUrl) {
    validateHttpUrl(config.expectedPublicBaseUrl, '--expected-public-base-url');
  }
  if (config.runDirectLive) {
    validateDirectLiveConfig(config);
  }
  validateEvidenceCommandArgs('状态快照命令参数', buildSnapshotArgs(config));
  if (config.runDirectLive) {
    validateEvidenceCommandArgs('direct live 命令参数', buildDirectLiveArgs(config));
  }
}

function validateDirectLiveConfig(config) {
  if (!config.directHttpsBaseUrl) {
    throw new Error('--run-direct-live 必须提供 --direct-https-base-url。');
  }
  validateHttpsUrl(config.directHttpsBaseUrl, '--direct-https-base-url');
  if (!config.directHttpBaseUrl) {
    throw new Error('--run-direct-live 必须提供 --direct-http-base-url。');
  }
  validateHttpOnlyUrl(config.directHttpBaseUrl, '--direct-http-base-url');
  if (!config.directHost) {
    throw new Error('--run-direct-live 必须提供 --direct-host。');
  }
  validateHostOption(config.directHost, '--direct-host');
  if (!config.directRedirectHost) {
    throw new Error('--run-direct-live 必须提供 --direct-redirect-host。');
  }
  validateHostOption(config.directRedirectHost, '--direct-redirect-host');
  if (config.directRedirectBaseUrl) {
    validateHttpsBaseUrl(config.directRedirectBaseUrl, '--direct-redirect-base-url');
  }
  if (!config.directSpacetimeDatabase) {
    throw new Error('--run-direct-live 必须提供 --direct-spacetime-database。');
  }
  validateNoControlCharacters(
    config.directSpacetimeDatabase,
    '--direct-spacetime-database',
  );
  if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(config.directSpacetimeDatabase)) {
    throw new Error(
      '--direct-spacetime-database 必须匹配 SpacetimeDB 数据库名规则 ^[a-z0-9]+(-[a-z0-9]+)*$。',
    );
  }
  if (config.directProbeToken) {
    validateNoControlCharacters(config.directProbeToken, '--direct-probe-token');
  }
  if (!config.directPingoraAccessLog) {
    throw new Error('--run-direct-live 必须提供 --direct-pingora-access-log。');
  }
  validateSafeAbsoluteFilePath(
    config.directPingoraAccessLog,
    '--direct-pingora-access-log',
  );
  validateNoControlCharacters(
    config.directAccessLogSinceLines,
    '--direct-access-log-since-lines',
  );
  if (!/^[1-9][0-9]*$/u.test(String(config.directAccessLogSinceLines))) {
    throw new Error('--direct-access-log-since-lines 必须是正整数。');
  }
}

function validatePhase(value) {
  validateSafeName(value, '--phase');
}

function validateSafeName(value, label) {
  const text = String(value || '');
  if (!/^[0-9A-Za-z._-]+$/u.test(text)) {
    throw new Error(
      `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`,
    );
  }
}

function validateEvidenceCommandArgs(label, args) {
  for (const arg of args) {
    if (/[\0\r\n]/u.test(String(arg))) {
      throw new Error(`${label}不能包含换行或 NUL 字符。`);
    }
  }
}

function validateSafeAbsoluteFilePath(value, label) {
  validateNoControlCharacters(value, label);
  if (!path.isAbsolute(value)) {
    throw new Error(`${label} 必须是绝对路径。`);
  }
  if (path.resolve(value) === path.parse(path.resolve(value)).root) {
    throw new Error(`${label} 不能是文件系统根目录。`);
  }
}

function validateNoControlCharacters(value, label) {
  if (/[\0\r\n]/u.test(String(value))) {
    throw new Error(`${label} 不能包含换行或 NUL 字符。`);
  }
}

function requireValue(argv, index, flag) {
  const value = argv[index];
  if (value === undefined || value.startsWith('--')) {
    throw new Error(`${flag} 缺少参数值`);
  }
  return value;
}

function readBoolEnv(name) {
  const value = process.env[name];
  if (value === undefined || value === null || String(value).trim() === '') {
    return false;
  }
  const normalized = String(value).trim().toLowerCase();
  if (['1', 'true', 'yes', 'on'].includes(normalized)) {
    return true;
  }
  if (['0', 'false', 'no', 'off'].includes(normalized)) {
    return false;
  }
  throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`);
}

function parseOptionalPositiveInt(raw, fallback, label) {
  if (raw === undefined || raw === null || String(raw).trim() === '') {
    return fallback;
  }
  return parseRequiredPositiveInt(raw, label);
}

function parseRequiredPositiveInt(raw, label) {
  validateNoControlCharacters(raw, label);
  const text = String(raw ?? '').trim();
  if (!/^[1-9][0-9]*$/u.test(text)) {
    throw new Error(`${label} 必须是正整数。`);
  }
  return Number.parseInt(text, 10);
}

function validateHttpUrl(value, label) {
  validateNoControlCharacters(value, label);
  try {
    const parsed = new URL(value);
    if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
      throw new Error('unsupported protocol');
    }
  } catch {
    throw new Error(`${label} 必须是合法 http(s) URL: ${value}`);
  }
}

function validateHttpsUrl(value, label) {
  validateHttpUrl(value, label);
  if (new URL(value).protocol !== 'https:') {
    throw new Error(`${label} 必须使用 https://。`);
  }
}

function validateHttpsBaseUrl(value, label) {
  validateHttpsUrl(value, label);
  const parsed = new URL(value);
  if (
    parsed.pathname !== '/' ||
    parsed.search ||
    parsed.hash ||
    parsed.username ||
    parsed.password
  ) {
    throw new Error(
      `${label} 只能是 HTTPS base URL，不能包含路径、查询、片段或认证信息。`,
    );
  }
}

function validateHttpOnlyUrl(value, label) {
  validateHttpUrl(value, label);
  if (new URL(value).protocol !== 'http:') {
    throw new Error(`${label} 必须使用 http://。`);
  }
}

function validateHostOption(value, label) {
  const raw = String(value);
  validateNoControlCharacters(raw, label);
  if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) {
    throw new Error(
      `${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符`,
    );
  }
  try {
    const parsed = new URL(`https://${raw}`);
    if (
      !parsed.hostname ||
      parsed.pathname !== '/' ||
      parsed.search ||
      parsed.hash ||
      parsed.username ||
      parsed.password
    ) {
      throw new Error('invalid host');
    }
  } catch {
    throw new Error(`${label} 不是合法的 host 或 host:port`);
  }
}

async function createBundleDir(outputRoot, phase, date) {
  await validateOutputRootForWriting(outputRoot);
  await mkdir(outputRoot, { recursive: true });
  await validateOutputRootForWriting(outputRoot);
  await access(outputRoot, fsConstants.W_OK);
  const baseName = `${formatTimestampForPath(date)}-${phase}`;
  for (let attempt = 0; attempt < 100; attempt += 1) {
    const suffix = attempt === 0 ? '' : `-${attempt}`;
    const bundleDir = path.join(outputRoot, `${baseName}${suffix}`);
    try {
      await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE });
      await chmod(bundleDir, BUNDLE_DIR_MODE);
      return bundleDir;
    } catch (error) {
      if (error?.code !== 'EEXIST') {
        throw error;
      }
    }
  }
  throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`);
}

async function validateOutputRootForWriting(outputRoot) {
  const target = path.resolve(outputRoot);
  const root = path.parse(target).root;
  let current = root;
  const segments = path
    .relative(root, target)
    .split(path.sep)
    .filter(Boolean);

  for (const segment of segments) {
    current = path.join(current, segment);
    let stats;
    try {
      stats = await lstat(current);
    } catch (error) {
      if (error?.code === 'ENOENT') {
        return;
      }
      throw error;
    }

    if (stats.isSymbolicLink()) {
      if (current === target) {
        throw new Error(`--output-root 不能是符号链接: ${current}`);
      }
      throw new Error(`--output-root 上级目录不能是符号链接: ${current}`);
    }

    if (!stats.isDirectory()) {
      if (current === target) {
        throw new Error(`--output-root 已存在但不是目录: ${current}`);
      }
      throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`);
    }
  }
}

async function writeEvidenceFile(filePath, content) {
  await writeFile(filePath, content, {
    encoding: 'utf8',
    flag: 'wx',
    mode: EVIDENCE_FILE_MODE,
  });
  await chmod(filePath, EVIDENCE_FILE_MODE);
}

async function buildEvidenceFileMetadata(filePath) {
  const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]);
  return {
    path: path.basename(filePath),
    sizeBytes: stats.size,
    sha256: createHash('sha256').update(content).digest('hex'),
  };
}

function formatTimestampForPath(date) {
  return date
    .toISOString()
    .replace(/[-:]/gu, '')
    .replace(/\.\d{3}Z$/u, 'Z');
}

function buildSnapshotArgs(config) {
  return [
    '--phase',
    config.phase,
    '--release-root',
    config.releaseRoot,
    '--health-patrol-env-file',
    config.healthPatrolEnvFile,
    '--pingora-env-file',
    config.pingoraEnvFile,
    ...(config.expectedGatewayMode
      ? ['--expected-gateway-mode', config.expectedGatewayMode]
      : []),
    ...(config.expectedPingoraEnvMode
      ? ['--expected-pingora-env-mode', config.expectedPingoraEnvMode]
      : []),
    ...(config.expectedPublicBaseUrl
      ? ['--expected-public-base-url', config.expectedPublicBaseUrl]
      : []),
    ...(config.expectedPublicHost !== null
      ? ['--expected-public-host', config.expectedPublicHost]
      : []),
    ...(config.requireEmptyPublicHost ? ['--require-empty-public-host'] : []),
    ...(config.runHealthPatrol ? ['--run-health-patrol'] : []),
    ...(config.requirePingoraGateway ? ['--require-pingora-gateway'] : []),
    '--timeout-ms',
    String(config.timeoutMs),
    ...(config.failOnCritical ? ['--fail-on-critical'] : []),
  ];
}

function buildDirectLiveArgs(config) {
  return [
    '--https-base-url',
    config.directHttpsBaseUrl,
    '--http-base-url',
    config.directHttpBaseUrl,
    '--host',
    config.directHost,
    '--redirect-host',
    config.directRedirectHost,
    ...(config.directRedirectBaseUrl
      ? ['--redirect-base-url', config.directRedirectBaseUrl]
      : []),
    ...(config.directProbeToken
      ? ['--probe-token', config.directProbeToken]
      : []),
    '--spacetime-database',
    config.directSpacetimeDatabase,
    '--require-wss-upgrade',
    '--pingora-access-log',
    config.directPingoraAccessLog,
    '--access-log-since-lines',
    config.directAccessLogSinceLines,
    '--json',
  ];
}

async function runEvidenceCommand({
  name,
  executable,
  args,
  stdoutFile,
  stderrFile,
  jsonFile,
  parseErrorFile,
  commandFile,
  parseJson,
}) {
  const startedAt = new Date();
  const result = await runCommand(executable, args);
  const finishedAt = new Date();
  const stdoutPath = path.join(bundleDir, stdoutFile);
  const stderrPath = path.join(bundleDir, stderrFile);
  const jsonPath = path.join(bundleDir, jsonFile);
  const parseErrorPath = path.join(bundleDir, parseErrorFile);
  const commandPath = path.join(bundleDir, commandFile);

  await writeEvidenceFile(stdoutPath, result.stdout);
  await writeEvidenceFile(stderrPath, result.stderr || result.error);

  const parsed = parseJson(result.stdout);
  let jsonMetadata = null;
  let parseErrorMetadata = null;
  if (parsed.ok) {
    await writeEvidenceFile(jsonPath, `${JSON.stringify(parsed.value, null, 2)}\n`);
    jsonMetadata = await buildEvidenceFileMetadata(jsonPath);
  } else {
    await writeEvidenceFile(parseErrorPath, `${parsed.error}\n`);
    parseErrorMetadata = await buildEvidenceFileMetadata(parseErrorPath);
  }

  const commandRecord = {
    name,
    executable,
    args: redactSecretArgs(args),
    command: formatCommand(executable, args),
    exitCode: result.code,
    startedAt: startedAt.toISOString(),
    finishedAt: finishedAt.toISOString(),
    durationMs: finishedAt.getTime() - startedAt.getTime(),
    stdoutPath: path.basename(stdoutPath),
    stderrPath: path.basename(stderrPath),
  };
  await writeEvidenceFile(
    commandPath,
    `${JSON.stringify(commandRecord, null, 2)}\n`,
  );
  const files = {
    stdout: await buildEvidenceFileMetadata(stdoutPath),
    stderr: await buildEvidenceFileMetadata(stderrPath),
    json: jsonMetadata,
    parseError: parseErrorMetadata,
    command: await buildEvidenceFileMetadata(commandPath),
  };

  return {
    result,
    parsed,
    stdoutPath,
    stderrPath,
    jsonPath,
    parseErrorPath,
    commandPath,
    commandRecord,
    files,
  };
}

function runCommand(command, args) {
  return new Promise((resolve) => {
    execFile(
      command,
      args,
      {
        env: process.env,
        windowsHide: true,
        maxBuffer: 2 * 1024 * 1024,
      },
      (error, stdout, stderr) => {
        resolve({
          code: typeof error?.code === 'number' ? error.code : error ? 1 : 0,
          stdout: String(stdout || ''),
          stderr: String(stderr || ''),
          error: error ? error.message : '',
        });
      },
    );
  });
}

function parseSnapshotJson(stdout) {
  try {
    return { ok: true, value: JSON.parse(stdout) };
  } catch (error) {
    return { ok: false, error: error.message };
  }
}

function parseDirectLiveJson(stdout) {
  const match = DIRECT_LIVE_JSON_PATTERN.exec(stdout);
  if (!match) {
    return {
      ok: false,
      error: 'direct live stdout 中未找到 JSON 结果对象。',
    };
  }
  const jsonText = extractJsonObject(stdout, match.index);
  if (!jsonText) {
    return {
      ok: false,
      error: 'direct live stdout 中未找到完整 JSON 结果对象。',
    };
  }
  try {
    return { ok: true, value: JSON.parse(jsonText) };
  } catch (error) {
    return { ok: false, error: error.message };
  }
}

function extractJsonObject(text, startIndex) {
  let depth = 0;
  let inString = false;
  let escaped = false;

  for (let index = startIndex; index < text.length; index += 1) {
    const char = text[index];
    if (inString) {
      if (escaped) {
        escaped = false;
      } else if (char === '\\') {
        escaped = true;
      } else if (char === '"') {
        inString = false;
      }
      continue;
    }

    if (char === '"') {
      inString = true;
      continue;
    }
    if (char === '{') {
      depth += 1;
      continue;
    }
    if (char === '}') {
      depth -= 1;
      if (depth === 0) {
        return text.slice(startIndex, index + 1);
      }
    }
  }

  return '';
}

function formatCommand(command, args) {
  return [command, ...redactSecretArgs(args)].join(' ');
}

function redactSecretArgs(args) {
  return args.map((arg, index) =>
    index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) ? '<redacted>' : arg,
  );
}