Genarrative/scripts/ops/pingora-direct-rehearsal-status.mjs

#!/usr/bin/env node

import { execFile } from 'node:child_process';
import { constants as fsConstants } from 'node:fs';
import { access, readFile, stat } from 'node:fs/promises';
import path from 'node:path';

const STATUS_RANK = {
  OK: 0,
  WARNING: 1,
  CRITICAL: 2,
};

const PUBLIC_GATEWAYS = new Set(['none', 'nginx', 'pingora-direct']);
const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']);
const DEFAULT_PORTS = [80, 443, 18081, 18083];
const SECRET_ENV_KEY_PATTERN =
  /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu;

const config = parseArgs(process.argv.slice(2));
const status = await buildStatus(config);

console.log(`${JSON.stringify(status, null, 2)}\n`);

if (config.failOnCritical && status.summary.status === 'CRITICAL') {
  process.exit(1);
}

function parseArgs(argv) {
  const result = {
    releaseRoot:
      process.env.GENARRATIVE_PINGORA_REHEARSAL_RELEASE_ROOT ||
      '/opt/genarrative/current',
    healthPatrolEnvFile:
      process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE ||
      '/etc/genarrative/health-patrol.env',
    pingoraEnvFile:
      process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE ||
      '/etc/genarrative/pingora-gateway.env',
    realpathConfigFile:
      process.env.GENARRATIVE_PINGORA_REALPATH_CANARY_CONFIG_FILE ||
      '/etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf',
    expectedPublicGateway:
      process.env.GENARRATIVE_PINGORA_REHEARSAL_EXPECT_PUBLIC_GATEWAY ||
      'none',
    expectedHealthPatrolGatewayMode:
      process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '',
    requireRealpathCanary: readBoolEnv(
      'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_REALPATH_CANARY',
    ),
    requirePingoraShadow: readBoolEnv(
      'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_PINGORA_SHADOW',
    ),
    requireCurrentReleaseGateway: readBoolEnv(
      'GENARRATIVE_PINGORA_REHEARSAL_REQUIRE_CURRENT_RELEASE_GATEWAY',
    ),
    timeoutMs: parseOptionalPositiveInt(
      process.env.GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS,
      5000,
      'GENARRATIVE_PINGORA_REHEARSAL_TIMEOUT_MS',
    ),
    failOnCritical: readBoolEnv(
      'GENARRATIVE_PINGORA_REHEARSAL_FAIL_ON_CRITICAL',
    ),
  };

  for (let index = 0; index < argv.length; index += 1) {
    const arg = argv[index];
    switch (arg) {
      case '-h':
      case '--help':
        printUsage();
        process.exit(0);
        break;
      case '--release-root':
        result.releaseRoot = requireValue(argv, ++index, arg);
        break;
      case '--health-patrol-env-file':
        result.healthPatrolEnvFile = requireValue(argv, ++index, arg);
        break;
      case '--pingora-env-file':
        result.pingoraEnvFile = requireValue(argv, ++index, arg);
        break;
      case '--realpath-config-file':
        result.realpathConfigFile = requireValue(argv, ++index, arg);
        break;
      case '--expect-public-gateway':
        result.expectedPublicGateway = requireValue(argv, ++index, arg);
        break;
      case '--expected-health-patrol-gateway-mode':
        result.expectedHealthPatrolGatewayMode = requireValue(
          argv,
          ++index,
          arg,
        );
        break;
      case '--require-realpath-canary':
        result.requireRealpathCanary = true;
        break;
      case '--require-pingora-shadow':
        result.requirePingoraShadow = true;
        break;
      case '--require-current-release-gateway':
        result.requireCurrentReleaseGateway = true;
        break;
      case '--timeout-ms':
        result.timeoutMs = parseRequiredPositiveInt(
          requireValue(argv, ++index, arg),
          arg,
        );
        break;
      case '--fail-on-critical':
        result.failOnCritical = true;
        break;
      default:
        throw new Error(`未知参数: ${arg}`);
    }
  }

  validateConfig(result);
  return result;
}

function printUsage() {
  console.log(`Usage:
  node scripts/ops/pingora-direct-rehearsal-status.mjs [options]

Options:
  --release-root <path>             current release 根目录，默认 /opt/genarrative/current。
  --health-patrol-env-file <path>   health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。
  --pingora-env-file <path>         pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。
  --realpath-config-file <path>     realpath canary Nginx 配置，默认 /etc/nginx/conf.d/zz-genarrative-pingora-realpath-canary.conf。
  --expect-public-gateway <mode>    可选，none / nginx / pingora-direct；nginx 模式会要求 80/443 仍由 Nginx 接流。
  --expected-health-patrol-gateway-mode <mode>
                                    可选，nginx 或 pingora-direct；不传时随 --expect-public-gateway 推导。
  --require-realpath-canary         要求 realpath canary 配置存在，且 127.0.0.1:18083 由 Nginx 监听。
  --require-pingora-shadow          要求 Pingora shadow 127.0.0.1:18081 正在监听。
  --require-current-release-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。
  --timeout-ms <ms>                 systemctl / ss / 子检查超时，默认 5000。
  --fail-on-critical                如果状态中出现 CRITICAL，则以退出码 1 结束。

该脚本只读采集 dev / release 上的 Pingora 直连切换彩排状态，不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。
`);
}

function validateConfig(input) {
  for (const [label, value] of [
    ['--release-root', input.releaseRoot],
    ['--health-patrol-env-file', input.healthPatrolEnvFile],
    ['--pingora-env-file', input.pingoraEnvFile],
    ['--realpath-config-file', input.realpathConfigFile],
  ]) {
    validateSafeAbsoluteFilePath(value, label);
  }

  if (!PUBLIC_GATEWAYS.has(input.expectedPublicGateway)) {
    throw new Error(
      `--expect-public-gateway 只支持 none / nginx / pingora-direct: ${input.expectedPublicGateway}`,
    );
  }
  if (
    input.expectedHealthPatrolGatewayMode &&
    !GATEWAY_MODES.has(input.expectedHealthPatrolGatewayMode)
  ) {
    throw new Error(
      `--expected-health-patrol-gateway-mode 只支持 nginx 或 pingora-direct: ${input.expectedHealthPatrolGatewayMode}`,
    );
  }
}

function requireValue(argv, index, flag) {
  const value = argv[index];
  if (value === undefined || value.startsWith('--')) {
    throw new Error(`${flag} 缺少参数值`);
  }
  return value;
}

function readBoolEnv(name) {
  const value = process.env[name];
  if (value === undefined || value === null || String(value).trim() === '') {
    return false;
  }
  const normalized = String(value).trim().toLowerCase();
  if (['1', 'true', 'yes', 'on'].includes(normalized)) {
    return true;
  }
  if (['0', 'false', 'no', 'off'].includes(normalized)) {
    return false;
  }
  throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`);
}

function parseOptionalPositiveInt(raw, fallback, label) {
  if (raw === undefined || raw === null || String(raw).trim() === '') {
    return fallback;
  }
  return parseRequiredPositiveInt(raw, label);
}

function parseRequiredPositiveInt(raw, label) {
  const text = String(raw ?? '').trim();
  if (!/^[1-9][0-9]*$/u.test(text)) {
    throw new Error(`${label} 必须是正整数。`);
  }
  return Number.parseInt(text, 10);
}

function validateSafeAbsoluteFilePath(value, label) {
  validateNoControlCharacters(value, label);
  if (!path.isAbsolute(value)) {
    throw new Error(`${label} 必须是绝对路径。`);
  }
  if (isFilesystemRootPath(value)) {
    throw new Error(`${label} 不能是文件系统根目录。`);
  }
}

function validateNoControlCharacters(value, label) {
  if (/[\0\r\n]/u.test(String(value))) {
    throw new Error(`${label} 不能包含换行或 NUL 字符。`);
  }
}

function isFilesystemRootPath(value) {
  const resolved = path.resolve(String(value));
  return resolved === path.parse(resolved).root;
}

async function buildStatus(input) {
  const healthPatrolEnv = await inspectHealthPatrolEnv(input);
  const pingoraEnv = await inspectPingoraEnv(input);
  const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv);
  delete healthPatrolEnv.secretValues;
  delete pingoraEnv.secretValues;
  const services = await inspectServices(input);
  const systemd = await inspectPingoraSystemd(input);
  const ports = await inspectPorts(input);
  const realpathCanary = await inspectRealpathCanary(input, ports);
  const currentReleaseAudit = await inspectCurrentRelease(input, secrets);
  const publicBoundary = inspectPublicBoundary(input, ports, systemd);
  const checks = [
    healthPatrolEnv.status,
    pingoraEnv.status,
    services.status,
    systemd.status,
    ports.status,
    realpathCanary.status,
    currentReleaseAudit.status,
    publicBoundary.status,
  ];

  return {
    schemaVersion: 1,
    generatedAt: new Date().toISOString(),
    summary: summarize(checks),
    releaseRoot: input.releaseRoot,
    expectations: {
      publicGateway: input.expectedPublicGateway,
      healthPatrolGatewayMode: expectedHealthPatrolGatewayMode(input),
      requireRealpathCanary: input.requireRealpathCanary,
      requirePingoraShadow: input.requirePingoraShadow,
      requireCurrentReleaseGateway: input.requireCurrentReleaseGateway,
    },
    publicBoundary,
    ports,
    services,
    systemd,
    healthPatrolEnv,
    pingoraEnv,
    realpathCanary,
    currentReleaseAudit,
  };
}

async function inspectHealthPatrolEnv(input) {
  const parsed = await readEnvFile(input.healthPatrolEnvFile);
  if (parsed.status === 'CRITICAL') {
    return parsed;
  }

  const values = parsed.values;
  const expectedMode = expectedHealthPatrolGatewayMode(input);
  const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || '';
  const diagnostics = [...parsed.diagnostics];
  let status = parsed.status;

  if (expectedMode && gatewayMode !== expectedMode) {
    diagnostics.push(
      `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${expectedMode}，实际 ${gatewayMode || '(空)'}`,
    );
    status = maxStatus(status, 'CRITICAL');
  }

  return {
    path: input.healthPatrolEnvFile,
    status,
    secretValues: collectSecretValuesFromEnv(values),
    values: {
      gatewayMode,
      publicBaseUrl:
        values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '',
      publicHost: values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '',
      apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '',
      pingoraBaseUrl:
        values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '',
      hasPingoraProbeToken: Boolean(
        values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN ||
          values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN,
      ),
    },
    diagnostics,
  };
}

function expectedHealthPatrolGatewayMode(input) {
  if (input.expectedHealthPatrolGatewayMode) {
    return input.expectedHealthPatrolGatewayMode;
  }
  if (input.expectedPublicGateway === 'nginx') {
    return 'nginx';
  }
  if (input.expectedPublicGateway === 'pingora-direct') {
    return 'pingora-direct';
  }
  return '';
}

async function inspectPingoraEnv(input) {
  const parsed = await readEnvFile(input.pingoraEnvFile);
  if (parsed.status === 'CRITICAL') {
    return {
      ...parsed,
      values: {},
    };
  }
  const values = parsed.values;
  return {
    path: input.pingoraEnvFile,
    status: parsed.status,
    secretValues: collectSecretValuesFromEnv(values),
    values: {
      listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '',
      tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '',
      httpRedirectListen:
        values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '',
      tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '',
      hasTlsKeyFile: Boolean(
        values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE,
      ),
      hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN),
      accessLogFile:
        values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '',
      compressionAlgorithms:
        values.GENARRATIVE_PINGORA_GATEWAY_COMPRESSION_ALGORITHMS || '',
      trustXForwardedFor:
        values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '',
    },
    diagnostics: parsed.diagnostics,
  };
}

async function readEnvFile(filePath) {
  let text;
  try {
    text = await readFile(filePath, 'utf8');
  } catch (error) {
    return {
      path: filePath,
      status: 'CRITICAL',
      values: {},
      diagnostics: [`无法读取 env 文件: ${error.message}`],
    };
  }

  const values = {};
  const diagnostics = [];
  for (const [index, rawLine] of text.split(/\r?\n/u).entries()) {
    let line = rawLine.trim();
    if (!line || line.startsWith('#')) {
      continue;
    }
    if (line.startsWith('export ')) {
      line = line.slice('export '.length).trim();
    }
    const equalsIndex = line.indexOf('=');
    if (equalsIndex <= 0) {
      diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`);
      continue;
    }
    const key = line.slice(0, equalsIndex).trim();
    values[key] = stripQuotes(line.slice(equalsIndex + 1).trim());
  }

  return {
    path: filePath,
    status: diagnostics.length > 0 ? 'WARNING' : 'OK',
    values,
    diagnostics,
  };
}

function stripQuotes(value) {
  if (value.length >= 2) {
    const first = value[0];
    const last = value[value.length - 1];
    if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
      return value.slice(1, -1);
    }
  }
  return value;
}

function collectSecretValuesFromEnv(values) {
  const secrets = [];
  for (const [key, value] of Object.entries(values || {})) {
    if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') {
      continue;
    }
    const trimmed = value.trim();
    if (trimmed.length > 0) {
      secrets.push(trimmed);
    }
  }
  return secrets;
}

function collectSecretValues(...envSnapshots) {
  const secrets = new Set();
  for (const snapshot of envSnapshots) {
    for (const value of snapshot?.secretValues || []) {
      if (typeof value === 'string' && value.trim()) {
        secrets.add(value.trim());
      }
    }
  }
  return [...secrets].sort((left, right) => right.length - left.length);
}

async function inspectServices(input) {
  const serviceNames = [
    'nginx.service',
    'genarrative-pingora-gateway.service',
    'genarrative-api.service',
    'spacetimedb.service',
  ];
  const services = [];
  for (const service of serviceNames) {
    services.push(await inspectService(service, input));
  }
  return {
    status: summarize(services.map((service) => service.status)).status,
    services,
  };
}

async function inspectService(service, input) {
  const result = await runCommand('systemctl', ['is-active', service], input);
  const state = result.stdout.trim() || result.stderr.trim() || result.error;
  let status = 'OK';
  if (service === 'nginx.service' && input.expectedPublicGateway === 'nginx') {
    status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL';
  } else if (
    service === 'genarrative-pingora-gateway.service' &&
    (input.requirePingoraShadow ||
      input.expectedPublicGateway === 'pingora-direct')
  ) {
    status = result.code === 0 && state === 'active' ? 'OK' : 'CRITICAL';
  } else if (
    (service === 'genarrative-api.service' ||
      service === 'spacetimedb.service') &&
    result.code !== 0
  ) {
    status = 'WARNING';
  }
  return {
    name: service,
    activeState: state || 'unknown',
    status,
    command: result.command,
  };
}

async function inspectPingoraSystemd(input) {
  const catResult = await runCommand(
    'systemctl',
    ['cat', 'genarrative-pingora-gateway.service'],
    input,
  );
  const showResult = await runCommand(
    'systemctl',
    [
      'show',
      'genarrative-pingora-gateway.service',
      '--property=FragmentPath',
      '--property=DropInPaths',
      '--property=User',
      '--property=ExecStart',
      '--no-pager',
    ],
    input,
  );
  const unitText = catResult.stdout;
  const hasAmbientCapability = unitText.includes(
    'AmbientCapabilities=CAP_NET_BIND_SERVICE',
  );
  const hasCapabilityBoundingSet = unitText.includes(
    'CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
  );
  const environmentFiles = [
    ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu),
  ].map((match) => match[1].trim());
  const environmentFileMatchesPingoraEnvFile = environmentFilesInclude(
    environmentFiles,
    input.pingoraEnvFile,
  );
  const show = parseSystemctlShow(showResult.stdout);
  const diagnostics = [];
  let status = 'OK';

  if (catResult.code !== 0) {
    diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`);
    status = 'CRITICAL';
  }
  if (showResult.code !== 0) {
    diagnostics.push(
      `systemctl show 失败: ${showResult.stderr || showResult.error}`,
    );
    status = 'CRITICAL';
  }
  if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) {
    diagnostics.push(
      `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`,
    );
    status = 'CRITICAL';
  }
  if (
    input.expectedPublicGateway === 'nginx' &&
    (hasAmbientCapability || hasCapabilityBoundingSet)
  ) {
    diagnostics.push(
      'Nginx 接公网阶段不应残留 Pingora 低端口 CAP_NET_BIND_SERVICE。',
    );
    status = 'CRITICAL';
  }
  if (
    input.expectedPublicGateway === 'pingora-direct' &&
    (!hasAmbientCapability || !hasCapabilityBoundingSet)
  ) {
    diagnostics.push(
      'pingora-direct 接公网阶段必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。',
    );
    status = 'CRITICAL';
  }

  return {
    status,
    hasAmbientCapability,
    hasCapabilityBoundingSet,
    environmentFiles,
    environmentFileMatchesPingoraEnvFile,
    fragmentPath: show.FragmentPath || '',
    dropInPaths: show.DropInPaths || '',
    user: show.User || '',
    execStart: show.ExecStart || '',
    diagnostics,
    commands: [catResult.command, showResult.command],
  };
}

function environmentFilesInclude(environmentFiles, expectedPath) {
  return environmentFiles.some((entry) =>
    splitSystemdEnvironmentFileEntry(entry).includes(expectedPath),
  );
}

function splitSystemdEnvironmentFileEntry(entry) {
  const files = [];
  for (const word of String(entry || '').split(/\s+/u)) {
    if (!word) {
      continue;
    }
    let normalized = word.trim();
    if (normalized.startsWith('-')) {
      normalized = normalized.slice(1);
    }
    if (
      (normalized.startsWith('"') && normalized.endsWith('"')) ||
      (normalized.startsWith("'") && normalized.endsWith("'"))
    ) {
      normalized = normalized.slice(1, -1);
    }
    if (normalized) {
      files.push(normalized);
    }
  }
  return files;
}

function parseSystemctlShow(text) {
  const result = {};
  for (const line of text.split(/\r?\n/u)) {
    const equalsIndex = line.indexOf('=');
    if (equalsIndex <= 0) {
      continue;
    }
    result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1);
  }
  return result;
}

async function inspectPorts(input) {
  const result = await runCommand('ss', ['-H', '-ltnp'], input);
  const listeners = result.code === 0 ? parseSsOutput(result.stdout) : [];
  const byPort = {};
  const statuses = [];
  for (const port of DEFAULT_PORTS) {
    const portListeners = listeners.filter((item) => item.port === port);
    const entry = inspectPort(port, portListeners, input);
    byPort[String(port)] = entry;
    statuses.push(entry.status);
  }
  if (result.code !== 0) {
    statuses.push('CRITICAL');
  }
  return {
    status: summarize(statuses).status,
    command: result.command,
    error: result.code === 0 ? '' : trimForJson(result.stderr || result.error),
    byPort,
  };
}

function parseSsOutput(stdout) {
  const listeners = [];
  for (const line of String(stdout || '').split(/\r?\n/u)) {
    const trimmed = line.trim();
    if (!trimmed) {
      continue;
    }
    const parts = trimmed.split(/\s+/u);
    const local = parts[3] || '';
    const port = parsePortFromLocalAddress(local);
    if (!port) {
      continue;
    }
    const processText = parts.slice(5).join(' ');
    const processNames = [
      ...new Set([...processText.matchAll(/"([^"]+)"/gu)].map((match) => match[1])),
    ];
    listeners.push({
      localAddress: local,
      port,
      loopback: isLoopbackLocalAddress(local),
      processNames,
    });
  }
  return listeners;
}

function parsePortFromLocalAddress(value) {
  const bracketMatch = String(value).match(/^\[[^\]]+\]:(\d+)$/u);
  const plainMatch = String(value).match(/:(\d+)$/u);
  const portText = bracketMatch?.[1] || plainMatch?.[1] || '';
  const port = Number.parseInt(portText, 10);
  return Number.isInteger(port) && port > 0 ? port : null;
}

function isLoopbackLocalAddress(value) {
  const text = String(value || '').toLowerCase();
  return (
    text.startsWith('127.') ||
    text.startsWith('[::1]') ||
    text.startsWith('::1') ||
    text.startsWith('localhost:')
  );
}

function inspectPort(port, listeners, input) {
  const hasNginx = listeners.some((listener) =>
    listener.processNames.some((name) => name.toLowerCase().includes('nginx')),
  );
  const hasPingora = listeners.some((listener) =>
    listener.processNames.some((name) => name.toLowerCase().includes('pingora')),
  );
  const hasUnknownProcess =
    listeners.length > 0 &&
    listeners.some((listener) => listener.processNames.length === 0);
  const loopbackOnly =
    listeners.length > 0 && listeners.every((listener) => listener.loopback);
  const diagnostics = [];
  let status = 'OK';

  if ([80, 443].includes(port)) {
    if (input.expectedPublicGateway === 'nginx') {
      if (listeners.length === 0) {
        diagnostics.push(`${port} 未监听，Nginx 公网入口未就绪。`);
        status = 'CRITICAL';
      }
      if (!hasNginx) {
        diagnostics.push(`${port} 未看到 Nginx 监听。`);
        status = 'CRITICAL';
      }
      if (hasPingora) {
        diagnostics.push(`${port} 已被 Pingora 监听，不能作为未切公网彩排状态。`);
        status = 'CRITICAL';
      }
    }
    if (input.expectedPublicGateway === 'pingora-direct') {
      if (!hasPingora) {
        diagnostics.push(`${port} 未看到 Pingora 监听。`);
        status = 'CRITICAL';
      }
      if (hasNginx) {
        diagnostics.push(`${port} 仍由 Nginx 监听。`);
        status = 'CRITICAL';
      }
    }
  }

  if (port === 18081 && input.requirePingoraShadow) {
    if (!hasPingora) {
      diagnostics.push('18081 未看到 Pingora shadow 监听。');
      status = 'CRITICAL';
    }
    if (!loopbackOnly) {
      diagnostics.push('Pingora shadow 18081 必须只监听 loopback。');
      status = 'CRITICAL';
    }
  }

  if (port === 18083 && input.requireRealpathCanary) {
    if (!hasNginx) {
      diagnostics.push('18083 未看到 Nginx realpath canary 监听。');
      status = 'CRITICAL';
    }
    if (!loopbackOnly) {
      diagnostics.push('Nginx realpath canary 18083 必须只监听 loopback。');
      status = 'CRITICAL';
    }
  }

  if (
    hasUnknownProcess &&
    ([80, 443].includes(port) ||
      (port === 18081 && input.requirePingoraShadow) ||
      (port === 18083 && input.requireRealpathCanary))
  ) {
    diagnostics.push('ss 未返回进程名，无法确认监听归属。');
    status = maxStatus(status, 'WARNING');
  }

  return {
    port,
    status,
    listening: listeners.length > 0,
    loopbackOnly,
    hasNginx,
    hasPingora,
    hasUnknownProcess,
    listeners,
    diagnostics,
  };
}

async function inspectRealpathCanary(input, ports) {
  const diagnostics = [];
  let status = 'OK';
  let exists = false;
  let templateLooksValid = false;
  try {
    const fileStat = await stat(input.realpathConfigFile);
    exists = fileStat.isFile();
  } catch {
    exists = false;
  }

  if (!exists) {
    diagnostics.push(`realpath canary 配置不存在: ${input.realpathConfigFile}`);
    status = input.requireRealpathCanary ? 'CRITICAL' : 'WARNING';
  } else {
    const content = await readFile(input.realpathConfigFile, 'utf8');
    const requiredSnippets = [
      'listen 127.0.0.1:18083',
      'genarrative-pingora-realpath-canary.access.log',
      'X-Genarrative-Nginx-Handoff pingora-realpath-canary',
    ];
    const missing = requiredSnippets.filter((snippet) => !content.includes(snippet));
    templateLooksValid = missing.length === 0;
    if (missing.length > 0) {
      diagnostics.push(`realpath canary 配置缺少关键片段: ${missing.join(', ')}`);
      status = 'CRITICAL';
    }
  }

  const portStatus = ports.byPort['18083'];
  if (input.requireRealpathCanary && portStatus?.status === 'CRITICAL') {
    status = 'CRITICAL';
  }

  return {
    path: input.realpathConfigFile,
    status,
    exists,
    templateLooksValid,
    port: portStatus,
    diagnostics,
  };
}

async function inspectCurrentRelease(input, secrets) {
  const script = path.join(
    input.releaseRoot,
    'scripts/ops/pingora-current-release-audit.mjs',
  );
  try {
    await access(script, fsConstants.R_OK);
  } catch (error) {
    return {
      status: 'CRITICAL',
      script,
      code: null,
      diagnostics: [`无法读取 current release 自审脚本: ${error.message}`],
    };
  }

  const args = [
    '--',
    script,
    '--release-root',
    input.releaseRoot,
    '--systemd-show',
    '--timeout-ms',
    String(input.timeoutMs),
  ];
  if (input.requireCurrentReleaseGateway) {
    args.push('--require-pingora-gateway');
  }
  const result = await runCommand('node', args, input);
  const parsed = parseJsonObject(result.stdout);
  let status = result.code === 0 ? 'OK' : 'CRITICAL';
  const diagnostics = [];
  if (!parsed.ok) {
    diagnostics.push(`无法解析 current release 自审 JSON: ${parsed.error}`);
    status = 'CRITICAL';
  }
  return {
    status,
    script,
    code: result.code,
    command: result.command,
    summary: parsed.ok ? parsed.value?.summary || null : null,
    pingoraGateway: parsed.ok ? parsed.value?.pingoraGateway || null : null,
    releaseManifest: parsed.ok ? parsed.value?.releaseManifest || null : null,
    stdout: trimForJson(redactSecrets(result.stdout, secrets)),
    stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)),
    diagnostics,
  };
}

function inspectPublicBoundary(input, ports, systemd) {
  const diagnostics = [];
  let status = 'OK';
  if (input.expectedPublicGateway === 'none') {
    return {
      status,
      expectedPublicGateway: input.expectedPublicGateway,
      diagnostics,
    };
  }

  for (const port of [80, 443]) {
    const portStatus = ports.byPort[String(port)];
    if (portStatus?.status === 'CRITICAL') {
      status = 'CRITICAL';
      diagnostics.push(...portStatus.diagnostics);
    }
  }
  if (systemd.status === 'CRITICAL') {
    status = 'CRITICAL';
    diagnostics.push(...systemd.diagnostics);
  }

  return {
    status,
    expectedPublicGateway: input.expectedPublicGateway,
    diagnostics,
  };
}

function runCommand(command, args, input, env = process.env) {
  validateNoControlCharacters(command, '子命令可执行文件');
  for (const arg of args) {
    validateNoControlCharacters(arg, '子命令参数');
  }
  return new Promise((resolve) => {
    execFile(
      command,
      args,
      {
        env,
        timeout: input.timeoutMs,
        windowsHide: true,
        maxBuffer: 1024 * 1024,
      },
      (error, stdout, stderr) => {
        resolve({
          command: formatCommand(command, args),
          code: typeof error?.code === 'number' ? error.code : error ? 1 : 0,
          stdout: String(stdout || ''),
          stderr: String(stderr || ''),
          timedOut: Boolean(error?.killed),
          error: error ? error.message : '',
        });
      },
    );
  });
}

function formatCommand(command, args) {
  return [command, ...args].join(' ');
}

function redactSecrets(value, secrets) {
  let text = String(value || '');
  for (const secret of secrets) {
    if (!secret) {
      continue;
    }
    text = text.split(secret).join('<redacted>');
  }
  return text;
}

function trimForJson(value) {
  const text = String(value || '').trim();
  if (text.length <= 1000) {
    return text;
  }
  return `${text.slice(0, 1000)}...<truncated>`;
}

function parseJsonObject(value) {
  try {
    return { ok: true, value: JSON.parse(value) };
  } catch (error) {
    return { ok: false, error: error.message };
  }
}

function summarize(statuses) {
  const normalized = statuses.filter(Boolean);
  const status = normalized.reduce(
    (current, item) => maxStatus(current, item),
    'OK',
  );
  return {
    status,
    criticalCount: normalized.filter((item) => item === 'CRITICAL').length,
    warningCount: normalized.filter((item) => item === 'WARNING').length,
  };
}

function maxStatus(left, right) {
  return STATUS_RANK[right] > STATUS_RANK[left] ? right : left;
}