#!/usr/bin/env node import { execFile } from 'node:child_process'; import { constants as fsConstants } from 'node:fs'; import { access, readFile, stat } from 'node:fs/promises'; import path from 'node:path'; const STATUS_RANK = { OK: 0, WARNING: 1, CRITICAL: 2, }; const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']); const PINGORA_ENV_MODES = new Set(['shadow', 'direct']); const SERVICES = [ 'genarrative-api.service', 'spacetimedb.service', 'nginx.service', 'genarrative-pingora-gateway.service', ]; const RELEASE_ARTIFACTS = [ 'scripts/ops/pingora-cutover-status-snapshot.mjs', 'scripts/ops/pingora-current-release-audit.mjs', 'scripts/ops/pingora-direct-rehearsal-status.mjs', 'scripts/ops/pingora-cutover-evidence-audit.mjs', 'scripts/ops/pingora-cutover-evidence-bundle.mjs', 'scripts/ops/pingora-cutover-command-evidence.mjs', 'scripts/ops/pingora-cutover-evidence-verify.mjs', 'scripts/ops/production-health-patrol.mjs', 'scripts/check-production-health-patrol-env.mjs', 'scripts/check-pingora-release-readiness.mjs', 'scripts/check-pingora-direct-preflight.mjs', 'scripts/check-pingora-direct-live.mjs', 'scripts/check-pingora-canary-live.mjs', 'scripts/check-pingora-canary-access-log-parity.mjs', 'scripts/deploy/pingora-direct-enable.sh', 'scripts/deploy/pingora-direct-rollback.sh', 'scripts/deploy/pingora-realpath-canary-enable.sh', 'scripts/deploy/pingora-realpath-canary-disable.sh', 'scripts/deploy/pingora-health-patrol-env-switch.mjs', 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', 'deploy/systemd/genarrative-pingora-gateway.service', 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf', 'deploy/nginx/snippets/genarrative-pingora-canary.conf', 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf', 'deploy/pingora/pingora-gateway.env.example', ]; const SECRET_ENV_KEY_PATTERN = /(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu; const config = parseArgs(process.argv.slice(2)); const snapshot = await buildSnapshot(config); console.log(`${JSON.stringify(snapshot, null, 2)}\n`); if (config.failOnCritical && snapshot.summary.status === 'CRITICAL') { process.exit(1); } function parseArgs(argv) { const result = { phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual', releaseRoot: process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || '/opt/genarrative/current', healthPatrolEnvFile: process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE || '/etc/genarrative/health-patrol.env', pingoraEnvFile: process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE || '/etc/genarrative/pingora-gateway.env', expectedPingoraEnvMode: process.env.GENARRATIVE_PINGORA_EXPECTED_ENV_MODE || '', expectedGatewayMode: process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '', expectedPublicBaseUrl: process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '', expectedPublicHost: process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null, requireEmptyPublicHost: readBoolEnv( 'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST', ), runHealthPatrol: readBoolEnv( 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL', ), requirePingoraGateway: readBoolEnv( 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY', ), timeoutMs: parseOptionalPositiveInt( process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS, 5000, 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS', ), failOnCritical: readBoolEnv( 'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL', ), }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; switch (arg) { case '-h': case '--help': printUsage(); process.exit(0); break; case '--phase': result.phase = requireValue(argv, ++index, arg); break; case '--release-root': result.releaseRoot = requireValue(argv, ++index, arg); break; case '--health-patrol-env-file': result.healthPatrolEnvFile = requireValue(argv, ++index, arg); break; case '--pingora-env-file': result.pingoraEnvFile = requireValue(argv, ++index, arg); break; case '--expected-pingora-env-mode': result.expectedPingoraEnvMode = requireValue(argv, ++index, arg); break; case '--expected-gateway-mode': result.expectedGatewayMode = requireValue(argv, ++index, arg); break; case '--expected-public-base-url': result.expectedPublicBaseUrl = requireValue(argv, ++index, arg); break; case '--expected-public-host': result.expectedPublicHost = requireValue(argv, ++index, arg); break; case '--require-empty-public-host': result.requireEmptyPublicHost = true; break; case '--run-health-patrol': result.runHealthPatrol = true; break; case '--require-pingora-gateway': result.requirePingoraGateway = true; break; case '--timeout-ms': result.timeoutMs = parseRequiredPositiveInt( requireValue(argv, ++index, arg), arg, ); break; case '--fail-on-critical': result.failOnCritical = true; break; default: throw new Error(`未知参数: ${arg}`); } } if (!path.isAbsolute(result.releaseRoot)) { throw new Error('--release-root 必须是绝对路径。'); } validateNoControlCharacters(result.releaseRoot, '--release-root'); if (isFilesystemRootPath(result.releaseRoot)) { throw new Error('--release-root 不能是文件系统根目录。'); } if (!path.isAbsolute(result.healthPatrolEnvFile)) { throw new Error('--health-patrol-env-file 必须是绝对路径。'); } validateNoControlCharacters( result.healthPatrolEnvFile, '--health-patrol-env-file', ); if (isFilesystemRootPath(result.healthPatrolEnvFile)) { throw new Error('--health-patrol-env-file 不能是文件系统根目录。'); } if (!path.isAbsolute(result.pingoraEnvFile)) { throw new Error('--pingora-env-file 必须是绝对路径。'); } validateNoControlCharacters(result.pingoraEnvFile, '--pingora-env-file'); if (isFilesystemRootPath(result.pingoraEnvFile)) { throw new Error('--pingora-env-file 不能是文件系统根目录。'); } if ( result.expectedGatewayMode && !GATEWAY_MODES.has(result.expectedGatewayMode) ) { throw new Error( `--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`, ); } if ( result.expectedPingoraEnvMode && !PINGORA_ENV_MODES.has(result.expectedPingoraEnvMode) ) { throw new Error( `--expected-pingora-env-mode 只支持 shadow 或 direct: ${result.expectedPingoraEnvMode}`, ); } if (result.expectedPublicHost !== null) { validateHostOption(result.expectedPublicHost, '--expected-public-host'); } if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) { throw new Error( '--expected-public-host 和 --require-empty-public-host 不能同时使用。', ); } if (result.expectedPublicBaseUrl) { validateHttpUrl(result.expectedPublicBaseUrl, '--expected-public-base-url'); } return result; } function printUsage() { console.log(`Usage: node scripts/ops/pingora-cutover-status-snapshot.mjs [options] Options: --phase 写入 JSON 的阶段标签,例如 pre-cutover / post-enable / post-rollback。 --release-root current release 根目录,默认 /opt/genarrative/current。 --health-patrol-env-file health-patrol env 文件,默认 /etc/genarrative/health-patrol.env。 --pingora-env-file pingora-gateway env 文件,默认 /etc/genarrative/pingora-gateway.env。 --expected-gateway-mode 可选,nginx 或 pingora-direct;提供后会检查 env 和 systemd capability 方向。 --expected-pingora-env-mode 可选,shadow 或 direct;提供后会检查 active Pingora env 姿态。 --expected-public-base-url 可选,要求 health-patrol public base URL 与该值一致。 --expected-public-host 可选,要求 health-patrol public Host 与该值一致。 --require-empty-public-host 可选,要求 health-patrol public Host 为空。 --run-health-patrol 读取 env 后执行 current release 随包生产巡检脚本并收录结果。 --require-pingora-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。 --timeout-ms systemctl / 子检查超时,默认 5000。 --fail-on-critical 如果快照中出现 CRITICAL,则以退出码 1 结束。 该脚本只读采集状态,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 `); } function requireValue(argv, index, flag) { const value = argv[index]; if (value === undefined || value.startsWith('--')) { throw new Error(`${flag} 缺少参数值`); } return value; } function readBoolEnv(name) { const value = process.env[name]; if (value === undefined || value === null || String(value).trim() === '') { return false; } const normalized = String(value).trim().toLowerCase(); if (['1', 'true', 'yes', 'on'].includes(normalized)) { return true; } if (['0', 'false', 'no', 'off'].includes(normalized)) { return false; } throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); } function parseOptionalPositiveInt(raw, fallback, label) { if (raw === undefined || raw === null || String(raw).trim() === '') { return fallback; } return parseRequiredPositiveInt(raw, label); } function parseRequiredPositiveInt(raw, label) { const text = String(raw ?? '').trim(); if (!/^[1-9][0-9]*$/u.test(text)) { throw new Error(`${label} 必须是正整数。`); } return Number.parseInt(text, 10); } function isFilesystemRootPath(value) { const resolved = path.resolve(String(value)); return resolved === path.parse(resolved).root; } function validateNoControlCharacters(value, label) { if (/[\0\r\n]/u.test(String(value))) { throw new Error(`${label} 不能包含换行或 NUL 字符。`); } } function validateHttpUrl(value, label) { try { const parsed = new URL(value); if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { throw new Error('unsupported protocol'); } } catch { throw new Error(`${label} 必须是合法 http(s) URL: ${value}`); } } function validateHostOption(value, label) { const raw = String(value); if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) { throw new Error( `${label} 只能是 host 或 host:port,不能包含 scheme、路径、查询、片段或空白字符`, ); } try { const parsed = new URL(`https://${raw}`); if ( !parsed.hostname || parsed.pathname !== '/' || parsed.search || parsed.hash || parsed.username || parsed.password ) { throw new Error('invalid host'); } } catch { throw new Error(`${label} 不是合法的 host 或 host:port`); } } async function buildSnapshot(input) { const context = { schemaVersion: 1, generatedAt: new Date().toISOString(), phase: input.phase, releaseRoot: input.releaseRoot, }; const healthPatrolEnv = await inspectHealthPatrolEnv(input); const pingoraEnv = await inspectPingoraEnv(input); const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv); delete healthPatrolEnv.secretValues; delete pingoraEnv.secretValues; const releaseArtifacts = await inspectReleaseArtifacts(input); const systemd = await inspectSystemd(input); const checks = await runChecks(input, healthPatrolEnv, secrets); const summary = summarize([ healthPatrolEnv.status, pingoraEnv.status, releaseArtifacts.status, systemd.status, ...checks.map((check) => check.status), ]); return { ...context, summary, healthPatrolEnv, pingoraEnv, releaseArtifacts, systemd, checks, }; } async function inspectHealthPatrolEnv(input) { const parsed = await readEnvFile(input.healthPatrolEnvFile); if (parsed.status === 'CRITICAL') { return parsed; } const values = parsed.values; const publicHost = values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || ''; const publicBaseUrl = values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || ''; const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || ''; const diagnostics = []; let status = 'OK'; if (input.expectedGatewayMode && gatewayMode !== input.expectedGatewayMode) { diagnostics.push( `GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${input.expectedGatewayMode},实际 ${gatewayMode || '(空)'}`, ); status = maxStatus(status, 'CRITICAL'); } if ( input.expectedPublicBaseUrl && normalizeBaseUrl(publicBaseUrl) !== normalizeBaseUrl(input.expectedPublicBaseUrl) ) { diagnostics.push( `GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${normalizeBaseUrl(input.expectedPublicBaseUrl)},实际 ${publicBaseUrl || '(空)'}`, ); status = maxStatus(status, 'CRITICAL'); } if (input.expectedPublicHost !== null && publicHost !== input.expectedPublicHost) { diagnostics.push( `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${input.expectedPublicHost},实际 ${publicHost || '(空)'}`, ); status = maxStatus(status, 'CRITICAL'); } if (input.requireEmptyPublicHost && publicHost) { diagnostics.push( `GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空,实际 ${publicHost}`, ); status = maxStatus(status, 'CRITICAL'); } return { path: input.healthPatrolEnvFile, status, secretValues: collectSecretValuesFromEnv(values), values: { gatewayMode, publicBaseUrl, publicHost, apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '', spacetimeBaseUrl: values.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL || '', pingoraBaseUrl: values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '', hasPingoraProbeToken: Boolean( values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN || values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN, ), }, diagnostics, }; } async function inspectPingoraEnv(input) { const parsed = await readEnvFile(input.pingoraEnvFile); if (parsed.status === 'CRITICAL') { return { ...parsed, values: {}, posture: { expectedMode: input.expectedPingoraEnvMode || '', mode: 'unknown', shadowReady: false, directReady: false, diagnostics: parsed.diagnostics || [], }, }; } const values = parsed.values; const envValues = { listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '', tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '', httpRedirectListen: values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '', tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '', tlsKeyFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE || '', forwardedProto: values.GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO || '', trustXForwardedFor: values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '', trustedFrontProxyConfirmed: values.GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED || '', protectionEnabled: values.GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED || '', instanceCount: values.GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT || '', sharedProtectionConfirmed: values.GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED || '', hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN), accessLogFile: values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '', }; const posture = summarizePingoraEnvPosture( envValues, input.expectedPingoraEnvMode, ); const diagnostics = [...parsed.diagnostics, ...posture.diagnostics]; const status = input.expectedPingoraEnvMode && posture.diagnostics.length > 0 ? 'CRITICAL' : parsed.status; return { path: input.pingoraEnvFile, status, secretValues: collectSecretValuesFromEnv(values), values: envValues, posture, diagnostics, }; } function summarizePingoraEnvPosture(values, expectedMode) { const shadowReady = values.listen === '127.0.0.1:18081' && values.tlsListen === '' && values.httpRedirectListen === '' && values.tlsCertFile === '' && values.tlsKeyFile === ''; const directReady = values.tlsListen !== '' && values.httpRedirectListen !== '' && values.tlsCertFile !== '' && values.tlsKeyFile !== '' && values.forwardedProto === 'https'; const mode = directReady ? 'direct' : shadowReady ? 'shadow' : 'mixed'; const diagnostics = []; if (expectedMode === 'shadow') { if (values.listen !== '127.0.0.1:18081') { diagnostics.push( `Pingora shadow env 要求 listen=127.0.0.1:18081,实际 ${values.listen || '-'}`, ); } for (const [label, value] of [ ['tlsListen', values.tlsListen], ['httpRedirectListen', values.httpRedirectListen], ['tlsCertFile', values.tlsCertFile], ['tlsKeyFile', values.tlsKeyFile], ]) { if (value !== '') { diagnostics.push( `Pingora shadow env 要求 ${label} 为空,实际 ${value}`, ); } } } if (expectedMode === 'direct') { for (const [label, value] of [ ['tlsListen', values.tlsListen], ['httpRedirectListen', values.httpRedirectListen], ['tlsCertFile', values.tlsCertFile], ['tlsKeyFile', values.tlsKeyFile], ]) { if (value === '') { diagnostics.push(`Pingora direct env 要求 ${label} 已配置。`); } } if (values.forwardedProto !== 'https') { diagnostics.push( `Pingora direct env 要求 forwardedProto=https,实际 ${values.forwardedProto || '-'}`, ); } } return { expectedMode: expectedMode || '', mode, shadowReady, directReady, diagnostics, }; } async function readEnvFile(filePath) { let text; try { text = await readFile(filePath, 'utf8'); } catch (error) { return { path: filePath, status: 'CRITICAL', values: {}, diagnostics: [`无法读取 env 文件: ${error.message}`], }; } const values = {}; const diagnostics = []; for (const [index, rawLine] of text.split(/\r?\n/u).entries()) { let line = rawLine.trim(); if (!line || line.startsWith('#')) { continue; } if (line.startsWith('export ')) { line = line.slice('export '.length).trim(); } const equalsIndex = line.indexOf('='); if (equalsIndex <= 0) { diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`); continue; } const key = line.slice(0, equalsIndex).trim(); values[key] = stripQuotes(line.slice(equalsIndex + 1).trim()); } return { path: filePath, status: diagnostics.length > 0 ? 'WARNING' : 'OK', values, diagnostics, }; } function collectSecretValuesFromEnv(values) { const secrets = []; for (const [key, value] of Object.entries(values || {})) { if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') { continue; } const trimmed = value.trim(); if (trimmed.length > 0) { secrets.push(trimmed); } } return secrets; } function collectSecretValues(...envSnapshots) { const secrets = new Set(); for (const snapshot of envSnapshots) { for (const value of snapshot?.secretValues || []) { if (typeof value === 'string' && value.trim()) { secrets.add(value.trim()); } } } return [...secrets].sort((left, right) => right.length - left.length); } function stripQuotes(value) { if (value.length >= 2) { const first = value[0]; const last = value[value.length - 1]; if ((first === '"' && last === '"') || (first === "'" && last === "'")) { return value.slice(1, -1); } } return value; } function normalizeBaseUrl(value) { if (!value) { return ''; } try { return new URL(value).toString().replace(/\/+$/u, ''); } catch { return value; } } async function inspectReleaseArtifacts(input) { const artifacts = []; for (const relativePath of RELEASE_ARTIFACTS) { const fullPath = path.join(input.releaseRoot, relativePath); const artifact = { path: relativePath, exists: false, executable: false, status: 'CRITICAL', }; try { const fileStat = await stat(fullPath); artifact.exists = fileStat.isFile() || fileStat.isDirectory(); if (fileStat.isFile()) { try { await access(fullPath, fsConstants.X_OK); artifact.executable = true; } catch { artifact.executable = false; } } artifact.status = artifact.exists ? 'OK' : 'CRITICAL'; } catch { artifact.status = 'CRITICAL'; } artifacts.push(artifact); } const status = artifacts.some((artifact) => artifact.status === 'CRITICAL') ? 'CRITICAL' : 'OK'; return { status, artifacts, }; } async function inspectSystemd(input) { const services = []; for (const service of SERVICES) { services.push(await inspectService(service, input)); } const pingoraUnit = await inspectPingoraUnit(input); return { status: summarize([ ...services.map((service) => service.status), pingoraUnit.status, ]).status, services, pingoraUnit, }; } async function inspectService(service, input) { const expected = isServiceExpected(service, input.expectedGatewayMode); const result = await runCommand('systemctl', ['is-active', service], input); const state = result.stdout.trim() || result.stderr.trim() || result.error; let status = 'OK'; let summary = state || 'unknown'; if (result.code === 0 && state === 'active') { summary = 'active'; } else if (expected) { status = 'CRITICAL'; } else { summary = state ? `${state} (not required)` : 'not required'; } return { name: service, expectedActive: expected, activeState: state, status, summary, command: result.command, }; } function isServiceExpected(service, gatewayMode) { if ( service === 'genarrative-api.service' || service === 'spacetimedb.service' ) { return Boolean(gatewayMode); } if (service === 'nginx.service') { return gatewayMode === 'nginx'; } if (service === 'genarrative-pingora-gateway.service') { return gatewayMode === 'pingora-direct'; } return false; } async function inspectPingoraUnit(input) { const catResult = await runCommand( 'systemctl', ['cat', 'genarrative-pingora-gateway.service'], input, ); const showResult = await runCommand( 'systemctl', [ 'show', 'genarrative-pingora-gateway.service', '--property=FragmentPath', '--property=DropInPaths', '--property=User', '--property=ExecStart', '--no-pager', ], input, ); const unitText = catResult.stdout; const hasAmbientCapability = unitText.includes( 'AmbientCapabilities=CAP_NET_BIND_SERVICE', ); const hasCapabilityBoundingSet = unitText.includes( 'CapabilityBoundingSet=CAP_NET_BIND_SERVICE', ); const environmentFiles = [ ...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu), ].map((match) => match[1].trim()); const environmentFileMatchesPingoraEnvFile = environmentFilesInclude( environmentFiles, input.pingoraEnvFile, ); const show = parseSystemctlShow(showResult.stdout); const diagnostics = []; let status = 'OK'; if (catResult.code !== 0) { diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`); status = 'CRITICAL'; } if (showResult.code !== 0) { diagnostics.push( `systemctl show 失败: ${showResult.stderr || showResult.error}`, ); status = 'CRITICAL'; } if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) { diagnostics.push( `systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`, ); status = 'CRITICAL'; } if (input.expectedGatewayMode === 'pingora-direct') { if (!hasAmbientCapability || !hasCapabilityBoundingSet) { diagnostics.push( 'pingora-direct 模式必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。', ); status = 'CRITICAL'; } } if (input.expectedGatewayMode === 'nginx') { if (hasAmbientCapability || hasCapabilityBoundingSet) { diagnostics.push( 'nginx 模式下 systemd 最终配置不应残留 CAP_NET_BIND_SERVICE。', ); status = 'CRITICAL'; } } return { status, hasAmbientCapability, hasCapabilityBoundingSet, environmentFiles, environmentFileMatchesPingoraEnvFile, fragmentPath: show.FragmentPath || '', dropInPaths: show.DropInPaths || '', user: show.User || '', execStart: show.ExecStart || '', diagnostics, commands: [catResult.command, showResult.command], }; } function environmentFilesInclude(environmentFiles, expectedPath) { return environmentFiles.some((entry) => splitSystemdEnvironmentFileEntry(entry).includes(expectedPath), ); } function splitSystemdEnvironmentFileEntry(entry) { const files = []; for (const word of String(entry || '').split(/\s+/u)) { if (!word) { continue; } let normalized = word.trim(); if (normalized.startsWith('-')) { normalized = normalized.slice(1); } if ( (normalized.startsWith('"') && normalized.endsWith('"')) || (normalized.startsWith("'") && normalized.endsWith("'")) ) { normalized = normalized.slice(1, -1); } if (normalized) { files.push(normalized); } } return files; } function parseSystemctlShow(text) { const result = {}; for (const line of text.split(/\r?\n/u)) { const equalsIndex = line.indexOf('='); if (equalsIndex <= 0) { continue; } result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); } return result; } async function runChecks(input, healthPatrolEnv, secrets) { const checks = []; checks.push(await runCurrentReleaseAudit(input, secrets)); if (input.expectedGatewayMode) { checks.push(await runHealthPatrolEnvCheck(input, secrets)); } if (input.runHealthPatrol) { checks.push(await runProductionHealthPatrol(input, healthPatrolEnv, secrets)); } return checks; } async function runCurrentReleaseAudit(input, secrets) { const script = path.join( input.releaseRoot, 'scripts/ops/pingora-current-release-audit.mjs', ); const result = await runCommand( 'node', [ '--', script, '--release-root', input.releaseRoot, ...(input.requirePingoraGateway ? ['--require-pingora-gateway'] : []), '--systemd-show', '--timeout-ms', String(input.timeoutMs), ], input, ); const parsed = parseJsonObject(result.stdout); const details = parsed.ok ? { summary: parsed.value.summary, pingoraGateway: parsed.value.pingoraGateway, checksums: parsed.value.checksums, releaseManifest: parsed.value.releaseManifest, systemd: parsed.value.systemd, } : { parseError: parsed.error, }; const check = commandCheck('current-release-audit', result, secrets, details); if (!parsed.ok) { check.status = 'CRITICAL'; check.stderr = trimForJson( [check.stderr, `无法解析 current release 自审 JSON: ${parsed.error}`] .filter(Boolean) .join('\n'), ); } return check; } async function runHealthPatrolEnvCheck(input, secrets) { const script = path.join( input.releaseRoot, 'scripts/check-production-health-patrol-env.mjs', ); const args = [ '--', script, '--env-file', input.healthPatrolEnvFile, '--expected-gateway-mode', input.expectedGatewayMode, ]; if (input.expectedPublicBaseUrl) { args.push('--expected-public-base-url', input.expectedPublicBaseUrl); } if (input.expectedPublicHost !== null) { args.push('--expected-public-host', input.expectedPublicHost); } if (input.requireEmptyPublicHost) { args.push('--require-empty-public-host'); } const result = await runCommand('node', args, input); return commandCheck('health-patrol-env-check', result, secrets); } async function runProductionHealthPatrol(input, healthPatrolEnv, secrets) { const script = path.join( input.releaseRoot, 'scripts/ops/production-health-patrol.mjs', ); const rawEnv = await readEnvFile(input.healthPatrolEnvFile); const env = { ...process.env, ...readEnvValuesForChild(rawEnv.status === 'CRITICAL' ? healthPatrolEnv : rawEnv), }; const result = await runCommand( 'node', ['--', script, '--json'], input, env, ); return commandCheck('production-health-patrol', result, secrets); } function readEnvValuesForChild(healthPatrolEnv) { if (!healthPatrolEnv.values) { return {}; } const result = {}; for (const [key, value] of Object.entries(healthPatrolEnv.values)) { if (typeof value === 'string') { result[key] = value; } } return result; } function commandCheck(name, result, secrets = [], details = undefined) { return { name, status: result.code === 0 ? 'OK' : 'CRITICAL', code: result.code, command: result.command, stdout: trimForJson(redactSecrets(result.stdout, secrets)), stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)), ...(details === undefined ? {} : { details }), }; } function redactSecrets(value, secrets) { let text = String(value || ''); for (const secret of secrets) { if (!secret) { continue; } text = text.split(secret).join(''); } return text; } function runCommand(command, args, input, env = process.env) { validateNoControlCharacters(command, '子命令可执行文件'); for (const arg of args) { validateNoControlCharacters(arg, '子命令参数'); } return new Promise((resolve) => { execFile( command, args, { env, timeout: input.timeoutMs, windowsHide: true, maxBuffer: 1024 * 1024, }, (error, stdout, stderr) => { resolve({ command: formatCommand(command, args), code: typeof error?.code === 'number' ? error.code : error ? 1 : 0, stdout: String(stdout || ''), stderr: String(stderr || ''), timedOut: Boolean(error?.killed), error: error ? error.message : '', }); }, ); }); } function formatCommand(command, args) { const secretValueFlags = new Set([ '--pingora-probe-token', '--probe-token', '--direct-probe-token', ]); const redactedArgs = args.map((arg, index) => index > 0 && secretValueFlags.has(args[index - 1]) ? '' : arg, ); return [command, ...redactedArgs].join(' '); } function trimForJson(value) { const text = String(value || '').trim(); if (text.length <= 1000) { return text; } return `${text.slice(0, 1000)}...`; } function parseJsonObject(value) { try { return { ok: true, value: JSON.parse(value) }; } catch (error) { return { ok: false, error: error.message }; } } function summarize(statuses) { const normalized = statuses.filter(Boolean); const status = normalized.reduce( (current, item) => maxStatus(current, item), 'OK', ); return { status, criticalCount: normalized.filter((item) => item === 'CRITICAL').length, warningCount: normalized.filter((item) => item === 'WARNING').length, }; } function maxStatus(left, right) { return STATUS_RANK[right] > STATUS_RANK[left] ? right : left; }