#!/usr/bin/env node import { execFile } from 'node:child_process'; import { createHash } from 'node:crypto'; import { constants as fsConstants } from 'node:fs'; import { access, readFile, realpath, stat } from 'node:fs/promises'; import path from 'node:path'; const REQUIRED_ARTIFACTS = [ { path: 'api-server', executable: true }, { path: 'scripts/database-backup-to-oss.mjs' }, { path: 'scripts/ops/production-health-patrol.mjs' }, { path: 'scripts/ops/pingora-current-release-audit.mjs' }, { path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' }, { path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' }, { path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' }, { path: 'scripts/check-production-health-patrol-env.mjs' }, { path: 'scripts/check-pingora-release-readiness.mjs' }, { path: 'scripts/check-pingora-direct-preflight.mjs' }, { path: 'scripts/check-pingora-direct-live.mjs' }, { path: 'scripts/check-pingora-canary-live.mjs' }, { path: 'scripts/check-pingora-canary-access-log-parity.mjs' }, { path: 'scripts/deploy/pingora-direct-enable.sh', executable: true }, { path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true }, { path: 'scripts/deploy/pingora-realpath-canary-enable.sh', executable: true, }, { path: 'scripts/deploy/pingora-realpath-canary-disable.sh', executable: true, }, { path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs', executable: true, }, { path: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs', executable: true, }, { path: 'scripts/deploy/pingora-tls-cert-sync.mjs', executable: true }, { path: 'deploy/systemd/genarrative-pingora-gateway.service' }, { path: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf' }, { path: 'deploy/nginx/snippets/genarrative-pingora-canary.conf' }, { path: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf' }, { path: 'deploy/env/health-patrol.env.example' }, { path: 'deploy/env/pingora-direct-live.env.example' }, { path: 'deploy/env/pingora-canary-live.env.example' }, { path: 'deploy/pingora/pingora-gateway.env.example' }, { path: 'deploy/pingora/nginx-route-parity.matrix.json' }, ]; const REQUIRED_DIRS = [ 'deploy/systemd', 'deploy/nginx', 'deploy/env', 'deploy/pingora', ]; const config = parseArgs(process.argv.slice(2)); const audit = await buildAudit(config); console.log(`${JSON.stringify(audit, null, 2)}\n`); if (audit.summary.status === 'CRITICAL' && !config.warnOnly) { process.exit(1); } function parseArgs(argv) { const result = { releaseRoot: process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT || '/opt/genarrative/current', requirePingoraGateway: readBoolEnv( 'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY', ), systemdShow: readBoolEnv( 'GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW', ), systemdService: process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SERVICE || 'genarrative-pingora-gateway.service', timeoutMs: parseOptionalPositiveInt( process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS, 5000, 'GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS', ), warnOnly: false, }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; switch (arg) { case '-h': case '--help': printUsage(); process.exit(0); break; case '--release-root': result.releaseRoot = requireValue(argv, ++index, arg); break; case '--require-pingora-gateway': result.requirePingoraGateway = true; break; case '--systemd-show': result.systemdShow = true; break; case '--systemd-service': result.systemdService = requireValue(argv, ++index, arg); break; case '--timeout-ms': result.timeoutMs = parseRequiredPositiveInt( requireValue(argv, ++index, arg), arg, ); break; case '--warn-only': result.warnOnly = true; break; default: throw new Error(`未知参数: ${arg}`); } } if (!path.isAbsolute(result.releaseRoot)) { throw new Error('--release-root 必须是绝对路径。'); } validateNoControlCharacters(result.releaseRoot, '--release-root'); if (isFilesystemRootPath(result.releaseRoot)) { throw new Error('--release-root 不能是文件系统根目录。'); } validateNoControlCharacters(result.systemdService, '--systemd-service'); if (!result.systemdService || /[\s/]/u.test(result.systemdService)) { throw new Error('--systemd-service 必须是 systemd unit 名称。'); } return result; } function printUsage() { console.log(`Usage: node scripts/ops/pingora-current-release-audit.mjs [options] Options: --release-root current release 根目录,默认 /opt/genarrative/current。 --require-pingora-gateway 要求 current release 已包含 pingora-gateway 且可执行。 --systemd-show 读取 systemctl show,确认 service ExecStart 指向 current release 的 pingora-gateway。 --systemd-service systemd service 名,默认 genarrative-pingora-gateway.service。 --timeout-ms systemctl 超时,默认 5000。 --warn-only 即使出现 CRITICAL 也以退出码 0 结束,只用于人工盘点。 该脚本只读检查 current release 自包含能力,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。 CRITICAL 默认会让进程以退出码 1 结束,适合作为切换窗口前置门禁。 `); } function requireValue(argv, index, flag) { const value = argv[index]; if (value === undefined || value.startsWith('--')) { throw new Error(`${flag} 缺少参数值`); } return value; } function readBoolEnv(name) { const value = process.env[name]; if (value === undefined || value === null || String(value).trim() === '') { return false; } const normalized = String(value).trim().toLowerCase(); if (['1', 'true', 'yes', 'on'].includes(normalized)) { return true; } if (['0', 'false', 'no', 'off'].includes(normalized)) { return false; } throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`); } function parseOptionalPositiveInt(raw, fallback, label) { if (raw === undefined || raw === null || String(raw).trim() === '') { return fallback; } return parseRequiredPositiveInt(raw, label); } function parseRequiredPositiveInt(raw, label) { const text = String(raw ?? '').trim(); if (!/^[1-9][0-9]*$/u.test(text)) { throw new Error(`${label} 必须是正整数。`); } return Number.parseInt(text, 10); } function isFilesystemRootPath(value) { const resolved = path.resolve(String(value)); return resolved === path.parse(resolved).root; } function validateNoControlCharacters(value, label) { if (/[\0\r\n]/u.test(String(value))) { throw new Error(`${label} 不能包含换行或 NUL 字符。`); } } async function buildAudit(input) { const artifacts = await inspectArtifacts(input); const directories = await inspectDirectories(input); const pingoraGateway = await inspectPingoraGateway(input); const checksums = await inspectChecksums(input, pingoraGateway); const releaseManifest = await inspectReleaseManifest(input, pingoraGateway); const systemd = input.systemdShow ? await inspectSystemd(input) : { checked: false, status: 'OK', diagnostics: [], }; const summary = summarize([ ...artifacts.map((artifact) => artifact.status), ...directories.map((directory) => directory.status), pingoraGateway.status, ...checksums.map((checksum) => checksum.status), releaseManifest.status, systemd.status, ]); return { schemaVersion: 1, generatedAt: new Date().toISOString(), releaseRoot: input.releaseRoot, summary, artifacts, directories, pingoraGateway, checksums, releaseManifest, systemd, }; } async function inspectArtifacts(input) { const artifacts = []; for (const artifact of REQUIRED_ARTIFACTS) { artifacts.push(await inspectArtifact(input.releaseRoot, artifact)); } return artifacts; } async function inspectDirectories(input) { const directories = []; for (const relativePath of REQUIRED_DIRS) { const fullPath = path.join(input.releaseRoot, relativePath); const diagnostics = []; let exists = false; let status = 'OK'; try { const fileStat = await stat(fullPath); exists = fileStat.isDirectory(); if (!exists) { diagnostics.push('路径存在但不是目录。'); status = 'CRITICAL'; } } catch (error) { diagnostics.push(`目录不存在或不可读: ${error.message}`); status = 'CRITICAL'; } directories.push({ path: relativePath, exists, status, diagnostics, }); } return directories; } async function inspectArtifact(releaseRoot, artifact) { const fullPath = path.join(releaseRoot, artifact.path); const diagnostics = []; let exists = false; let type = 'missing'; let executable = false; let status = 'OK'; try { const fileStat = await stat(fullPath); exists = true; type = fileStat.isFile() ? 'file' : fileStat.isDirectory() ? 'directory' : 'other'; if (type !== 'file') { diagnostics.push('路径存在但不是文件。'); status = 'CRITICAL'; } } catch (error) { diagnostics.push(`文件不存在或不可读: ${error.message}`); status = 'CRITICAL'; } if (exists && artifact.executable) { try { await access(fullPath, fsConstants.X_OK); executable = true; } catch (error) { diagnostics.push(`文件不可执行: ${error.message}`); status = 'CRITICAL'; } } return { path: artifact.path, expectedExecutable: Boolean(artifact.executable), exists, type, executable, status, diagnostics, }; } async function inspectPingoraGateway(input) { const binaryPath = path.join(input.releaseRoot, 'pingora-gateway'); const diagnostics = []; let included = false; let executable = false; let status = 'OK'; try { const fileStat = await stat(binaryPath); included = fileStat.isFile(); if (!included) { diagnostics.push('pingora-gateway 路径存在但不是文件。'); status = 'CRITICAL'; } } catch (error) { if (input.requirePingoraGateway) { diagnostics.push(`缺少 pingora-gateway: ${error.message}`); status = 'CRITICAL'; } } if (included) { try { await access(binaryPath, fsConstants.X_OK); executable = true; } catch (error) { diagnostics.push(`pingora-gateway 不可执行: ${error.message}`); status = 'CRITICAL'; } } return { required: input.requirePingoraGateway, included, path: 'pingora-gateway', absolutePath: binaryPath, executable, status, diagnostics, }; } async function inspectChecksums(input, pingoraGateway) { const checksumTargets = [ { path: 'api-server', checksumPath: 'api-server.sha256', required: true, }, { path: 'pingora-gateway', checksumPath: 'pingora-gateway.sha256', required: input.requirePingoraGateway || pingoraGateway.included, }, ]; const checksums = []; for (const target of checksumTargets) { checksums.push(await inspectChecksum(input.releaseRoot, target)); } return checksums; } async function inspectChecksum(releaseRoot, target) { const diagnostics = []; const artifactPath = path.join(releaseRoot, target.path); const checksumPath = path.join(releaseRoot, target.checksumPath); let status = 'OK'; let checked = false; let exists = false; let expectedSha256 = ''; let computedSha256 = ''; let checksumFileTarget = ''; if (!target.required) { return { path: target.path, checksumPath: target.checksumPath, required: false, checked, exists, checksumFileTarget, expectedSha256, computedSha256, matches: null, status, diagnostics, }; } try { const checksumText = await readFile(checksumPath, 'utf8'); exists = true; const parsed = parseSha256File(checksumText); expectedSha256 = parsed.sha256; checksumFileTarget = parsed.fileName; if (path.basename(checksumFileTarget) !== path.basename(target.path)) { diagnostics.push( `${target.checksumPath} 指向了非预期文件: ${checksumFileTarget}`, ); status = 'CRITICAL'; } } catch (error) { diagnostics.push(`checksum 文件不存在、不可读或格式错误: ${error.message}`); status = 'CRITICAL'; } try { computedSha256 = await sha256File(artifactPath); checked = true; } catch (error) { diagnostics.push(`无法读取待校验文件: ${error.message}`); status = 'CRITICAL'; } const matches = checked && Boolean(expectedSha256) && computedSha256 === expectedSha256; if (checked && expectedSha256 && !matches) { diagnostics.push( `${target.path} sha256 不匹配: expected ${expectedSha256}, actual ${computedSha256}`, ); status = 'CRITICAL'; } return { path: target.path, checksumPath: target.checksumPath, required: target.required, checked, exists, checksumFileTarget, expectedSha256, computedSha256, matches, status, diagnostics, }; } function parseSha256File(content) { const firstLine = content.split(/\r?\n/u).find((line) => line.trim()); if (!firstLine) { throw new Error('checksum 文件为空'); } const match = firstLine.match(/^([0-9a-fA-F]{64})\s+(.+)$/u); if (!match) { throw new Error('checksum 行必须是 sha256sum 输出格式'); } return { sha256: match[1].toLowerCase(), fileName: match[2].replace(/^\*/u, '').trim(), }; } async function sha256File(filePath) { const content = await readFile(filePath); return createHash('sha256').update(content).digest('hex'); } async function inspectReleaseManifest(input, pingoraGateway) { const candidates = [ 'release-manifest.api-server.json', 'release-manifest.json', ]; const diagnostics = []; let status = 'OK'; let pathUsed = ''; let manifest = null; for (const candidate of candidates) { try { const content = await readFile( path.join(input.releaseRoot, candidate), 'utf8', ); manifest = JSON.parse(content); pathUsed = candidate; break; } catch (error) { diagnostics.push(`${candidate} 不可用: ${error.message}`); } } if (!manifest) { return { checked: true, path: '', candidates, componentType: '', artifacts: [], status: 'CRITICAL', diagnostics, }; } diagnostics.length = 0; const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : []; const apiArtifact = artifacts.find( (artifact) => artifact?.path === 'api-server', ); const pingoraArtifact = artifacts.find( (artifact) => artifact?.path === 'pingora-gateway', ); if (manifest.component_type !== 'api-server') { diagnostics.push( `release manifest component_type 应为 api-server,实际 ${manifest.component_type || '(空)'}`, ); status = 'CRITICAL'; } if (!apiArtifact) { diagnostics.push('release manifest 缺少 api-server artifact。'); status = 'CRITICAL'; } else if (apiArtifact.checksum_path !== 'api-server.sha256') { diagnostics.push( `api-server artifact checksum_path 应为 api-server.sha256,实际 ${apiArtifact.checksum_path || '(空)'}`, ); status = 'CRITICAL'; } if (input.requirePingoraGateway || pingoraGateway.included) { if (!pingoraArtifact) { diagnostics.push('release manifest 缺少 pingora-gateway artifact。'); status = 'CRITICAL'; } else if (pingoraArtifact.checksum_path !== 'pingora-gateway.sha256') { diagnostics.push( `pingora-gateway artifact checksum_path 应为 pingora-gateway.sha256,实际 ${pingoraArtifact.checksum_path || '(空)'}`, ); status = 'CRITICAL'; } } else if (pingoraArtifact) { diagnostics.push( 'release manifest 登记了 pingora-gateway,但 current release 未包含该二进制。', ); status = 'CRITICAL'; } return { checked: true, path: pathUsed, candidates, componentType: manifest.component_type || '', artifacts: artifacts.map((artifact) => ({ component: artifact?.component || '', path: artifact?.path || '', checksumPath: artifact?.checksum_path || '', })), status, diagnostics, }; } async function inspectSystemd(input) { const expectedBinary = path.join(input.releaseRoot, 'pingora-gateway'); const expectedRealpath = await resolveOptionalRealpath(expectedBinary); const result = await runCommand( 'systemctl', [ 'show', input.systemdService, '--property=FragmentPath', '--property=DropInPaths', '--property=User', '--property=ExecStart', '--no-pager', ], input, ); const show = parseSystemctlShow(result.stdout); const diagnostics = []; let status = 'OK'; if (result.code !== 0) { diagnostics.push(`systemctl show 失败: ${result.stderr || result.error}`); status = 'CRITICAL'; } const execStart = show.ExecStart || ''; const execStartBinary = parseExecStartBinary(execStart); const execStartRealpath = execStartBinary ? await resolveOptionalRealpath(execStartBinary) : null; const execStartMatches = execStartBinary === expectedBinary || (expectedRealpath && execStartRealpath && expectedRealpath === execStartRealpath); if (result.code === 0 && !execStartMatches) { diagnostics.push( `ExecStart 未指向 current release 网关二进制: expected ${expectedBinary}, actual ${execStart || '(空)'}`, ); status = 'CRITICAL'; } return { checked: true, service: input.systemdService, status, expectedBinary, expectedRealpath, execStartBinary, execStartRealpath, fragmentPath: show.FragmentPath || '', dropInPaths: show.DropInPaths || '', user: show.User || '', execStart, command: result.command, diagnostics, }; } async function resolveOptionalRealpath(filePath) { try { return await realpath(filePath); } catch { return null; } } function parseExecStartBinary(execStart) { const pathMatch = execStart.match(/\bpath=([^ ;]+)(?:\s|;|$)/u); if (pathMatch) { return pathMatch[1]; } const argvMatch = execStart.match(/\bargv\[\]=([^ ;]+)(?:\s|;|$)/u); if (argvMatch) { return argvMatch[1]; } return ''; } function parseSystemctlShow(text) { const result = {}; for (const line of text.split(/\r?\n/u)) { const equalsIndex = line.indexOf('='); if (equalsIndex <= 0) { continue; } result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1); } return result; } function runCommand(command, args, input) { validateNoControlCharacters(command, '子命令可执行文件'); for (const arg of args) { validateNoControlCharacters(arg, '子命令参数'); } return new Promise((resolve) => { const startedAt = new Date(); const child = execFile( command, args, { encoding: 'utf8', timeout: input.timeoutMs, maxBuffer: 1024 * 1024, }, (error, stdout, stderr) => { const finishedAt = new Date(); resolve({ command: [command, ...args], code: error?.code ?? 0, signal: error?.signal ?? null, error: error ? error.message : '', stdout: stdout || '', stderr: stderr || '', startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), }); }, ); child.on('error', (error) => { const finishedAt = new Date(); resolve({ command: [command, ...args], code: 1, signal: null, error: error.message, stdout: '', stderr: '', startedAt: startedAt.toISOString(), finishedAt: finishedAt.toISOString(), durationMs: finishedAt.getTime() - startedAt.getTime(), }); }); }); } function summarize(statuses) { const criticalCount = statuses.filter( (status) => status === 'CRITICAL', ).length; const warningCount = statuses.filter((status) => status === 'WARNING').length; return { status: criticalCount > 0 ? 'CRITICAL' : warningCount > 0 ? 'WARNING' : 'OK', criticalCount, warningCount, }; }