#!/usr/bin/env node import { readFileSync } from 'node:fs'; import path from 'node:path'; const DEFAULT_CANARY_PREFIX = '/__genarrative_pingora_canary'; const REALPATH_HEALTHZ_PATH = '/__genarrative_pingora_realpath_canary/healthz'; const SHADOW_HEALTHZ_PATH = '/__genarrative_pingora/healthz'; const config = parseArgs(process.argv.slice(2)); const failures = []; const nginxRecords = readLogRecords(config.nginxLogFile, parseNginxAccessLine); const pingoraRecords = readLogRecords( config.pingoraLogFile, parseKeyValueAccessLine, ); const parity = buildParity(nginxRecords, pingoraRecords); if (config.json) { console.log(`${JSON.stringify(parity, null, 2)}\n`); } if (failures.length > 0) { console.error('[pingora-canary-access-log-parity] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } if (!config.json) { console.log( `[pingora-canary-access-log-parity] OK (${parity.summary.matchedCount}/${parity.summary.nginxCanaryCount} matched)`, ); } function parseArgs(argv) { const result = { nginxLogFile: process.env.GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG || '/var/log/nginx/genarrative.access.log', pingoraLogFile: process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG || '/var/log/genarrative/pingora-gateway.access.log', prefix: process.env.GENARRATIVE_PINGORA_CANARY_PREFIX || DEFAULT_CANARY_PREFIX, mode: normalizeMode( process.env.GENARRATIVE_PINGORA_CANARY_MODE || 'prefix', 'GENARRATIVE_PINGORA_CANARY_MODE', ), sinceLines: parseOptionalPositiveInt( process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES, 2000, 'GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES', ), requiredPaths: [], json: false, }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; switch (arg) { case '-h': case '--help': usage(); process.exit(0); break; case '--nginx-log-file': result.nginxLogFile = requireValue(argv, ++index, arg); break; case '--pingora-log-file': result.pingoraLogFile = requireValue(argv, ++index, arg); break; case '--prefix': result.prefix = normalizePrefix(requireValue(argv, ++index, arg), arg); break; case '--mode': result.mode = normalizeMode(requireValue(argv, ++index, arg), arg); break; case '--realpath': result.mode = 'realpath'; break; case '--since-lines': result.sinceLines = parseRequiredPositiveInt( requireValue(argv, ++index, arg), '--since-lines', ); break; case '--path': result.requiredPaths.push( normalizeRequestPath(requireValue(argv, ++index, arg), arg), ); break; case '--json': result.json = true; break; default: throw new Error(`未知参数: ${arg}`); } } result.prefix = normalizePrefix(result.prefix, '--prefix'); for (const [label, file] of [ ['--nginx-log-file', result.nginxLogFile], ['--pingora-log-file', result.pingoraLogFile], ]) { validateSafeAbsoluteFilePath(file, label); } return result; } function usage() { console.log(`Usage: node scripts/check-pingora-canary-access-log-parity.mjs [options] Options: --nginx-log-file Nginx access log,默认 /var/log/nginx/genarrative.access.log。 --pingora-log-file Pingora access log,默认 /var/log/genarrative/pingora-gateway.access.log。 --prefix canary 前缀,默认 /__genarrative_pingora_canary。 --mode 对账模式,默认 prefix;realpath 要求 Nginx path 与 Pingora path 对齐。 --realpath Shortcut for --mode realpath。 --since-lines 只读取日志尾部行数,默认 2000。 --path 必须出现并完成对照的原始 canary 路径,可重复。 --json 输出 JSON。 该脚本只读比较 Nginx canary handoff access log 和 Pingora access log,不修改日志、不 reload Nginx 或 Pingora。 Nginx canary exact healthz 会转发到 Pingora shadow healthz,其余前缀路径按 rewrite 后路径对照。 真实路径 canary 使用独立 Nginx access log;除 realpath healthz 探针外,Nginx 与 Pingora path 必须一致。 `); } function requireValue(argv, index, flag) { const value = argv[index]; if (!value || value.startsWith('--')) { throw new Error(`${flag} 缺少参数值`); } return value; } function parseOptionalPositiveInt(raw, fallback, label) { if (raw === undefined || raw === null || String(raw).trim() === '') { return fallback; } return parseRequiredPositiveInt(raw, label); } function parseRequiredPositiveInt(raw, label) { const rawText = String(raw ?? ''); validateNoControlCharacters(rawText, label); const text = rawText.trim(); if (!/^[1-9]\d*$/.test(text)) { throw new Error(`${label} 必须是正整数。`); } return Number.parseInt(text, 10); } function normalizeMode(raw, label) { validateNoControlCharacters(raw, label); const value = String(raw || '').trim(); if (!value || value === 'prefix') { return 'prefix'; } if (value === 'realpath') { return 'realpath'; } throw new Error(`${label} 必须是 prefix 或 realpath。`); } function validateSafeAbsoluteFilePath(value, flag) { validateNoControlCharacters(value, flag); if (!path.isAbsolute(value)) { throw new Error(`${flag} 必须是绝对路径。`); } if (isFilesystemRootPath(value)) { throw new Error(`${flag} 不能是文件系统根目录。`); } } function isFilesystemRootPath(value) { const resolved = path.resolve(String(value)); return resolved === path.parse(resolved).root; } function normalizePrefix(prefix, label) { validateNoControlCharacters(prefix, label); if (!prefix || prefix === '/') { return DEFAULT_CANARY_PREFIX; } const withLeadingSlash = prefix.startsWith('/') ? prefix : `/${prefix}`; return withLeadingSlash.endsWith('/') ? withLeadingSlash.slice(0, -1) : withLeadingSlash; } function normalizeRequestPath(value, label = '--path') { validateNoControlCharacters(value, label); const raw = String(value || '').trim(); if (!raw) { return '/'; } const pathOnly = raw.includes('://') ? new URL(raw).pathname : raw.split('?')[0]; return pathOnly.startsWith('/') ? pathOnly : `/${pathOnly}`; } function validateNoControlCharacters(value, label) { if (/[\0\r\n]/u.test(String(value ?? ''))) { throw new Error(`${label} 不能包含换行或 NUL 字符。`); } } function readLogRecords(filePath, parser) { let text; try { text = readFileSync(filePath, 'utf8'); } catch (error) { failures.push(`无法读取日志文件 ${filePath}: ${error.message}`); return []; } return text .split(/\r?\n/u) .filter(Boolean) .slice(-config.sinceLines) .map((line, index) => parseLogRecord(parser, line, index + 1, filePath)) .filter(Boolean); } function parseLogRecord(parser, line, lineNumber, filePath) { try { return parser(line, lineNumber); } catch (error) { failures.push( `日志文件 ${filePath} 第 ${lineNumber} 行解析失败: ${error.message}`, ); return null; } } function parseNginxAccessLine(line, lineNumber) { const requestMatch = line.match(/"([A-Z]+)\s+([^"\s]+)\s+HTTP\/[^"]+"/u); const statusMatch = line.match(/"\s+(\d{3})\s+/u); const requestIdMatch = line.match(/(?:^|\s)request_id=([^\s]+)/u); if (!requestMatch || !statusMatch || !requestIdMatch) { return null; } const uri = requestMatch[2]; return { source: 'nginx', lineNumber, requestId: requestIdMatch[1], method: requestMatch[1], uri, path: normalizeRequestPath(uri, `Nginx access log 第 ${lineNumber} 行 URI`), status: Number.parseInt(statusMatch[1], 10), raw: line, }; } function parseKeyValueAccessLine(line, lineNumber) { const fields = {}; for (const part of line.split(/\t/u)) { const separator = part.indexOf('='); if (separator <= 0) { continue; } fields[part.slice(0, separator)] = unescapeAccessLogValue( part.slice(separator + 1), ); } if (!fields.request_id || !fields.path || !fields.status) { return null; } return { source: 'pingora', lineNumber, requestId: fields.request_id, method: fields.method || '', uri: fields.uri || fields.path, path: normalizeRequestPath( fields.path, `Pingora access log 第 ${lineNumber} 行 path`, ), status: Number.parseInt(fields.status, 10), route: fields.route || '', proxyTarget: fields.proxy_target || '', upstream: fields.upstream || '', raw: line, }; } function unescapeAccessLogValue(value) { return String(value) .replaceAll('%09', '\t') .replaceAll('%0A', '\n') .replaceAll('%0D', '\r'); } function buildParity(nginxRecords, pingoraRecords) { const pingoraByRequestId = new Map(); for (const record of pingoraRecords) { if (!pingoraByRequestId.has(record.requestId)) { pingoraByRequestId.set(record.requestId, []); } pingoraByRequestId.get(record.requestId).push(record); } const nginxCanary = nginxRecords.filter((record) => isCanaryRecord(record)); if (nginxCanary.length === 0) { failures.push(noCanaryRecordsMessage()); } const matches = []; const missing = []; const mismatches = []; const seenCanaryPaths = new Set(); for (const nginxRecord of nginxCanary) { seenCanaryPaths.add(nginxRecord.path); const expectedPath = stripCanaryPrefix(nginxRecord.path); const candidates = pingoraByRequestId.get(nginxRecord.requestId) || []; const match = candidates.find((record) => record.path === expectedPath); if (!match) { missing.push({ requestId: nginxRecord.requestId, nginxPath: nginxRecord.path, expectedPingoraPath: expectedPath, }); continue; } const mismatch = { requestId: nginxRecord.requestId, nginxPath: nginxRecord.path, pingoraPath: match.path, problems: [], }; if (nginxRecord.method && match.method && nginxRecord.method !== match.method) { mismatch.problems.push(`method ${nginxRecord.method} != ${match.method}`); } if (nginxRecord.status !== match.status) { mismatch.problems.push(`status ${nginxRecord.status} != ${match.status}`); } if (mismatch.problems.length > 0) { mismatches.push(mismatch); continue; } matches.push({ requestId: nginxRecord.requestId, nginxPath: nginxRecord.path, pingoraPath: match.path, status: nginxRecord.status, route: match.route, proxyTarget: match.proxyTarget, }); } for (const requiredPath of config.requiredPaths) { if (!seenCanaryPaths.has(requiredPath)) { failures.push(`Nginx canary 日志缺少必需路径: ${requiredPath}`); } } for (const item of missing) { failures.push( `缺少对应 Pingora access log: request_id=${item.requestId} nginx_path=${item.nginxPath} expected_pingora_path=${item.expectedPingoraPath}`, ); } for (const item of mismatches) { failures.push( `Nginx/Pingora access log 不一致: request_id=${item.requestId} nginx_path=${item.nginxPath} pingora_path=${item.pingoraPath} ${item.problems.join(', ')}`, ); } return { ok: failures.length === 0, mode: config.mode, prefix: config.prefix, summary: { nginxCanaryCount: nginxCanary.length, pingoraCount: pingoraRecords.length, matchedCount: matches.length, missingCount: missing.length, mismatchCount: mismatches.length, }, matches, missing, mismatches, }; } function isCanaryRecord(record) { if (config.mode === 'realpath') { return ( record.path === REALPATH_HEALTHZ_PATH || record.path === '/api/assets/history' || record.path.startsWith('/v1/database/') || record.path.startsWith('/v1/identity') || record.path === '/assets/app.js' || record.path === '/generated-pingora-canary-smoke' ); } return ( record.path.startsWith(`${config.prefix}/`) || record.path === config.prefix ); } function noCanaryRecordsMessage() { if (config.mode === 'realpath') { return `Nginx 日志尾部 ${config.sinceLines} 行中没有真实路径 canary 请求`; } return `Nginx 日志尾部 ${config.sinceLines} 行中没有 canary 前缀请求: ${config.prefix}`; } function stripCanaryPrefix(canaryPath) { if (config.mode === 'realpath') { if (canaryPath === REALPATH_HEALTHZ_PATH) { return SHADOW_HEALTHZ_PATH; } return canaryPath; } if (canaryPath === `${config.prefix}/healthz`) { return SHADOW_HEALTHZ_PATH; } if (canaryPath === config.prefix) { return '/'; } const stripped = canaryPath.slice(config.prefix.length); return stripped || '/'; }