kdletters
071faa482c
纳入 AGC Cargo workspace 的统一 rustfmt 检查与格式化入口 完成项目 TypeScript/Prettier 与 Rust 全量格式化 修复 Pingora expected executable 门禁的空白敏感误报 同步开发运维文档与 AGC skill pack 格式化忽略规则
434 lines
13 KiB
JavaScript
434 lines
13 KiB
JavaScript
#!/usr/bin/env node
|
||
|
||
import { readFileSync } from 'node:fs';
|
||
import path from 'node:path';
|
||
|
||
const DEFAULT_CANARY_PREFIX = '/__genarrative_pingora_canary';
|
||
const REALPATH_HEALTHZ_PATH = '/__genarrative_pingora_realpath_canary/healthz';
|
||
const SHADOW_HEALTHZ_PATH = '/__genarrative_pingora/healthz';
|
||
|
||
const config = parseArgs(process.argv.slice(2));
|
||
const failures = [];
|
||
|
||
const nginxRecords = readLogRecords(config.nginxLogFile, parseNginxAccessLine);
|
||
const pingoraRecords = readLogRecords(
|
||
config.pingoraLogFile,
|
||
parseKeyValueAccessLine,
|
||
);
|
||
const parity = buildParity(nginxRecords, pingoraRecords);
|
||
|
||
if (config.json) {
|
||
console.log(`${JSON.stringify(parity, null, 2)}\n`);
|
||
}
|
||
|
||
if (failures.length > 0) {
|
||
console.error('[pingora-canary-access-log-parity] FAILED');
|
||
for (const failure of failures) {
|
||
console.error(`- ${failure}`);
|
||
}
|
||
process.exit(1);
|
||
}
|
||
|
||
if (!config.json) {
|
||
console.log(
|
||
`[pingora-canary-access-log-parity] OK (${parity.summary.matchedCount}/${parity.summary.nginxCanaryCount} matched)`,
|
||
);
|
||
}
|
||
|
||
function parseArgs(argv) {
|
||
const result = {
|
||
nginxLogFile:
|
||
process.env.GENARRATIVE_PINGORA_CANARY_NGINX_ACCESS_LOG ||
|
||
'/var/log/nginx/genarrative.access.log',
|
||
pingoraLogFile:
|
||
process.env.GENARRATIVE_PINGORA_CANARY_PINGORA_ACCESS_LOG ||
|
||
'/var/log/genarrative/pingora-gateway.access.log',
|
||
prefix:
|
||
process.env.GENARRATIVE_PINGORA_CANARY_PREFIX || DEFAULT_CANARY_PREFIX,
|
||
mode: normalizeMode(
|
||
process.env.GENARRATIVE_PINGORA_CANARY_MODE || 'prefix',
|
||
'GENARRATIVE_PINGORA_CANARY_MODE',
|
||
),
|
||
sinceLines: parseOptionalPositiveInt(
|
||
process.env.GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES,
|
||
2000,
|
||
'GENARRATIVE_PINGORA_CANARY_ACCESS_LOG_SINCE_LINES',
|
||
),
|
||
requiredPaths: [],
|
||
json: false,
|
||
};
|
||
|
||
for (let index = 0; index < argv.length; index += 1) {
|
||
const arg = argv[index];
|
||
switch (arg) {
|
||
case '-h':
|
||
case '--help':
|
||
usage();
|
||
process.exit(0);
|
||
break;
|
||
case '--nginx-log-file':
|
||
result.nginxLogFile = requireValue(argv, ++index, arg);
|
||
break;
|
||
case '--pingora-log-file':
|
||
result.pingoraLogFile = requireValue(argv, ++index, arg);
|
||
break;
|
||
case '--prefix':
|
||
result.prefix = normalizePrefix(requireValue(argv, ++index, arg), arg);
|
||
break;
|
||
case '--mode':
|
||
result.mode = normalizeMode(requireValue(argv, ++index, arg), arg);
|
||
break;
|
||
case '--realpath':
|
||
result.mode = 'realpath';
|
||
break;
|
||
case '--since-lines':
|
||
result.sinceLines = parseRequiredPositiveInt(
|
||
requireValue(argv, ++index, arg),
|
||
'--since-lines',
|
||
);
|
||
break;
|
||
case '--path':
|
||
result.requiredPaths.push(
|
||
normalizeRequestPath(requireValue(argv, ++index, arg), arg),
|
||
);
|
||
break;
|
||
case '--json':
|
||
result.json = true;
|
||
break;
|
||
default:
|
||
throw new Error(`未知参数: ${arg}`);
|
||
}
|
||
}
|
||
|
||
result.prefix = normalizePrefix(result.prefix, '--prefix');
|
||
for (const [label, file] of [
|
||
['--nginx-log-file', result.nginxLogFile],
|
||
['--pingora-log-file', result.pingoraLogFile],
|
||
]) {
|
||
validateSafeAbsoluteFilePath(file, label);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
function usage() {
|
||
console.log(`Usage:
|
||
node scripts/check-pingora-canary-access-log-parity.mjs [options]
|
||
|
||
Options:
|
||
--nginx-log-file <path> Nginx access log，默认 /var/log/nginx/genarrative.access.log。
|
||
--pingora-log-file <path> Pingora access log，默认 /var/log/genarrative/pingora-gateway.access.log。
|
||
--prefix <path> canary 前缀，默认 /__genarrative_pingora_canary。
|
||
--mode <prefix|realpath> 对账模式，默认 prefix；realpath 要求 Nginx path 与 Pingora path 对齐。
|
||
--realpath Shortcut for --mode realpath。
|
||
--since-lines <count> 只读取日志尾部行数，默认 2000。
|
||
--path <path> 必须出现并完成对照的原始 canary 路径，可重复。
|
||
--json 输出 JSON。
|
||
|
||
该脚本只读比较 Nginx canary handoff access log 和 Pingora access log，不修改日志、不 reload Nginx 或 Pingora。
|
||
Nginx canary exact healthz 会转发到 Pingora shadow healthz，其余前缀路径按 rewrite 后路径对照。
|
||
真实路径 canary 使用独立 Nginx access log；除 realpath healthz 探针外，Nginx 与 Pingora path 必须一致。
|
||
`);
|
||
}
|
||
|
||
function requireValue(argv, index, flag) {
|
||
const value = argv[index];
|
||
if (!value || value.startsWith('--')) {
|
||
throw new Error(`${flag} 缺少参数值`);
|
||
}
|
||
return value;
|
||
}
|
||
|
||
function parseOptionalPositiveInt(raw, fallback, label) {
|
||
if (raw === undefined || raw === null || String(raw).trim() === '') {
|
||
return fallback;
|
||
}
|
||
return parseRequiredPositiveInt(raw, label);
|
||
}
|
||
|
||
function parseRequiredPositiveInt(raw, label) {
|
||
const rawText = String(raw ?? '');
|
||
validateNoControlCharacters(rawText, label);
|
||
const text = rawText.trim();
|
||
if (!/^[1-9]\d*$/.test(text)) {
|
||
throw new Error(`${label} 必须是正整数。`);
|
||
}
|
||
return Number.parseInt(text, 10);
|
||
}
|
||
|
||
function normalizeMode(raw, label) {
|
||
validateNoControlCharacters(raw, label);
|
||
const value = String(raw || '').trim();
|
||
if (!value || value === 'prefix') {
|
||
return 'prefix';
|
||
}
|
||
if (value === 'realpath') {
|
||
return 'realpath';
|
||
}
|
||
throw new Error(`${label} 必须是 prefix 或 realpath。`);
|
||
}
|
||
|
||
function validateSafeAbsoluteFilePath(value, flag) {
|
||
validateNoControlCharacters(value, flag);
|
||
if (!path.isAbsolute(value)) {
|
||
throw new Error(`${flag} 必须是绝对路径。`);
|
||
}
|
||
if (isFilesystemRootPath(value)) {
|
||
throw new Error(`${flag} 不能是文件系统根目录。`);
|
||
}
|
||
}
|
||
|
||
function isFilesystemRootPath(value) {
|
||
const resolved = path.resolve(String(value));
|
||
return resolved === path.parse(resolved).root;
|
||
}
|
||
|
||
function normalizePrefix(prefix, label) {
|
||
validateNoControlCharacters(prefix, label);
|
||
if (!prefix || prefix === '/') {
|
||
return DEFAULT_CANARY_PREFIX;
|
||
}
|
||
const withLeadingSlash = prefix.startsWith('/') ? prefix : `/${prefix}`;
|
||
return withLeadingSlash.endsWith('/')
|
||
? withLeadingSlash.slice(0, -1)
|
||
: withLeadingSlash;
|
||
}
|
||
|
||
function normalizeRequestPath(value, label = '--path') {
|
||
validateNoControlCharacters(value, label);
|
||
const raw = String(value || '').trim();
|
||
if (!raw) {
|
||
return '/';
|
||
}
|
||
const pathOnly = raw.includes('://')
|
||
? new URL(raw).pathname
|
||
: raw.split('?')[0];
|
||
return pathOnly.startsWith('/') ? pathOnly : `/${pathOnly}`;
|
||
}
|
||
|
||
function validateNoControlCharacters(value, label) {
|
||
if (/[\0\r\n]/u.test(String(value ?? ''))) {
|
||
throw new Error(`${label} 不能包含换行或 NUL 字符。`);
|
||
}
|
||
}
|
||
|
||
function readLogRecords(filePath, parser) {
|
||
let text;
|
||
try {
|
||
text = readFileSync(filePath, 'utf8');
|
||
} catch (error) {
|
||
failures.push(`无法读取日志文件 ${filePath}: ${error.message}`);
|
||
return [];
|
||
}
|
||
return text
|
||
.split(/\r?\n/u)
|
||
.filter(Boolean)
|
||
.slice(-config.sinceLines)
|
||
.map((line, index) => parseLogRecord(parser, line, index + 1, filePath))
|
||
.filter(Boolean);
|
||
}
|
||
|
||
function parseLogRecord(parser, line, lineNumber, filePath) {
|
||
try {
|
||
return parser(line, lineNumber);
|
||
} catch (error) {
|
||
failures.push(
|
||
`日志文件 ${filePath} 第 ${lineNumber} 行解析失败: ${error.message}`,
|
||
);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
function parseNginxAccessLine(line, lineNumber) {
|
||
const requestMatch = line.match(/"([A-Z]+)\s+([^"\s]+)\s+HTTP\/[^"]+"/u);
|
||
const statusMatch = line.match(/"\s+(\d{3})\s+/u);
|
||
const requestIdMatch = line.match(/(?:^|\s)request_id=([^\s]+)/u);
|
||
if (!requestMatch || !statusMatch || !requestIdMatch) {
|
||
return null;
|
||
}
|
||
const uri = requestMatch[2];
|
||
return {
|
||
source: 'nginx',
|
||
lineNumber,
|
||
requestId: requestIdMatch[1],
|
||
method: requestMatch[1],
|
||
uri,
|
||
path: normalizeRequestPath(uri, `Nginx access log 第 ${lineNumber} 行 URI`),
|
||
status: Number.parseInt(statusMatch[1], 10),
|
||
raw: line,
|
||
};
|
||
}
|
||
|
||
function parseKeyValueAccessLine(line, lineNumber) {
|
||
const fields = {};
|
||
for (const part of line.split(/\t/u)) {
|
||
const separator = part.indexOf('=');
|
||
if (separator <= 0) {
|
||
continue;
|
||
}
|
||
fields[part.slice(0, separator)] = unescapeAccessLogValue(
|
||
part.slice(separator + 1),
|
||
);
|
||
}
|
||
if (!fields.request_id || !fields.path || !fields.status) {
|
||
return null;
|
||
}
|
||
return {
|
||
source: 'pingora',
|
||
lineNumber,
|
||
requestId: fields.request_id,
|
||
method: fields.method || '',
|
||
uri: fields.uri || fields.path,
|
||
path: normalizeRequestPath(
|
||
fields.path,
|
||
`Pingora access log 第 ${lineNumber} 行 path`,
|
||
),
|
||
status: Number.parseInt(fields.status, 10),
|
||
route: fields.route || '',
|
||
proxyTarget: fields.proxy_target || '',
|
||
upstream: fields.upstream || '',
|
||
raw: line,
|
||
};
|
||
}
|
||
|
||
function unescapeAccessLogValue(value) {
|
||
return String(value)
|
||
.replaceAll('%09', '\t')
|
||
.replaceAll('%0A', '\n')
|
||
.replaceAll('%0D', '\r');
|
||
}
|
||
|
||
function buildParity(nginxRecords, pingoraRecords) {
|
||
const pingoraByRequestId = new Map();
|
||
for (const record of pingoraRecords) {
|
||
if (!pingoraByRequestId.has(record.requestId)) {
|
||
pingoraByRequestId.set(record.requestId, []);
|
||
}
|
||
pingoraByRequestId.get(record.requestId).push(record);
|
||
}
|
||
|
||
const nginxCanary = nginxRecords.filter((record) => isCanaryRecord(record));
|
||
if (nginxCanary.length === 0) {
|
||
failures.push(noCanaryRecordsMessage());
|
||
}
|
||
|
||
const matches = [];
|
||
const missing = [];
|
||
const mismatches = [];
|
||
const seenCanaryPaths = new Set();
|
||
|
||
for (const nginxRecord of nginxCanary) {
|
||
seenCanaryPaths.add(nginxRecord.path);
|
||
const expectedPath = stripCanaryPrefix(nginxRecord.path);
|
||
const candidates = pingoraByRequestId.get(nginxRecord.requestId) || [];
|
||
const match = candidates.find((record) => record.path === expectedPath);
|
||
if (!match) {
|
||
missing.push({
|
||
requestId: nginxRecord.requestId,
|
||
nginxPath: nginxRecord.path,
|
||
expectedPingoraPath: expectedPath,
|
||
});
|
||
continue;
|
||
}
|
||
const mismatch = {
|
||
requestId: nginxRecord.requestId,
|
||
nginxPath: nginxRecord.path,
|
||
pingoraPath: match.path,
|
||
problems: [],
|
||
};
|
||
if (
|
||
nginxRecord.method &&
|
||
match.method &&
|
||
nginxRecord.method !== match.method
|
||
) {
|
||
mismatch.problems.push(`method ${nginxRecord.method} != ${match.method}`);
|
||
}
|
||
if (nginxRecord.status !== match.status) {
|
||
mismatch.problems.push(`status ${nginxRecord.status} != ${match.status}`);
|
||
}
|
||
if (mismatch.problems.length > 0) {
|
||
mismatches.push(mismatch);
|
||
continue;
|
||
}
|
||
matches.push({
|
||
requestId: nginxRecord.requestId,
|
||
nginxPath: nginxRecord.path,
|
||
pingoraPath: match.path,
|
||
status: nginxRecord.status,
|
||
route: match.route,
|
||
proxyTarget: match.proxyTarget,
|
||
});
|
||
}
|
||
|
||
for (const requiredPath of config.requiredPaths) {
|
||
if (!seenCanaryPaths.has(requiredPath)) {
|
||
failures.push(`Nginx canary 日志缺少必需路径: ${requiredPath}`);
|
||
}
|
||
}
|
||
for (const item of missing) {
|
||
failures.push(
|
||
`缺少对应 Pingora access log: request_id=${item.requestId} nginx_path=${item.nginxPath} expected_pingora_path=${item.expectedPingoraPath}`,
|
||
);
|
||
}
|
||
for (const item of mismatches) {
|
||
failures.push(
|
||
`Nginx/Pingora access log 不一致: request_id=${item.requestId} nginx_path=${item.nginxPath} pingora_path=${item.pingoraPath} ${item.problems.join(', ')}`,
|
||
);
|
||
}
|
||
|
||
return {
|
||
ok: failures.length === 0,
|
||
mode: config.mode,
|
||
prefix: config.prefix,
|
||
summary: {
|
||
nginxCanaryCount: nginxCanary.length,
|
||
pingoraCount: pingoraRecords.length,
|
||
matchedCount: matches.length,
|
||
missingCount: missing.length,
|
||
mismatchCount: mismatches.length,
|
||
},
|
||
matches,
|
||
missing,
|
||
mismatches,
|
||
};
|
||
}
|
||
|
||
function isCanaryRecord(record) {
|
||
if (config.mode === 'realpath') {
|
||
return (
|
||
record.path === REALPATH_HEALTHZ_PATH ||
|
||
record.path === '/api/assets/history' ||
|
||
record.path.startsWith('/v1/database/') ||
|
||
record.path.startsWith('/v1/identity') ||
|
||
record.path === '/assets/app.js' ||
|
||
record.path === '/generated-pingora-canary-smoke'
|
||
);
|
||
}
|
||
return (
|
||
record.path.startsWith(`${config.prefix}/`) || record.path === config.prefix
|
||
);
|
||
}
|
||
|
||
function noCanaryRecordsMessage() {
|
||
if (config.mode === 'realpath') {
|
||
return `Nginx 日志尾部 ${config.sinceLines} 行中没有真实路径 canary 请求`;
|
||
}
|
||
return `Nginx 日志尾部 ${config.sinceLines} 行中没有 canary 前缀请求: ${config.prefix}`;
|
||
}
|
||
|
||
function stripCanaryPrefix(canaryPath) {
|
||
if (config.mode === 'realpath') {
|
||
if (canaryPath === REALPATH_HEALTHZ_PATH) {
|
||
return SHADOW_HEALTHZ_PATH;
|
||
}
|
||
return canaryPath;
|
||
}
|
||
if (canaryPath === `${config.prefix}/healthz`) {
|
||
return SHADOW_HEALTHZ_PATH;
|
||
}
|
||
if (canaryPath === config.prefix) {
|
||
return '/';
|
||
}
|
||
const stripped = canaryPath.slice(config.prefix.length);
|
||
return stripped || '/';
|
||
}
|