Files
Genarrative/scripts/ops/pingora-current-release-audit.mjs
kdletters 071faa482c 统一 Rust 与 TypeScript 格式化门禁
纳入 AGC Cargo workspace 的统一 rustfmt 检查与格式化入口

完成项目 TypeScript/Prettier 与 Rust 全量格式化

修复 Pingora expected executable 门禁的空白敏感误报

同步开发运维文档与 AGC skill pack 格式化忽略规则
2026-09-01 16:28:34 +08:00

726 lines
20 KiB
JavaScript

#!/usr/bin/env node
import { execFile } from 'node:child_process';
import { createHash } from 'node:crypto';
import { constants as fsConstants } from 'node:fs';
import { access, readFile, realpath, stat } from 'node:fs/promises';
import path from 'node:path';
const REQUIRED_ARTIFACTS = [
{ path: 'api-server', executable: true },
{ path: 'scripts/database-backup-to-oss.mjs' },
{ path: 'scripts/ops/production-health-patrol.mjs' },
{ path: 'scripts/ops/pingora-current-release-audit.mjs' },
{ path: 'scripts/ops/pingora-direct-rehearsal-status.mjs' },
{ path: 'scripts/ops/pingora-cutover-status-snapshot.mjs' },
{ path: 'scripts/ops/pingora-cutover-evidence-bundle.mjs' },
{ path: 'scripts/check-production-health-patrol-env.mjs' },
{ path: 'scripts/check-pingora-release-readiness.mjs' },
{ path: 'scripts/check-pingora-direct-preflight.mjs' },
{ path: 'scripts/check-pingora-direct-live.mjs' },
{ path: 'scripts/check-pingora-canary-live.mjs' },
{ path: 'scripts/check-pingora-canary-access-log-parity.mjs' },
{ path: 'scripts/deploy/pingora-direct-enable.sh', executable: true },
{ path: 'scripts/deploy/pingora-direct-rollback.sh', executable: true },
{
path: 'scripts/deploy/pingora-realpath-canary-enable.sh',
executable: true,
},
{
path: 'scripts/deploy/pingora-realpath-canary-disable.sh',
executable: true,
},
{
path: 'scripts/deploy/pingora-health-patrol-env-switch.mjs',
executable: true,
},
{
path: 'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
executable: true,
},
{ path: 'scripts/deploy/pingora-tls-cert-sync.mjs', executable: true },
{ path: 'deploy/systemd/genarrative-pingora-gateway.service' },
{ path: 'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf' },
{ path: 'deploy/nginx/snippets/genarrative-pingora-canary.conf' },
{ path: 'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf' },
{ path: 'deploy/env/health-patrol.env.example' },
{ path: 'deploy/env/pingora-direct-live.env.example' },
{ path: 'deploy/env/pingora-canary-live.env.example' },
{ path: 'deploy/pingora/pingora-gateway.env.example' },
{ path: 'deploy/pingora/nginx-route-parity.matrix.json' },
];
const REQUIRED_DIRS = [
'deploy/systemd',
'deploy/nginx',
'deploy/env',
'deploy/pingora',
];
const config = parseArgs(process.argv.slice(2));
const audit = await buildAudit(config);
console.log(`${JSON.stringify(audit, null, 2)}\n`);
if (audit.summary.status === 'CRITICAL' && !config.warnOnly) {
process.exit(1);
}
function parseArgs(argv) {
const result = {
releaseRoot:
process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT ||
'/opt/genarrative/current',
requirePingoraGateway: readBoolEnv(
'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY',
),
systemdShow: readBoolEnv(
'GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW',
),
systemdService:
process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SERVICE ||
'genarrative-pingora-gateway.service',
timeoutMs: parseOptionalPositiveInt(
process.env.GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS,
5000,
'GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS',
),
warnOnly: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
switch (arg) {
case '-h':
case '--help':
printUsage();
process.exit(0);
break;
case '--release-root':
result.releaseRoot = requireValue(argv, ++index, arg);
break;
case '--require-pingora-gateway':
result.requirePingoraGateway = true;
break;
case '--systemd-show':
result.systemdShow = true;
break;
case '--systemd-service':
result.systemdService = requireValue(argv, ++index, arg);
break;
case '--timeout-ms':
result.timeoutMs = parseRequiredPositiveInt(
requireValue(argv, ++index, arg),
arg,
);
break;
case '--warn-only':
result.warnOnly = true;
break;
default:
throw new Error(`未知参数: ${arg}`);
}
}
if (!path.isAbsolute(result.releaseRoot)) {
throw new Error('--release-root 必须是绝对路径。');
}
validateNoControlCharacters(result.releaseRoot, '--release-root');
if (isFilesystemRootPath(result.releaseRoot)) {
throw new Error('--release-root 不能是文件系统根目录。');
}
validateNoControlCharacters(result.systemdService, '--systemd-service');
if (!result.systemdService || /[\s/]/u.test(result.systemdService)) {
throw new Error('--systemd-service 必须是 systemd unit 名称。');
}
return result;
}
function printUsage() {
console.log(`Usage:
node scripts/ops/pingora-current-release-audit.mjs [options]
Options:
--release-root <path> current release 根目录,默认 /opt/genarrative/current。
--require-pingora-gateway 要求 current release 已包含 pingora-gateway 且可执行。
--systemd-show 读取 systemctl show,确认 service ExecStart 指向 current release 的 pingora-gateway。
--systemd-service <name> systemd service 名,默认 genarrative-pingora-gateway.service。
--timeout-ms <ms> systemctl 超时,默认 5000。
--warn-only 即使出现 CRITICAL 也以退出码 0 结束,只用于人工盘点。
该脚本只读检查 current release 自包含能力,不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。
CRITICAL 默认会让进程以退出码 1 结束,适合作为切换窗口前置门禁。
`);
}
function requireValue(argv, index, flag) {
const value = argv[index];
if (value === undefined || value.startsWith('--')) {
throw new Error(`${flag} 缺少参数值`);
}
return value;
}
function readBoolEnv(name) {
const value = process.env[name];
if (value === undefined || value === null || String(value).trim() === '') {
return false;
}
const normalized = String(value).trim().toLowerCase();
if (['1', 'true', 'yes', 'on'].includes(normalized)) {
return true;
}
if (['0', 'false', 'no', 'off'].includes(normalized)) {
return false;
}
throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`);
}
function parseOptionalPositiveInt(raw, fallback, label) {
if (raw === undefined || raw === null || String(raw).trim() === '') {
return fallback;
}
return parseRequiredPositiveInt(raw, label);
}
function parseRequiredPositiveInt(raw, label) {
const text = String(raw ?? '').trim();
if (!/^[1-9][0-9]*$/u.test(text)) {
throw new Error(`${label} 必须是正整数。`);
}
return Number.parseInt(text, 10);
}
function isFilesystemRootPath(value) {
const resolved = path.resolve(String(value));
return resolved === path.parse(resolved).root;
}
function validateNoControlCharacters(value, label) {
if (/[\0\r\n]/u.test(String(value))) {
throw new Error(`${label} 不能包含换行或 NUL 字符。`);
}
}
async function buildAudit(input) {
const artifacts = await inspectArtifacts(input);
const directories = await inspectDirectories(input);
const pingoraGateway = await inspectPingoraGateway(input);
const checksums = await inspectChecksums(input, pingoraGateway);
const releaseManifest = await inspectReleaseManifest(input, pingoraGateway);
const systemd = input.systemdShow
? await inspectSystemd(input)
: {
checked: false,
status: 'OK',
diagnostics: [],
};
const summary = summarize([
...artifacts.map((artifact) => artifact.status),
...directories.map((directory) => directory.status),
pingoraGateway.status,
...checksums.map((checksum) => checksum.status),
releaseManifest.status,
systemd.status,
]);
return {
schemaVersion: 1,
generatedAt: new Date().toISOString(),
releaseRoot: input.releaseRoot,
summary,
artifacts,
directories,
pingoraGateway,
checksums,
releaseManifest,
systemd,
};
}
async function inspectArtifacts(input) {
const artifacts = [];
for (const artifact of REQUIRED_ARTIFACTS) {
artifacts.push(await inspectArtifact(input.releaseRoot, artifact));
}
return artifacts;
}
async function inspectDirectories(input) {
const directories = [];
for (const relativePath of REQUIRED_DIRS) {
const fullPath = path.join(input.releaseRoot, relativePath);
const diagnostics = [];
let exists = false;
let status = 'OK';
try {
const fileStat = await stat(fullPath);
exists = fileStat.isDirectory();
if (!exists) {
diagnostics.push('路径存在但不是目录。');
status = 'CRITICAL';
}
} catch (error) {
diagnostics.push(`目录不存在或不可读: ${error.message}`);
status = 'CRITICAL';
}
directories.push({
path: relativePath,
exists,
status,
diagnostics,
});
}
return directories;
}
async function inspectArtifact(releaseRoot, artifact) {
const fullPath = path.join(releaseRoot, artifact.path);
const diagnostics = [];
let exists = false;
let type = 'missing';
let executable = false;
let status = 'OK';
try {
const fileStat = await stat(fullPath);
exists = true;
type = fileStat.isFile()
? 'file'
: fileStat.isDirectory()
? 'directory'
: 'other';
if (type !== 'file') {
diagnostics.push('路径存在但不是文件。');
status = 'CRITICAL';
}
} catch (error) {
diagnostics.push(`文件不存在或不可读: ${error.message}`);
status = 'CRITICAL';
}
if (exists && artifact.executable) {
try {
await access(fullPath, fsConstants.X_OK);
executable = true;
} catch (error) {
diagnostics.push(`文件不可执行: ${error.message}`);
status = 'CRITICAL';
}
}
return {
path: artifact.path,
expectedExecutable: Boolean(artifact.executable),
exists,
type,
executable,
status,
diagnostics,
};
}
async function inspectPingoraGateway(input) {
const binaryPath = path.join(input.releaseRoot, 'pingora-gateway');
const diagnostics = [];
let included = false;
let executable = false;
let status = 'OK';
try {
const fileStat = await stat(binaryPath);
included = fileStat.isFile();
if (!included) {
diagnostics.push('pingora-gateway 路径存在但不是文件。');
status = 'CRITICAL';
}
} catch (error) {
if (input.requirePingoraGateway) {
diagnostics.push(`缺少 pingora-gateway: ${error.message}`);
status = 'CRITICAL';
}
}
if (included) {
try {
await access(binaryPath, fsConstants.X_OK);
executable = true;
} catch (error) {
diagnostics.push(`pingora-gateway 不可执行: ${error.message}`);
status = 'CRITICAL';
}
}
return {
required: input.requirePingoraGateway,
included,
path: 'pingora-gateway',
absolutePath: binaryPath,
executable,
status,
diagnostics,
};
}
async function inspectChecksums(input, pingoraGateway) {
const checksumTargets = [
{
path: 'api-server',
checksumPath: 'api-server.sha256',
required: true,
},
{
path: 'pingora-gateway',
checksumPath: 'pingora-gateway.sha256',
required: input.requirePingoraGateway || pingoraGateway.included,
},
];
const checksums = [];
for (const target of checksumTargets) {
checksums.push(await inspectChecksum(input.releaseRoot, target));
}
return checksums;
}
async function inspectChecksum(releaseRoot, target) {
const diagnostics = [];
const artifactPath = path.join(releaseRoot, target.path);
const checksumPath = path.join(releaseRoot, target.checksumPath);
let status = 'OK';
let checked = false;
let exists = false;
let expectedSha256 = '';
let computedSha256 = '';
let checksumFileTarget = '';
if (!target.required) {
return {
path: target.path,
checksumPath: target.checksumPath,
required: false,
checked,
exists,
checksumFileTarget,
expectedSha256,
computedSha256,
matches: null,
status,
diagnostics,
};
}
try {
const checksumText = await readFile(checksumPath, 'utf8');
exists = true;
const parsed = parseSha256File(checksumText);
expectedSha256 = parsed.sha256;
checksumFileTarget = parsed.fileName;
if (path.basename(checksumFileTarget) !== path.basename(target.path)) {
diagnostics.push(
`${target.checksumPath} 指向了非预期文件: ${checksumFileTarget}`,
);
status = 'CRITICAL';
}
} catch (error) {
diagnostics.push(`checksum 文件不存在、不可读或格式错误: ${error.message}`);
status = 'CRITICAL';
}
try {
computedSha256 = await sha256File(artifactPath);
checked = true;
} catch (error) {
diagnostics.push(`无法读取待校验文件: ${error.message}`);
status = 'CRITICAL';
}
const matches =
checked && Boolean(expectedSha256) && computedSha256 === expectedSha256;
if (checked && expectedSha256 && !matches) {
diagnostics.push(
`${target.path} sha256 不匹配: expected ${expectedSha256}, actual ${computedSha256}`,
);
status = 'CRITICAL';
}
return {
path: target.path,
checksumPath: target.checksumPath,
required: target.required,
checked,
exists,
checksumFileTarget,
expectedSha256,
computedSha256,
matches,
status,
diagnostics,
};
}
function parseSha256File(content) {
const firstLine = content.split(/\r?\n/u).find((line) => line.trim());
if (!firstLine) {
throw new Error('checksum 文件为空');
}
const match = firstLine.match(/^([0-9a-fA-F]{64})\s+(.+)$/u);
if (!match) {
throw new Error('checksum 行必须是 sha256sum 输出格式');
}
return {
sha256: match[1].toLowerCase(),
fileName: match[2].replace(/^\*/u, '').trim(),
};
}
async function sha256File(filePath) {
const content = await readFile(filePath);
return createHash('sha256').update(content).digest('hex');
}
async function inspectReleaseManifest(input, pingoraGateway) {
const candidates = [
'release-manifest.api-server.json',
'release-manifest.json',
];
const diagnostics = [];
let status = 'OK';
let pathUsed = '';
let manifest = null;
for (const candidate of candidates) {
try {
const content = await readFile(
path.join(input.releaseRoot, candidate),
'utf8',
);
manifest = JSON.parse(content);
pathUsed = candidate;
break;
} catch (error) {
diagnostics.push(`${candidate} 不可用: ${error.message}`);
}
}
if (!manifest) {
return {
checked: true,
path: '',
candidates,
componentType: '',
artifacts: [],
status: 'CRITICAL',
diagnostics,
};
}
diagnostics.length = 0;
const artifacts = Array.isArray(manifest.artifacts) ? manifest.artifacts : [];
const apiArtifact = artifacts.find(
(artifact) => artifact?.path === 'api-server',
);
const pingoraArtifact = artifacts.find(
(artifact) => artifact?.path === 'pingora-gateway',
);
if (manifest.component_type !== 'api-server') {
diagnostics.push(
`release manifest component_type 应为 api-server,实际 ${manifest.component_type || '(空)'}`,
);
status = 'CRITICAL';
}
if (!apiArtifact) {
diagnostics.push('release manifest 缺少 api-server artifact。');
status = 'CRITICAL';
} else if (apiArtifact.checksum_path !== 'api-server.sha256') {
diagnostics.push(
`api-server artifact checksum_path 应为 api-server.sha256,实际 ${apiArtifact.checksum_path || '(空)'}`,
);
status = 'CRITICAL';
}
if (input.requirePingoraGateway || pingoraGateway.included) {
if (!pingoraArtifact) {
diagnostics.push('release manifest 缺少 pingora-gateway artifact。');
status = 'CRITICAL';
} else if (pingoraArtifact.checksum_path !== 'pingora-gateway.sha256') {
diagnostics.push(
`pingora-gateway artifact checksum_path 应为 pingora-gateway.sha256,实际 ${pingoraArtifact.checksum_path || '(空)'}`,
);
status = 'CRITICAL';
}
} else if (pingoraArtifact) {
diagnostics.push(
'release manifest 登记了 pingora-gateway,但 current release 未包含该二进制。',
);
status = 'CRITICAL';
}
return {
checked: true,
path: pathUsed,
candidates,
componentType: manifest.component_type || '',
artifacts: artifacts.map((artifact) => ({
component: artifact?.component || '',
path: artifact?.path || '',
checksumPath: artifact?.checksum_path || '',
})),
status,
diagnostics,
};
}
async function inspectSystemd(input) {
const expectedBinary = path.join(input.releaseRoot, 'pingora-gateway');
const expectedRealpath = await resolveOptionalRealpath(expectedBinary);
const result = await runCommand(
'systemctl',
[
'show',
input.systemdService,
'--property=FragmentPath',
'--property=DropInPaths',
'--property=User',
'--property=ExecStart',
'--no-pager',
],
input,
);
const show = parseSystemctlShow(result.stdout);
const diagnostics = [];
let status = 'OK';
if (result.code !== 0) {
diagnostics.push(`systemctl show 失败: ${result.stderr || result.error}`);
status = 'CRITICAL';
}
const execStart = show.ExecStart || '';
const execStartBinary = parseExecStartBinary(execStart);
const execStartRealpath = execStartBinary
? await resolveOptionalRealpath(execStartBinary)
: null;
const execStartMatches =
execStartBinary === expectedBinary ||
(expectedRealpath &&
execStartRealpath &&
expectedRealpath === execStartRealpath);
if (result.code === 0 && !execStartMatches) {
diagnostics.push(
`ExecStart 未指向 current release 网关二进制: expected ${expectedBinary}, actual ${execStart || '(空)'}`,
);
status = 'CRITICAL';
}
return {
checked: true,
service: input.systemdService,
status,
expectedBinary,
expectedRealpath,
execStartBinary,
execStartRealpath,
fragmentPath: show.FragmentPath || '',
dropInPaths: show.DropInPaths || '',
user: show.User || '',
execStart,
command: result.command,
diagnostics,
};
}
async function resolveOptionalRealpath(filePath) {
try {
return await realpath(filePath);
} catch {
return null;
}
}
function parseExecStartBinary(execStart) {
const pathMatch = execStart.match(/\bpath=([^ ;]+)(?:\s|;|$)/u);
if (pathMatch) {
return pathMatch[1];
}
const argvMatch = execStart.match(/\bargv\[\]=([^ ;]+)(?:\s|;|$)/u);
if (argvMatch) {
return argvMatch[1];
}
return '';
}
function parseSystemctlShow(text) {
const result = {};
for (const line of text.split(/\r?\n/u)) {
const equalsIndex = line.indexOf('=');
if (equalsIndex <= 0) {
continue;
}
result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1);
}
return result;
}
function runCommand(command, args, input) {
validateNoControlCharacters(command, '子命令可执行文件');
for (const arg of args) {
validateNoControlCharacters(arg, '子命令参数');
}
return new Promise((resolve) => {
const startedAt = new Date();
const child = execFile(
command,
args,
{
encoding: 'utf8',
timeout: input.timeoutMs,
maxBuffer: 1024 * 1024,
},
(error, stdout, stderr) => {
const finishedAt = new Date();
resolve({
command: [command, ...args],
code: error?.code ?? 0,
signal: error?.signal ?? null,
error: error ? error.message : '',
stdout: stdout || '',
stderr: stderr || '',
startedAt: startedAt.toISOString(),
finishedAt: finishedAt.toISOString(),
durationMs: finishedAt.getTime() - startedAt.getTime(),
});
},
);
child.on('error', (error) => {
const finishedAt = new Date();
resolve({
command: [command, ...args],
code: 1,
signal: null,
error: error.message,
stdout: '',
stderr: '',
startedAt: startedAt.toISOString(),
finishedAt: finishedAt.toISOString(),
durationMs: finishedAt.getTime() - startedAt.getTime(),
});
});
});
}
function summarize(statuses) {
const criticalCount = statuses.filter(
(status) => status === 'CRITICAL',
).length;
const warningCount = statuses.filter((status) => status === 'WARNING').length;
return {
status:
criticalCount > 0 ? 'CRITICAL' : warningCount > 0 ? 'WARNING' : 'OK',
criticalCount,
warningCount,
};
}