Files
Genarrative/scripts/check-pingora-current-release-audit.mjs
kdletters 071faa482c 统一 Rust 与 TypeScript 格式化门禁
纳入 AGC Cargo workspace 的统一 rustfmt 检查与格式化入口

完成项目 TypeScript/Prettier 与 Rust 全量格式化

修复 Pingora expected executable 门禁的空白敏感误报

同步开发运维文档与 AGC skill pack 格式化忽略规则
2026-09-01 16:28:34 +08:00

705 lines
22 KiB
JavaScript

#!/usr/bin/env node
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import {
chmodSync,
copyFileSync,
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
symlinkSync,
writeFileSync,
} from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
const AUDIT_SCRIPT = 'scripts/ops/pingora-current-release-audit.mjs';
const failures = [];
const tmpRoot = mkdtempSync(
path.join(tmpdir(), 'genarrative-pingora-current-release-audit-'),
);
try {
main();
} finally {
rmSync(tmpRoot, { recursive: true, force: true });
}
if (failures.length > 0) {
console.error('[check:pingora-current-release-audit] FAILED');
for (const failure of failures) {
console.error(`- ${failure}`);
}
process.exit(1);
}
console.log('[check:pingora-current-release-audit] OK');
function main() {
assertScriptShape();
assertCompleteReleasePasses();
assertChecksumMismatchFails();
assertRequirePingoraGatewayFailsWhenChecksumMissing();
assertManifestMustRegisterIncludedPingoraGateway();
assertMissingCutoverScriptFails();
assertRequirePingoraGatewayFailsWhenMissing();
assertIncludedPingoraGatewayMustBeExecutable();
assertSystemdShowRequiresCurrentReleaseExecStart();
assertSystemdShowAcceptsCurrentSymlinkExecStart();
assertRejectsRelativeReleaseRoot();
assertRejectsFilesystemRootReleaseRoot();
assertRejectsPathArgsWithControlCharacters();
assertRejectsInvalidTimeout();
assertRejectsInvalidBoolEnv();
}
function assertScriptShape() {
const content = readFileSync(AUDIT_SCRIPT, 'utf8');
assertIncludes(
content,
'该脚本只读检查 current release 自包含能力',
'current release 自审脚本 usage 必须说明只读边界。',
);
assertIncludes(
content,
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
'current release 自审必须检查切换证据包脚本。',
);
assertIncludes(
content,
'scripts/deploy/pingora-tls-cert-sync.mjs',
'current release 自审必须检查 TLS 证书同步脚本。',
);
assertIncludes(
content,
'scripts/ops/pingora-current-release-audit.mjs',
'current release 自审必须检查自身已随包发布。',
);
assertIncludes(
content,
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'current release 自审必须检查直连彩排状态脚本已随包发布。',
);
assertIncludes(
content,
'scripts/check-pingora-release-readiness.mjs',
'current release 自审必须检查 release readiness 聚合门禁脚本。',
);
assertIncludes(
content,
'scripts/check-pingora-canary-live.mjs',
'current release 自审必须检查 canary live smoke 脚本。',
);
assertIncludes(
content,
'scripts/check-pingora-canary-access-log-parity.mjs',
'current release 自审必须检查 canary access log 对账脚本。',
);
if (
content.includes('writeFile') ||
content.includes('rmSync(') ||
content.includes('systemctl reload') ||
content.includes('daemon-reload')
) {
failures.push(
'current release 自审脚本不应写文件、删除文件或 reload systemd。',
);
}
}
function assertCompleteReleasePasses() {
const fixture = prepareFixture('complete-release');
const result = runAudit(fixture, ['--require-pingora-gateway']);
assertStatus(result, 0, '完整 current release 自审应通过。');
if (result.status !== 0) {
return;
}
const payload = parseJson(result.stdout, '完整 release 自审输出');
assertEqual(payload.summary.status, 'OK', '完整 release summary 应为 OK。');
assertEqual(
payload.pingoraGateway.included,
true,
'完整 release 应识别已包含 pingora-gateway。',
);
assertEqual(
payload.pingoraGateway.executable,
true,
'完整 release 应识别 pingora-gateway 可执行。',
);
const pingoraChecksum = payload.checksums.find(
(checksum) => checksum.path === 'pingora-gateway',
);
if (!pingoraChecksum || pingoraChecksum.matches !== true) {
failures.push('完整 release 自审必须确认 pingora-gateway checksum 匹配。');
}
assertEqual(
payload.releaseManifest.status,
'OK',
'完整 release manifest 自审应通过。',
);
if (
!payload.releaseManifest.artifacts?.some(
(artifact) => artifact.path === 'pingora-gateway',
)
) {
failures.push('完整 release manifest 必须登记 pingora-gateway。');
}
const bundleArtifact = payload.artifacts.find(
(artifact) =>
artifact.path === 'scripts/ops/pingora-cutover-evidence-bundle.mjs',
);
if (!bundleArtifact || bundleArtifact.status !== 'OK') {
failures.push('完整 release 自审必须确认切换证据包脚本存在。');
}
const canaryLogParityArtifact = payload.artifacts.find(
(artifact) =>
artifact.path === 'scripts/check-pingora-canary-access-log-parity.mjs',
);
if (!canaryLogParityArtifact || canaryLogParityArtifact.status !== 'OK') {
failures.push('完整 release 自审必须确认 canary access log 对账脚本存在。');
}
const canaryLiveArtifact = payload.artifacts.find(
(artifact) => artifact.path === 'scripts/check-pingora-canary-live.mjs',
);
if (!canaryLiveArtifact || canaryLiveArtifact.status !== 'OK') {
failures.push('完整 release 自审必须确认 canary live smoke 脚本存在。');
}
const tlsCertSyncArtifact = payload.artifacts.find(
(artifact) => artifact.path === 'scripts/deploy/pingora-tls-cert-sync.mjs',
);
if (!tlsCertSyncArtifact || tlsCertSyncArtifact.status !== 'OK') {
failures.push('完整 release 自审必须确认 TLS 证书同步脚本存在且可执行。');
}
}
function assertChecksumMismatchFails() {
const fixture = prepareFixture('checksum-mismatch');
writeFileSync(
path.join(fixture.releaseRoot, 'pingora-gateway.sha256'),
`${'0'.repeat(64)} pingora-gateway\n`,
'utf8',
);
const result = runAudit(fixture, ['--require-pingora-gateway']);
assertStatus(result, 1, 'pingora-gateway checksum 不匹配时自审必须失败。');
const payload = parseJson(result.stdout, 'checksum 不匹配自审输出');
const checksum = payload.checksums?.find(
(item) => item.path === 'pingora-gateway',
);
if (!checksum || checksum.status !== 'CRITICAL') {
failures.push(
'checksum 不匹配时 pingora-gateway checksum 必须标记 CRITICAL。',
);
}
}
function assertRequirePingoraGatewayFailsWhenChecksumMissing() {
const fixture = prepareFixture('missing-gateway-checksum');
rmSync(path.join(fixture.releaseRoot, 'pingora-gateway.sha256'));
const result = runAudit(fixture, ['--require-pingora-gateway']);
assertStatus(result, 1, '要求 Pingora 但 checksum 缺失时自审必须失败。');
const payload = parseJson(result.stdout, '缺少 Pingora checksum 自审输出');
const checksum = payload.checksums?.find(
(item) => item.path === 'pingora-gateway',
);
if (!checksum || checksum.status !== 'CRITICAL') {
failures.push('缺少 Pingora checksum 时必须标记 CRITICAL。');
}
}
function assertManifestMustRegisterIncludedPingoraGateway() {
const fixture = prepareFixture('manifest-missing-gateway', {
includePingoraInManifest: false,
});
const result = runAudit(fixture, ['--require-pingora-gateway']);
assertStatus(result, 1, 'manifest 未登记 Pingora 时自审必须失败。');
const payload = parseJson(result.stdout, 'manifest 缺少 Pingora 自审输出');
assertEqual(
payload.releaseManifest?.status,
'CRITICAL',
'manifest 缺少 Pingora 时 releaseManifest 必须标记 CRITICAL。',
);
assertIncludes(
payload.releaseManifest?.diagnostics?.join('\n') || '',
'release manifest 缺少 pingora-gateway artifact',
'manifest 缺少 Pingora 时必须给出明确诊断。',
);
}
function assertMissingCutoverScriptFails() {
const fixture = prepareFixture('missing-evidence-bundle');
rmSync(
path.join(
fixture.releaseRoot,
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
),
);
const result = runAudit(fixture);
assertStatus(result, 1, '缺少切换证据包脚本时自审必须失败。');
const payload = parseJson(result.stdout, '缺少证据包自审输出');
assertEqual(
payload.summary.status,
'CRITICAL',
'缺少证据包时 summary 必须是 CRITICAL。',
);
const artifact = payload.artifacts.find(
(item) => item.path === 'scripts/ops/pingora-cutover-evidence-bundle.mjs',
);
if (!artifact || artifact.status !== 'CRITICAL') {
failures.push('缺少证据包时对应 artifact 必须标记 CRITICAL。');
}
}
function assertRequirePingoraGatewayFailsWhenMissing() {
const fixture = prepareFixture('missing-required-gateway', {
includePingoraGateway: false,
});
const result = runAudit(fixture, ['--require-pingora-gateway']);
assertStatus(result, 1, '要求 Pingora 二进制但 release 未包含时必须失败。');
const payload = parseJson(result.stdout, '缺少 Pingora 二进制自审输出');
assertEqual(
payload.pingoraGateway.status,
'CRITICAL',
'缺少必需 Pingora 二进制必须标记 CRITICAL。',
);
}
function assertIncludedPingoraGatewayMustBeExecutable() {
const fixture = prepareFixture('non-executable-gateway');
chmodSync(path.join(fixture.releaseRoot, 'pingora-gateway'), 0o644);
const result = runAudit(fixture);
assertStatus(result, 1, '包含不可执行 pingora-gateway 时必须失败。');
const payload = parseJson(result.stdout, '不可执行 Pingora 二进制自审输出');
assertEqual(
payload.pingoraGateway.status,
'CRITICAL',
'不可执行 Pingora 二进制必须标记 CRITICAL。',
);
}
function assertSystemdShowRequiresCurrentReleaseExecStart() {
const fixture = prepareFixture('systemd-old-path');
const result = runAudit(fixture, ['--systemd-show'], {
systemdExecStart: '/opt/genarrative/old-release/pingora-gateway',
});
assertStatus(result, 1, 'systemd ExecStart 指向旧 release 时必须失败。');
const payload = parseJson(result.stdout, '旧 ExecStart 自审输出');
assertEqual(
payload.systemd.status,
'CRITICAL',
'旧 ExecStart 必须让 systemd 自审标记 CRITICAL。',
);
assertIncludes(
payload.systemd.diagnostics.join('\n'),
'ExecStart 未指向 current release 网关二进制',
'旧 ExecStart 必须给出明确诊断。',
);
}
function assertSystemdShowAcceptsCurrentSymlinkExecStart() {
const fixture = prepareFixture('systemd-current-symlink', {
releaseDirectoryName: 'releases/current-build',
});
const currentRoot = path.join(fixture.root, 'current');
symlinkSync(fixture.releaseRoot, currentRoot, 'dir');
const currentBinary = path.join(currentRoot, 'pingora-gateway');
const result = runAudit(fixture, ['--systemd-show'], {
systemdExecStart: currentBinary,
});
assertStatus(
result,
0,
'systemd ExecStart 指向 current symlink 且解析到本次 release 时必须通过。',
);
if (result.status !== 0) {
return;
}
const payload = parseJson(
result.stdout,
'current symlink ExecStart 自审输出',
);
assertEqual(
payload.systemd.status,
'OK',
'current symlink ExecStart 必须让 systemd 自审通过。',
);
assertEqual(
payload.systemd.execStartBinary,
currentBinary,
'systemd 自审必须记录 ExecStart 中的原始 current symlink 路径。',
);
assertEqual(
payload.systemd.expectedRealpath,
payload.systemd.execStartRealpath,
'systemd 自审必须用真实路径确认 current symlink 指向本次 release 二进制。',
);
}
function assertRejectsRelativeReleaseRoot() {
const result = spawnSync(
'node',
[AUDIT_SCRIPT, '--release-root', 'build/current'],
{
cwd: process.cwd(),
encoding: 'utf8',
},
);
if ((result.status ?? 0) === 0) {
failures.push('current release 自审必须拒绝相对 release root。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--release-root 必须是绝对路径',
'相对 release root 必须给出明确错误。',
);
}
function assertRejectsFilesystemRootReleaseRoot() {
const result = spawnSync('node', [AUDIT_SCRIPT, '--release-root', '/'], {
cwd: process.cwd(),
encoding: 'utf8',
});
if ((result.status ?? 0) === 0) {
failures.push('current release 自审必须拒绝文件系统根目录 release root。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--release-root 不能是文件系统根目录',
'文件系统根目录 release root 必须给出明确错误。',
);
}
function assertRejectsPathArgsWithControlCharacters() {
const fixture = prepareFixture('control-character-release-root');
const result = runAudit(fixture, [
'--systemd-show',
'--release-root',
`${fixture.releaseRoot}\n--fake-flag`,
]);
if ((result.status ?? 0) === 0) {
failures.push('current release 自审必须拒绝带换行的 release root。');
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
'--release-root 不能包含换行或 NUL 字符',
'带换行的 release root 必须给出明确错误。',
);
const commandsLog = readFileSync(fixture.commandsLog, 'utf8');
if (commandsLog.includes('systemctl')) {
failures.push('release root 含控制字符时必须在执行 systemctl 前失败。');
}
}
function assertRejectsInvalidTimeout() {
const fixture = prepareFixture('invalid-timeout');
const cliResult = runAudit(fixture, ['--timeout-ms', '0']);
if ((cliResult.status ?? 0) === 0) {
failures.push('current release 自审必须拒绝非正数 --timeout-ms。');
}
assertIncludes(
`${cliResult.stdout}\n${cliResult.stderr}`,
'--timeout-ms 必须是正整数',
'非法 --timeout-ms 必须给出明确错误。',
);
const envResult = runAudit(fixture, [], {
timeoutEnv: 'abc',
});
if ((envResult.status ?? 0) === 0) {
failures.push('current release 自审必须拒绝非法 timeout env。');
}
assertIncludes(
`${envResult.stdout}\n${envResult.stderr}`,
'GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS 必须是正整数',
'非法 timeout env 必须给出明确错误。',
);
}
function assertRejectsInvalidBoolEnv() {
const fixture = prepareFixture('invalid-bool-env');
const cases = [
{
env: { GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY: 'ture' },
expected:
'GENARRATIVE_PINGORA_CURRENT_RELEASE_REQUIRE_GATEWAY 必须是布尔值',
reason: 'current release 自审必须拒绝拼写错误的 require gateway env。',
},
{
env: { GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW: 'enabled' },
expected: 'GENARRATIVE_PINGORA_CURRENT_RELEASE_SYSTEMD_SHOW 必须是布尔值',
reason: 'current release 自审必须拒绝非法 systemd show env。',
},
];
for (const testCase of cases) {
const result = runAudit(fixture, [], { extraEnv: testCase.env });
if ((result.status ?? 0) === 0) {
failures.push(testCase.reason);
}
assertIncludes(
`${result.stdout}\n${result.stderr}`,
testCase.expected,
`${testCase.reason} 必须给出明确错误。`,
);
}
}
function prepareFixture(name, options = {}) {
const root = path.join(tmpRoot, name);
const releaseRoot = path.join(
root,
options.releaseDirectoryName || 'current',
);
const fakeBin = path.join(root, 'bin');
const commandsLog = path.join(root, 'commands.log');
const includePingoraGateway = options.includePingoraGateway !== false;
mkdirSync(fakeBin, { recursive: true });
writeFileSync(commandsLog, '', 'utf8');
prepareReleaseRoot(releaseRoot, {
includePingoraGateway,
includePingoraInManifest: options.includePingoraInManifest !== false,
});
writeFakeSystemctl(fakeBin, commandsLog, releaseRoot);
return {
root,
releaseRoot,
fakeBin,
commandsLog,
};
}
function prepareReleaseRoot(releaseRoot, options) {
for (const dir of [
'scripts',
'scripts/ops',
'scripts/deploy',
'deploy/systemd',
'deploy/nginx',
'deploy/env',
'deploy/pingora',
]) {
mkdirSync(path.join(releaseRoot, dir), { recursive: true });
}
writeFileSync(
path.join(releaseRoot, 'api-server'),
'#!/usr/bin/env bash\n',
'utf8',
);
chmodExecutable(path.join(releaseRoot, 'api-server'));
writeChecksum(releaseRoot, 'api-server');
for (const file of [
'scripts/database-backup-to-oss.mjs',
'scripts/ops/production-health-patrol.mjs',
'scripts/ops/pingora-current-release-audit.mjs',
'scripts/ops/pingora-direct-rehearsal-status.mjs',
'scripts/ops/pingora-cutover-status-snapshot.mjs',
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
'scripts/check-production-health-patrol-env.mjs',
'scripts/check-pingora-release-readiness.mjs',
'scripts/check-pingora-direct-preflight.mjs',
'scripts/check-pingora-direct-live.mjs',
'scripts/check-pingora-canary-live.mjs',
'scripts/check-pingora-canary-access-log-parity.mjs',
'scripts/deploy/pingora-direct-enable.sh',
'scripts/deploy/pingora-direct-rollback.sh',
'scripts/deploy/pingora-realpath-canary-enable.sh',
'scripts/deploy/pingora-realpath-canary-disable.sh',
'scripts/deploy/pingora-health-patrol-env-switch.mjs',
'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
'scripts/deploy/pingora-tls-cert-sync.mjs',
'deploy/systemd/genarrative-pingora-gateway.service',
'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf',
'deploy/nginx/snippets/genarrative-pingora-canary.conf',
'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf',
'deploy/env/health-patrol.env.example',
'deploy/env/pingora-direct-live.env.example',
'deploy/env/pingora-canary-live.env.example',
'deploy/pingora/pingora-gateway.env.example',
'deploy/pingora/nginx-route-parity.matrix.json',
]) {
const target = path.join(releaseRoot, file);
mkdirSync(path.dirname(target), { recursive: true });
if (existsSync(file)) {
copyFileSync(file, target);
} else {
writeFileSync(target, '', 'utf8');
}
}
for (const file of [
'scripts/deploy/pingora-direct-enable.sh',
'scripts/deploy/pingora-direct-rollback.sh',
'scripts/deploy/pingora-realpath-canary-enable.sh',
'scripts/deploy/pingora-realpath-canary-disable.sh',
'scripts/deploy/pingora-health-patrol-env-switch.mjs',
'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
'scripts/deploy/pingora-tls-cert-sync.mjs',
]) {
chmodExecutable(path.join(releaseRoot, file));
}
if (options.includePingoraGateway) {
writeFileSync(
path.join(releaseRoot, 'pingora-gateway'),
'#!/usr/bin/env bash\n',
'utf8',
);
chmodExecutable(path.join(releaseRoot, 'pingora-gateway'));
writeChecksum(releaseRoot, 'pingora-gateway');
}
writeReleaseManifest(releaseRoot, {
includePingoraGateway: options.includePingoraGateway,
includePingoraInManifest: options.includePingoraInManifest !== false,
});
}
function writeFakeSystemctl(fakeBin, commandsLog, releaseRoot) {
writeFileSync(
path.join(fakeBin, 'systemctl'),
[
'#!/usr/bin/env bash',
`printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`,
'if [[ "$1" == "show" ]]; then',
' exec_start="${FAKE_SYSTEMD_EXEC_START:-' +
shellEscapeForDoubleQuote(path.join(releaseRoot, 'pingora-gateway')) +
'}"',
' cat <<SHOW',
'FragmentPath=/etc/systemd/system/genarrative-pingora-gateway.service',
'DropInPaths=',
'User=genarrative',
'SHOW',
' printf "ExecStart={ path=%s ; argv[]=%s ; }\\n" "$exec_start" "$exec_start"',
' exit 0',
'fi',
'exit 1',
'',
].join('\n'),
'utf8',
);
chmodExecutable(path.join(fakeBin, 'systemctl'));
}
function runAudit(fixture, args = [], options = {}) {
return spawnSync(
'node',
[AUDIT_SCRIPT, '--release-root', fixture.releaseRoot, ...args],
{
cwd: process.cwd(),
encoding: 'utf8',
env: {
...process.env,
PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`,
...(options.timeoutEnv
? {
GENARRATIVE_PINGORA_CURRENT_RELEASE_TIMEOUT_MS:
options.timeoutEnv,
}
: {}),
...(options.extraEnv || {}),
FAKE_SYSTEMD_EXEC_START:
options.systemdExecStart ||
path.join(fixture.releaseRoot, 'pingora-gateway'),
},
},
);
}
function chmodExecutable(filePath) {
chmodSync(filePath, 0o755);
}
function writeChecksum(directory, fileName) {
const content = readFileSync(path.join(directory, fileName));
const checksum = createHash('sha256').update(content).digest('hex');
writeFileSync(
path.join(directory, `${fileName}.sha256`),
`${checksum} ${fileName}\n`,
'utf8',
);
}
function writeReleaseManifest(releaseRoot, options) {
const artifacts = [
{
component: 'api-server',
path: 'api-server',
checksum_path: 'api-server.sha256',
},
];
if (options.includePingoraGateway && options.includePingoraInManifest) {
artifacts.push({
component: 'pingora-gateway',
path: 'pingora-gateway',
checksum_path: 'pingora-gateway.sha256',
});
}
writeFileSync(
path.join(releaseRoot, 'release-manifest.api-server.json'),
`${JSON.stringify(
{
version: 'check-current-release',
component_type: 'api-server',
artifacts,
},
null,
2,
)}\n`,
'utf8',
);
}
function shellQuote(value) {
return `'${String(value).replace(/'/g, "'\\''")}'`;
}
function shellEscapeForDoubleQuote(value) {
return String(value).replace(/["\\$`]/g, '\\$&');
}
function parseJson(text, label) {
try {
return JSON.parse(text);
} catch (error) {
failures.push(`${label} 不是合法 JSON: ${error.message}`);
return {};
}
}
function assertStatus(result, expected, reason) {
if ((result.status ?? 0) !== expected) {
failures.push(
`${reason} 实际退出码 ${result.status}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`,
);
}
}
function assertEqual(actual, expected, reason) {
if (actual !== expected) {
failures.push(`${reason} 实际 ${actual},预期 ${expected}。`);
}
}
function assertIncludes(value, expected, reason) {
const haystack = Array.isArray(value) ? value.join('\n') : String(value);
if (!haystack.includes(expected)) {
failures.push(`${reason} 缺少: ${expected}`);
}
}