#!/usr/bin/env node import { spawn, spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from 'node:fs'; import http from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; const failures = []; const tmpRoot = mkdtempSync( path.join(tmpdir(), 'genarrative-health-patrol-check-'), ); try { await main(); } finally { rmSync(tmpRoot, { recursive: true, force: true }); } if (failures.length > 0) { console.error('[check:production-health-patrol] FAILED'); for (const failure of failures) { console.error(`- ${failure}`); } process.exit(1); } console.log('[check:production-health-patrol] OK'); async function main() { assertPublicBaseUrlDefaultsToGatewayEntry(); await assertNginxModeChecksNginxService(); await assertPingoraDirectModeChecksPingoraServiceAndPublicHost(); assertRejectsInvalidGatewayMode(); assertRejectsInvalidPublicHost(); assertRejectsInvalidTimeoutAndSlowThreshold(); assertRejectsInvalidBoolEnv(); } function assertPublicBaseUrlDefaultsToGatewayEntry() { const script = readFileSync( 'scripts/ops/production-health-patrol.mjs', 'utf8', ); assertIncludes( script, "process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n 'http://127.0.0.1'", 'publicBaseUrl 默认必须指向本机网关入口,不能回落到 API 直连端口。', ); assertIncludes( script, "process.env.GENARRATIVE_HEALTH_PATROL_BGFILTER_BASE_URL ||\n 'http://127.0.0.1:8083'", 'BgFilter worker 巡检默认必须指向唯一实例的 loopback 端口。', ); if ( script.includes( 'process.env.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL ||\n process.env.GENARRATIVE_HEALTH_PATROL_API_BASE_URL', ) ) { failures.push('publicBaseUrl 默认不应回落到 API base URL。'); } } async function assertNginxModeChecksNginxService() { const fixture = await prepareFixture('nginx-mode'); const result = await runPatrol(fixture, ['--gateway-mode', 'nginx']); assertStatus(result, 0, 'nginx gateway mode 巡检应成功。'); if (result.status !== 0) { return; } const payload = JSON.parse(result.stdout); assertEqual( payload.gatewayMode, 'nginx', '巡检 JSON 必须记录 gatewayMode=nginx。', ); const commandsLog = readFileSync(fixture.commandsLog, 'utf8'); assertIncludes( commandsLog, 'systemctl is-active nginx.service', 'nginx gateway mode 必须检查 nginx.service。', ); assertIncludes( commandsLog, 'systemctl is-active genarrative-bgfilter-worker.service', '生产巡检必须检查唯一 BgFilter worker service。', ); if (!payload.checks.some((check) => check.name === 'bgfilter:/readyz')) { failures.push('生产巡检必须探测 BgFilter worker /readyz。'); } if (commandsLog.includes('genarrative-pingora-gateway.service')) { failures.push( 'nginx gateway mode 不应要求 Pingora gateway service active。', ); } } async function assertPingoraDirectModeChecksPingoraServiceAndPublicHost() { const fixture = await prepareFixture('pingora-direct-mode'); const result = await runPatrol(fixture, [ '--gateway-mode', 'pingora-direct', '--public-host', 'genarrative.example', ]); assertStatus(result, 0, 'pingora-direct gateway mode 巡检应成功。'); if (result.status !== 0) { return; } const payload = JSON.parse(result.stdout); assertEqual( payload.gatewayMode, 'pingora-direct', '巡检 JSON 必须记录 gatewayMode=pingora-direct。', ); const commandsLog = readTextFile(fixture.commandsLog, 'systemctl 命令日志'); assertIncludes( commandsLog, 'systemctl is-active genarrative-pingora-gateway.service', 'pingora-direct gateway mode 必须检查 Pingora gateway service。', ); if (commandsLog.includes('systemctl is-active nginx.service')) { failures.push( 'pingora-direct gateway mode 不应要求 nginx.service active。', ); } const requestsLog = readTextFile( fixture.requestsLog, 'public probe 请求日志', ); assertIncludes( requestsLog, 'host=genarrative.example path=/', 'public probe 必须带正式域名 Host header。', ); } function assertRejectsInvalidGatewayMode() { const result = spawnSync( 'node', [ 'scripts/ops/production-health-patrol.mjs', '--gateway-mode', 'direct', '--skip-journal', '--json', ], { cwd: process.cwd(), encoding: 'utf8', }, ); if ((result.status ?? 0) === 0) { failures.push('非法 gateway mode 必须失败。'); } assertIncludes( `${result.stdout}\n${result.stderr}`, '--gateway-mode 只支持 nginx 或 pingora-direct', '非法 gateway mode 必须给出明确错误。', ); } function assertRejectsInvalidPublicHost() { const result = spawnSync( 'node', [ 'scripts/ops/production-health-patrol.mjs', '--public-host', 'https://genarrative.example', '--skip-journal', '--json', ], { cwd: process.cwd(), encoding: 'utf8', }, ); if ((result.status ?? 0) === 0) { failures.push('非法 public host 必须失败。'); } assertIncludes( `${result.stdout}\n${result.stderr}`, '--public-host 只接受域名或 host:port', '非法 public host 必须给出明确错误。', ); } function assertRejectsInvalidTimeoutAndSlowThreshold() { const cases = [ { args: ['--timeout-ms', '0'], env: {}, expected: '--timeout-ms 必须是正整数', reason: '非法 CLI timeout 必须失败。', }, { args: ['--slow-ms', 'abc'], env: {}, expected: '--slow-ms 必须是正整数', reason: '非法 CLI slow threshold 必须失败。', }, { args: [], env: { GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS: 'abc', }, expected: 'GENARRATIVE_HEALTH_PATROL_TIMEOUT_MS 必须是正整数', reason: '非法 env timeout 必须失败。', }, { args: [], env: { GENARRATIVE_HEALTH_PATROL_SLOW_MS: '0', }, expected: 'GENARRATIVE_HEALTH_PATROL_SLOW_MS 必须是正整数', reason: '非法 env slow threshold 必须失败。', }, ]; for (const testCase of cases) { const result = spawnSync( 'node', [ 'scripts/ops/production-health-patrol.mjs', '--skip-journal', '--json', ...testCase.args, ], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, ...testCase.env, }, }, ); if ((result.status ?? 0) === 0) { failures.push(testCase.reason); } assertIncludes( `${result.stdout}\n${result.stderr}`, testCase.expected, `${testCase.reason} 必须给出明确错误。`, ); } } function assertRejectsInvalidBoolEnv() { const cases = [ { env: { GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING: 'ture', }, expected: 'GENARRATIVE_HEALTH_PATROL_FAIL_ON_WARNING 必须是布尔值', reason: '生产健康巡检必须拒绝拼写错误的 fail-on-warning env。', }, { env: { GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL: 'maybe', }, expected: 'GENARRATIVE_HEALTH_PATROL_SKIP_JOURNAL 必须是布尔值', reason: '生产健康巡检必须拒绝非法 skip-journal env。', }, ]; for (const testCase of cases) { const result = spawnSync( 'node', ['scripts/ops/production-health-patrol.mjs', '--json'], { cwd: process.cwd(), encoding: 'utf8', env: { ...process.env, ...testCase.env, }, }, ); if ((result.status ?? 0) === 0) { failures.push(testCase.reason); } assertIncludes( `${result.stdout}\n${result.stderr}`, testCase.expected, `${testCase.reason} 必须给出明确错误。`, ); } } async function prepareFixture(name) { const root = path.join(tmpRoot, name); const fakeBin = path.join(root, 'bin'); const commandsLog = path.join(root, 'commands.log'); const requestsLog = path.join(root, 'requests.log'); mkdirSync(fakeBin, { recursive: true }); writeFileSync( path.join(fakeBin, 'systemctl'), [ '#!/usr/bin/env bash', `printf 'systemctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'if [[ "$1" == "list-units" && "$2" == "genarrative-external-generation-worker@*.service" ]]; then', ' echo "genarrative-external-generation-worker@1.service loaded active running Genarrative external generation worker 1"', ' exit 0', 'fi', 'echo active', 'exit 0', '', ].join('\n'), 'utf8', ); writeFileSync( path.join(fakeBin, 'journalctl'), [ '#!/usr/bin/env bash', `printf 'journalctl %s\\n' "$*" >> ${shellQuote(commandsLog)}`, 'echo "-- No entries --"', 'exit 0', '', ].join('\n'), 'utf8', ); chmodExecutable(path.join(fakeBin, 'systemctl')); chmodExecutable(path.join(fakeBin, 'journalctl')); const server = http.createServer((request, response) => { writeFileSync( requestsLog, `host=${request.headers.host || ''} path=${request.url || ''}\n`, { encoding: 'utf8', flag: 'a', }, ); response.writeHead(200, { 'Content-Type': 'application/json', }); response.end(JSON.stringify({ ok: true })); }); await listen(server, '127.0.0.1', 0); const address = server.address(); if (!address || typeof address === 'string') { throw new Error('测试 HTTP server 未返回 TCP 端口。'); } return { root, fakeBin, commandsLog, requestsLog, server, baseUrl: `http://127.0.0.1:${address.port}`, }; } async function runPatrol(fixture, args) { try { return await spawnProcess( 'node', [ 'scripts/ops/production-health-patrol.mjs', '--api-base-url', fixture.baseUrl, '--bgfilter-base-url', fixture.baseUrl, '--spacetime-base-url', fixture.baseUrl, '--public-base-url', fixture.baseUrl, '--skip-journal', '--json', ...args, ], { cwd: process.cwd(), env: { ...process.env, PATH: `${fixture.fakeBin}:${process.env.PATH || ''}`, }, }, ); } finally { await closeServer(fixture.server); } } function spawnProcess(command, args, options) { return new Promise((resolve) => { const child = spawn(command, args, { ...options, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; let stderr = ''; child.stdout.setEncoding('utf8'); child.stderr.setEncoding('utf8'); child.stdout.on('data', (chunk) => { stdout += chunk; }); child.stderr.on('data', (chunk) => { stderr += chunk; }); child.on('error', (error) => { resolve({ status: 1, stdout, stderr: `${stderr}${error.message}`, }); }); child.on('close', (code, signal) => { resolve({ status: code ?? 1, signal, stdout, stderr, }); }); }); } function listen(server, host, port) { return new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, host, resolve); }); } function closeServer(server) { return new Promise((resolve) => { server.close(() => resolve()); }); } function readTextFile(filePath, label) { if (!existsSync(filePath)) { failures.push(`${label} 未生成:${filePath}`); return ''; } return readFileSync(filePath, 'utf8'); } function chmodExecutable(filePath) { spawnSync('chmod', ['0755', filePath], { cwd: process.cwd(), encoding: 'utf8', }); } function shellQuote(value) { return `'${String(value).replace(/'/g, "'\\''")}'`; } function assertStatus(result, expected, reason) { const actual = result.status ?? 0; if (actual !== expected) { failures.push( `${reason} 预期退出码 ${expected},实际 ${actual}。\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ); } } function assertIncludes(content, needle, reason) { if (!content.includes(needle)) { failures.push(`${reason} 缺少: ${needle}`); } } function assertEqual(actual, expected, reason) { if (actual !== expected) { failures.push(`${reason} 实际 ${actual},预期 ${expected}。`); } }