kdletters
e3258f6f1d
新增 Pingora shadow env 回切脚本与对应检查。 补齐直连证据包时间线和 cutoverRunId 审计门禁。 支持 Gitea Host 透传并更新直连多域名文档。 修复百分号编码静态图标路径并补 smoke 覆盖。 更新生产发布与运维护栏对 Pingora 发布包的校验。
1053 lines
32 KiB
JavaScript
1053 lines
32 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { execFile } from 'node:child_process';
|
|
import { constants as fsConstants } from 'node:fs';
|
|
import { access, readFile, stat } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
const STATUS_RANK = {
|
|
OK: 0,
|
|
WARNING: 1,
|
|
CRITICAL: 2,
|
|
};
|
|
|
|
const GATEWAY_MODES = new Set(['nginx', 'pingora-direct']);
|
|
const PINGORA_ENV_MODES = new Set(['shadow', 'direct']);
|
|
const SERVICES = [
|
|
'genarrative-api.service',
|
|
'spacetimedb.service',
|
|
'nginx.service',
|
|
'genarrative-pingora-gateway.service',
|
|
];
|
|
const RELEASE_ARTIFACTS = [
|
|
'scripts/ops/pingora-cutover-status-snapshot.mjs',
|
|
'scripts/ops/pingora-current-release-audit.mjs',
|
|
'scripts/ops/pingora-direct-rehearsal-status.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-audit.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-bundle.mjs',
|
|
'scripts/ops/pingora-cutover-command-evidence.mjs',
|
|
'scripts/ops/pingora-cutover-evidence-verify.mjs',
|
|
'scripts/ops/production-health-patrol.mjs',
|
|
'scripts/check-production-health-patrol-env.mjs',
|
|
'scripts/check-pingora-release-readiness.mjs',
|
|
'scripts/check-pingora-direct-preflight.mjs',
|
|
'scripts/check-pingora-direct-live.mjs',
|
|
'scripts/check-pingora-canary-live.mjs',
|
|
'scripts/check-pingora-canary-access-log-parity.mjs',
|
|
'scripts/deploy/pingora-direct-enable.sh',
|
|
'scripts/deploy/pingora-direct-rollback.sh',
|
|
'scripts/deploy/pingora-realpath-canary-enable.sh',
|
|
'scripts/deploy/pingora-realpath-canary-disable.sh',
|
|
'scripts/deploy/pingora-health-patrol-env-switch.mjs',
|
|
'scripts/deploy/pingora-gateway-env-shadow-switch.mjs',
|
|
'deploy/systemd/genarrative-pingora-gateway.service',
|
|
'deploy/systemd/genarrative-pingora-gateway-direct-entry.conf',
|
|
'deploy/nginx/snippets/genarrative-pingora-canary.conf',
|
|
'deploy/nginx/snippets/genarrative-pingora-realpath-canary.conf',
|
|
'deploy/pingora/pingora-gateway.env.example',
|
|
];
|
|
const SECRET_ENV_KEY_PATTERN =
|
|
/(?:TOKEN|SECRET|PASSWORD|PRIVATE_KEY|ACCESS_KEY|API_KEY|AUTH|CREDENTIAL)/iu;
|
|
|
|
const config = parseArgs(process.argv.slice(2));
|
|
const snapshot = await buildSnapshot(config);
|
|
|
|
console.log(`${JSON.stringify(snapshot, null, 2)}\n`);
|
|
|
|
if (config.failOnCritical && snapshot.summary.status === 'CRITICAL') {
|
|
process.exit(1);
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const result = {
|
|
phase: process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_PHASE || 'manual',
|
|
releaseRoot:
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_RELEASE_ROOT ||
|
|
'/opt/genarrative/current',
|
|
healthPatrolEnvFile:
|
|
process.env.GENARRATIVE_HEALTH_PATROL_ENV_FILE ||
|
|
'/etc/genarrative/health-patrol.env',
|
|
pingoraEnvFile:
|
|
process.env.GENARRATIVE_PINGORA_DIRECT_PREFLIGHT_ENV_FILE ||
|
|
'/etc/genarrative/pingora-gateway.env',
|
|
expectedPingoraEnvMode:
|
|
process.env.GENARRATIVE_PINGORA_EXPECTED_ENV_MODE || '',
|
|
expectedGatewayMode:
|
|
process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_GATEWAY_MODE || '',
|
|
expectedPublicBaseUrl:
|
|
process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_BASE_URL || '',
|
|
expectedPublicHost:
|
|
process.env.GENARRATIVE_HEALTH_PATROL_EXPECTED_PUBLIC_HOST || null,
|
|
requireEmptyPublicHost: readBoolEnv(
|
|
'GENARRATIVE_HEALTH_PATROL_REQUIRE_EMPTY_PUBLIC_HOST',
|
|
),
|
|
runHealthPatrol: readBoolEnv(
|
|
'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_RUN_HEALTH_PATROL',
|
|
),
|
|
requirePingoraGateway: readBoolEnv(
|
|
'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_REQUIRE_PINGORA_GATEWAY',
|
|
),
|
|
timeoutMs: parseOptionalPositiveInt(
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS,
|
|
5000,
|
|
'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_TIMEOUT_MS',
|
|
),
|
|
failOnCritical: readBoolEnv(
|
|
'GENARRATIVE_PINGORA_CUTOVER_SNAPSHOT_FAIL_ON_CRITICAL',
|
|
),
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
switch (arg) {
|
|
case '-h':
|
|
case '--help':
|
|
printUsage();
|
|
process.exit(0);
|
|
break;
|
|
case '--phase':
|
|
result.phase = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--release-root':
|
|
result.releaseRoot = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--health-patrol-env-file':
|
|
result.healthPatrolEnvFile = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--pingora-env-file':
|
|
result.pingoraEnvFile = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--expected-pingora-env-mode':
|
|
result.expectedPingoraEnvMode = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--expected-gateway-mode':
|
|
result.expectedGatewayMode = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--expected-public-base-url':
|
|
result.expectedPublicBaseUrl = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--expected-public-host':
|
|
result.expectedPublicHost = requireValue(argv, ++index, arg);
|
|
break;
|
|
case '--require-empty-public-host':
|
|
result.requireEmptyPublicHost = true;
|
|
break;
|
|
case '--run-health-patrol':
|
|
result.runHealthPatrol = true;
|
|
break;
|
|
case '--require-pingora-gateway':
|
|
result.requirePingoraGateway = true;
|
|
break;
|
|
case '--timeout-ms':
|
|
result.timeoutMs = parseRequiredPositiveInt(
|
|
requireValue(argv, ++index, arg),
|
|
arg,
|
|
);
|
|
break;
|
|
case '--fail-on-critical':
|
|
result.failOnCritical = true;
|
|
break;
|
|
default:
|
|
throw new Error(`未知参数: ${arg}`);
|
|
}
|
|
}
|
|
|
|
if (!path.isAbsolute(result.releaseRoot)) {
|
|
throw new Error('--release-root 必须是绝对路径。');
|
|
}
|
|
validateNoControlCharacters(result.releaseRoot, '--release-root');
|
|
if (isFilesystemRootPath(result.releaseRoot)) {
|
|
throw new Error('--release-root 不能是文件系统根目录。');
|
|
}
|
|
if (!path.isAbsolute(result.healthPatrolEnvFile)) {
|
|
throw new Error('--health-patrol-env-file 必须是绝对路径。');
|
|
}
|
|
validateNoControlCharacters(
|
|
result.healthPatrolEnvFile,
|
|
'--health-patrol-env-file',
|
|
);
|
|
if (isFilesystemRootPath(result.healthPatrolEnvFile)) {
|
|
throw new Error('--health-patrol-env-file 不能是文件系统根目录。');
|
|
}
|
|
if (!path.isAbsolute(result.pingoraEnvFile)) {
|
|
throw new Error('--pingora-env-file 必须是绝对路径。');
|
|
}
|
|
validateNoControlCharacters(result.pingoraEnvFile, '--pingora-env-file');
|
|
if (isFilesystemRootPath(result.pingoraEnvFile)) {
|
|
throw new Error('--pingora-env-file 不能是文件系统根目录。');
|
|
}
|
|
if (
|
|
result.expectedGatewayMode &&
|
|
!GATEWAY_MODES.has(result.expectedGatewayMode)
|
|
) {
|
|
throw new Error(
|
|
`--expected-gateway-mode 只支持 nginx 或 pingora-direct: ${result.expectedGatewayMode}`,
|
|
);
|
|
}
|
|
if (
|
|
result.expectedPingoraEnvMode &&
|
|
!PINGORA_ENV_MODES.has(result.expectedPingoraEnvMode)
|
|
) {
|
|
throw new Error(
|
|
`--expected-pingora-env-mode 只支持 shadow 或 direct: ${result.expectedPingoraEnvMode}`,
|
|
);
|
|
}
|
|
if (result.expectedPublicHost !== null) {
|
|
validateHostOption(result.expectedPublicHost, '--expected-public-host');
|
|
}
|
|
if (result.expectedPublicHost !== null && result.requireEmptyPublicHost) {
|
|
throw new Error(
|
|
'--expected-public-host 和 --require-empty-public-host 不能同时使用。',
|
|
);
|
|
}
|
|
if (result.expectedPublicBaseUrl) {
|
|
validateHttpUrl(result.expectedPublicBaseUrl, '--expected-public-base-url');
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function printUsage() {
|
|
console.log(`Usage:
|
|
node scripts/ops/pingora-cutover-status-snapshot.mjs [options]
|
|
|
|
Options:
|
|
--phase <name> 写入 JSON 的阶段标签，例如 pre-cutover / post-enable / post-rollback。
|
|
--release-root <path> current release 根目录，默认 /opt/genarrative/current。
|
|
--health-patrol-env-file <path> health-patrol env 文件，默认 /etc/genarrative/health-patrol.env。
|
|
--pingora-env-file <path> pingora-gateway env 文件，默认 /etc/genarrative/pingora-gateway.env。
|
|
--expected-gateway-mode <mode> 可选，nginx 或 pingora-direct；提供后会检查 env 和 systemd capability 方向。
|
|
--expected-pingora-env-mode <mode>
|
|
可选，shadow 或 direct；提供后会检查 active Pingora env 姿态。
|
|
--expected-public-base-url <url> 可选，要求 health-patrol public base URL 与该值一致。
|
|
--expected-public-host <host> 可选，要求 health-patrol public Host 与该值一致。
|
|
--require-empty-public-host 可选，要求 health-patrol public Host 为空。
|
|
--run-health-patrol 读取 env 后执行 current release 随包生产巡检脚本并收录结果。
|
|
--require-pingora-gateway 要求 current release 自审确认 pingora-gateway、checksum 和 manifest。
|
|
--timeout-ms <ms> systemctl / 子检查超时，默认 5000。
|
|
--fail-on-critical 如果快照中出现 CRITICAL，则以退出码 1 结束。
|
|
|
|
该脚本只读采集状态，不写 /etc、不 reload systemd、不修改 Nginx 或 Pingora。
|
|
`);
|
|
}
|
|
|
|
function requireValue(argv, index, flag) {
|
|
const value = argv[index];
|
|
if (value === undefined || value.startsWith('--')) {
|
|
throw new Error(`${flag} 缺少参数值`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function readBoolEnv(name) {
|
|
const value = process.env[name];
|
|
if (value === undefined || value === null || String(value).trim() === '') {
|
|
return false;
|
|
}
|
|
const normalized = String(value).trim().toLowerCase();
|
|
if (['1', 'true', 'yes', 'on'].includes(normalized)) {
|
|
return true;
|
|
}
|
|
if (['0', 'false', 'no', 'off'].includes(normalized)) {
|
|
return false;
|
|
}
|
|
throw new Error(`${name} 必须是布尔值 true/false 或 1/0。`);
|
|
}
|
|
|
|
function parseOptionalPositiveInt(raw, fallback, label) {
|
|
if (raw === undefined || raw === null || String(raw).trim() === '') {
|
|
return fallback;
|
|
}
|
|
return parseRequiredPositiveInt(raw, label);
|
|
}
|
|
|
|
function parseRequiredPositiveInt(raw, label) {
|
|
const text = String(raw ?? '').trim();
|
|
if (!/^[1-9][0-9]*$/u.test(text)) {
|
|
throw new Error(`${label} 必须是正整数。`);
|
|
}
|
|
return Number.parseInt(text, 10);
|
|
}
|
|
|
|
function isFilesystemRootPath(value) {
|
|
const resolved = path.resolve(String(value));
|
|
return resolved === path.parse(resolved).root;
|
|
}
|
|
|
|
function validateNoControlCharacters(value, label) {
|
|
if (/[\0\r\n]/u.test(String(value))) {
|
|
throw new Error(`${label} 不能包含换行或 NUL 字符。`);
|
|
}
|
|
}
|
|
|
|
function validateHttpUrl(value, label) {
|
|
try {
|
|
const parsed = new URL(value);
|
|
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
throw new Error('unsupported protocol');
|
|
}
|
|
} catch {
|
|
throw new Error(`${label} 必须是合法 http(s) URL: ${value}`);
|
|
}
|
|
}
|
|
|
|
function validateHostOption(value, label) {
|
|
const raw = String(value);
|
|
if (raw !== raw.trim() || raw.includes('://') || /[\s/?#@]/.test(raw)) {
|
|
throw new Error(
|
|
`${label} 只能是 host 或 host:port，不能包含 scheme、路径、查询、片段或空白字符`,
|
|
);
|
|
}
|
|
try {
|
|
const parsed = new URL(`https://${raw}`);
|
|
if (
|
|
!parsed.hostname ||
|
|
parsed.pathname !== '/' ||
|
|
parsed.search ||
|
|
parsed.hash ||
|
|
parsed.username ||
|
|
parsed.password
|
|
) {
|
|
throw new Error('invalid host');
|
|
}
|
|
} catch {
|
|
throw new Error(`${label} 不是合法的 host 或 host:port`);
|
|
}
|
|
}
|
|
|
|
async function buildSnapshot(input) {
|
|
const context = {
|
|
schemaVersion: 1,
|
|
generatedAt: new Date().toISOString(),
|
|
phase: input.phase,
|
|
releaseRoot: input.releaseRoot,
|
|
};
|
|
const healthPatrolEnv = await inspectHealthPatrolEnv(input);
|
|
const pingoraEnv = await inspectPingoraEnv(input);
|
|
const secrets = collectSecretValues(healthPatrolEnv, pingoraEnv);
|
|
delete healthPatrolEnv.secretValues;
|
|
delete pingoraEnv.secretValues;
|
|
const releaseArtifacts = await inspectReleaseArtifacts(input);
|
|
const systemd = await inspectSystemd(input);
|
|
const checks = await runChecks(input, healthPatrolEnv, secrets);
|
|
const summary = summarize([
|
|
healthPatrolEnv.status,
|
|
pingoraEnv.status,
|
|
releaseArtifacts.status,
|
|
systemd.status,
|
|
...checks.map((check) => check.status),
|
|
]);
|
|
|
|
return {
|
|
...context,
|
|
summary,
|
|
healthPatrolEnv,
|
|
pingoraEnv,
|
|
releaseArtifacts,
|
|
systemd,
|
|
checks,
|
|
};
|
|
}
|
|
|
|
async function inspectHealthPatrolEnv(input) {
|
|
const parsed = await readEnvFile(input.healthPatrolEnvFile);
|
|
if (parsed.status === 'CRITICAL') {
|
|
return parsed;
|
|
}
|
|
|
|
const values = parsed.values;
|
|
const publicHost = values.GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST || '';
|
|
const publicBaseUrl =
|
|
values.GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL || '';
|
|
const gatewayMode = values.GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE || '';
|
|
const diagnostics = [];
|
|
let status = 'OK';
|
|
|
|
if (input.expectedGatewayMode && gatewayMode !== input.expectedGatewayMode) {
|
|
diagnostics.push(
|
|
`GENARRATIVE_HEALTH_PATROL_GATEWAY_MODE 应为 ${input.expectedGatewayMode}，实际 ${gatewayMode || '(空)'}`,
|
|
);
|
|
status = maxStatus(status, 'CRITICAL');
|
|
}
|
|
if (
|
|
input.expectedPublicBaseUrl &&
|
|
normalizeBaseUrl(publicBaseUrl) !== normalizeBaseUrl(input.expectedPublicBaseUrl)
|
|
) {
|
|
diagnostics.push(
|
|
`GENARRATIVE_HEALTH_PATROL_PUBLIC_BASE_URL 应为 ${normalizeBaseUrl(input.expectedPublicBaseUrl)}，实际 ${publicBaseUrl || '(空)'}`,
|
|
);
|
|
status = maxStatus(status, 'CRITICAL');
|
|
}
|
|
if (input.expectedPublicHost !== null && publicHost !== input.expectedPublicHost) {
|
|
diagnostics.push(
|
|
`GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为 ${input.expectedPublicHost}，实际 ${publicHost || '(空)'}`,
|
|
);
|
|
status = maxStatus(status, 'CRITICAL');
|
|
}
|
|
if (input.requireEmptyPublicHost && publicHost) {
|
|
diagnostics.push(
|
|
`GENARRATIVE_HEALTH_PATROL_PUBLIC_HOST 应为空，实际 ${publicHost}`,
|
|
);
|
|
status = maxStatus(status, 'CRITICAL');
|
|
}
|
|
|
|
return {
|
|
path: input.healthPatrolEnvFile,
|
|
status,
|
|
secretValues: collectSecretValuesFromEnv(values),
|
|
values: {
|
|
gatewayMode,
|
|
publicBaseUrl,
|
|
publicHost,
|
|
apiBaseUrl: values.GENARRATIVE_HEALTH_PATROL_API_BASE_URL || '',
|
|
spacetimeBaseUrl:
|
|
values.GENARRATIVE_HEALTH_PATROL_SPACETIME_BASE_URL || '',
|
|
pingoraBaseUrl:
|
|
values.GENARRATIVE_HEALTH_PATROL_PINGORA_BASE_URL || '',
|
|
hasPingoraProbeToken: Boolean(
|
|
values.GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN ||
|
|
values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN,
|
|
),
|
|
},
|
|
diagnostics,
|
|
};
|
|
}
|
|
|
|
async function inspectPingoraEnv(input) {
|
|
const parsed = await readEnvFile(input.pingoraEnvFile);
|
|
if (parsed.status === 'CRITICAL') {
|
|
return {
|
|
...parsed,
|
|
values: {},
|
|
posture: {
|
|
expectedMode: input.expectedPingoraEnvMode || '',
|
|
mode: 'unknown',
|
|
shadowReady: false,
|
|
directReady: false,
|
|
diagnostics: parsed.diagnostics || [],
|
|
},
|
|
};
|
|
}
|
|
const values = parsed.values;
|
|
const envValues = {
|
|
listen: values.GENARRATIVE_PINGORA_GATEWAY_LISTEN || '',
|
|
tlsListen: values.GENARRATIVE_PINGORA_GATEWAY_TLS_LISTEN || '',
|
|
httpRedirectListen:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_HTTP_REDIRECT_LISTEN || '',
|
|
tlsCertFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_CERT_FILE || '',
|
|
tlsKeyFile: values.GENARRATIVE_PINGORA_GATEWAY_TLS_KEY_FILE || '',
|
|
forwardedProto:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_FORWARDED_PROTO || '',
|
|
trustXForwardedFor:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_TRUST_X_FORWARDED_FOR || '',
|
|
trustedFrontProxyConfirmed:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_TRUSTED_FRONT_PROXY_CONFIRMED || '',
|
|
protectionEnabled:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_PROTECTION_ENABLED || '',
|
|
instanceCount:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_INSTANCE_COUNT || '',
|
|
sharedProtectionConfirmed:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_SHARED_PROTECTION_CONFIRMED || '',
|
|
hasProbeToken: Boolean(values.GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN),
|
|
accessLogFile:
|
|
values.GENARRATIVE_PINGORA_GATEWAY_ACCESS_LOG_FILE || '',
|
|
};
|
|
const posture = summarizePingoraEnvPosture(
|
|
envValues,
|
|
input.expectedPingoraEnvMode,
|
|
);
|
|
const diagnostics = [...parsed.diagnostics, ...posture.diagnostics];
|
|
const status =
|
|
input.expectedPingoraEnvMode && posture.diagnostics.length > 0
|
|
? 'CRITICAL'
|
|
: parsed.status;
|
|
return {
|
|
path: input.pingoraEnvFile,
|
|
status,
|
|
secretValues: collectSecretValuesFromEnv(values),
|
|
values: envValues,
|
|
posture,
|
|
diagnostics,
|
|
};
|
|
}
|
|
|
|
function summarizePingoraEnvPosture(values, expectedMode) {
|
|
const shadowReady =
|
|
values.listen === '127.0.0.1:18081' &&
|
|
values.tlsListen === '' &&
|
|
values.httpRedirectListen === '' &&
|
|
values.tlsCertFile === '' &&
|
|
values.tlsKeyFile === '';
|
|
const directReady =
|
|
values.tlsListen !== '' &&
|
|
values.httpRedirectListen !== '' &&
|
|
values.tlsCertFile !== '' &&
|
|
values.tlsKeyFile !== '' &&
|
|
values.forwardedProto === 'https';
|
|
const mode = directReady ? 'direct' : shadowReady ? 'shadow' : 'mixed';
|
|
const diagnostics = [];
|
|
|
|
if (expectedMode === 'shadow') {
|
|
if (values.listen !== '127.0.0.1:18081') {
|
|
diagnostics.push(
|
|
`Pingora shadow env 要求 listen=127.0.0.1:18081，实际 ${values.listen || '-'}`,
|
|
);
|
|
}
|
|
for (const [label, value] of [
|
|
['tlsListen', values.tlsListen],
|
|
['httpRedirectListen', values.httpRedirectListen],
|
|
['tlsCertFile', values.tlsCertFile],
|
|
['tlsKeyFile', values.tlsKeyFile],
|
|
]) {
|
|
if (value !== '') {
|
|
diagnostics.push(
|
|
`Pingora shadow env 要求 ${label} 为空，实际 ${value}`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (expectedMode === 'direct') {
|
|
for (const [label, value] of [
|
|
['tlsListen', values.tlsListen],
|
|
['httpRedirectListen', values.httpRedirectListen],
|
|
['tlsCertFile', values.tlsCertFile],
|
|
['tlsKeyFile', values.tlsKeyFile],
|
|
]) {
|
|
if (value === '') {
|
|
diagnostics.push(`Pingora direct env 要求 ${label} 已配置。`);
|
|
}
|
|
}
|
|
if (values.forwardedProto !== 'https') {
|
|
diagnostics.push(
|
|
`Pingora direct env 要求 forwardedProto=https，实际 ${values.forwardedProto || '-'}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
return {
|
|
expectedMode: expectedMode || '',
|
|
mode,
|
|
shadowReady,
|
|
directReady,
|
|
diagnostics,
|
|
};
|
|
}
|
|
|
|
async function readEnvFile(filePath) {
|
|
let text;
|
|
try {
|
|
text = await readFile(filePath, 'utf8');
|
|
} catch (error) {
|
|
return {
|
|
path: filePath,
|
|
status: 'CRITICAL',
|
|
values: {},
|
|
diagnostics: [`无法读取 env 文件: ${error.message}`],
|
|
};
|
|
}
|
|
|
|
const values = {};
|
|
const diagnostics = [];
|
|
for (const [index, rawLine] of text.split(/\r?\n/u).entries()) {
|
|
let line = rawLine.trim();
|
|
if (!line || line.startsWith('#')) {
|
|
continue;
|
|
}
|
|
if (line.startsWith('export ')) {
|
|
line = line.slice('export '.length).trim();
|
|
}
|
|
const equalsIndex = line.indexOf('=');
|
|
if (equalsIndex <= 0) {
|
|
diagnostics.push(`第 ${index + 1} 行不是 KEY=VALUE 格式`);
|
|
continue;
|
|
}
|
|
const key = line.slice(0, equalsIndex).trim();
|
|
values[key] = stripQuotes(line.slice(equalsIndex + 1).trim());
|
|
}
|
|
|
|
return {
|
|
path: filePath,
|
|
status: diagnostics.length > 0 ? 'WARNING' : 'OK',
|
|
values,
|
|
diagnostics,
|
|
};
|
|
}
|
|
|
|
function collectSecretValuesFromEnv(values) {
|
|
const secrets = [];
|
|
for (const [key, value] of Object.entries(values || {})) {
|
|
if (!SECRET_ENV_KEY_PATTERN.test(key) || typeof value !== 'string') {
|
|
continue;
|
|
}
|
|
const trimmed = value.trim();
|
|
if (trimmed.length > 0) {
|
|
secrets.push(trimmed);
|
|
}
|
|
}
|
|
return secrets;
|
|
}
|
|
|
|
function collectSecretValues(...envSnapshots) {
|
|
const secrets = new Set();
|
|
for (const snapshot of envSnapshots) {
|
|
for (const value of snapshot?.secretValues || []) {
|
|
if (typeof value === 'string' && value.trim()) {
|
|
secrets.add(value.trim());
|
|
}
|
|
}
|
|
}
|
|
return [...secrets].sort((left, right) => right.length - left.length);
|
|
}
|
|
|
|
function stripQuotes(value) {
|
|
if (value.length >= 2) {
|
|
const first = value[0];
|
|
const last = value[value.length - 1];
|
|
if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
|
|
return value.slice(1, -1);
|
|
}
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function normalizeBaseUrl(value) {
|
|
if (!value) {
|
|
return '';
|
|
}
|
|
try {
|
|
return new URL(value).toString().replace(/\/+$/u, '');
|
|
} catch {
|
|
return value;
|
|
}
|
|
}
|
|
|
|
async function inspectReleaseArtifacts(input) {
|
|
const artifacts = [];
|
|
for (const relativePath of RELEASE_ARTIFACTS) {
|
|
const fullPath = path.join(input.releaseRoot, relativePath);
|
|
const artifact = {
|
|
path: relativePath,
|
|
exists: false,
|
|
executable: false,
|
|
status: 'CRITICAL',
|
|
};
|
|
try {
|
|
const fileStat = await stat(fullPath);
|
|
artifact.exists = fileStat.isFile() || fileStat.isDirectory();
|
|
if (fileStat.isFile()) {
|
|
try {
|
|
await access(fullPath, fsConstants.X_OK);
|
|
artifact.executable = true;
|
|
} catch {
|
|
artifact.executable = false;
|
|
}
|
|
}
|
|
artifact.status = artifact.exists ? 'OK' : 'CRITICAL';
|
|
} catch {
|
|
artifact.status = 'CRITICAL';
|
|
}
|
|
artifacts.push(artifact);
|
|
}
|
|
|
|
const status = artifacts.some((artifact) => artifact.status === 'CRITICAL')
|
|
? 'CRITICAL'
|
|
: 'OK';
|
|
return {
|
|
status,
|
|
artifacts,
|
|
};
|
|
}
|
|
|
|
async function inspectSystemd(input) {
|
|
const services = [];
|
|
for (const service of SERVICES) {
|
|
services.push(await inspectService(service, input));
|
|
}
|
|
const pingoraUnit = await inspectPingoraUnit(input);
|
|
return {
|
|
status: summarize([
|
|
...services.map((service) => service.status),
|
|
pingoraUnit.status,
|
|
]).status,
|
|
services,
|
|
pingoraUnit,
|
|
};
|
|
}
|
|
|
|
async function inspectService(service, input) {
|
|
const expected = isServiceExpected(service, input.expectedGatewayMode);
|
|
const result = await runCommand('systemctl', ['is-active', service], input);
|
|
const state = result.stdout.trim() || result.stderr.trim() || result.error;
|
|
let status = 'OK';
|
|
let summary = state || 'unknown';
|
|
if (result.code === 0 && state === 'active') {
|
|
summary = 'active';
|
|
} else if (expected) {
|
|
status = 'CRITICAL';
|
|
} else {
|
|
summary = state ? `${state} (not required)` : 'not required';
|
|
}
|
|
return {
|
|
name: service,
|
|
expectedActive: expected,
|
|
activeState: state,
|
|
status,
|
|
summary,
|
|
command: result.command,
|
|
};
|
|
}
|
|
|
|
function isServiceExpected(service, gatewayMode) {
|
|
if (
|
|
service === 'genarrative-api.service' ||
|
|
service === 'spacetimedb.service'
|
|
) {
|
|
return Boolean(gatewayMode);
|
|
}
|
|
if (service === 'nginx.service') {
|
|
return gatewayMode === 'nginx';
|
|
}
|
|
if (service === 'genarrative-pingora-gateway.service') {
|
|
return gatewayMode === 'pingora-direct';
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async function inspectPingoraUnit(input) {
|
|
const catResult = await runCommand(
|
|
'systemctl',
|
|
['cat', 'genarrative-pingora-gateway.service'],
|
|
input,
|
|
);
|
|
const showResult = await runCommand(
|
|
'systemctl',
|
|
[
|
|
'show',
|
|
'genarrative-pingora-gateway.service',
|
|
'--property=FragmentPath',
|
|
'--property=DropInPaths',
|
|
'--property=User',
|
|
'--property=ExecStart',
|
|
'--no-pager',
|
|
],
|
|
input,
|
|
);
|
|
|
|
const unitText = catResult.stdout;
|
|
const hasAmbientCapability = unitText.includes(
|
|
'AmbientCapabilities=CAP_NET_BIND_SERVICE',
|
|
);
|
|
const hasCapabilityBoundingSet = unitText.includes(
|
|
'CapabilityBoundingSet=CAP_NET_BIND_SERVICE',
|
|
);
|
|
const environmentFiles = [
|
|
...unitText.matchAll(/^\s*EnvironmentFile=(.+)$/gmu),
|
|
].map((match) => match[1].trim());
|
|
const environmentFileMatchesPingoraEnvFile = environmentFilesInclude(
|
|
environmentFiles,
|
|
input.pingoraEnvFile,
|
|
);
|
|
const show = parseSystemctlShow(showResult.stdout);
|
|
const diagnostics = [];
|
|
let status = 'OK';
|
|
|
|
if (catResult.code !== 0) {
|
|
diagnostics.push(`systemctl cat 失败: ${catResult.stderr || catResult.error}`);
|
|
status = 'CRITICAL';
|
|
}
|
|
if (showResult.code !== 0) {
|
|
diagnostics.push(
|
|
`systemctl show 失败: ${showResult.stderr || showResult.error}`,
|
|
);
|
|
status = 'CRITICAL';
|
|
}
|
|
if (catResult.code === 0 && !environmentFileMatchesPingoraEnvFile) {
|
|
diagnostics.push(
|
|
`systemctl cat genarrative-pingora-gateway.service EnvironmentFile 未包含本次 --pingora-env-file: ${input.pingoraEnvFile}`,
|
|
);
|
|
status = 'CRITICAL';
|
|
}
|
|
if (input.expectedGatewayMode === 'pingora-direct') {
|
|
if (!hasAmbientCapability || !hasCapabilityBoundingSet) {
|
|
diagnostics.push(
|
|
'pingora-direct 模式必须在 systemd 最终配置中包含 CAP_NET_BIND_SERVICE。',
|
|
);
|
|
status = 'CRITICAL';
|
|
}
|
|
}
|
|
if (input.expectedGatewayMode === 'nginx') {
|
|
if (hasAmbientCapability || hasCapabilityBoundingSet) {
|
|
diagnostics.push(
|
|
'nginx 模式下 systemd 最终配置不应残留 CAP_NET_BIND_SERVICE。',
|
|
);
|
|
status = 'CRITICAL';
|
|
}
|
|
}
|
|
|
|
return {
|
|
status,
|
|
hasAmbientCapability,
|
|
hasCapabilityBoundingSet,
|
|
environmentFiles,
|
|
environmentFileMatchesPingoraEnvFile,
|
|
fragmentPath: show.FragmentPath || '',
|
|
dropInPaths: show.DropInPaths || '',
|
|
user: show.User || '',
|
|
execStart: show.ExecStart || '',
|
|
diagnostics,
|
|
commands: [catResult.command, showResult.command],
|
|
};
|
|
}
|
|
|
|
function environmentFilesInclude(environmentFiles, expectedPath) {
|
|
return environmentFiles.some((entry) =>
|
|
splitSystemdEnvironmentFileEntry(entry).includes(expectedPath),
|
|
);
|
|
}
|
|
|
|
function splitSystemdEnvironmentFileEntry(entry) {
|
|
const files = [];
|
|
for (const word of String(entry || '').split(/\s+/u)) {
|
|
if (!word) {
|
|
continue;
|
|
}
|
|
let normalized = word.trim();
|
|
if (normalized.startsWith('-')) {
|
|
normalized = normalized.slice(1);
|
|
}
|
|
if (
|
|
(normalized.startsWith('"') && normalized.endsWith('"')) ||
|
|
(normalized.startsWith("'") && normalized.endsWith("'"))
|
|
) {
|
|
normalized = normalized.slice(1, -1);
|
|
}
|
|
if (normalized) {
|
|
files.push(normalized);
|
|
}
|
|
}
|
|
return files;
|
|
}
|
|
|
|
function parseSystemctlShow(text) {
|
|
const result = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const equalsIndex = line.indexOf('=');
|
|
if (equalsIndex <= 0) {
|
|
continue;
|
|
}
|
|
result[line.slice(0, equalsIndex)] = line.slice(equalsIndex + 1);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
async function runChecks(input, healthPatrolEnv, secrets) {
|
|
const checks = [];
|
|
checks.push(await runCurrentReleaseAudit(input, secrets));
|
|
if (input.expectedGatewayMode) {
|
|
checks.push(await runHealthPatrolEnvCheck(input, secrets));
|
|
}
|
|
if (input.runHealthPatrol) {
|
|
checks.push(await runProductionHealthPatrol(input, healthPatrolEnv, secrets));
|
|
}
|
|
return checks;
|
|
}
|
|
|
|
async function runCurrentReleaseAudit(input, secrets) {
|
|
const script = path.join(
|
|
input.releaseRoot,
|
|
'scripts/ops/pingora-current-release-audit.mjs',
|
|
);
|
|
const result = await runCommand(
|
|
'node',
|
|
[
|
|
'--',
|
|
script,
|
|
'--release-root',
|
|
input.releaseRoot,
|
|
...(input.requirePingoraGateway ? ['--require-pingora-gateway'] : []),
|
|
'--systemd-show',
|
|
'--timeout-ms',
|
|
String(input.timeoutMs),
|
|
],
|
|
input,
|
|
);
|
|
const parsed = parseJsonObject(result.stdout);
|
|
const details = parsed.ok
|
|
? {
|
|
summary: parsed.value.summary,
|
|
pingoraGateway: parsed.value.pingoraGateway,
|
|
checksums: parsed.value.checksums,
|
|
releaseManifest: parsed.value.releaseManifest,
|
|
systemd: parsed.value.systemd,
|
|
}
|
|
: {
|
|
parseError: parsed.error,
|
|
};
|
|
const check = commandCheck('current-release-audit', result, secrets, details);
|
|
if (!parsed.ok) {
|
|
check.status = 'CRITICAL';
|
|
check.stderr = trimForJson(
|
|
[check.stderr, `无法解析 current release 自审 JSON: ${parsed.error}`]
|
|
.filter(Boolean)
|
|
.join('\n'),
|
|
);
|
|
}
|
|
return check;
|
|
}
|
|
|
|
async function runHealthPatrolEnvCheck(input, secrets) {
|
|
const script = path.join(
|
|
input.releaseRoot,
|
|
'scripts/check-production-health-patrol-env.mjs',
|
|
);
|
|
const args = [
|
|
'--',
|
|
script,
|
|
'--env-file',
|
|
input.healthPatrolEnvFile,
|
|
'--expected-gateway-mode',
|
|
input.expectedGatewayMode,
|
|
];
|
|
if (input.expectedPublicBaseUrl) {
|
|
args.push('--expected-public-base-url', input.expectedPublicBaseUrl);
|
|
}
|
|
if (input.expectedPublicHost !== null) {
|
|
args.push('--expected-public-host', input.expectedPublicHost);
|
|
}
|
|
if (input.requireEmptyPublicHost) {
|
|
args.push('--require-empty-public-host');
|
|
}
|
|
const result = await runCommand('node', args, input);
|
|
return commandCheck('health-patrol-env-check', result, secrets);
|
|
}
|
|
|
|
async function runProductionHealthPatrol(input, healthPatrolEnv, secrets) {
|
|
const script = path.join(
|
|
input.releaseRoot,
|
|
'scripts/ops/production-health-patrol.mjs',
|
|
);
|
|
const rawEnv = await readEnvFile(input.healthPatrolEnvFile);
|
|
const env = {
|
|
...process.env,
|
|
...readEnvValuesForChild(rawEnv.status === 'CRITICAL' ? healthPatrolEnv : rawEnv),
|
|
};
|
|
const result = await runCommand(
|
|
'node',
|
|
['--', script, '--json'],
|
|
input,
|
|
env,
|
|
);
|
|
return commandCheck('production-health-patrol', result, secrets);
|
|
}
|
|
|
|
function readEnvValuesForChild(healthPatrolEnv) {
|
|
if (!healthPatrolEnv.values) {
|
|
return {};
|
|
}
|
|
const result = {};
|
|
for (const [key, value] of Object.entries(healthPatrolEnv.values)) {
|
|
if (typeof value === 'string') {
|
|
result[key] = value;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function commandCheck(name, result, secrets = [], details = undefined) {
|
|
return {
|
|
name,
|
|
status: result.code === 0 ? 'OK' : 'CRITICAL',
|
|
code: result.code,
|
|
command: result.command,
|
|
stdout: trimForJson(redactSecrets(result.stdout, secrets)),
|
|
stderr: trimForJson(redactSecrets(result.stderr || result.error, secrets)),
|
|
...(details === undefined ? {} : { details }),
|
|
};
|
|
}
|
|
|
|
function redactSecrets(value, secrets) {
|
|
let text = String(value || '');
|
|
for (const secret of secrets) {
|
|
if (!secret) {
|
|
continue;
|
|
}
|
|
text = text.split(secret).join('<redacted>');
|
|
}
|
|
return text;
|
|
}
|
|
|
|
function runCommand(command, args, input, env = process.env) {
|
|
validateNoControlCharacters(command, '子命令可执行文件');
|
|
for (const arg of args) {
|
|
validateNoControlCharacters(arg, '子命令参数');
|
|
}
|
|
return new Promise((resolve) => {
|
|
execFile(
|
|
command,
|
|
args,
|
|
{
|
|
env,
|
|
timeout: input.timeoutMs,
|
|
windowsHide: true,
|
|
maxBuffer: 1024 * 1024,
|
|
},
|
|
(error, stdout, stderr) => {
|
|
resolve({
|
|
command: formatCommand(command, args),
|
|
code: typeof error?.code === 'number' ? error.code : error ? 1 : 0,
|
|
stdout: String(stdout || ''),
|
|
stderr: String(stderr || ''),
|
|
timedOut: Boolean(error?.killed),
|
|
error: error ? error.message : '',
|
|
});
|
|
},
|
|
);
|
|
});
|
|
}
|
|
|
|
function formatCommand(command, args) {
|
|
const secretValueFlags = new Set([
|
|
'--pingora-probe-token',
|
|
'--probe-token',
|
|
'--direct-probe-token',
|
|
]);
|
|
const redactedArgs = args.map((arg, index) =>
|
|
index > 0 && secretValueFlags.has(args[index - 1]) ? '<redacted>' : arg,
|
|
);
|
|
return [command, ...redactedArgs].join(' ');
|
|
}
|
|
|
|
function trimForJson(value) {
|
|
const text = String(value || '').trim();
|
|
if (text.length <= 1000) {
|
|
return text;
|
|
}
|
|
return `${text.slice(0, 1000)}...<truncated>`;
|
|
}
|
|
|
|
function parseJsonObject(value) {
|
|
try {
|
|
return { ok: true, value: JSON.parse(value) };
|
|
} catch (error) {
|
|
return { ok: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
function summarize(statuses) {
|
|
const normalized = statuses.filter(Boolean);
|
|
const status = normalized.reduce(
|
|
(current, item) => maxStatus(current, item),
|
|
'OK',
|
|
);
|
|
return {
|
|
status,
|
|
criticalCount: normalized.filter((item) => item === 'CRITICAL').length,
|
|
warningCount: normalized.filter((item) => item === 'WARNING').length,
|
|
};
|
|
}
|
|
|
|
function maxStatus(left, right) {
|
|
return STATUS_RANK[right] > STATUS_RANK[left] ? right : left;
|
|
}
|