kdletters
a7711d2dc3
新增 pingora-gateway 独立二进制 crate，覆盖路由、静态资源、压缩、接流保护、TLS 直连和访问日志能力。 新增 Nginx canary、realpath canary、direct preflight、direct live、direct enable 和 rollback 脚本。 新增 Pingora 切流证据包、命令证据、manifest 验真、根目录总审计和 release readiness 聚合门禁。 完善 API release、Jenkins、systemd、health patrol、生产部署和发布包自包含校验。 更新 Pingora 试点文档、Nginx README 与 Hermes 共享记忆。
519 lines
15 KiB
JavaScript
519 lines
15 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { spawn } from 'node:child_process';
|
|
import { createHash } from 'node:crypto';
|
|
import { constants as fsConstants } from 'node:fs';
|
|
import {
|
|
access,
|
|
chmod,
|
|
lstat,
|
|
mkdir,
|
|
readFile,
|
|
stat,
|
|
writeFile,
|
|
} from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
const BUNDLE_DIR_MODE = 0o750;
|
|
const EVIDENCE_FILE_MODE = 0o640;
|
|
const SECRET_VALUE_FLAGS = new Set([
|
|
'--direct-probe-token',
|
|
'--probe-token',
|
|
'--pingora-shadow-probe-token',
|
|
'--rollback-pingora-shadow-probe-token',
|
|
]);
|
|
const SECRET_ENV_KEYS = [
|
|
'GENARRATIVE_PINGORA_DIRECT_PROBE_TOKEN',
|
|
'GENARRATIVE_PINGORA_ROLLBACK_SHADOW_PROBE_TOKEN',
|
|
'GENARRATIVE_HEALTH_PATROL_PINGORA_PROBE_TOKEN',
|
|
'GENARRATIVE_PINGORA_GATEWAY_PROBE_TOKEN',
|
|
];
|
|
|
|
const config = parseArgs(process.argv.slice(2));
|
|
const secretValues = collectSecretValues(config.commandArgs, process.env);
|
|
const startedAt = new Date();
|
|
const bundleDir = await createBundleDir(
|
|
config.outputRoot,
|
|
config.phase,
|
|
config.commandName,
|
|
startedAt,
|
|
);
|
|
const run = await runCommand(config.command, config.commandArgs, secretValues);
|
|
const finishedAt = new Date();
|
|
|
|
const stdoutPath = path.join(bundleDir, 'command.stdout.txt');
|
|
const stderrPath = path.join(bundleDir, 'command.stderr.txt');
|
|
const recordPath = path.join(bundleDir, 'command-record.json');
|
|
const manifestPath = path.join(bundleDir, 'manifest.json');
|
|
const exitCode =
|
|
run.signal
|
|
? run.exitCode ?? 1
|
|
: run.exitCode === null || run.exitCode === undefined
|
|
? run.error
|
|
? 1
|
|
: 0
|
|
: run.exitCode;
|
|
const status = exitCode === 0 && !run.signal && !run.error ? 'OK' : 'FAILED';
|
|
|
|
await writeEvidenceFile(stdoutPath, redactSecrets(run.stdout, secretValues));
|
|
await writeEvidenceFile(stderrPath, redactSecrets(run.stderr, secretValues));
|
|
|
|
const commandRecord = {
|
|
schemaVersion: 1,
|
|
name: config.commandName,
|
|
phase: config.phase,
|
|
...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}),
|
|
...(config.expectedExecutable
|
|
? { expectedExecutable: config.expectedExecutable }
|
|
: {}),
|
|
executable: redactSecrets(config.command, secretValues),
|
|
args: redactSecretArgs(config.commandArgs, secretValues),
|
|
command: formatCommand(config.command, config.commandArgs, secretValues),
|
|
cwd: process.cwd(),
|
|
exitCode,
|
|
signal: run.signal,
|
|
error: run.error || null,
|
|
startedAt: startedAt.toISOString(),
|
|
finishedAt: finishedAt.toISOString(),
|
|
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
|
stdoutPath: path.basename(stdoutPath),
|
|
stderrPath: path.basename(stderrPath),
|
|
};
|
|
await writeEvidenceFile(
|
|
recordPath,
|
|
`${JSON.stringify(commandRecord, null, 2)}\n`,
|
|
);
|
|
const artifactFiles = {
|
|
stdout: await buildEvidenceFileMetadata(stdoutPath),
|
|
stderr: await buildEvidenceFileMetadata(stderrPath),
|
|
commandRecord: await buildEvidenceFileMetadata(recordPath),
|
|
};
|
|
|
|
const manifest = {
|
|
schemaVersion: 1,
|
|
generatedAt: new Date().toISOString(),
|
|
phase: config.phase,
|
|
commandName: config.commandName,
|
|
...(config.cutoverRunId ? { cutoverRunId: config.cutoverRunId } : {}),
|
|
...(config.expectedExecutable
|
|
? { expectedExecutable: config.expectedExecutable }
|
|
: {}),
|
|
outputRoot: config.outputRoot,
|
|
bundleDir,
|
|
summary: {
|
|
status,
|
|
exitCode,
|
|
signal: run.signal,
|
|
},
|
|
files: {
|
|
manifest: path.basename(manifestPath),
|
|
...artifactFiles,
|
|
},
|
|
command: commandRecord,
|
|
};
|
|
await writeEvidenceFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
|
|
|
console.log(
|
|
`${JSON.stringify(
|
|
{
|
|
ok: status === 'OK',
|
|
phase: config.phase,
|
|
commandName: config.commandName,
|
|
cutoverRunId: config.cutoverRunId || null,
|
|
expectedExecutable: config.expectedExecutable || null,
|
|
status,
|
|
exitCode,
|
|
bundleDir,
|
|
manifestPath,
|
|
stdoutPath,
|
|
stderrPath,
|
|
commandRecordPath: recordPath,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
|
|
if (exitCode !== 0) {
|
|
process.exit(exitCode);
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const separatorIndex = argv.indexOf('--');
|
|
const optionArgs =
|
|
separatorIndex < 0 ? argv : argv.slice(0, separatorIndex);
|
|
if (optionArgs.includes('-h') || optionArgs.includes('--help')) {
|
|
printUsage();
|
|
process.exit(0);
|
|
}
|
|
if (separatorIndex < 0) {
|
|
throw new Error('必须使用 -- 分隔证据参数和真实命令。');
|
|
}
|
|
|
|
const command = argv[separatorIndex + 1] || '';
|
|
const commandArgs = argv.slice(separatorIndex + 2);
|
|
const result = {
|
|
phase:
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_PHASE || 'manual',
|
|
commandName:
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_COMMAND_NAME || 'command',
|
|
cutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '',
|
|
expectedExecutable:
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_EXPECTED_EXECUTABLE || '',
|
|
requiredArgs: parseRequiredArgsEnv(
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_REQUIRED_ARGS || '',
|
|
),
|
|
outputRoot:
|
|
process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT ||
|
|
'/var/log/genarrative/pingora-cutover-evidence',
|
|
command,
|
|
commandArgs,
|
|
};
|
|
|
|
for (let index = 0; index < optionArgs.length; index += 1) {
|
|
const arg = optionArgs[index];
|
|
switch (arg) {
|
|
case '-h':
|
|
case '--help':
|
|
printUsage();
|
|
process.exit(0);
|
|
break;
|
|
case '--phase':
|
|
result.phase = requireValue(optionArgs, ++index, arg);
|
|
break;
|
|
case '--command-name':
|
|
result.commandName = requireValue(optionArgs, ++index, arg);
|
|
break;
|
|
case '--cutover-run-id':
|
|
result.cutoverRunId = requireValue(optionArgs, ++index, arg);
|
|
break;
|
|
case '--expected-executable':
|
|
result.expectedExecutable = requireValue(optionArgs, ++index, arg);
|
|
break;
|
|
case '--require-arg':
|
|
result.requiredArgs.push(requireAnyValue(optionArgs, ++index, arg));
|
|
break;
|
|
case '--output-root':
|
|
result.outputRoot = requireValue(optionArgs, ++index, arg);
|
|
break;
|
|
default:
|
|
throw new Error(`未知参数: ${arg}`);
|
|
}
|
|
}
|
|
|
|
validateConfig(result);
|
|
return result;
|
|
}
|
|
|
|
function printUsage() {
|
|
console.log(`Usage:
|
|
node scripts/ops/pingora-cutover-command-evidence.mjs [options] -- <command> [args...]
|
|
|
|
Options:
|
|
--phase <name> 写入 manifest 的阶段标签，例如 enable-apply / rollback-apply。
|
|
--command-name <name> 写入 manifest 的命令名，例如 pingora-direct-rollback-apply。
|
|
--cutover-run-id <id> 可选，写入 manifest 的本次切换批次 ID；正式 runbook 会用同一 ID 串联所有阶段和命令证据。
|
|
--expected-executable <path>
|
|
可选，要求 -- 后面的真实命令精确等于该绝对路径；正式 runbook 用于绑定 apply 证据和 current release 随包脚本。
|
|
--require-arg <arg> 可选，要求 -- 后面的真实命令参数包含该值；正式 runbook 用于在执行前确认 apply 证据包含 --apply。
|
|
--output-root <path> 证据输出根目录，默认 /var/log/genarrative/pingora-cutover-evidence。
|
|
|
|
该脚本会执行 -- 后面的真实命令，并把 stdout、stderr、退出码、脱敏后的命令记录和 manifest 写入 --output-root 下的新证据目录；失败时仍保留证据并返回真实命令的退出码。
|
|
`);
|
|
}
|
|
|
|
function validateConfig(config) {
|
|
validateSafeName(config.phase, '--phase');
|
|
validateSafeName(config.commandName, '--command-name');
|
|
if (config.cutoverRunId) {
|
|
validateSafeName(config.cutoverRunId, '--cutover-run-id');
|
|
}
|
|
if (config.expectedExecutable) {
|
|
validateSafeExecutablePath(config.expectedExecutable, '--expected-executable');
|
|
}
|
|
if (!path.isAbsolute(config.outputRoot)) {
|
|
throw new Error('--output-root 必须是绝对路径。');
|
|
}
|
|
if (path.resolve(config.outputRoot) === path.parse(path.resolve(config.outputRoot)).root) {
|
|
throw new Error('--output-root 不能是文件系统根目录。');
|
|
}
|
|
validateNoControlCharacters(config.outputRoot, '--output-root');
|
|
if (!config.command) {
|
|
throw new Error('-- 后必须提供真实命令。');
|
|
}
|
|
validateCommandPath(config.command);
|
|
for (const commandArg of config.commandArgs) {
|
|
validateCommandArg(commandArg);
|
|
}
|
|
if (
|
|
config.expectedExecutable &&
|
|
config.command !== config.expectedExecutable
|
|
) {
|
|
throw new Error(
|
|
`真实命令与 --expected-executable 不一致: expected ${config.expectedExecutable}, got ${config.command}`,
|
|
);
|
|
}
|
|
for (const requiredArg of config.requiredArgs) {
|
|
validateRequiredArg(requiredArg, '--require-arg');
|
|
if (!config.commandArgs.includes(requiredArg)) {
|
|
throw new Error(`真实命令参数缺少 --require-arg 要求的 ${requiredArg}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
function validateCommandPath(command) {
|
|
validateSafeExecutablePath(command, '真实命令');
|
|
}
|
|
|
|
function validateCommandArg(value) {
|
|
if (/[\0\r\n]/u.test(String(value))) {
|
|
throw new Error('真实命令参数不能包含换行或 NUL 字符。');
|
|
}
|
|
}
|
|
|
|
function validateNoControlCharacters(value, label) {
|
|
if (/[\0\r\n]/u.test(String(value))) {
|
|
throw new Error(`${label} 不能包含换行或 NUL 字符。`);
|
|
}
|
|
}
|
|
|
|
function validateSafeExecutablePath(command, label) {
|
|
const prefix = label.startsWith('--') ? `${label} ` : label;
|
|
if (!path.isAbsolute(command)) {
|
|
throw new Error(`${prefix}必须是绝对路径。`);
|
|
}
|
|
if (path.resolve(command) === path.parse(path.resolve(command)).root) {
|
|
throw new Error(`${prefix}不能是文件系统根目录。`);
|
|
}
|
|
if (/[\0\r\n]/u.test(command)) {
|
|
throw new Error(`${prefix}不能包含换行或 NUL 字符。`);
|
|
}
|
|
}
|
|
|
|
function parseRequiredArgsEnv(value) {
|
|
return String(value || '')
|
|
.split(',')
|
|
.map((item) => item.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function validateRequiredArg(value, label) {
|
|
if (
|
|
typeof value !== 'string' ||
|
|
value.length === 0 ||
|
|
/[\0\r\n]/u.test(value)
|
|
) {
|
|
throw new Error(`${label} 必须是非空且不包含换行的字符串。`);
|
|
}
|
|
}
|
|
|
|
function validateSafeName(value, label) {
|
|
const text = String(value || '');
|
|
if (!/^[0-9A-Za-z._-]+$/u.test(text)) {
|
|
throw new Error(
|
|
`${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function requireValue(argv, index, flag) {
|
|
const value = argv[index];
|
|
if (value === undefined || value.startsWith('--')) {
|
|
throw new Error(`${flag} 缺少参数值`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function requireAnyValue(argv, index, flag) {
|
|
const value = argv[index];
|
|
if (value === undefined) {
|
|
throw new Error(`${flag} 缺少参数值`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
async function createBundleDir(outputRoot, phase, commandName, date) {
|
|
await validateOutputRootForWriting(outputRoot);
|
|
await mkdir(outputRoot, { recursive: true });
|
|
await validateOutputRootForWriting(outputRoot);
|
|
await access(outputRoot, fsConstants.W_OK);
|
|
const baseName = `${formatTimestampForPath(date)}-${phase}-${commandName}`;
|
|
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
const suffix = attempt === 0 ? '' : `-${attempt}`;
|
|
const bundleDir = path.join(outputRoot, `${baseName}${suffix}`);
|
|
try {
|
|
await mkdir(bundleDir, { recursive: false, mode: BUNDLE_DIR_MODE });
|
|
await chmod(bundleDir, BUNDLE_DIR_MODE);
|
|
return bundleDir;
|
|
} catch (error) {
|
|
if (error?.code !== 'EEXIST') {
|
|
throw error;
|
|
}
|
|
}
|
|
}
|
|
throw new Error(`无法创建唯一证据目录: ${path.join(outputRoot, baseName)}`);
|
|
}
|
|
|
|
async function validateOutputRootForWriting(outputRoot) {
|
|
const target = path.resolve(outputRoot);
|
|
const root = path.parse(target).root;
|
|
let current = root;
|
|
const segments = path
|
|
.relative(root, target)
|
|
.split(path.sep)
|
|
.filter(Boolean);
|
|
|
|
for (const segment of segments) {
|
|
current = path.join(current, segment);
|
|
let stats;
|
|
try {
|
|
stats = await lstat(current);
|
|
} catch (error) {
|
|
if (error?.code === 'ENOENT') {
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
if (stats.isSymbolicLink()) {
|
|
if (current === target) {
|
|
throw new Error(`--output-root 不能是符号链接: ${current}`);
|
|
}
|
|
throw new Error(`--output-root 上级目录不能是符号链接: ${current}`);
|
|
}
|
|
|
|
if (!stats.isDirectory()) {
|
|
if (current === target) {
|
|
throw new Error(`--output-root 已存在但不是目录: ${current}`);
|
|
}
|
|
throw new Error(`--output-root 上级路径已存在但不是目录: ${current}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function writeEvidenceFile(filePath, content) {
|
|
await writeFile(filePath, content, {
|
|
encoding: 'utf8',
|
|
flag: 'wx',
|
|
mode: EVIDENCE_FILE_MODE,
|
|
});
|
|
await chmod(filePath, EVIDENCE_FILE_MODE);
|
|
}
|
|
|
|
async function buildEvidenceFileMetadata(filePath) {
|
|
const [stats, content] = await Promise.all([stat(filePath), readFile(filePath)]);
|
|
return {
|
|
path: path.basename(filePath),
|
|
sizeBytes: stats.size,
|
|
sha256: createHash('sha256').update(content).digest('hex'),
|
|
};
|
|
}
|
|
|
|
function runCommand(command, args, secrets) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, {
|
|
cwd: process.cwd(),
|
|
env: process.env,
|
|
shell: false,
|
|
windowsHide: true,
|
|
});
|
|
let stdout = '';
|
|
let stderr = '';
|
|
let spawnError = '';
|
|
|
|
child.stdout?.on('data', (chunk) => {
|
|
const text = String(chunk);
|
|
stdout += text;
|
|
process.stdout.write(redactSecrets(text, secrets));
|
|
});
|
|
child.stderr?.on('data', (chunk) => {
|
|
const text = String(chunk);
|
|
stderr += text;
|
|
process.stderr.write(redactSecrets(text, secrets));
|
|
});
|
|
child.on('error', (error) => {
|
|
spawnError = error.message;
|
|
});
|
|
child.on('close', (exitCode, signal) => {
|
|
resolve({
|
|
exitCode,
|
|
signal: signal || null,
|
|
error: spawnError,
|
|
stdout,
|
|
stderr,
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function collectSecretValues(args, env) {
|
|
const values = [];
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const inlineValue = getInlineSecretFlagValue(args[index]);
|
|
if (inlineValue) {
|
|
values.push(inlineValue);
|
|
continue;
|
|
}
|
|
if (index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1]) && args[index]) {
|
|
values.push(args[index]);
|
|
}
|
|
}
|
|
for (const key of SECRET_ENV_KEYS) {
|
|
const value = String(env[key] || '');
|
|
if (value) {
|
|
values.push(value);
|
|
}
|
|
}
|
|
return [...new Set(values)].filter((value) => value.length >= 4);
|
|
}
|
|
|
|
function redactSecrets(text, secrets) {
|
|
let redacted = String(text || '');
|
|
for (const secret of secrets) {
|
|
redacted = redacted.split(secret).join('<redacted>');
|
|
}
|
|
return redacted;
|
|
}
|
|
|
|
function formatCommand(command, args, secrets) {
|
|
return [redactSecrets(command, secrets), ...redactSecretArgs(args, secrets)].join(' ');
|
|
}
|
|
|
|
function redactSecretArgs(args, secrets = []) {
|
|
return args.map((arg, index) => {
|
|
const redacted =
|
|
index > 0 && SECRET_VALUE_FLAGS.has(args[index - 1])
|
|
? '<redacted>'
|
|
: redactInlineSecretArg(arg);
|
|
return redactSecrets(redacted, secrets);
|
|
});
|
|
}
|
|
|
|
function getInlineSecretFlagValue(arg) {
|
|
const text = String(arg || '');
|
|
for (const flag of SECRET_VALUE_FLAGS) {
|
|
const prefix = `${flag}=`;
|
|
if (text.startsWith(prefix)) {
|
|
return text.slice(prefix.length);
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
function redactInlineSecretArg(arg) {
|
|
const text = String(arg || '');
|
|
for (const flag of SECRET_VALUE_FLAGS) {
|
|
if (text.startsWith(`${flag}=`)) {
|
|
return `${flag}=<redacted>`;
|
|
}
|
|
}
|
|
return arg;
|
|
}
|
|
|
|
function formatTimestampForPath(date) {
|
|
return date
|
|
.toISOString()
|
|
.replace(/[-:]/gu, '')
|
|
.replace(/\.\d{3}Z$/u, 'Z');
|
|
}
|