#!/usr/bin/env node import { execFile } from 'node:child_process'; import { constants as fsConstants } from 'node:fs'; import { access, lstat, readdir, readFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const DEFAULT_EVIDENCE_ROOT = '/var/log/genarrative/pingora-cutover-evidence'; const DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS = 24 * 60 * 60 * 1000; const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); const CUTOVER_TIMELINE_ORDER = [ { type: 'phase', phase: 'pre-cutover' }, { type: 'command', phase: 'enable-apply', commandName: 'pingora-direct-enable-apply', }, { type: 'command', phase: 'post-enable', commandName: 'pingora-health-patrol-direct-env-switch', }, { type: 'phase', phase: 'post-enable' }, { type: 'command', phase: 'rollback-prep', commandName: 'pingora-gateway-shadow-env-switch', }, { type: 'command', phase: 'rollback-prep', commandName: 'pingora-health-patrol-nginx-env-switch', }, { type: 'command', phase: 'rollback-apply', commandName: 'pingora-direct-rollback-apply', }, { type: 'phase', phase: 'post-rollback' }, ]; const config = parseArgs(process.argv.slice(2)); await validateReadOnlyDirectory(config.evidenceRoot, '--evidence-root'); await validateVerifierScript(config.verifyScript); const discovery = await discoverEvidence(config.evidenceRoot, { allowExtraRootEntries: config.allowExtraRootEntries, }); const audit = await buildAudit(config, discovery); console.log(`${JSON.stringify(audit, null, 2)}\n`); if (!audit.ok) { process.exit(1); } function parseArgs(argv) { const result = { evidenceRoot: process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_ROOT || DEFAULT_EVIDENCE_ROOT, verifyScript: process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_VERIFY_SCRIPT || path.join(SCRIPT_DIR, 'pingora-cutover-evidence-verify.mjs'), timelineMaxSpanMs: parsePositiveInteger( process.env.GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS || DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS, 'GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS', ), requiredCutoverRunId: process.env.GENARRATIVE_PINGORA_CUTOVER_RUN_ID || '', allowExtraRootEntries: false, requiredPhases: [], requiredPhaseDirectLiveAccessLog: [], requiredPhaseDirectLiveStaticHeaders: [], requiredPhasePingoraEnvShadow: [], requiredCommands: [], requiredCommandExecutables: [], requiredCommandArgs: [], }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; switch (arg) { case '-h': case '--help': printUsage(); process.exit(0); break; case '--evidence-root': result.evidenceRoot = requireValue(argv, ++index, arg); break; case '--verify-script': result.verifyScript = requireValue(argv, ++index, arg); break; case '--phase': case '--require-phase': result.requiredPhases.push(requireValue(argv, ++index, arg)); break; case '--require-command': result.requiredCommands.push(requireValue(argv, ++index, arg)); break; case '--require-phase-direct-live-access-log': result.requiredPhaseDirectLiveAccessLog.push( requireValue(argv, ++index, arg), ); break; case '--require-phase-direct-live-static-headers': result.requiredPhaseDirectLiveStaticHeaders.push( requireValue(argv, ++index, arg), ); break; case '--require-phase-pingora-env-shadow': result.requiredPhasePingoraEnvShadow.push( requireValue(argv, ++index, arg), ); break; case '--require-command-executable': result.requiredCommandExecutables.push(requireValue(argv, ++index, arg)); break; case '--require-command-arg': result.requiredCommandArgs.push(requireValue(argv, ++index, arg)); break; case '--timeline-max-span-ms': result.timelineMaxSpanMs = parsePositiveInteger( requireValue(argv, ++index, arg), '--timeline-max-span-ms', ); break; case '--require-cutover-run-id': result.requiredCutoverRunId = requireValue(argv, ++index, arg); break; case '--allow-extra-root-entries': result.allowExtraRootEntries = true; break; default: throw new Error(`未知参数: ${arg}`); } } for (const [label, value] of [ ['--evidence-root', result.evidenceRoot], ['--verify-script', result.verifyScript], ]) { if (!path.isAbsolute(value)) { throw new Error(`${label} 必须是绝对路径。`); } if (isFilesystemRootPath(value)) { throw new Error(`${label} 不能是文件系统根目录。`); } validateNoControlCharacters(value, label); } result.requiredPhases = normalizeRequiredPhases(result.requiredPhases); result.requiredPhaseDirectLiveAccessLog = normalizeRequiredPhases( result.requiredPhaseDirectLiveAccessLog, ); result.requiredPhaseDirectLiveStaticHeaders = normalizeRequiredPhases( result.requiredPhaseDirectLiveStaticHeaders, ); result.requiredPhasePingoraEnvShadow = normalizeRequiredPhases( result.requiredPhasePingoraEnvShadow, ); const requiredCommands = normalizeRequiredCommands(result.requiredCommands); result.requiredCommandExecutables = normalizeRequiredCommandExecutables( result.requiredCommandExecutables, ); result.requiredCommandArgs = normalizeRequiredCommandArgs( result.requiredCommandArgs, ); result.requiredCommands = mergeRequiredCommands( requiredCommands, result.requiredCommandExecutables, result.requiredCommandArgs, ); if (result.requiredCutoverRunId) { validateSafeCutoverRunId( result.requiredCutoverRunId, '--require-cutover-run-id', ); } return result; } function printUsage() { console.log(`Usage: node scripts/ops/pingora-cutover-evidence-audit.mjs [options] Options: --evidence-root 证据根目录,默认 ${DEFAULT_EVIDENCE_ROOT}。 --verify-script 证据验真脚本,默认同目录 pingora-cutover-evidence-verify.mjs。 --require-phase 要求并验真某个阶段的最新证据;可重复。--phase 是别名。 --require-command : 要求并验真某个切换命令证据;可重复。 --require-phase-direct-live-access-log 要求某个阶段 manifest.summary.directLiveAccessLog 已存在且 direct live request_id access log 对账成功;正式切换默认用于 post-enable。 --require-phase-direct-live-static-headers 要求某个阶段 manifest.summary.directLiveStaticHeaders 已存在且包含可判定的 direct live 静态响应头摘要;正式切换默认用于 post-enable。 --require-phase-pingora-env-shadow 要求某个阶段 manifest.summary.pingoraEnvShadow 已存在且证明 Pingora env 已恢复 shadow 高端口;正式切换默认用于 post-rollback。 --require-command-executable :: 要求某个命令证据的 manifest.expectedExecutable、manifest.command.executable 和 command-record.json executable 与该绝对路径一致;可重复。 --require-command-arg :: 要求某个命令证据的 manifest.command.args 和 command-record.json args 都包含该参数;可重复。 --timeline-max-span-ms 标准五段切换时间线允许的最大跨度,默认 ${DEFAULT_CUTOVER_TIMELINE_MAX_SPAN_MS}ms,可用 GENARRATIVE_PINGORA_CUTOVER_EVIDENCE_TIMELINE_MAX_SPAN_MS 覆盖。 --require-cutover-run-id 只接受 manifest.cutoverRunId 与该 ID 一致的阶段和命令证据,避免混入其它切换批次。 --allow-extra-root-entries 允许证据根目录中存在非证据目录条目;默认拒绝,正式切换归档不应使用。 该脚本只读扫描 Pingora 直连切换证据根目录,按 manifest.phase 找到每个阶段的最新证据目录,也可按 manifest.phase + manifest.commandName 找到真实切换命令证据,并调用随包证据验真脚本的 --require-summary-ok 严格模式校验 schemaVersion=1 manifest.files 的 sizeBytes / sha256 与 manifest.summary.status=OK;所有候选证据都必须带合法 manifest.generatedAt,最新证据选择和标准时间线证明只使用该字段,不用目录 mtime 兜底;若同一阶段或同一命令的最新 manifest.generatedAt 重复,则拒绝按目录名打平并要求重新归档或清理证据根目录;命令证据的 manifest.command 与 command-record.json 也必须是 schemaVersion=1;可用 --require-phase-direct-live-access-log 要求指定阶段必须带 direct live access log 对账摘要,防止旧 post-enable 证据包缺少 request_id 反查复盘入口;可用 --require-phase-direct-live-static-headers 要求指定阶段必须带 direct live 静态响应头摘要,防止旧 post-enable 证据包缺少 Cache-Control、ETag、Last-Modified、Range 和 304 复盘入口;可用 --require-phase-pingora-env-shadow 要求指定阶段必须带 Pingora env shadow 摘要,防止旧 post-rollback 证据包无法证明 active env 已从低端口 direct 恢复到 127.0.0.1:18081;证据根目录默认只能包含带 manifest.json 的证据目录,不允许夹带普通文件、无 manifest 目录或符号链接;不会修改证据目录、不会 reload systemd、不会访问 Nginx 或 Pingora。 `); } function requireValue(argv, index, flag) { const value = argv[index]; if (value === undefined || value.startsWith('--')) { throw new Error(`${flag} 缺少参数值`); } return value; } function normalizeRequiredPhases(phases) { const seen = new Set(); const normalized = []; for (const phase of phases) { const text = String(phase || '').trim(); validateSafePhase(text, '--require-phase'); if (!seen.has(text)) { seen.add(text); normalized.push(text); } } return normalized; } function normalizeRequiredCommands(commands) { const seen = new Set(); const normalized = []; for (const command of commands) { const text = String(command || '').trim(); const parts = text.split(':'); if (parts.length !== 2 || !parts[0] || !parts[1]) { throw new Error( '--require-command 必须使用 : 格式。', ); } const [phase, commandName] = parts; validateSafePhase(phase, '--require-command phase'); validateSafeCommandName(commandName, '--require-command commandName'); const key = `${phase}:${commandName}`; if (!seen.has(key)) { seen.add(key); normalized.push({ phase, commandName }); } } return normalized; } function normalizeRequiredCommandExecutables(commandExecutables) { const seen = new Map(); const normalized = []; for (const item of commandExecutables) { const text = String(item || '').trim(); const parts = text.split(':'); if (parts.length < 3 || !parts[0] || !parts[1]) { throw new Error( '--require-command-executable 必须使用 :: 格式。', ); } const [phase, commandName, ...pathParts] = parts; const executable = pathParts.join(':'); validateSafePhase(phase, '--require-command-executable phase'); validateSafeCommandName( commandName, '--require-command-executable commandName', ); if (!path.isAbsolute(executable)) { throw new Error('--require-command-executable executable 必须是绝对路径。'); } if (isFilesystemRootPath(executable)) { throw new Error( '--require-command-executable executable 不能是文件系统根目录。', ); } if (/[\0\r\n]/u.test(executable)) { throw new Error( '--require-command-executable executable 不能包含换行或 NUL 字符。', ); } const key = `${phase}:${commandName}`; const previousExecutable = seen.get(key); if (previousExecutable && previousExecutable !== executable) { throw new Error( '--require-command-executable 不能为同一个 : 指定多个不同路径。', ); } if (!previousExecutable) { seen.set(key, executable); normalized.push({ phase, commandName, executable }); } } return normalized; } function normalizeRequiredCommandArgs(commandArgs) { const seen = new Set(); const normalized = []; for (const item of commandArgs) { const text = String(item || '').trim(); const parts = text.split(':'); if (parts.length < 3 || !parts[0] || !parts[1]) { throw new Error( '--require-command-arg 必须使用 :: 格式。', ); } const [phase, commandName, ...argParts] = parts; const requiredArg = argParts.join(':'); validateSafePhase(phase, '--require-command-arg phase'); validateSafeCommandName(commandName, '--require-command-arg commandName'); validateRequiredCommandArg(requiredArg, '--require-command-arg arg'); const key = `${phase}:${commandName}:${requiredArg}`; if (!seen.has(key)) { seen.add(key); normalized.push({ phase, commandName, arg: requiredArg }); } } return normalized; } function mergeRequiredCommands( requiredCommands, requiredCommandExecutables, requiredCommandArgs, ) { const seen = new Set(); const merged = []; for (const command of [ ...requiredCommands, ...requiredCommandExecutables, ...requiredCommandArgs, ]) { const key = `${command.phase}:${command.commandName}`; if (!seen.has(key)) { seen.add(key); merged.push({ phase: command.phase, commandName: command.commandName, }); } } return merged; } function findRequiredCommandExecutable(requiredCommandExecutables, command) { return ( requiredCommandExecutables.find( (item) => item.phase === command.phase && item.commandName === command.commandName, )?.executable || null ); } function findRequiredCommandArgs(requiredCommandArgs, command) { return requiredCommandArgs .filter( (item) => item.phase === command.phase && item.commandName === command.commandName, ) .map((item) => item.arg); } function validateSafePhase(value, label) { if (!isSafePhase(value)) { throw new Error( `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, ); } } function validateSafeCommandName(value, label) { if (!isSafePhase(value)) { throw new Error( `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, ); } } function validateSafeCutoverRunId(value, label) { if (!isSafePhase(value)) { throw new Error( `${label} 只能包含 ASCII 字母、数字、点、下划线或短横线。`, ); } } function validateRequiredCommandArg(value, label) { if ( typeof value !== 'string' || value.length === 0 || /[\0\r\n]/u.test(value) ) { throw new Error(`${label} 必须是非空且不包含换行的字符串。`); } } function isSafePhase(value) { return /^[0-9A-Za-z._-]+$/u.test(String(value || '')); } function parsePositiveInteger(value, label) { const text = String(value || '').trim(); if (!/^[1-9]\d*$/u.test(text)) { throw new Error(`${label} 必须是正整数。`); } const parsed = Number(text); if (!Number.isSafeInteger(parsed)) { throw new Error(`${label} 超出 JavaScript 安全整数范围。`); } return parsed; } async function validateReadOnlyDirectory(dirPath, label) { const stats = await lstat(dirPath); if (stats.isSymbolicLink()) { throw new Error(`${label} 不能是符号链接: ${dirPath}`); } if (!stats.isDirectory()) { throw new Error(`${label} 必须是目录: ${dirPath}`); } await access(dirPath, fsConstants.R_OK | fsConstants.X_OK); } async function validateVerifierScript(scriptPath) { const stats = await lstat(scriptPath); if (stats.isSymbolicLink()) { throw new Error(`--verify-script 不能是符号链接: ${scriptPath}`); } if (!stats.isFile()) { throw new Error(`--verify-script 必须是文件: ${scriptPath}`); } await access(scriptPath, fsConstants.R_OK); } async function discoverEvidence(evidenceRoot, options = {}) { const allowExtraRootEntries = Boolean(options.allowExtraRootEntries); const entries = await readdir(evidenceRoot, { withFileTypes: true }); const candidates = []; const diagnostics = []; for (const entry of entries) { const entryPath = path.join(evidenceRoot, entry.name); const linkStats = await lstat(entryPath); if (linkStats.isSymbolicLink()) { if (!allowExtraRootEntries) { diagnostics.push({ path: entry.name, status: 'CRITICAL', reason: '证据根目录下不能包含符号链接条目。', }); } continue; } if (!linkStats.isDirectory()) { if (!allowExtraRootEntries) { diagnostics.push({ path: entry.name, status: 'CRITICAL', reason: linkStats.isFile() ? '证据根目录只能包含证据目录,发现普通文件。' : '证据根目录只能包含证据目录,发现非目录条目。', }); } continue; } const manifestPath = path.join(entryPath, 'manifest.json'); let manifestStats; try { manifestStats = await lstat(manifestPath); } catch (error) { if (error?.code === 'ENOENT') { if (!allowExtraRootEntries) { diagnostics.push({ path: entry.name, status: 'CRITICAL', reason: '证据根目录只能包含带 manifest.json 的证据目录。', }); } } else { diagnostics.push({ path: entry.name, status: 'CRITICAL', reason: `读取 manifest 状态失败: ${error.message}`, }); } continue; } if (manifestStats.isSymbolicLink()) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest 不能是符号链接。', }); continue; } if (!manifestStats.isFile()) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest 必须是普通文件。', }); continue; } const manifest = await readJsonManifest(manifestPath, entry.name, diagnostics); if (!manifest) { continue; } if (manifest.schemaVersion !== 1) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.schemaVersion 必须是 1。', }); continue; } if (!manifest.phase || typeof manifest.phase !== 'string') { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest 缺少 phase 字符串。', }); continue; } if (!isSafePhase(manifest.phase)) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.phase 只能包含 ASCII 字母、数字、点、下划线或短横线。', }); continue; } const topLevelCommandName = manifest.commandName === undefined ? null : manifest.commandName; const embeddedCommandName = manifest.command?.name === undefined ? null : manifest.command.name; if ( topLevelCommandName !== null && (typeof topLevelCommandName !== 'string' || !isSafePhase(topLevelCommandName)) ) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.commandName 只能包含 ASCII 字母、数字、点、下划线或短横线。', }); continue; } if ( embeddedCommandName !== null && (typeof embeddedCommandName !== 'string' || !isSafePhase(embeddedCommandName)) ) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.command.name 只能包含 ASCII 字母、数字、点、下划线或短横线。', }); continue; } if ( topLevelCommandName !== null && embeddedCommandName !== null && topLevelCommandName !== embeddedCommandName ) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: `manifest.commandName 与 manifest.command.name 必须一致,manifest.commandName=${formatNullable(topLevelCommandName)},manifest.command.name=${formatNullable(embeddedCommandName)}。`, }); continue; } const commandName = topLevelCommandName || embeddedCommandName; const cutoverRunId = manifest.cutoverRunId === undefined ? null : manifest.cutoverRunId; if ( cutoverRunId !== null && (typeof cutoverRunId !== 'string' || !isSafePhase(cutoverRunId)) ) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.cutoverRunId 只能包含 ASCII 字母、数字、点、下划线或短横线。', }); continue; } const topLevelExpectedExecutable = manifest.expectedExecutable === undefined ? null : manifest.expectedExecutable; const embeddedExpectedExecutable = manifest.command?.expectedExecutable === undefined ? null : manifest.command.expectedExecutable; const expectedExecutableDiagnostics = [ ...validateOptionalAbsolutePath( topLevelExpectedExecutable, 'manifest.expectedExecutable', ), ...validateOptionalAbsolutePath( embeddedExpectedExecutable, 'manifest.command.expectedExecutable', ), ]; if (expectedExecutableDiagnostics.length > 0) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: expectedExecutableDiagnostics.join(' '), }); continue; } const generatedAt = normalizeIsoTime(manifest.generatedAt); if (!generatedAt) { diagnostics.push({ path: entry.name, manifestPath, status: 'CRITICAL', reason: 'manifest.generatedAt 必须是合法 ISO 时间;证据最新选择和时间线证明不能依赖目录 mtime。', }); continue; } candidates.push({ phase: manifest.phase, cutoverRunId, bundleDir: entryPath, manifestPath, directoryName: entry.name, generatedAt, summaryStatus: manifest.summary?.status || null, summaryExitCode: manifest.summary?.exitCode ?? null, summarySignal: manifest.summary?.signal ?? null, summaryDirectLiveAccessLog: manifest.summary?.directLiveAccessLog ?? null, summaryDirectLiveStaticHeaders: manifest.summary?.directLiveStaticHeaders ?? null, summaryPingoraEnvShadow: manifest.summary?.pingoraEnvShadow ?? null, commandName, manifestCommand: manifest.command || null, commandRecordFileName: manifest.files?.commandRecord?.path || null, stdoutFileName: manifest.files?.stdout?.path || null, stderrFileName: manifest.files?.stderr?.path || null, commandStartedAt: normalizeIsoTime(manifest.command?.startedAt), commandFinishedAt: normalizeIsoTime(manifest.command?.finishedAt), commandDurationMs: manifest.command?.durationMs ?? null, expectedExecutable: topLevelExpectedExecutable ?? embeddedExpectedExecutable ?? null, executable: manifest.command?.executable || null, }); } return { candidates, diagnostics }; } function validateOptionalAbsolutePath(value, label) { if (value === null) { return []; } return validateRequiredAbsolutePath(value, label); } function validateRequiredAbsolutePath(value, label) { if (typeof value !== 'string' || value.length === 0) { return [`${label} 必须是绝对路径字符串。`]; } if (!path.isAbsolute(value)) { return [`${label} 必须是绝对路径。`]; } if (isFilesystemRootPath(value)) { return [`${label} 不能是文件系统根目录。`]; } if (/[\0\r\n]/u.test(value)) { return [`${label} 不能包含换行或 NUL 字符。`]; } return []; } async function readJsonManifest(manifestPath, directoryName, diagnostics) { let content; try { content = await readFile(manifestPath, 'utf8'); } catch (error) { diagnostics.push({ path: directoryName, manifestPath, status: 'CRITICAL', reason: `读取 manifest 失败: ${error.message}`, }); return null; } try { const manifest = JSON.parse(content); if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { diagnostics.push({ path: directoryName, manifestPath, status: 'CRITICAL', reason: 'manifest 顶层必须是 JSON object。', }); return null; } return manifest; } catch (error) { diagnostics.push({ path: directoryName, manifestPath, status: 'CRITICAL', reason: `manifest 不是合法 JSON: ${error.message}`, }); return null; } } async function buildAudit(config, discovery) { const phasesToCheck = config.requiredPhases.length > 0 ? config.requiredPhases : config.requiredCommands.length > 0 ? [] : sortedUnique( discovery.candidates .filter((candidate) => candidate.commandName === null) .map((candidate) => candidate.phase), ); const phases = []; for (const phase of phasesToCheck) { const requiredAccessLog = config.requiredPhaseDirectLiveAccessLog.includes( phase, ); const requiredStaticHeaders = config.requiredPhaseDirectLiveStaticHeaders.includes( phase, ); const requiredPingoraEnvShadow = config.requiredPhasePingoraEnvShadow.includes( phase, ); const allPhaseCandidates = discovery.candidates.filter( (candidate) => candidate.phase === phase && candidate.commandName === null, ); const candidates = filterCandidatesByCutoverRunId( allPhaseCandidates, config.requiredCutoverRunId, ) .sort(compareEvidenceCandidates); if (candidates.length === 0) { phases.push({ phase, status: 'MISSING', candidateCount: 0, allCandidateCount: allPhaseCandidates.length, requiredCutoverRunId: config.requiredCutoverRunId || null, requiredDirectLiveAccessLog: requiredAccessLog, requiredDirectLiveStaticHeaders: requiredStaticHeaders, requiredPingoraEnvShadow, directLiveAccessLog: null, directLiveStaticHeaders: null, pingoraEnvShadow: null, diagnostics: [ missingEvidenceDiagnostic( '该阶段', config.requiredCutoverRunId, allPhaseCandidates.length, ), ], }); continue; } const latestSelection = selectUniqueLatestEvidence(candidates); if (!latestSelection.ok) { phases.push({ phase, status: 'AMBIGUOUS_LATEST', candidateCount: candidates.length, allCandidateCount: allPhaseCandidates.length, requiredCutoverRunId: config.requiredCutoverRunId || null, generatedAt: latestSelection.generatedAt, requiredDirectLiveAccessLog: requiredAccessLog, requiredDirectLiveStaticHeaders: requiredStaticHeaders, requiredPingoraEnvShadow, directLiveAccessLog: null, directLiveStaticHeaders: null, pingoraEnvShadow: null, ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, diagnostics: latestSelection.diagnostics, }); continue; } const latest = latestSelection.latest; const verify = await runVerifier(config.verifyScript, latest.bundleDir); const manifestCheck = checkPhaseManifest(latest, { requiredAccessLog, requiredStaticHeaders, requiredPingoraEnvShadow, }); phases.push({ phase, status: phaseStatus(verify, manifestCheck), candidateCount: candidates.length, latestBundleDir: latest.bundleDir, manifestPath: latest.manifestPath, cutoverRunId: latest.cutoverRunId, requiredCutoverRunId: config.requiredCutoverRunId || null, generatedAt: latest.generatedAt, summaryStatus: latest.summaryStatus, requiredDirectLiveAccessLog: requiredAccessLog, requiredDirectLiveStaticHeaders: requiredStaticHeaders, requiredPingoraEnvShadow, directLiveAccessLog: latest.summaryDirectLiveAccessLog, directLiveStaticHeaders: latest.summaryDirectLiveStaticHeaders, pingoraEnvShadow: latest.summaryPingoraEnvShadow, commandName: latest.commandName, verify, manifestCheck, diagnostics: [ ...(verify.ok ? [] : ['最新证据目录 manifest 验真失败。']), ...manifestCheck.diagnostics, ], }); } const commands = []; for (const requiredCommand of config.requiredCommands) { const allCommandCandidates = discovery.candidates.filter( (candidate) => candidate.phase === requiredCommand.phase && candidate.commandName === requiredCommand.commandName, ); const candidates = filterCandidatesByCutoverRunId( allCommandCandidates, config.requiredCutoverRunId, ) .sort(compareEvidenceCandidates); if (candidates.length === 0) { commands.push({ phase: requiredCommand.phase, commandName: requiredCommand.commandName, status: 'MISSING', candidateCount: 0, allCandidateCount: allCommandCandidates.length, requiredCutoverRunId: config.requiredCutoverRunId || null, diagnostics: [ missingEvidenceDiagnostic( '该切换命令', config.requiredCutoverRunId, allCommandCandidates.length, ), ], }); continue; } const latestSelection = selectUniqueLatestEvidence(candidates); if (!latestSelection.ok) { commands.push({ phase: requiredCommand.phase, commandName: requiredCommand.commandName, status: 'AMBIGUOUS_LATEST', candidateCount: candidates.length, allCandidateCount: allCommandCandidates.length, requiredCutoverRunId: config.requiredCutoverRunId || null, generatedAt: latestSelection.generatedAt, ambiguousBundleDirs: latestSelection.ambiguousBundleDirs, diagnostics: latestSelection.diagnostics, }); continue; } const latest = latestSelection.latest; const verify = await runVerifier(config.verifyScript, latest.bundleDir); const expectedExecutable = findRequiredCommandExecutable( config.requiredCommandExecutables, requiredCommand, ); const requiredArgs = findRequiredCommandArgs( config.requiredCommandArgs, requiredCommand, ); const manifestCheck = await checkCommandManifest( latest, expectedExecutable, requiredArgs, ); commands.push({ phase: requiredCommand.phase, commandName: requiredCommand.commandName, status: commandStatus(verify, manifestCheck), candidateCount: candidates.length, latestBundleDir: latest.bundleDir, manifestPath: latest.manifestPath, cutoverRunId: latest.cutoverRunId, requiredCutoverRunId: config.requiredCutoverRunId || null, generatedAt: latest.generatedAt, summaryStatus: latest.summaryStatus, summaryExitCode: latest.summaryExitCode, summarySignal: latest.summarySignal, expectedExecutable: latest.expectedExecutable, executable: latest.executable, commandRecordPath: manifestCheck.commandRecordPath, stdoutPath: manifestCheck.stdoutPath, stderrPath: manifestCheck.stderrPath, commandStartedAt: manifestCheck.commandRecordStartedAt, commandFinishedAt: manifestCheck.commandRecordFinishedAt, commandDurationMs: manifestCheck.commandRecordDurationMs, requiredExecutable: expectedExecutable, requiredArgs, verify, manifestCheck, diagnostics: [ ...(verify.ok ? [] : ['最新命令证据目录 manifest 验真失败。']), ...manifestCheck.diagnostics, ], }); } const criticalDiagnostics = discovery.diagnostics.filter( (item) => item.status === 'CRITICAL', ); const timeline = buildTimelineAudit( phases, commands, config.timelineMaxSpanMs, ); const failedPhases = phases.filter((phase) => phase.status !== 'OK'); const failedCommands = commands.filter((command) => command.status !== 'OK'); const ok = phases.length + commands.length > 0 && failedPhases.length === 0 && failedCommands.length === 0 && criticalDiagnostics.length === 0 && timeline.ok; return { ok, generatedAt: new Date().toISOString(), evidenceRoot: config.evidenceRoot, verifyScript: config.verifyScript, requiredCutoverRunId: config.requiredCutoverRunId || null, allowExtraRootEntries: config.allowExtraRootEntries, timelineMaxSpanMs: config.timelineMaxSpanMs, requiredPhases: config.requiredPhases, requiredCommands: config.requiredCommands, requiredCommandExecutables: config.requiredCommandExecutables, requiredCommandArgs: config.requiredCommandArgs, discoveredCount: discovery.candidates.length, checkedCount: phases.length + commands.length, failedCount: failedPhases.length + failedCommands.length + criticalDiagnostics.length + timeline.failedCount, diagnostics: discovery.diagnostics, summary: buildOperatorSummary({ ok, phases, commands, criticalDiagnostics, timeline, checkedCount: phases.length + commands.length, failedCount: failedPhases.length + failedCommands.length + criticalDiagnostics.length + timeline.failedCount, }), timeline, phases, commands, }; } function buildOperatorSummary(input) { const pingoraEnvShadowEvidence = input.phases .filter((phase) => phase.requiredPingoraEnvShadow) .map((phase) => { const summary = phase.pingoraEnvShadow ?? null; const firstDiagnostic = phase.diagnostics?.[0] ?? null; return { phase: phase.phase, status: phase.status, latestBundleDir: phase.latestBundleDir ?? null, manifestPath: phase.manifestPath ?? null, cutoverRunId: phase.cutoverRunId ?? null, generatedAt: phase.generatedAt ?? null, shadow: { required: true, ok: phase.manifestCheck?.pingoraEnvShadowCheck?.ok === true, reason: phase.manifestCheck?.pingoraEnvShadowCheck?.reason ?? firstDiagnostic, summary, }, }; }); const directLiveEvidence = input.phases .filter( (phase) => phase.requiredDirectLiveAccessLog || phase.requiredDirectLiveStaticHeaders, ) .map((phase) => { const accessLogRequired = phase.requiredDirectLiveAccessLog === true; const staticHeadersRequired = phase.requiredDirectLiveStaticHeaders === true; const accessLogSummary = phase.directLiveAccessLog ?? null; const staticHeadersSummary = phase.directLiveStaticHeaders ?? null; const accessLogChecked = accessLogRequired || accessLogSummary !== null; const staticHeadersChecked = staticHeadersRequired || staticHeadersSummary !== null; const firstDiagnostic = phase.diagnostics?.[0] ?? null; return { phase: phase.phase, status: phase.status, latestBundleDir: phase.latestBundleDir ?? null, manifestPath: phase.manifestPath ?? null, cutoverRunId: phase.cutoverRunId ?? null, generatedAt: phase.generatedAt ?? null, accessLog: { required: accessLogRequired, ok: accessLogChecked ? phase.manifestCheck?.directLiveAccessLogCheck?.ok === true : null, reason: accessLogChecked ? phase.manifestCheck?.directLiveAccessLogCheck?.reason ?? firstDiagnostic : null, summary: accessLogSummary, }, staticHeaders: { required: staticHeadersRequired, ok: staticHeadersChecked ? phase.manifestCheck?.directLiveStaticHeadersCheck?.ok === true : null, reason: staticHeadersChecked ? phase.manifestCheck?.directLiveStaticHeadersCheck?.reason ?? firstDiagnostic : null, summary: staticHeadersSummary, }, }; }); return { status: input.ok ? 'OK' : 'CRITICAL', checkedCount: input.checkedCount, failedCount: input.failedCount, failedItems: [ ...input.criticalDiagnostics.map((diagnostic) => ({ type: 'evidence-root', status: diagnostic.status, path: diagnostic.path ?? null, manifestPath: diagnostic.manifestPath ?? null, reason: diagnostic.reason ?? null, })), ...input.phases .filter((phase) => phase.status !== 'OK') .map((phase) => ({ type: 'phase', phase: phase.phase, status: phase.status, latestBundleDir: phase.latestBundleDir ?? null, manifestPath: phase.manifestPath ?? null, cutoverRunId: phase.cutoverRunId ?? null, generatedAt: phase.generatedAt ?? null, diagnostics: phase.diagnostics ?? [], })), ...input.commands .filter((command) => command.status !== 'OK') .map((command) => ({ type: 'command', phase: command.phase, commandName: command.commandName, status: command.status, latestBundleDir: command.latestBundleDir ?? null, manifestPath: command.manifestPath ?? null, cutoverRunId: command.cutoverRunId ?? null, generatedAt: command.generatedAt ?? null, diagnostics: command.diagnostics ?? [], })), ...(input.timeline.ok ? [] : [ { type: 'timeline', status: 'FAILED', diagnostics: input.timeline.diagnostics ?? [], failureBreakdown: input.timeline.failureBreakdown ?? null, spanMs: input.timeline.spanMs ?? null, maxSpanMs: input.timeline.maxSpanMs ?? null, }, ]), ], pingoraEnvShadowEvidence, directLiveEvidence, timeline: { checked: input.timeline.checked, ok: input.timeline.ok, failedCount: input.timeline.failedCount, spanMs: input.timeline.spanMs, maxSpanMs: input.timeline.maxSpanMs, firstGeneratedAt: input.timeline.firstGeneratedAt, lastGeneratedAt: input.timeline.lastGeneratedAt, }, }; } function filterCandidatesByCutoverRunId(candidates, requiredCutoverRunId) { if (!requiredCutoverRunId) { return candidates; } return candidates.filter( (candidate) => candidate.cutoverRunId === requiredCutoverRunId, ); } function missingEvidenceDiagnostic(label, requiredCutoverRunId, allCandidateCount) { if (!requiredCutoverRunId) { return `没有找到${label}的证据 manifest。`; } if (allCandidateCount > 0) { return `没有找到${label}且 cutoverRunId=${requiredCutoverRunId} 的证据 manifest;已有同名证据属于其它切换批次或缺少 cutoverRunId。`; } return `没有找到${label}的证据 manifest,无法满足 cutoverRunId=${requiredCutoverRunId}。`; } function selectUniqueLatestEvidence(candidates) { const latest = candidates.at(-1); const latestGeneratedAt = latest?.generatedAt || null; const ambiguousCandidates = candidates.filter( (candidate) => candidate.generatedAt === latestGeneratedAt, ); if (ambiguousCandidates.length > 1) { const ambiguousBundleDirs = ambiguousCandidates.map( (candidate) => candidate.bundleDir, ); return { ok: false, latest: null, generatedAt: latestGeneratedAt, ambiguousBundleDirs, diagnostics: [ `最新证据 manifest.generatedAt=${latestGeneratedAt} 出现重复,不能按目录名打平选择;请重新归档或清理证据根目录。`, `重复 generatedAt 的证据目录: ${ambiguousBundleDirs.join(', ')}`, ], }; } return { ok: true, latest, generatedAt: latestGeneratedAt, ambiguousBundleDirs: [], diagnostics: [], }; } function checkPhaseManifest(candidate, options = {}) { const { requiredAccessLog = false, requiredStaticHeaders = false, requiredPingoraEnvShadow = false, } = options; const diagnostics = []; if (candidate.commandName !== null) { diagnostics.push( `最新阶段证据不能带 manifest.commandName,实际为 ${formatNullable(candidate.commandName)}。`, ); } if (candidate.summaryStatus !== 'OK') { diagnostics.push( `最新阶段证据 manifest.summary.status 必须是 OK,实际为 ${formatNullable(candidate.summaryStatus)}。`, ); } const accessLogCheck = checkDirectLiveAccessLogSummary( candidate.summaryDirectLiveAccessLog, ); if (requiredAccessLog && !accessLogCheck.ok) { diagnostics.push( `最新阶段证据 manifest.summary.directLiveAccessLog 必须包含 direct live access log 对账摘要:${accessLogCheck.reason}`, ); } const staticHeadersCheck = checkDirectLiveStaticHeadersSummary( candidate.summaryDirectLiveStaticHeaders, ); if (requiredStaticHeaders && !staticHeadersCheck.ok) { diagnostics.push( `最新阶段证据 manifest.summary.directLiveStaticHeaders 必须包含 direct live 静态响应头摘要:${staticHeadersCheck.reason}`, ); } const pingoraEnvShadowCheck = checkPingoraEnvShadowSummary( candidate.summaryPingoraEnvShadow, ); if (requiredPingoraEnvShadow && !pingoraEnvShadowCheck.ok) { diagnostics.push( `最新阶段证据 manifest.summary.pingoraEnvShadow 必须证明 Pingora env 已恢复 shadow:${pingoraEnvShadowCheck.reason}`, ); } return { ok: diagnostics.length === 0, expectedSummaryStatus: 'OK', actualSummaryStatus: candidate.summaryStatus, requiredDirectLiveAccessLog: requiredAccessLog, requiredDirectLiveStaticHeaders: requiredStaticHeaders, requiredPingoraEnvShadow, directLiveAccessLogCheck: accessLogCheck, directLiveStaticHeadersCheck: staticHeadersCheck, pingoraEnvShadowCheck, diagnostics, }; } function checkPingoraEnvShadowSummary(summary) { if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { return { ok: false, reason: '字段缺失或不是对象。' }; } const failures = []; if (summary.present !== true) { failures.push('present 必须为 true'); } if (summary.ok !== true) { failures.push('ok 必须为 true'); } if (summary.listen !== '127.0.0.1:18081') { failures.push( `listen 必须是 127.0.0.1:18081,实际为 ${formatNullable(summary.listen)}`, ); } if (summary.tlsListen !== '') { failures.push(`tlsListen 必须为空,实际为 ${formatNullable(summary.tlsListen)}`); } if (summary.httpRedirectListen !== '') { failures.push( `httpRedirectListen 必须为空,实际为 ${formatNullable(summary.httpRedirectListen)}`, ); } if (summary.tlsCertFile !== '') { failures.push( `tlsCertFile 必须为空,实际为 ${formatNullable(summary.tlsCertFile)}`, ); } if (summary.tlsKeyFile !== '') { failures.push( `tlsKeyFile 必须为空,实际为 ${formatNullable(summary.tlsKeyFile)}`, ); } if (summary.mode && summary.mode !== 'shadow') { failures.push(`mode 必须是 shadow,实际为 ${formatNullable(summary.mode)}`); } if (summary.shadowReady === false) { failures.push('shadowReady 必须为 true'); } return failures.length === 0 ? { ok: true, reason: null } : { ok: false, reason: failures.join(';') }; } function checkDirectLiveAccessLogSummary(summary) { if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { return { ok: false, reason: '字段缺失或不是对象。' }; } if (summary.present !== true) { return { ok: false, reason: 'present 必须为 true。' }; } const checked = Number(summary.checked); const matchedCount = Number(summary.matchedCount); const missingCount = Number(summary.missingCount); const mismatchCount = Number(summary.mismatchCount); const failures = []; if (!Number.isInteger(checked) || checked <= 0) { failures.push('checked 必须是正整数'); } if (!Number.isInteger(matchedCount) || matchedCount !== checked) { failures.push('matchedCount 必须等于 checked'); } if (!Number.isInteger(missingCount) || missingCount !== 0) { failures.push('missingCount 必须为 0'); } if (!Number.isInteger(mismatchCount) || mismatchCount !== 0) { failures.push('mismatchCount 必须为 0'); } return failures.length === 0 ? { ok: true, reason: null } : { ok: false, reason: failures.join(';') }; } function checkDirectLiveStaticHeadersSummary(summary) { if (!summary || typeof summary !== 'object' || Array.isArray(summary)) { return { ok: false, reason: '字段缺失或不是对象。' }; } if (summary.present !== true) { return { ok: false, reason: 'present 必须为 true。' }; } if (summary.skipped) { return { ok: false, reason: `静态检查被跳过:${formatNullable(summary.skipReason)}`, }; } const targets = [summary.normal, summary.fingerprinted].filter( (target) => target && target.present, ); if (targets.length === 0) { return { ok: false, reason: '缺少 normal 或 fingerprinted 静态资产摘要。' }; } const failures = []; for (const [label, target] of [ ['normal', summary.normal], ['fingerprinted', summary.fingerprinted], ]) { if (!target || !target.present) { continue; } if (target.ok !== true) { failures.push(`${label}.ok 不是 true`); } for (const key of [ 'cacheControl', 'etag', 'lastModified', 'acceptRanges', 'rangeContentRange', ]) { if (typeof target[key] !== 'string' || target[key].length === 0) { failures.push(`${label}.${key} 缺失`); } } if (target.rangeStatusCode !== 206) { failures.push(`${label}.rangeStatusCode 不是 206`); } if (target.etag304StatusCode !== 304) { failures.push(`${label}.etag304StatusCode 不是 304`); } if (target.lastModified304StatusCode !== 304) { failures.push(`${label}.lastModified304StatusCode 不是 304`); } } return failures.length === 0 ? { ok: true, reason: null } : { ok: false, reason: failures.join(';') }; } async function checkCommandManifest( candidate, expectedExecutable = null, requiredArgs = [], ) { const diagnostics = []; if (candidate.summaryStatus !== 'OK') { diagnostics.push( `最新命令证据 manifest.summary.status 必须是 OK,实际为 ${formatNullable(candidate.summaryStatus)}。`, ); } if (candidate.summaryExitCode !== 0) { diagnostics.push( `最新命令证据 manifest.summary.exitCode 必须是 0,实际为 ${formatNullable(candidate.summaryExitCode)}。`, ); } if (candidate.summarySignal !== null) { diagnostics.push( `最新命令证据 manifest.summary.signal 必须为空,实际为 ${formatNullable(candidate.summarySignal)}。`, ); } if (expectedExecutable) { if (candidate.expectedExecutable !== expectedExecutable) { diagnostics.push( `最新命令证据 manifest.expectedExecutable 必须是 ${expectedExecutable},实际为 ${formatNullable(candidate.expectedExecutable)}。`, ); } if (candidate.executable !== expectedExecutable) { diagnostics.push( `最新命令证据 command.executable 必须是 ${expectedExecutable},实际为 ${formatNullable(candidate.executable)}。`, ); } } diagnostics.push(...checkCommandOutputEvidenceFileNames(candidate)); const embeddedCommandCheck = checkCommandRecordObject( candidate.manifestCommand, 'manifest.command', candidate, expectedExecutable, requiredArgs, ); diagnostics.push(...embeddedCommandCheck.diagnostics); const commandRecordCheck = await readCommandRecord(candidate); diagnostics.push(...commandRecordCheck.diagnostics); if (commandRecordCheck.record) { const fileCommandCheck = checkCommandRecordObject( commandRecordCheck.record, 'command-record.json', candidate, expectedExecutable, requiredArgs, ); diagnostics.push(...fileCommandCheck.diagnostics); if (embeddedCommandCheck.record) { diagnostics.push( ...compareCommandRecords( embeddedCommandCheck.record, commandRecordCheck.record, ), ); } } return { ok: diagnostics.length === 0, expectedSummaryStatus: 'OK', actualSummaryStatus: candidate.summaryStatus, expectedExitCode: 0, actualExitCode: candidate.summaryExitCode, expectedSignal: null, actualSignal: candidate.summarySignal, expectedExecutable, actualExpectedExecutable: candidate.expectedExecutable, actualExecutable: candidate.executable, stdoutPath: candidate.stdoutFileName, stderrPath: candidate.stderrFileName, commandRecordStdoutPath: commandRecordCheck.record?.stdoutPath ?? null, commandRecordStderrPath: commandRecordCheck.record?.stderrPath ?? null, commandRecordPath: commandRecordCheck.path, commandRecordReadOk: commandRecordCheck.ok, commandRecordExpectedExecutable: commandRecordCheck.record?.expectedExecutable ?? null, commandRecordExecutable: commandRecordCheck.record?.executable ?? null, commandRecordStartedAt: normalizeIsoTime(commandRecordCheck.record?.startedAt), commandRecordFinishedAt: normalizeIsoTime(commandRecordCheck.record?.finishedAt), commandRecordDurationMs: commandRecordCheck.record?.durationMs ?? null, requiredArgs, diagnostics, }; } function checkCommandOutputEvidenceFileNames(candidate) { const diagnostics = []; for (const [field, fileName] of [ ['stdout', candidate.stdoutFileName], ['stderr', candidate.stderrFileName], ]) { if (!fileName) { diagnostics.push( `最新命令证据 manifest.files.${field}.path 必须存在。`, ); continue; } if (!isSafeEvidenceFileName(fileName)) { diagnostics.push( `最新命令证据 manifest.files.${field}.path 只能是证据目录内的普通文件名,实际为 ${formatNullable(fileName)}。`, ); } } return diagnostics; } async function readCommandRecord(candidate) { const fileName = candidate.commandRecordFileName; if (!fileName) { return { ok: false, path: null, record: null, diagnostics: ['最新命令证据 manifest.files.commandRecord.path 必须存在。'], }; } if (!isSafeEvidenceFileName(fileName)) { return { ok: false, path: null, record: null, diagnostics: [ `最新命令证据 manifest.files.commandRecord.path 只能是证据目录内的普通文件名,实际为 ${formatNullable(fileName)}。`, ], }; } const recordPath = path.join(candidate.bundleDir, fileName); try { const stats = await lstat(recordPath); if (stats.isSymbolicLink()) { return { ok: false, path: recordPath, record: null, diagnostics: ['最新命令证据 command-record.json 不能是符号链接。'], }; } if (!stats.isFile()) { return { ok: false, path: recordPath, record: null, diagnostics: ['最新命令证据 command-record.json 必须是普通文件。'], }; } const content = await readFile(recordPath, 'utf8'); const record = JSON.parse(content); if (!record || typeof record !== 'object' || Array.isArray(record)) { return { ok: false, path: recordPath, record: null, diagnostics: ['最新命令证据 command-record.json 顶层必须是 JSON object。'], }; } return { ok: true, path: recordPath, record, diagnostics: [] }; } catch (error) { return { ok: false, path: recordPath, record: null, diagnostics: [ `读取最新命令证据 command-record.json 失败: ${error.message}`, ], }; } } function checkCommandRecordObject( record, label, candidate, expectedExecutable, requiredArgs = [], ) { const diagnostics = []; if (!record || typeof record !== 'object' || Array.isArray(record)) { return { ok: false, record: null, diagnostics: [`最新命令证据 ${label} 必须是 JSON object。`], }; } const checks = [ ['schemaVersion', 1], ['phase', candidate.phase], ['name', candidate.commandName], ['exitCode', candidate.summaryExitCode], ['signal', candidate.summarySignal], ]; if (candidate.cutoverRunId !== null || record.cutoverRunId !== undefined) { checks.push(['cutoverRunId', candidate.cutoverRunId]); } if ( candidate.expectedExecutable !== null || record.expectedExecutable !== undefined ) { checks.push(['expectedExecutable', candidate.expectedExecutable]); } for (const [field, expected] of checks) { const actual = record[field] ?? null; if (actual !== expected) { diagnostics.push( `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)},实际为 ${formatNullable(actual)}。`, ); } } for (const [field, expected] of [ ['stdoutPath', candidate.stdoutFileName], ['stderrPath', candidate.stderrFileName], ]) { const actual = record[field] ?? null; if (actual !== expected) { diagnostics.push( `最新命令证据 ${label}.${field} 必须是 ${formatNullable(expected)},实际为 ${formatNullable(actual)}。`, ); } } diagnostics.push(...checkCommandRecordInvocation(record, label)); diagnostics.push( ...validateRequiredAbsolutePath(record.executable, `${label}.executable`), ); const effectiveExpectedExecutable = expectedExecutable || candidate.expectedExecutable; if (effectiveExpectedExecutable) { if (record.expectedExecutable !== effectiveExpectedExecutable) { diagnostics.push( `最新命令证据 ${label}.expectedExecutable 必须是 ${effectiveExpectedExecutable},实际为 ${formatNullable(record.expectedExecutable)}。`, ); } if (record.executable !== effectiveExpectedExecutable) { diagnostics.push( `最新命令证据 ${label}.executable 必须是 ${effectiveExpectedExecutable},实际为 ${formatNullable(record.executable)}。`, ); } } diagnostics.push(...checkRequiredCommandArgs(record, label, requiredArgs)); diagnostics.push(...checkCommandRecordTimes(record, label, candidate)); return { ok: diagnostics.length === 0, record, diagnostics, }; } function checkRequiredCommandArgs(record, label, requiredArgs) { const diagnostics = []; if (requiredArgs.length === 0) { return diagnostics; } if (!Array.isArray(record.args)) { diagnostics.push( `最新命令证据 ${label}.args 必须是字符串数组,才能校验必需参数 ${formatNullable(requiredArgs)}。`, ); return diagnostics; } for (const requiredArg of requiredArgs) { if (!record.args.includes(requiredArg)) { diagnostics.push( `最新命令证据 ${label}.args 必须包含 ${formatNullable(requiredArg)}。`, ); } } return diagnostics; } function checkCommandRecordInvocation(record, label) { const diagnostics = []; if (!Array.isArray(record.args)) { diagnostics.push(`最新命令证据 ${label}.args 必须是字符串数组。`); } else if (record.args.some((item) => typeof item !== 'string')) { diagnostics.push(`最新命令证据 ${label}.args 必须只包含字符串。`); } else if (record.args.some((item) => /[\0\r\n]/u.test(item))) { diagnostics.push( `最新命令证据 ${label}.args 不能包含换行或 NUL 字符。`, ); } if (typeof record.command !== 'string' || record.command.length === 0) { diagnostics.push(`最新命令证据 ${label}.command 必须是非空字符串。`); } if (typeof record.cwd !== 'string' || !path.isAbsolute(record.cwd)) { diagnostics.push(`最新命令证据 ${label}.cwd 必须是绝对路径字符串。`); } if (record.error !== null && typeof record.error !== 'string') { diagnostics.push(`最新命令证据 ${label}.error 必须是字符串或 null。`); } return diagnostics; } function checkCommandRecordTimes(record, label, candidate) { const diagnostics = []; const startedAt = normalizeIsoTime(record.startedAt); const finishedAt = normalizeIsoTime(record.finishedAt); if (!startedAt) { diagnostics.push(`最新命令证据 ${label}.startedAt 必须是合法 ISO 时间。`); } if (!finishedAt) { diagnostics.push(`最新命令证据 ${label}.finishedAt 必须是合法 ISO 时间。`); } if (startedAt && finishedAt) { const startedMs = Date.parse(startedAt); const finishedMs = Date.parse(finishedAt); if (finishedMs < startedMs) { diagnostics.push( `最新命令证据 ${label}.finishedAt 不能早于 startedAt。`, ); } if (candidate.generatedAt) { const generatedMs = Date.parse(candidate.generatedAt); if (generatedMs < finishedMs) { diagnostics.push( `最新命令证据 manifest.generatedAt 不能早于 ${label}.finishedAt。`, ); } } const durationMs = record.durationMs; if (!Number.isSafeInteger(durationMs) || durationMs < 0) { diagnostics.push( `最新命令证据 ${label}.durationMs 必须是非负安全整数。`, ); } else { const actualDurationMs = finishedMs - startedMs; if (durationMs !== actualDurationMs) { diagnostics.push( `最新命令证据 ${label}.durationMs 必须等于 finishedAt - startedAt,实际记录 ${durationMs}ms,计算值 ${actualDurationMs}ms。`, ); } } } return diagnostics; } function compareCommandRecords(embeddedRecord, fileRecord) { const diagnostics = []; const scalarFields = [ 'schemaVersion', 'name', 'phase', 'cutoverRunId', 'expectedExecutable', 'executable', 'command', 'cwd', 'error', 'stdoutPath', 'stderrPath', 'exitCode', 'signal', ]; for (const field of scalarFields) { const embedded = embeddedRecord[field] ?? null; const file = fileRecord[field] ?? null; if (embedded !== file) { diagnostics.push( `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致,manifest=${formatNullable(embedded)},command-record=${formatNullable(file)}。`, ); } } for (const field of ['args']) { const embedded = embeddedRecord[field] ?? null; const file = fileRecord[field] ?? null; if (!jsonValuesEqual(embedded, file)) { diagnostics.push( `最新命令证据 manifest.command.${field} 与 command-record.json.${field} 必须一致,manifest=${formatNullable(embedded)},command-record=${formatNullable(file)}。`, ); } } return diagnostics; } function jsonValuesEqual(left, right) { return JSON.stringify(left) === JSON.stringify(right); } function phaseStatus(verify, manifestCheck) { if (!manifestCheck.ok) { return 'MANIFEST_FAILED'; } if (!verify.ok) { return 'VERIFY_FAILED'; } return 'OK'; } function commandStatus(verify, manifestCheck) { if (!manifestCheck.ok) { return 'MANIFEST_FAILED'; } if (!verify.ok) { return 'VERIFY_FAILED'; } return 'OK'; } function formatNullable(value) { return value === null || value === undefined ? '' : JSON.stringify(value); } function isSafeEvidenceFileName(value) { return ( typeof value === 'string' && value.length > 0 && !/[\0\r\n]/u.test(value) && !path.isAbsolute(value) && !value.includes('/') && !value.includes('\\') && value !== '.' && value !== '..' ); } function buildTimelineAudit(phases, commands, timelineMaxSpanMs) { const selected = []; const missing = []; for (const expected of CUTOVER_TIMELINE_ORDER) { const item = expected.type === 'phase' ? phases.find((phase) => phase.phase === expected.phase) : commands.find( (command) => command.phase === expected.phase && command.commandName === expected.commandName, ); if (!item) { missing.push(expected); continue; } selected.push({ ...expected, generatedAt: item.generatedAt, latestBundleDir: item.latestBundleDir, manifestPath: item.manifestPath, cutoverRunId: item.cutoverRunId, status: item.status, }); } if (selected.length === 0 || missing.length > 0) { return { ok: true, checked: false, failedCount: 0, reason: selected.length === 0 ? '未要求标准直连切换时间线证据。' : '仅在标准直连切换时间线的阶段和命令证据都被要求时检查顺序。', maxSpanMs: timelineMaxSpanMs, spanMs: null, firstGeneratedAt: null, lastGeneratedAt: null, expectedOrder: CUTOVER_TIMELINE_ORDER, missing, items: selected, diagnostics: [], }; } const diagnostics = []; const failureBreakdown = { nonOkItems: 0, missingGeneratedAt: 0, cutoverRunIdMismatch: 0, outOfOrder: 0, spanExceeded: 0, }; const nonOkItems = selected.filter((item) => item.status !== 'OK'); failureBreakdown.nonOkItems = nonOkItems.length; if (nonOkItems.length > 0) { diagnostics.push( `标准切换时间线包含非 OK 证据: ${nonOkItems .map((item) => `${formatTimelineItem(item)}=${item.status}`) .join(', ')}。`, ); } const timelineCutoverRunIds = selected.map((item) => ({ item: formatTimelineItem(item), cutoverRunId: item.cutoverRunId, })); const hasDeclaredCutoverRunId = timelineCutoverRunIds.some( (item) => item.cutoverRunId !== null, ); const distinctCutoverRunIds = new Set( timelineCutoverRunIds.map((item) => item.cutoverRunId ?? ''), ); if (hasDeclaredCutoverRunId && distinctCutoverRunIds.size > 1) { failureBreakdown.cutoverRunIdMismatch = 1; diagnostics.push( `标准切换时间线包含不同 cutoverRunId 或缺少 cutoverRunId: ${timelineCutoverRunIds .map((item) => `${item.item}=${formatNullable(item.cutoverRunId)}`) .join(', ')},疑似混入不同切换批次证据。`, ); } const parsedTimes = []; for (const item of selected) { if (!item.generatedAt) { failureBreakdown.missingGeneratedAt += 1; diagnostics.push( `${formatTimelineItem(item)} 缺少合法 manifest.generatedAt,无法证明切换时间线。`, ); parsedTimes.push(null); continue; } parsedTimes.push(Date.parse(item.generatedAt)); } for (let index = 1; index < selected.length; index += 1) { const previous = selected[index - 1]; const current = selected[index]; const previousTime = parsedTimes[index - 1]; const currentTime = parsedTimes[index]; if ( previousTime !== null && currentTime !== null && currentTime < previousTime ) { failureBreakdown.outOfOrder += 1; diagnostics.push( `${formatTimelineItem(current)} 的 manifest.generatedAt 早于 ${formatTimelineItem(previous)},疑似混入不同切换窗口证据。`, ); } } let firstGeneratedAt = null; let lastGeneratedAt = null; let spanMs = null; if (parsedTimes.every((time) => time !== null && !Number.isNaN(time))) { const firstTime = parsedTimes[0]; const lastTime = parsedTimes.at(-1); firstGeneratedAt = selected[0].generatedAt; lastGeneratedAt = selected.at(-1).generatedAt; spanMs = lastTime - firstTime; if (spanMs > timelineMaxSpanMs) { failureBreakdown.spanExceeded = 1; diagnostics.push( `标准切换时间线从 ${firstGeneratedAt} 到 ${lastGeneratedAt} 的时间跨度 ${spanMs}ms 超过 ${timelineMaxSpanMs}ms,疑似混入不同切换窗口证据。`, ); } } const timelineFailedCount = Object.values(failureBreakdown).reduce( (sum, count) => sum + count, 0, ); return { ok: diagnostics.length === 0, checked: true, failedCount: timelineFailedCount, failureBreakdown, maxSpanMs: timelineMaxSpanMs, spanMs, firstGeneratedAt, lastGeneratedAt, cutoverRunIds: timelineCutoverRunIds, expectedOrder: CUTOVER_TIMELINE_ORDER, items: selected, diagnostics, }; } function formatTimelineItem(item) { if (item.type === 'command') { return `${item.phase}:${item.commandName}`; } return item.phase; } function compareEvidenceCandidates(left, right) { const leftTime = Date.parse(left.generatedAt); const rightTime = Date.parse(right.generatedAt); if (leftTime !== rightTime) { return leftTime - rightTime; } return left.directoryName.localeCompare(right.directoryName); } function sortedUnique(values) { return [...new Set(values)].sort((left, right) => left.localeCompare(right)); } function normalizeIsoTime(value) { if ( !value || typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u.test(value) ) { return null; } const time = Date.parse(value); if (Number.isNaN(time)) { return null; } const normalized = new Date(time).toISOString(); return normalized === value ? normalized : null; } async function runVerifier(verifyScript, bundleDir) { const result = await execFileJson(process.execPath, [ '--', verifyScript, '--bundle-dir', bundleDir, '--require-summary-ok', ]); const parsed = parseJsonOutput(result.stdout); return { ok: result.exitCode === 0 && parsed.ok && parsed.value?.ok === true && parsed.value?.requireSummaryOk === true, exitCode: result.exitCode, stdoutJson: parsed.ok, checkedCount: parsed.value?.checkedCount ?? null, failedCount: parsed.value?.failedCount ?? null, requireSummaryOk: parsed.value?.requireSummaryOk ?? null, summary: parsed.value?.summary ?? null, stderr: result.stderr, error: result.error, }; } function execFileJson(command, args) { return new Promise((resolve) => { execFile( command, args, { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024, shell: false, }, (error, stdout, stderr) => { resolve({ exitCode: typeof error?.code === 'number' ? error.code : error ? 1 : 0, stdout: stdout || '', stderr: stderr || '', error: error?.message || null, }); }, ); }); } function parseJsonOutput(stdout) { try { return { ok: true, value: JSON.parse(stdout) }; } catch { return { ok: false, value: null }; } } function isFilesystemRootPath(value) { const resolved = path.resolve(String(value)); return resolved === path.parse(resolved).root; } function validateNoControlCharacters(value, label) { if (/[\0\r\n]/u.test(String(value))) { throw new Error(`${label} 不能包含换行或 NUL 字符。`); } }