Files
Genarrative/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e/entry.mjs
T
AIGameCreator App be89296492 拆分 AI 游戏创作客户端大型模块
拆分 App 认证、壳层、运行配置与项目摘要模块
拆分 Tauri 项目能力与 Rust 测试领域模块
拆分界面测试与 Agent Runtime 真实 E2E 套件
补充源码扫描和客户端模块化文档约定
2026-07-21 22:53:29 +08:00

1255 lines
48 KiB
JavaScript

import {
assert,
hashValue,
prerequisiteLabel,
recordError,
summarizeRecordedError,
throwIfShutdownRequested,
} from './assertions/core.mjs';
import {
absolutePathVariants,
countExactSecrets,
disposableProjectPathVariants,
formalConfigPathVariants,
isNonEmptyString,
} from './assertions/runtime.mjs';
import {
closeOwnedRunnerKillHandle,
closeSourceAppDataDirectoryGuard,
removeIsolatedSuiteAppData,
stopOwnedIsolatedRunner,
} from './harness/app-data.mjs';
import { loadConfig, parseArguments } from './harness/config.mjs';
import { closeInteractiveCli } from './harness/process.mjs';
import { checkPrerequisites } from './harness/project.mjs';
import {
buildSummary,
emptyEvidence,
isIsolatedRunnerSuite,
removeDisposableProject,
} from './harness/reporting.mjs';
import { countSecretsInProject } from './harness/runtime.mjs';
import {
activeCommandChildren,
BlockedError,
commandRootErrorMarker,
contextCompactionConstraintCanary,
processStoppedMarker,
responseStreamThinkingMarkers,
shutdownWaiters,
state,
steerInstruction,
StreamingSecretScanner,
userInputAnswerCanary,
userInputAnswerText,
} from './runtime-state.mjs';
import {
collectPartialContextCompactionEvidence,
emptyContextCompactionEvidence,
isContextCompactionSuite,
runContextCompactionE2e,
} from './suites/context-compaction.mjs';
import { runRealE2e } from './suites/full.mjs';
import {
collectPartialGoalEvidence,
emptyGoalEvidence,
goalPrivateBodyValues,
isGoalRuntimeSuite,
runGoalRuntimeE2e,
} from './suites/goal.mjs';
import {
collectPartialMcpEvidence,
emptyMcpEvidence,
isMcpRuntimeSuite,
mcpPrivateValues,
runMcpRuntimeE2e,
stopMcpHttpFixture,
} from './suites/mcp.mjs';
import {
collectPartialParallelReadEvidence,
emptyParallelReadEvidence,
isParallelReadSuite,
runParallelReadE2e,
} from './suites/parallel-read.mjs';
import {
emptyProcessEvidence,
isProcessSessionSuite,
runProcessSessionE2e,
} from './suites/process-session.mjs';
import {
collectPartialProjectSkillEvidence,
emptyProjectSkillEvidence,
isProjectSkillSuite,
runProjectSkillE2e,
} from './suites/project-skill.mjs';
import {
collectPartialResponseStreamEvidence,
emptyResponseStreamEvidence,
isResponseStreamSuite,
runResponseStreamE2e,
} from './suites/response-stream.mjs';
import {
collectPartialScopedAgentsEvidence,
emptyScopedAgentsEvidence,
isScopedAgentsSuite,
runScopedAgentsE2e,
} from './suites/scoped-agents.mjs';
import { runAgentRuntimeRealE2eSelfTests } from './suites/self-test.mjs';
import {
collectPartialSteerRunnerKillEvidence,
emptySteerRunnerKillEvidence,
isSteerRunnerKillSuite,
runSteerRunnerKillE2e,
} from './suites/steer-runner-kill.mjs';
import {
collectPartialSupervisorAutonomousPlayableEvidence,
emptySupervisorAutonomousPlayableEvidence,
isSupervisorAutonomousPlayableLaneDefenseSuite,
runSupervisorAutonomousPlayableLaneDefenseE2e,
} from './suites/supervisor-autonomous-playable.mjs';
import {
collectPartialSupervisorSwarmEvidence,
emptySupervisorSwarmEvidence,
isSupervisorSwarmInteractiveChatSuite,
isSupervisorSwarmSuite,
isSupervisorSwarmTransientRetrySuite,
recordSupervisorSwarmChatSessionFailureDiagnostic,
runSupervisorSwarmE2e,
supervisorSwarmChatSessionFailureEvidence,
supervisorSwarmToolPlanHandoffProxyNeedsCleanup,
} from './suites/supervisor-swarm.mjs';
import {
collectPartialUserInputEvidence,
emptyUserInputEvidence,
isUserInputRuntimeSuite,
runUserInputRuntimeE2e,
} from './suites/user-input.mjs';
import {
collectPartialWebSearchEvidence,
emptyWebSearchEvidence,
isWebSearchSuite,
runWebSearchE2e,
webSearchPrivateLeakValues,
} from './suites/web-search.mjs';
state.evidence = emptyEvidence();
export function requestShutdown(signal) {
if (state.shutdownSignal) return;
state.shutdownSignal = signal;
state.status = 'FAIL';
recordError(`interrupted-${signal.toLowerCase()}`);
if (state.cleanupInProgress) return;
for (const waiter of shutdownWaiters) waiter();
for (const child of activeCommandChildren) {
if (child.exitCode !== null || child.signalCode !== null) continue;
child.kill('SIGTERM');
const forceTimer = setTimeout(() => {
if (child.exitCode === null && child.signalCode === null) {
child.kill('SIGKILL');
}
}, 1_000);
forceTimer.unref();
}
}
for (const signal of ['SIGINT', 'SIGTERM']) {
process.on(signal, () => requestShutdown(signal));
}
export const selfTestRequested =
process.argv.length === 3 && process.argv[2] === '--self-test';
if (selfTestRequested) {
const selfTestEvidence = await runAgentRuntimeRealE2eSelfTests();
process.stdout.write(`${JSON.stringify(selfTestEvidence, null, 2)}\n`);
} else {
try {
state.options = parseArguments(process.argv.slice(2));
state.suite = state.options.suite;
state.runtimeConfigDir = state.options.configDir;
if (isProcessSessionSuite()) state.evidence = emptyProcessEvidence();
if (isGoalRuntimeSuite()) state.evidence = emptyGoalEvidence();
if (isResponseStreamSuite()) state.evidence = emptyResponseStreamEvidence();
if (isWebSearchSuite()) state.evidence = emptyWebSearchEvidence();
if (isContextCompactionSuite()) {
state.evidence = emptyContextCompactionEvidence();
}
if (isMcpRuntimeSuite()) state.evidence = emptyMcpEvidence();
if (isUserInputRuntimeSuite()) state.evidence = emptyUserInputEvidence();
if (isScopedAgentsSuite()) state.evidence = emptyScopedAgentsEvidence();
if (isProjectSkillSuite()) state.evidence = emptyProjectSkillEvidence();
if (isParallelReadSuite()) state.evidence = emptyParallelReadEvidence();
if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
state.evidence = emptySupervisorAutonomousPlayableEvidence();
}
if (isSupervisorSwarmSuite()) {
state.evidence = emptySupervisorSwarmEvidence();
}
if (isSteerRunnerKillSuite()) {
state.evidence = emptySteerRunnerKillEvidence();
}
const loaded = await loadConfig(state.options.configDir);
if (
isWebSearchSuite() ||
isContextCompactionSuite() ||
isMcpRuntimeSuite() ||
isUserInputRuntimeSuite() ||
isScopedAgentsSuite() ||
isProjectSkillSuite() ||
isParallelReadSuite() ||
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
isSupervisorSwarmSuite() ||
isSteerRunnerKillSuite()
) {
state.formalConfigPathTranscriptScanner = new StreamingSecretScanner(
absolutePathVariants(state.options.configDir, loaded.realConfigDir),
);
}
state.secrets = loaded.secrets;
state.transcriptScanner = new StreamingSecretScanner(state.secrets);
state.config = await checkPrerequisites(loaded.config);
const required = isSupervisorAutonomousPlayableLaneDefenseSuite()
? ['llmConfigured', 'chromeAvailable']
: isProcessSessionSuite() || isIsolatedRunnerSuite()
? ['llmConfigured']
: ['llmConfigured', 'chromeAvailable'];
if (state.suite === 'full') {
required.push('editorApiConfigured');
}
state.blocked = required
.filter((name) => !state.config[name])
.map((name) => prerequisiteLabel(name));
if (state.blocked.length > 0) {
state.status = 'BLOCKED';
} else {
if (isSteerRunnerKillSuite()) {
await runSteerRunnerKillE2e();
} else if (isGoalRuntimeSuite()) {
await runGoalRuntimeE2e();
} else if (isResponseStreamSuite()) {
await runResponseStreamE2e();
} else if (isWebSearchSuite()) {
await runWebSearchE2e();
} else if (isContextCompactionSuite()) {
await runContextCompactionE2e();
} else if (isMcpRuntimeSuite()) {
await runMcpRuntimeE2e();
} else if (isUserInputRuntimeSuite()) {
await runUserInputRuntimeE2e();
} else if (isScopedAgentsSuite()) {
await runScopedAgentsE2e();
} else if (isProjectSkillSuite()) {
await runProjectSkillE2e();
} else if (isParallelReadSuite()) {
await runParallelReadE2e();
} else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
await runSupervisorAutonomousPlayableLaneDefenseE2e();
} else if (isSupervisorSwarmSuite()) {
await runSupervisorSwarmE2e();
} else if (isProcessSessionSuite()) {
await runProcessSessionE2e();
} else {
await runRealE2e();
}
state.status = 'PASS';
}
throwIfShutdownRequested();
} catch (error) {
if (error instanceof BlockedError) {
state.status = 'BLOCKED';
state.blocked = [...new Set([...state.blocked, ...error.components])];
} else {
state.status = 'FAIL';
}
recordError(error?.code ?? 'unexpected-error', error);
} finally {
state.cleanupInProgress = true;
if (isUserInputRuntimeSuite() && state.userInputCliSession) {
try {
await closeInteractiveCli(state.userInputCliSession);
} catch (error) {
state.status = 'FAIL';
recordError('user-input-cli-cleanup-failed', error);
}
state.userInputCliSession = null;
}
if (
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
state.supervisorAutonomousPlayableCliSession
) {
try {
await closeInteractiveCli(state.supervisorAutonomousPlayableCliSession);
} catch (error) {
state.status = 'FAIL';
recordError('supervisor-autonomous-playable-cli-cleanup-failed', error);
}
state.supervisorAutonomousPlayableCliSession = null;
}
if (
isSupervisorSwarmInteractiveChatSuite() &&
state.supervisorSwarmCliSession
) {
try {
if (
state.status !== 'PASS' &&
state.supervisorSwarmCliSession.closed &&
!state.supervisorSwarm.chatSessionFailureDiagnostic
) {
recordSupervisorSwarmChatSessionFailureDiagnostic(
state.supervisorSwarmCliSession,
);
}
await closeInteractiveCli(state.supervisorSwarmCliSession);
} catch (error) {
state.status = 'FAIL';
recordError(
'supervisor-swarm-autonomous-chat-cli-cleanup-failed',
error,
);
}
state.supervisorSwarmCliSession = null;
}
if (isMcpRuntimeSuite() && state.mcp.httpFixture) {
try {
await stopMcpHttpFixture();
state.evidence.httpFixtureStopped = true;
} catch (error) {
state.status = 'FAIL';
recordError('mcp-http-fixture-cleanup-failed', error);
}
}
if (isIsolatedRunnerSuite() && state.isolatedRunner.appDataDir) {
try {
await stopOwnedIsolatedRunner();
state.isolatedRunner.stopped = true;
state.isolatedRunner.cleanupPerformed =
await removeIsolatedSuiteAppData();
if (!state.isolatedRunner.cleanupPerformed) {
state.status = 'FAIL';
recordError('isolated-appdata-cleanup-sentinel-missing');
}
} catch (error) {
state.status = 'FAIL';
const safeCleanupErrorCode =
isNonEmptyString(error?.code) &&
/^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code)
? error.code
: 'isolated-owned-runner-cleanup-failed';
recordError(safeCleanupErrorCode, error);
await closeOwnedRunnerKillHandle(
state.isolatedRunner.current?.killHandle,
).catch(() => {});
}
const killMethod =
state.isolatedRunner.pidfdClaimCount > 0 ? 'linux-pidfd' : null;
if (isSteerRunnerKillSuite()) {
state.evidence.steerRunnerStopped = state.isolatedRunner.stopped;
state.evidence.steerAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.steerRunnerKillMethod = killMethod;
state.evidence.steerRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.steerRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigHardlinkCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigLinksVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('steer-runner-kill-formal-config-cli-call-detected');
}
} else if (isGoalRuntimeSuite()) {
state.evidence.goalRunnerStopped = state.isolatedRunner.stopped;
state.evidence.goalAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.goalRunnerKillMethod = killMethod;
state.evidence.goalRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.goalRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
} else if (isResponseStreamSuite()) {
state.evidence.responseStreamRunnerStopped =
state.isolatedRunner.stopped;
state.evidence.responseStreamAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.responseStreamRunnerKillMethod = killMethod;
state.evidence.responseStreamRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.responseStreamRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigHardlinkCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigLinksVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('response-stream-formal-config-cli-call-detected');
}
} else if (isWebSearchSuite()) {
state.evidence.webSearchRunnerStopped = state.isolatedRunner.stopped;
state.evidence.webSearchAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.webSearchRunnerKillMethod = killMethod;
state.evidence.webSearchRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.webSearchRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('web-search-formal-config-cli-call-detected');
}
} else if (isMcpRuntimeSuite()) {
state.evidence.mcpRunnerStopped = state.isolatedRunner.stopped;
state.evidence.mcpAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.mcpRunnerKillMethod = killMethod;
state.evidence.mcpRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.mcpRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('mcp-formal-config-cli-call-detected');
}
} else if (isUserInputRuntimeSuite()) {
state.evidence.userInputRunnerStopped = state.isolatedRunner.stopped;
state.evidence.userInputAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.userInputRunnerKillMethod = killMethod;
state.evidence.userInputRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.userInputRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigHardlinkCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigLinksVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('user-input-formal-config-cli-call-detected');
}
} else if (isScopedAgentsSuite()) {
state.evidence.scopedAgentsRunnerStopped = state.isolatedRunner.stopped;
state.evidence.scopedAgentsAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.scopedAgentsRunnerKillMethod = killMethod;
state.evidence.scopedAgentsRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.scopedAgentsRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('scoped-agents-formal-config-cli-call-detected');
}
} else if (isProjectSkillSuite()) {
state.evidence.projectSkillRunnerStopped = state.isolatedRunner.stopped;
state.evidence.projectSkillAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.projectSkillRunnerKillMethod = killMethod;
state.evidence.projectSkillRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.projectSkillRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('project-skill-formal-config-cli-call-detected');
}
} else if (isParallelReadSuite()) {
state.evidence.parallelReadRunnerStopped = state.isolatedRunner.stopped;
state.evidence.parallelReadAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.parallelReadRunnerKillMethod = killMethod;
state.evidence.parallelReadRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.parallelReadRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('parallel-read-formal-config-cli-call-detected');
}
} else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
state.evidence.supervisorAutonomousPlayableRunnerStopped =
state.isolatedRunner.stopped;
state.evidence.supervisorAutonomousPlayableAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceAppDataDirectoryUntouched =
state.isolatedRunner.sourceAppDataDirectoryUntouched;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError(
'supervisor-autonomous-playable-formal-config-cli-call-detected',
);
}
} else if (isSupervisorSwarmSuite()) {
state.evidence.supervisorSwarmRunnerStopped =
state.isolatedRunner.stopped;
state.evidence.supervisorSwarmAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.supervisorSwarmRunnerKillMethod = killMethod;
state.evidence.supervisorSwarmRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.supervisorSwarmRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceAppDataDirectoryUntouched =
state.isolatedRunner.sourceAppDataDirectoryUntouched;
state.evidence.sourceConfigReplicaCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigReplicasVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('supervisor-swarm-formal-config-cli-call-detected');
}
} else {
assert(
isContextCompactionSuite(),
'unknown-isolated-suite-cleanup-profile',
);
state.evidence.contextCompactionRunnerStopped =
state.isolatedRunner.stopped;
state.evidence.contextCompactionAppDataCleanupPerformed =
state.isolatedRunner.cleanupPerformed;
state.evidence.contextCompactionRunnerKillMethod = killMethod;
state.evidence.contextCompactionRunnerPidfdClaimCount =
state.isolatedRunner.pidfdClaimCount;
state.evidence.contextCompactionRunnerPidfdSignalCount =
state.isolatedRunner.pidfdSignalCount;
state.evidence.formalConfigCliCallCount =
state.isolatedRunner.sourceConfigCliCallCount;
state.evidence.sourceRunnerEndpointUnchanged =
state.isolatedRunner.sourceRunnerEndpointUnchanged;
state.evidence.sourceConfigHardlinkCount =
state.isolatedRunner.configLinks.length;
state.evidence.sourceConfigLinksVerified =
state.isolatedRunner.sourceConfigLinksVerified;
state.evidence.isolatedAppDataUsed = true;
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
state.status = 'FAIL';
recordError('context-compaction-formal-config-cli-call-detected');
}
}
}
if (state.isolatedRunner.sourceAppDataDirectoryWatcher) {
closeSourceAppDataDirectoryGuard();
state.status = 'FAIL';
recordError('source-appdata-directory-guard-not-verified');
}
if (
isSupervisorSwarmTransientRetrySuite() &&
state.supervisorSwarm.transientFaultProxy
) {
try {
await state.supervisorSwarm.transientFaultProxy.stop();
const proxyStats = state.supervisorSwarm.transientFaultProxy.getStats();
state.evidence.transientFaultRequestCount = proxyStats.requestCount;
state.evidence.transientFaultInjectedCount =
proxyStats.faultInjectedCount;
state.evidence.transientFaultHeldRequestCount =
proxyStats.heldRequestCount;
state.evidence.transientFaultForwardedRequestCount =
proxyStats.forwardedRequestCount;
state.evidence.transientFaultProxyStopped = proxyStats.stopped === true;
if (!state.evidence.transientFaultProxyStopped) {
state.status = 'FAIL';
recordError('supervisor-swarm-transient-retry-proxy-not-stopped');
}
} catch (error) {
state.status = 'FAIL';
recordError(
'supervisor-swarm-transient-retry-proxy-cleanup-failed',
error,
);
}
}
if (supervisorSwarmToolPlanHandoffProxyNeedsCleanup()) {
try {
await state.supervisorSwarm.toolPlanHandoffProxy.stop();
const proxyStats =
state.supervisorSwarm.toolPlanHandoffProxy.getStats();
state.evidence.toolPlanHandoffProxyRequestCount =
proxyStats.requestCount;
state.evidence.toolPlanHandoffProxyForwardedRequestCount =
proxyStats.forwardedRequestCount;
state.evidence.toolPlanHandoffProxyStopped =
proxyStats.stopped === true;
if (!state.evidence.toolPlanHandoffProxyStopped) {
state.status = 'FAIL';
recordError('supervisor-swarm-tool-plan-handoff-proxy-not-stopped');
}
} catch (error) {
state.status = 'FAIL';
recordError(
'supervisor-swarm-tool-plan-handoff-proxy-cleanup-failed',
error,
);
}
}
if (
isSteerRunnerKillSuite() &&
state.projectRoot &&
state.status !== 'PASS'
) {
try {
state.evidence = {
...state.evidence,
...(await collectPartialSteerRunnerKillEvidence()),
};
} catch (error) {
recordError('steer-runner-kill-partial-evidence-read-failed', error);
}
}
if (isGoalRuntimeSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialGoalEvidence()),
};
} catch (error) {
recordError('goal-partial-evidence-read-failed', error);
}
}
if (
isResponseStreamSuite() &&
state.projectRoot &&
state.status !== 'PASS'
) {
try {
state.evidence = {
...state.evidence,
...(await collectPartialResponseStreamEvidence()),
};
} catch (error) {
recordError('response-stream-partial-evidence-read-failed', error);
}
}
if (isWebSearchSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialWebSearchEvidence()),
};
} catch (error) {
recordError('web-search-partial-evidence-read-failed', error);
}
}
if (
isContextCompactionSuite() &&
state.projectRoot &&
state.status !== 'PASS'
) {
try {
state.evidence = {
...state.evidence,
...(await collectPartialContextCompactionEvidence()),
};
} catch (error) {
recordError('context-compaction-partial-evidence-read-failed', error);
}
}
if (isMcpRuntimeSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialMcpEvidence()),
};
} catch (error) {
recordError('mcp-partial-evidence-read-failed', error);
}
}
if (
isUserInputRuntimeSuite() &&
state.projectRoot &&
state.status !== 'PASS'
) {
try {
state.evidence = {
...state.evidence,
...(await collectPartialUserInputEvidence()),
};
} catch (error) {
recordError('user-input-partial-evidence-read-failed', error);
}
}
if (isScopedAgentsSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialScopedAgentsEvidence()),
};
} catch (error) {
recordError('scoped-agents-partial-evidence-read-failed', error);
}
}
if (isProjectSkillSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialProjectSkillEvidence()),
};
} catch (error) {
recordError('project-skill-partial-evidence-read-failed', error);
}
}
if (isParallelReadSuite() && state.projectRoot && state.status !== 'PASS') {
try {
state.evidence = {
...state.evidence,
...(await collectPartialParallelReadEvidence()),
};
} catch (error) {
recordError('parallel-read-partial-evidence-read-failed', error);
}
}
if (
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
state.projectRoot &&
state.status !== 'PASS' &&
state.evidence.evidenceCompleteness !== 'complete'
) {
try {
state.evidence = {
...state.evidence,
...(await collectPartialSupervisorAutonomousPlayableEvidence()),
};
} catch (error) {
recordError(
'supervisor-autonomous-playable-partial-evidence-read-failed',
error,
);
}
}
if (
isSupervisorSwarmSuite() &&
state.projectRoot &&
state.status !== 'PASS'
) {
try {
state.evidence = await collectPartialSupervisorSwarmEvidence(
state.evidence,
);
} catch (error) {
recordError('supervisor-swarm-partial-evidence-read-failed', error);
}
}
if (state.projectRoot && state.secrets.length > 0) {
try {
state.projectLeakCount = await countSecretsInProject(
state.projectRoot,
state.secrets,
);
} catch (error) {
state.status = 'FAIL';
recordError('project-secret-scan-failed', error);
}
}
state.transcriptLeakCount = state.transcriptScanner?.count ?? 0;
state.projectPathTranscriptLeakCount =
state.projectPathTranscriptScanner?.count ?? 0;
state.formalConfigPathTranscriptLeakCount =
state.formalConfigPathTranscriptScanner?.count ?? 0;
if (state.transcriptLeakCount + state.projectLeakCount > 0) {
state.status = 'FAIL';
recordError('loaded-key-leak-detected');
}
if (state.projectPathTranscriptLeakCount > 0) {
state.status = 'FAIL';
recordError('disposable-project-path-transcript-leak-detected');
}
if (state.formalConfigPathTranscriptLeakCount > 0) {
state.status = 'FAIL';
recordError('formal-config-path-transcript-leak-detected');
}
const isolatedRunnerAllowsProjectCleanup =
!isIsolatedRunnerSuite() ||
!state.isolatedRunner.appDataDir ||
state.isolatedRunner.stopped;
if (
state.projectRoot &&
!state.options?.keepProject &&
isolatedRunnerAllowsProjectCleanup
) {
try {
state.cleanupPerformed = await removeDisposableProject();
if (!state.cleanupPerformed) {
state.status = 'FAIL';
recordError('cleanup-sentinel-missing');
}
} catch (error) {
state.status = 'FAIL';
recordError('cleanup-failed', error);
}
}
let summary = buildSummary();
let report = JSON.stringify(summary, null, 2);
if (isProcessSessionSuite() && state.process.challenge) {
state.process.reportLeakCount = countExactSecrets(
Buffer.from(report),
[
state.process.challenge,
state.process.readyLine,
state.process.echoLine,
processStoppedMarker,
].filter(Boolean),
);
state.evidence.processReportLeakCount = state.process.reportLeakCount;
if (state.process.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('process-private-output-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
state.commandMarkerReportLeakCount = countExactSecrets(
Buffer.from(report),
[commandRootErrorMarker],
);
if (state.commandMarkerReportLeakCount > 0) {
state.status = 'FAIL';
recordError('command-output-marker-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
state.steerInstructionReportLeakCount = countExactSecrets(
Buffer.from(report),
[steerInstruction],
);
state.evidence.steerInstructionReportLeakCount =
state.steerInstructionReportLeakCount;
if (state.steerInstructionReportLeakCount > 0) {
state.status = 'FAIL';
recordError('steer-instruction-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
if (isGoalRuntimeSuite()) {
state.goalBodyReportLeakCount = countExactSecrets(
Buffer.from(report),
goalPrivateBodyValues(),
);
state.evidence.goalBodyReportLeakCount = state.goalBodyReportLeakCount;
if (state.goalBodyReportLeakCount > 0) {
state.status = 'FAIL';
recordError('goal-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isResponseStreamSuite()) {
state.responseStream.reportLeakCount = countExactSecrets(
Buffer.from(report),
[
state.responseStream.finalText,
...responseStreamThinkingMarkers,
].filter(isNonEmptyString),
);
state.evidence.responseStreamReportLeakCount =
state.responseStream.reportLeakCount;
if (state.responseStream.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('response-stream-private-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isWebSearchSuite()) {
state.webSearch.reportLeakCount = countExactSecrets(
Buffer.from(report),
webSearchPrivateLeakValues(),
);
state.evidence.webSearchReportLeakCount = state.webSearch.reportLeakCount;
if (state.webSearch.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('web-search-private-context-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isContextCompactionSuite()) {
state.contextCompaction.reportLeakCount = countExactSecrets(
Buffer.from(report),
[
contextCompactionConstraintCanary,
...state.contextCompaction.privateSummaries,
].filter(isNonEmptyString),
);
state.evidence.contextCompactionReportLeakCount =
state.contextCompaction.reportLeakCount;
if (state.contextCompaction.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('context-compaction-private-context-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isMcpRuntimeSuite()) {
state.mcp.reportLeakCount = countExactSecrets(
Buffer.from(report),
mcpPrivateValues(),
);
state.evidence.mcpReportLeakCount = state.mcp.reportLeakCount;
if (state.mcp.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('mcp-private-context-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isUserInputRuntimeSuite()) {
state.userInput.reportLeakCount = countExactSecrets(
Buffer.from(report),
[
userInputAnswerCanary,
userInputAnswerText,
...state.userInput.privateValues,
].filter(isNonEmptyString),
);
state.evidence.userInputReportLeakCount = state.userInput.reportLeakCount;
if (state.userInput.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('user-input-private-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isScopedAgentsSuite()) {
state.scopedAgents.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.scopedAgents.privateValues,
);
state.evidence.scopedAgentsReportLeakCount =
state.scopedAgents.reportLeakCount;
if (state.scopedAgents.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('scoped-agents-private-instruction-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isProjectSkillSuite()) {
state.projectSkill.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.projectSkill.privateValues,
);
state.evidence.projectSkillReportLeakCount =
state.projectSkill.reportLeakCount;
if (state.projectSkill.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('project-skill-private-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isParallelReadSuite()) {
state.parallelRead.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.parallelRead.privateValues,
);
state.evidence.parallelReadReportLeakCount =
state.parallelRead.reportLeakCount;
if (state.parallelRead.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('parallel-read-private-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
state.supervisorAutonomousPlayable.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.supervisorAutonomousPlayable.privateValues,
);
state.evidence.supervisorAutonomousPlayableReportLeakCount =
state.supervisorAutonomousPlayable.reportLeakCount;
if (state.supervisorAutonomousPlayable.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('supervisor-autonomous-playable-report-body-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
if (isSupervisorSwarmSuite()) {
state.supervisorSwarm.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.supervisorSwarm.privateValues,
);
state.evidence.supervisorSwarmReportLeakCount =
state.supervisorSwarm.reportLeakCount;
if (state.supervisorSwarm.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('supervisor-swarm-private-body-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
state.projectPathReportLeakCount = countExactSecrets(
Buffer.from(report),
disposableProjectPathVariants(),
);
state.evidence.projectPathReportLeakCount =
state.projectPathReportLeakCount;
if (state.projectPathReportLeakCount > 0) {
state.status = 'FAIL';
recordError('disposable-project-path-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
if (
isWebSearchSuite() ||
isContextCompactionSuite() ||
isMcpRuntimeSuite() ||
isUserInputRuntimeSuite() ||
isScopedAgentsSuite() ||
isProjectSkillSuite() ||
isParallelReadSuite() ||
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
isSupervisorSwarmSuite() ||
isSteerRunnerKillSuite()
) {
state.formalConfigPathReportLeakCount = countExactSecrets(
Buffer.from(report),
formalConfigPathVariants(),
);
state.evidence.formalConfigPathReportLeakCount =
state.formalConfigPathReportLeakCount;
if (state.formalConfigPathReportLeakCount > 0) {
state.status = 'FAIL';
recordError('formal-config-path-report-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
}
state.reportLeakCount = countExactSecrets(
Buffer.from(report),
state.secrets,
);
if (state.reportLeakCount > 0) {
state.status = 'FAIL';
recordError('report-key-leak-detected');
summary = buildSummary();
report = JSON.stringify(summary, null, 2);
}
const remainingProjectPathReportLeakCount = countExactSecrets(
Buffer.from(report),
disposableProjectPathVariants(),
);
const remainingResponseStreamReportLeakCount = isResponseStreamSuite()
? countExactSecrets(
Buffer.from(report),
[
state.responseStream.finalText,
...responseStreamThinkingMarkers,
].filter(isNonEmptyString),
)
: 0;
const remainingWebSearchReportLeakCount = isWebSearchSuite()
? countExactSecrets(Buffer.from(report), webSearchPrivateLeakValues())
: 0;
const remainingMcpReportLeakCount = isMcpRuntimeSuite()
? countExactSecrets(Buffer.from(report), mcpPrivateValues())
: 0;
const remainingUserInputReportLeakCount = isUserInputRuntimeSuite()
? countExactSecrets(
Buffer.from(report),
[
userInputAnswerCanary,
userInputAnswerText,
...state.userInput.privateValues,
].filter(isNonEmptyString),
)
: 0;
const remainingScopedAgentsReportLeakCount = isScopedAgentsSuite()
? countExactSecrets(Buffer.from(report), state.scopedAgents.privateValues)
: 0;
const remainingProjectSkillReportLeakCount = isProjectSkillSuite()
? countExactSecrets(Buffer.from(report), state.projectSkill.privateValues)
: 0;
const remainingParallelReadReportLeakCount = isParallelReadSuite()
? countExactSecrets(Buffer.from(report), state.parallelRead.privateValues)
: 0;
const remainingSupervisorAutonomousPlayableReportLeakCount =
isSupervisorAutonomousPlayableLaneDefenseSuite()
? countExactSecrets(
Buffer.from(report),
state.supervisorAutonomousPlayable.privateValues,
)
: 0;
const remainingSupervisorSwarmReportLeakCount = isSupervisorSwarmSuite()
? countExactSecrets(
Buffer.from(report),
state.supervisorSwarm.privateValues,
)
: 0;
const remainingFormalConfigPathReportLeakCount =
isWebSearchSuite() ||
isContextCompactionSuite() ||
isMcpRuntimeSuite() ||
isUserInputRuntimeSuite() ||
isScopedAgentsSuite() ||
isProjectSkillSuite() ||
isParallelReadSuite() ||
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
isSupervisorSwarmSuite() ||
isSteerRunnerKillSuite()
? countExactSecrets(Buffer.from(report), formalConfigPathVariants())
: 0;
if (
remainingProjectPathReportLeakCount > 0 ||
remainingResponseStreamReportLeakCount > 0 ||
remainingWebSearchReportLeakCount > 0 ||
remainingMcpReportLeakCount > 0 ||
remainingUserInputReportLeakCount > 0 ||
remainingScopedAgentsReportLeakCount > 0 ||
remainingProjectSkillReportLeakCount > 0 ||
remainingParallelReadReportLeakCount > 0 ||
remainingSupervisorAutonomousPlayableReportLeakCount > 0 ||
remainingSupervisorSwarmReportLeakCount > 0 ||
remainingFormalConfigPathReportLeakCount > 0
) {
state.status = 'FAIL';
recordError(
remainingProjectPathReportLeakCount > 0
? 'disposable-project-path-report-redaction-required'
: remainingResponseStreamReportLeakCount > 0
? 'response-stream-report-redaction-required'
: remainingMcpReportLeakCount > 0
? 'mcp-report-redaction-required'
: remainingUserInputReportLeakCount > 0
? 'user-input-report-redaction-required'
: remainingScopedAgentsReportLeakCount > 0
? 'scoped-agents-report-redaction-required'
: remainingProjectSkillReportLeakCount > 0
? 'project-skill-report-redaction-required'
: remainingParallelReadReportLeakCount > 0
? 'parallel-read-report-redaction-required'
: remainingSupervisorAutonomousPlayableReportLeakCount > 0
? 'supervisor-autonomous-playable-report-redaction-required'
: remainingSupervisorSwarmReportLeakCount > 0
? 'supervisor-swarm-report-redaction-required'
: remainingFormalConfigPathReportLeakCount > 0
? 'formal-config-path-report-redaction-required'
: 'web-search-report-redaction-required',
);
const safeSummary = {
status: state.status,
suite: state.suite,
blocked: state.blocked,
cleanup: {
performed: state.cleanupPerformed,
kept: Boolean(state.options?.keepProject),
},
evidence: {
projectPathReportLeakCount: remainingProjectPathReportLeakCount,
responseStreamReportLeakCount: remainingResponseStreamReportLeakCount,
webSearchReportLeakCount: remainingWebSearchReportLeakCount,
mcpReportLeakCount: remainingMcpReportLeakCount,
userInputReportLeakCount: remainingUserInputReportLeakCount,
scopedAgentsReportLeakCount: remainingScopedAgentsReportLeakCount,
projectSkillReportLeakCount: remainingProjectSkillReportLeakCount,
parallelReadReportLeakCount: remainingParallelReadReportLeakCount,
supervisorAutonomousPlayableReportLeakCount:
remainingSupervisorAutonomousPlayableReportLeakCount,
supervisorSwarmReportLeakCount:
remainingSupervisorSwarmReportLeakCount,
formalConfigPathReportLeakCount:
remainingFormalConfigPathReportLeakCount,
...(isSupervisorSwarmSuite()
? supervisorSwarmChatSessionFailureEvidence()
: {}),
},
errorCount: state.errors.length,
errorHashes: state.errors.map(summarizeRecordedError),
};
safeSummary.summaryHash = hashValue(JSON.stringify(safeSummary));
report = JSON.stringify(safeSummary, null, 2);
}
process.stdout.write(`${report}\n`);
process.exitCode = state.shutdownSignal
? state.shutdownSignal === 'SIGINT'
? 130
: 143
: state.status === 'PASS'
? 0
: state.status === 'BLOCKED'
? 2
: 1;
}
}