import { assert, hashValue, prerequisiteLabel, recordError, summarizeRecordedError, throwIfShutdownRequested, } from './assertions/core.mjs'; import { absolutePathVariants, countExactSecrets, disposableProjectPathVariants, formalConfigPathVariants, isNonEmptyString, } from './assertions/runtime.mjs'; import { closeOwnedRunnerKillHandle, closeSourceAppDataDirectoryGuard, removeIsolatedSuiteAppData, stopOwnedIsolatedRunner, } from './harness/app-data.mjs'; import { loadConfig, parseArguments } from './harness/config.mjs'; import { activeInteractiveCliSessions, captureOwnedProcessCleanupSnapshot, closeInteractiveCli, destroyInteractiveCliOutputStreams, interactiveCliOutput, verifyOwnedProcessCleanupSnapshot, waitForInteractiveCliStdioClose, } from './harness/process.mjs'; import { checkPrerequisites } from './harness/project.mjs'; import { buildSummary, emptyEvidence, isIsolatedRunnerSuite, removeDisposableProject, } from './harness/reporting.mjs'; import { countSecretsInProject } from './harness/runtime.mjs'; import { activeCommandChildren, BlockedError, commandRootErrorMarker, contextCompactionConstraintCanary, processStoppedMarker, responseStreamThinkingMarkers, shutdownWaiters, state, steerInstruction, StreamingSecretScanner, userInputAnswerCanary, userInputAnswerText, } from './runtime-state.mjs'; import { collectPartialContextCompactionEvidence, emptyContextCompactionEvidence, isContextCompactionSuite, runContextCompactionE2e, } from './suites/context-compaction.mjs'; import { runRealE2e } from './suites/full.mjs'; import { collectPartialGoalEvidence, emptyGoalEvidence, goalPrivateBodyValues, isGoalRuntimeSuite, runGoalRuntimeE2e, } from './suites/goal.mjs'; import { collectPartialParallelReadEvidence, emptyParallelReadEvidence, isParallelReadSuite, runParallelReadE2e, } from './suites/parallel-read.mjs'; import { emptyProcessEvidence, isProcessSessionSuite, runProcessSessionE2e, } from './suites/process-session.mjs'; import { collectPartialProjectSkillEvidence, emptyProjectSkillEvidence, isProjectSkillSuite, runProjectSkillE2e, } from './suites/project-skill.mjs'; import { collectPartialResponseStreamEvidence, emptyResponseStreamEvidence, isResponseStreamSuite, runResponseStreamE2e, } from './suites/response-stream.mjs'; import { collectPartialScopedAgentsEvidence, emptyScopedAgentsEvidence, isScopedAgentsSuite, runScopedAgentsE2e, } from './suites/scoped-agents.mjs'; import { runAgentRuntimeRealE2eSelfTests } from './suites/self-test.mjs'; import { collectPartialSteerRunnerKillEvidence, emptySteerRunnerKillEvidence, isSteerRunnerKillSuite, runSteerRunnerKillE2e, } from './suites/steer-runner-kill.mjs'; import { collectPartialSupervisorAutonomousPlayableEvidence, emptySupervisorAutonomousPlayableEvidence, isSupervisorAutonomousPlayableLaneDefenseSuite, runSupervisorAutonomousPlayableLaneDefenseE2e, } from './suites/supervisor-autonomous-playable.mjs'; import { collectPartialSupervisorSwarmEvidence, emptySupervisorSwarmEvidence, isSupervisorSwarmInteractiveChatSuite, isSupervisorSwarmSuite, isSupervisorSwarmTransientRetrySuite, recordSupervisorSwarmChatSessionFailureDiagnostic, runSupervisorSwarmE2e, supervisorSwarmChatSessionFailureEvidence, supervisorSwarmToolPlanHandoffProxyNeedsCleanup, } from './suites/supervisor-swarm.mjs'; import { collectPartialUserInputEvidence, emptyUserInputEvidence, isUserInputRuntimeSuite, runUserInputRuntimeE2e, } from './suites/user-input.mjs'; import { collectPartialWebSearchEvidence, emptyWebSearchEvidence, isWebSearchSuite, runWebSearchE2e, webSearchPrivateLeakValues, } from './suites/web-search.mjs'; state.evidence = emptyEvidence(); export function requestShutdown(signal) { if (state.shutdownSignal) return; state.shutdownSignal = signal; state.status = 'FAIL'; recordError(`interrupted-${signal.toLowerCase()}`); if (state.cleanupInProgress) return; for (const waiter of shutdownWaiters) waiter(); for (const child of activeCommandChildren) { if (child.exitCode !== null || child.signalCode !== null) continue; child.kill('SIGTERM'); const forceTimer = setTimeout(() => { if (child.exitCode === null && child.signalCode === null) { child.kill('SIGKILL'); } }, 1_000); forceTimer.unref(); } } for (const signal of ['SIGINT', 'SIGTERM']) { process.on(signal, () => requestShutdown(signal)); } export const selfTestRequested = process.argv.length === 3 && process.argv[2] === '--self-test'; if (selfTestRequested) { const selfTestEvidence = await runAgentRuntimeRealE2eSelfTests(); process.stdout.write(`${JSON.stringify(selfTestEvidence, null, 2)}\n`); } else { try { state.options = parseArguments(process.argv.slice(2)); state.suite = state.options.suite; state.runtimeConfigDir = state.options.configDir; if (isProcessSessionSuite()) state.evidence = emptyProcessEvidence(); if (isGoalRuntimeSuite()) state.evidence = emptyGoalEvidence(); if (isResponseStreamSuite()) state.evidence = emptyResponseStreamEvidence(); if (isWebSearchSuite()) state.evidence = emptyWebSearchEvidence(); if (isContextCompactionSuite()) { state.evidence = emptyContextCompactionEvidence(); } if (isUserInputRuntimeSuite()) state.evidence = emptyUserInputEvidence(); if (isScopedAgentsSuite()) state.evidence = emptyScopedAgentsEvidence(); if (isProjectSkillSuite()) state.evidence = emptyProjectSkillEvidence(); if (isParallelReadSuite()) state.evidence = emptyParallelReadEvidence(); if (isSupervisorAutonomousPlayableLaneDefenseSuite()) { state.evidence = emptySupervisorAutonomousPlayableEvidence(); } if (isSupervisorSwarmSuite()) { state.evidence = emptySupervisorSwarmEvidence(); } if (isSteerRunnerKillSuite()) { state.evidence = emptySteerRunnerKillEvidence(); } const loaded = await loadConfig(state.options.configDir); if ( isWebSearchSuite() || isContextCompactionSuite() || isUserInputRuntimeSuite() || isScopedAgentsSuite() || isProjectSkillSuite() || isParallelReadSuite() || isSupervisorAutonomousPlayableLaneDefenseSuite() || isSupervisorSwarmSuite() || isSteerRunnerKillSuite() ) { state.formalConfigPathTranscriptScanner = new StreamingSecretScanner( absolutePathVariants(state.options.configDir, loaded.realConfigDir), ); } state.secrets = loaded.secrets; state.transcriptScanner = new StreamingSecretScanner(state.secrets); state.config = await checkPrerequisites(loaded.config); const required = isSupervisorAutonomousPlayableLaneDefenseSuite() ? ['llmConfigured', 'chromeAvailable'] : isProcessSessionSuite() || isIsolatedRunnerSuite() ? ['llmConfigured'] : ['llmConfigured', 'chromeAvailable']; if (state.suite === 'full') { required.push('editorApiConfigured'); } state.blocked = required .filter((name) => !state.config[name]) .map((name) => prerequisiteLabel(name)); if (state.blocked.length > 0) { state.status = 'BLOCKED'; } else { if (isSteerRunnerKillSuite()) { await runSteerRunnerKillE2e(); } else if (isGoalRuntimeSuite()) { await runGoalRuntimeE2e(); } else if (isResponseStreamSuite()) { await runResponseStreamE2e(); } else if (isWebSearchSuite()) { await runWebSearchE2e(); } else if (isContextCompactionSuite()) { await runContextCompactionE2e(); } else if (isUserInputRuntimeSuite()) { await runUserInputRuntimeE2e(); } else if (isScopedAgentsSuite()) { await runScopedAgentsE2e(); } else if (isProjectSkillSuite()) { await runProjectSkillE2e(); } else if (isParallelReadSuite()) { await runParallelReadE2e(); } else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) { await runSupervisorAutonomousPlayableLaneDefenseE2e(); } else if (isSupervisorSwarmSuite()) { await runSupervisorSwarmE2e(); } else if (isProcessSessionSuite()) { await runProcessSessionE2e(); } else { await runRealE2e(); } state.status = 'PASS'; } throwIfShutdownRequested(); } catch (error) { if (error instanceof BlockedError) { state.status = 'BLOCKED'; state.blocked = [...new Set([...state.blocked, ...error.components])]; } else { state.status = 'FAIL'; } recordError(error?.code ?? 'unexpected-error', error); } finally { state.cleanupInProgress = true; const stateTrackedInteractiveCliSessions = new Set( [ state.userInputCliSession, state.supervisorAutonomousPlayableCliSession, state.supervisorSwarmCliSession, ].filter(Boolean), ); const interactiveCliSessions = [ ...new Set([ ...stateTrackedInteractiveCliSessions, ...activeInteractiveCliSessions, ]), ]; const supervisorAutonomousPlayableCliSession = state.supervisorAutonomousPlayableCliSession; if (isUserInputRuntimeSuite() && state.userInputCliSession) { try { await closeInteractiveCli(state.userInputCliSession); } catch (error) { state.status = 'FAIL'; recordError('user-input-cli-cleanup-failed', error); } state.userInputCliSession = null; } if ( isSupervisorAutonomousPlayableLaneDefenseSuite() && state.supervisorAutonomousPlayableCliSession ) { try { await closeInteractiveCli(state.supervisorAutonomousPlayableCliSession); } catch (error) { state.status = 'FAIL'; recordError('supervisor-autonomous-playable-cli-cleanup-failed', error); } state.supervisorAutonomousPlayableCliSession = null; } if ( isSupervisorSwarmInteractiveChatSuite() && state.supervisorSwarmCliSession ) { try { if ( state.status !== 'PASS' && state.supervisorSwarmCliSession.closed && !state.supervisorSwarm.chatSessionFailureDiagnostic ) { recordSupervisorSwarmChatSessionFailureDiagnostic( state.supervisorSwarmCliSession, ); } await closeInteractiveCli(state.supervisorSwarmCliSession); } catch (error) { state.status = 'FAIL'; recordError( 'supervisor-swarm-autonomous-chat-cli-cleanup-failed', error, ); } state.supervisorSwarmCliSession = null; } for (const session of interactiveCliSessions) { if (stateTrackedInteractiveCliSessions.has(session)) continue; try { await closeInteractiveCli(session); } catch (error) { state.status = 'FAIL'; recordError('interactive-cli-cleanup-failed', error); } } if ( isSupervisorAutonomousPlayableLaneDefenseSuite() && state.isolatedRunner.appDataDir ) { try { const runnerPid = state.isolatedRunner.current?.pid ?? null; const helperPid = state.isolatedRunner.current?.killHandle?.child?.pid ?? null; state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot = await captureOwnedProcessCleanupSnapshot({ runnerPid, helperPids: Number.isSafeInteger(helperPid) ? [helperPid] : [], rootPids: [ ...interactiveCliSessions.map((session) => session.child?.pid), ...[...activeCommandChildren].map((child) => child.pid), ].filter((pid) => Number.isSafeInteger(pid) && pid > 0), }); const observed = state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot .observedCounts; assert( observed.runner === 1 && observed.helper === 1, 'supervisor-autonomous-playable-owned-process-snapshot-incomplete', ); } catch (error) { state.status = 'FAIL'; recordError( 'supervisor-autonomous-playable-owned-process-snapshot-failed', error, ); } } if (isIsolatedRunnerSuite() && state.isolatedRunner.appDataDir) { try { await stopOwnedIsolatedRunner(); state.isolatedRunner.stopped = true; } catch (error) { state.status = 'FAIL'; const safeCleanupErrorCode = isNonEmptyString(error?.code) && /^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code) ? error.code : 'isolated-owned-runner-cleanup-failed'; recordError(safeCleanupErrorCode, error); await closeOwnedRunnerKillHandle( state.isolatedRunner.current?.killHandle, ).catch(() => {}); } } for (const session of interactiveCliSessions) { try { await waitForInteractiveCliStdioClose(session, 10_000); } catch (error) { destroyInteractiveCliOutputStreams(session); state.status = 'FAIL'; recordError( error?.code === 'interactive-cli-stdio-close-timeout' ? error.code : 'interactive-cli-stdio-cleanup-failed', error, ); } } if (supervisorAutonomousPlayableCliSession) { state.supervisorAutonomousPlayable.cliOutput = interactiveCliOutput( supervisorAutonomousPlayableCliSession, ); } if (isIsolatedRunnerSuite() && state.isolatedRunner.appDataDir) { if (state.isolatedRunner.stopped) { try { state.isolatedRunner.cleanupPerformed = await removeIsolatedSuiteAppData(); if (!state.isolatedRunner.cleanupPerformed) { state.status = 'FAIL'; recordError('isolated-appdata-cleanup-sentinel-missing'); } } catch (error) { state.status = 'FAIL'; const safeCleanupErrorCode = isNonEmptyString(error?.code) && /^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code) ? error.code : 'isolated-appdata-cleanup-failed'; recordError(safeCleanupErrorCode, error); } } const killMethod = state.isolatedRunner.pidfdClaimCount > 0 ? process.platform === 'win32' ? 'windows-process-handle' : 'linux-pidfd' : null; if (isSteerRunnerKillSuite()) { state.evidence.steerRunnerStopped = state.isolatedRunner.stopped; state.evidence.steerAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.steerRunnerKillMethod = killMethod; state.evidence.steerRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.steerRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigHardlinkCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigLinksVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('steer-runner-kill-formal-config-cli-call-detected'); } } else if (isGoalRuntimeSuite()) { state.evidence.goalRunnerStopped = state.isolatedRunner.stopped; state.evidence.goalAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.goalRunnerKillMethod = killMethod; state.evidence.goalRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.goalRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; } else if (isResponseStreamSuite()) { state.evidence.responseStreamRunnerStopped = state.isolatedRunner.stopped; state.evidence.responseStreamAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.responseStreamRunnerKillMethod = killMethod; state.evidence.responseStreamRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.responseStreamRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigHardlinkCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigLinksVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('response-stream-formal-config-cli-call-detected'); } } else if (isWebSearchSuite()) { state.evidence.webSearchRunnerStopped = state.isolatedRunner.stopped; state.evidence.webSearchAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.webSearchRunnerKillMethod = killMethod; state.evidence.webSearchRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.webSearchRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('web-search-formal-config-cli-call-detected'); } } else if (isUserInputRuntimeSuite()) { state.evidence.userInputRunnerStopped = state.isolatedRunner.stopped; state.evidence.userInputAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.userInputRunnerKillMethod = killMethod; state.evidence.userInputRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.userInputRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigHardlinkCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigLinksVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('user-input-formal-config-cli-call-detected'); } } else if (isScopedAgentsSuite()) { state.evidence.scopedAgentsRunnerStopped = state.isolatedRunner.stopped; state.evidence.scopedAgentsAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.scopedAgentsRunnerKillMethod = killMethod; state.evidence.scopedAgentsRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.scopedAgentsRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('scoped-agents-formal-config-cli-call-detected'); } } else if (isProjectSkillSuite()) { state.evidence.projectSkillRunnerStopped = state.isolatedRunner.stopped; state.evidence.projectSkillAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.projectSkillRunnerKillMethod = killMethod; state.evidence.projectSkillRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.projectSkillRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('project-skill-formal-config-cli-call-detected'); } } else if (isParallelReadSuite()) { state.evidence.parallelReadRunnerStopped = state.isolatedRunner.stopped; state.evidence.parallelReadAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.parallelReadRunnerKillMethod = killMethod; state.evidence.parallelReadRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.parallelReadRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('parallel-read-formal-config-cli-call-detected'); } } else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) { state.evidence.supervisorAutonomousPlayableRunnerStopped = state.isolatedRunner.stopped; state.evidence.supervisorAutonomousPlayableAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceAppDataDirectoryUntouched = state.isolatedRunner.sourceAppDataDirectoryUntouched; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError( 'supervisor-autonomous-playable-formal-config-cli-call-detected', ); } } else if (isSupervisorSwarmSuite()) { state.evidence.supervisorSwarmRunnerStopped = state.isolatedRunner.stopped; state.evidence.supervisorSwarmAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.supervisorSwarmRunnerKillMethod = killMethod; state.evidence.supervisorSwarmRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.supervisorSwarmRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceAppDataDirectoryUntouched = state.isolatedRunner.sourceAppDataDirectoryUntouched; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('supervisor-swarm-formal-config-cli-call-detected'); } } else { assert( isContextCompactionSuite(), 'unknown-isolated-suite-cleanup-profile', ); state.evidence.contextCompactionRunnerStopped = state.isolatedRunner.stopped; state.evidence.contextCompactionAppDataCleanupPerformed = state.isolatedRunner.cleanupPerformed; state.evidence.contextCompactionRunnerKillMethod = killMethod; state.evidence.contextCompactionRunnerPidfdClaimCount = state.isolatedRunner.pidfdClaimCount; state.evidence.contextCompactionRunnerPidfdSignalCount = state.isolatedRunner.pidfdSignalCount; state.evidence.formalConfigCliCallCount = state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; state.evidence.sourceConfigHardlinkCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigLinksVerified = state.isolatedRunner.sourceConfigLinksVerified; state.evidence.isolatedAppDataUsed = true; if (state.isolatedRunner.sourceConfigCliCallCount > 0) { state.status = 'FAIL'; recordError('context-compaction-formal-config-cli-call-detected'); } } } if (state.isolatedRunner.sourceAppDataDirectoryWatcher) { closeSourceAppDataDirectoryGuard(); state.status = 'FAIL'; recordError('source-appdata-directory-guard-not-verified'); } if ( isSupervisorSwarmTransientRetrySuite() && state.supervisorSwarm.transientFaultProxy ) { try { await state.supervisorSwarm.transientFaultProxy.stop(); const proxyStats = state.supervisorSwarm.transientFaultProxy.getStats(); state.evidence.transientFaultRequestCount = proxyStats.requestCount; state.evidence.transientFaultInjectedCount = proxyStats.faultInjectedCount; state.evidence.transientFaultHeldRequestCount = proxyStats.heldRequestCount; state.evidence.transientFaultForwardedRequestCount = proxyStats.forwardedRequestCount; state.evidence.transientFaultProxyStopped = proxyStats.stopped === true; if (!state.evidence.transientFaultProxyStopped) { state.status = 'FAIL'; recordError('supervisor-swarm-transient-retry-proxy-not-stopped'); } } catch (error) { state.status = 'FAIL'; recordError( 'supervisor-swarm-transient-retry-proxy-cleanup-failed', error, ); } } if (supervisorSwarmToolPlanHandoffProxyNeedsCleanup()) { try { await state.supervisorSwarm.toolPlanHandoffProxy.stop(); const proxyStats = state.supervisorSwarm.toolPlanHandoffProxy.getStats(); state.evidence.toolPlanHandoffProxyRequestCount = proxyStats.requestCount; state.evidence.toolPlanHandoffProxyForwardedRequestCount = proxyStats.forwardedRequestCount; state.evidence.toolPlanHandoffProxyStopped = proxyStats.stopped === true; if (!state.evidence.toolPlanHandoffProxyStopped) { state.status = 'FAIL'; recordError('supervisor-swarm-tool-plan-handoff-proxy-not-stopped'); } } catch (error) { state.status = 'FAIL'; recordError( 'supervisor-swarm-tool-plan-handoff-proxy-cleanup-failed', error, ); } } if ( isSupervisorAutonomousPlayableLaneDefenseSuite() && state.isolatedRunner.appDataDir ) { const snapshot = state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot; if (snapshot) { try { const cleanup = await verifyOwnedProcessCleanupSnapshot(snapshot); state.evidence.ownedProcessIdentityCaptured = true; state.evidence.ownedRunnerObservedCount = snapshot.observedCounts.runner; state.evidence.ownedHelperObservedCount = snapshot.observedCounts.helper; state.evidence.ownedNodeDescendantObservedCount = snapshot.observedCounts.node; state.evidence.ownedBrowserDescendantObservedCount = snapshot.observedCounts.browser; state.evidence.ownedCommandDescendantObservedCount = snapshot.observedCounts.command; state.evidence.ownedRunnerResidualCount = cleanup.residualCounts.runner; state.evidence.ownedHelperResidualCount = cleanup.residualCounts.helper; state.evidence.ownedNodeDescendantResidualCount = cleanup.residualCounts.node; state.evidence.ownedBrowserDescendantResidualCount = cleanup.residualCounts.browser; state.evidence.ownedCommandDescendantResidualCount = cleanup.residualCounts.command; state.evidence.activeCommandChildrenAfterCleanup = cleanup.activeCommandChildCount; state.evidence.activeInteractiveCliSessionsAfterCleanup = cleanup.activeInteractiveCliSessionCount; state.evidence.ownedProcessCleanupPassed = cleanup.clean; if (!cleanup.clean) { state.status = 'FAIL'; recordError( 'supervisor-autonomous-playable-owned-process-residual-detected', ); } } catch (error) { state.status = 'FAIL'; recordError( 'supervisor-autonomous-playable-owned-process-verification-failed', error, ); } } else { state.status = 'FAIL'; recordError( 'supervisor-autonomous-playable-owned-process-snapshot-missing', ); } } if ( isSteerRunnerKillSuite() && state.projectRoot && state.status !== 'PASS' ) { try { state.evidence = { ...state.evidence, ...(await collectPartialSteerRunnerKillEvidence()), }; } catch (error) { recordError('steer-runner-kill-partial-evidence-read-failed', error); } } if (isGoalRuntimeSuite() && state.projectRoot && state.status !== 'PASS') { try { state.evidence = { ...state.evidence, ...(await collectPartialGoalEvidence()), }; } catch (error) { recordError('goal-partial-evidence-read-failed', error); } } if ( isResponseStreamSuite() && state.projectRoot && state.status !== 'PASS' ) { try { state.evidence = { ...state.evidence, ...(await collectPartialResponseStreamEvidence()), }; } catch (error) { recordError('response-stream-partial-evidence-read-failed', error); } } if (isWebSearchSuite() && state.projectRoot && state.status !== 'PASS') { try { state.evidence = { ...state.evidence, ...(await collectPartialWebSearchEvidence()), }; } catch (error) { recordError('web-search-partial-evidence-read-failed', error); } } if ( isContextCompactionSuite() && state.projectRoot && state.status !== 'PASS' ) { try { state.evidence = { ...state.evidence, ...(await collectPartialContextCompactionEvidence()), }; } catch (error) { recordError('context-compaction-partial-evidence-read-failed', error); } } if ( isUserInputRuntimeSuite() && state.projectRoot && state.status !== 'PASS' ) { try { state.evidence = { ...state.evidence, ...(await collectPartialUserInputEvidence()), }; } catch (error) { recordError('user-input-partial-evidence-read-failed', error); } } if (isScopedAgentsSuite() && state.projectRoot && state.status !== 'PASS') { try { state.evidence = { ...state.evidence, ...(await collectPartialScopedAgentsEvidence()), }; } catch (error) { recordError('scoped-agents-partial-evidence-read-failed', error); } } if (isProjectSkillSuite() && state.projectRoot && state.status !== 'PASS') { try { state.evidence = { ...state.evidence, ...(await collectPartialProjectSkillEvidence()), }; } catch (error) { recordError('project-skill-partial-evidence-read-failed', error); } } if (isParallelReadSuite() && state.projectRoot && state.status !== 'PASS') { try { state.evidence = { ...state.evidence, ...(await collectPartialParallelReadEvidence()), }; } catch (error) { recordError('parallel-read-partial-evidence-read-failed', error); } } if ( isSupervisorAutonomousPlayableLaneDefenseSuite() && state.projectRoot && state.status !== 'PASS' && state.evidence.evidenceCompleteness !== 'complete' ) { try { state.evidence = { ...state.evidence, ...(await collectPartialSupervisorAutonomousPlayableEvidence()), }; } catch (error) { recordError( 'supervisor-autonomous-playable-partial-evidence-read-failed', error, ); } } if ( isSupervisorSwarmSuite() && state.projectRoot && state.status !== 'PASS' ) { try { state.evidence = await collectPartialSupervisorSwarmEvidence( state.evidence, ); } catch (error) { recordError('supervisor-swarm-partial-evidence-read-failed', error); } } if (state.projectRoot && state.secrets.length > 0) { try { state.projectLeakCount = await countSecretsInProject( state.projectRoot, state.secrets, ); } catch (error) { state.status = 'FAIL'; recordError('project-secret-scan-failed', error); } } state.transcriptLeakCount = state.transcriptScanner?.count ?? 0; state.projectPathTranscriptLeakCount = state.projectPathTranscriptScanner?.count ?? 0; state.formalConfigPathTranscriptLeakCount = state.formalConfigPathTranscriptScanner?.count ?? 0; if (state.transcriptLeakCount + state.projectLeakCount > 0) { state.status = 'FAIL'; recordError('loaded-key-leak-detected'); } if (state.projectPathTranscriptLeakCount > 0) { state.status = 'FAIL'; recordError('disposable-project-path-transcript-leak-detected'); } if (state.formalConfigPathTranscriptLeakCount > 0) { state.status = 'FAIL'; recordError('formal-config-path-transcript-leak-detected'); } const isolatedRunnerAllowsProjectCleanup = !isIsolatedRunnerSuite() || !state.isolatedRunner.appDataDir || state.isolatedRunner.stopped; if ( state.projectRoot && !state.options?.keepProject && isolatedRunnerAllowsProjectCleanup ) { try { state.cleanupPerformed = await removeDisposableProject(); if (!state.cleanupPerformed) { state.status = 'FAIL'; recordError('cleanup-sentinel-missing'); } } catch (error) { state.status = 'FAIL'; recordError('cleanup-failed', error); } } let summary = buildSummary(); let report = JSON.stringify(summary, null, 2); if (isProcessSessionSuite() && state.process.challenge) { state.process.reportLeakCount = countExactSecrets( Buffer.from(report), [ state.process.challenge, state.process.readyLine, state.process.echoLine, processStoppedMarker, ].filter(Boolean), ); state.evidence.processReportLeakCount = state.process.reportLeakCount; if (state.process.reportLeakCount > 0) { state.status = 'FAIL'; recordError('process-private-output-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } state.commandMarkerReportLeakCount = countExactSecrets( Buffer.from(report), [commandRootErrorMarker], ); if (state.commandMarkerReportLeakCount > 0) { state.status = 'FAIL'; recordError('command-output-marker-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } state.steerInstructionReportLeakCount = countExactSecrets( Buffer.from(report), [steerInstruction], ); state.evidence.steerInstructionReportLeakCount = state.steerInstructionReportLeakCount; if (state.steerInstructionReportLeakCount > 0) { state.status = 'FAIL'; recordError('steer-instruction-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } if (isGoalRuntimeSuite()) { state.goalBodyReportLeakCount = countExactSecrets( Buffer.from(report), goalPrivateBodyValues(), ); state.evidence.goalBodyReportLeakCount = state.goalBodyReportLeakCount; if (state.goalBodyReportLeakCount > 0) { state.status = 'FAIL'; recordError('goal-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isResponseStreamSuite()) { state.responseStream.reportLeakCount = countExactSecrets( Buffer.from(report), [ state.responseStream.finalText, ...responseStreamThinkingMarkers, ].filter(isNonEmptyString), ); state.evidence.responseStreamReportLeakCount = state.responseStream.reportLeakCount; if (state.responseStream.reportLeakCount > 0) { state.status = 'FAIL'; recordError('response-stream-private-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isWebSearchSuite()) { state.webSearch.reportLeakCount = countExactSecrets( Buffer.from(report), webSearchPrivateLeakValues(), ); state.evidence.webSearchReportLeakCount = state.webSearch.reportLeakCount; if (state.webSearch.reportLeakCount > 0) { state.status = 'FAIL'; recordError('web-search-private-context-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isContextCompactionSuite()) { state.contextCompaction.reportLeakCount = countExactSecrets( Buffer.from(report), [ contextCompactionConstraintCanary, ...state.contextCompaction.privateSummaries, ].filter(isNonEmptyString), ); state.evidence.contextCompactionReportLeakCount = state.contextCompaction.reportLeakCount; if (state.contextCompaction.reportLeakCount > 0) { state.status = 'FAIL'; recordError('context-compaction-private-context-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isUserInputRuntimeSuite()) { state.userInput.reportLeakCount = countExactSecrets( Buffer.from(report), [ userInputAnswerCanary, userInputAnswerText, ...state.userInput.privateValues, ].filter(isNonEmptyString), ); state.evidence.userInputReportLeakCount = state.userInput.reportLeakCount; if (state.userInput.reportLeakCount > 0) { state.status = 'FAIL'; recordError('user-input-private-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isScopedAgentsSuite()) { state.scopedAgents.reportLeakCount = countExactSecrets( Buffer.from(report), state.scopedAgents.privateValues, ); state.evidence.scopedAgentsReportLeakCount = state.scopedAgents.reportLeakCount; if (state.scopedAgents.reportLeakCount > 0) { state.status = 'FAIL'; recordError('scoped-agents-private-instruction-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isProjectSkillSuite()) { state.projectSkill.reportLeakCount = countExactSecrets( Buffer.from(report), state.projectSkill.privateValues, ); state.evidence.projectSkillReportLeakCount = state.projectSkill.reportLeakCount; if (state.projectSkill.reportLeakCount > 0) { state.status = 'FAIL'; recordError('project-skill-private-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isParallelReadSuite()) { state.parallelRead.reportLeakCount = countExactSecrets( Buffer.from(report), state.parallelRead.privateValues, ); state.evidence.parallelReadReportLeakCount = state.parallelRead.reportLeakCount; if (state.parallelRead.reportLeakCount > 0) { state.status = 'FAIL'; recordError('parallel-read-private-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isSupervisorAutonomousPlayableLaneDefenseSuite()) { state.supervisorAutonomousPlayable.reportLeakCount = countExactSecrets( Buffer.from(report), state.supervisorAutonomousPlayable.privateValues, ); state.evidence.supervisorAutonomousPlayableReportLeakCount = state.supervisorAutonomousPlayable.reportLeakCount; if (state.supervisorAutonomousPlayable.reportLeakCount > 0) { state.status = 'FAIL'; recordError('supervisor-autonomous-playable-report-body-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } if (isSupervisorSwarmSuite()) { state.supervisorSwarm.reportLeakCount = countExactSecrets( Buffer.from(report), state.supervisorSwarm.privateValues, ); state.evidence.supervisorSwarmReportLeakCount = state.supervisorSwarm.reportLeakCount; if (state.supervisorSwarm.reportLeakCount > 0) { state.status = 'FAIL'; recordError('supervisor-swarm-private-body-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } state.projectPathReportLeakCount = countExactSecrets( Buffer.from(report), disposableProjectPathVariants(), ); state.evidence.projectPathReportLeakCount = state.projectPathReportLeakCount; if (state.projectPathReportLeakCount > 0) { state.status = 'FAIL'; recordError('disposable-project-path-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } if ( isWebSearchSuite() || isContextCompactionSuite() || isUserInputRuntimeSuite() || isScopedAgentsSuite() || isProjectSkillSuite() || isParallelReadSuite() || isSupervisorAutonomousPlayableLaneDefenseSuite() || isSupervisorSwarmSuite() || isSteerRunnerKillSuite() ) { state.formalConfigPathReportLeakCount = countExactSecrets( Buffer.from(report), formalConfigPathVariants(), ); state.evidence.formalConfigPathReportLeakCount = state.formalConfigPathReportLeakCount; if (state.formalConfigPathReportLeakCount > 0) { state.status = 'FAIL'; recordError('formal-config-path-report-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } } state.reportLeakCount = countExactSecrets( Buffer.from(report), state.secrets, ); if (state.reportLeakCount > 0) { state.status = 'FAIL'; recordError('report-key-leak-detected'); summary = buildSummary(); report = JSON.stringify(summary, null, 2); } const remainingProjectPathReportLeakCount = countExactSecrets( Buffer.from(report), disposableProjectPathVariants(), ); const remainingResponseStreamReportLeakCount = isResponseStreamSuite() ? countExactSecrets( Buffer.from(report), [ state.responseStream.finalText, ...responseStreamThinkingMarkers, ].filter(isNonEmptyString), ) : 0; const remainingWebSearchReportLeakCount = isWebSearchSuite() ? countExactSecrets(Buffer.from(report), webSearchPrivateLeakValues()) : 0; const remainingUserInputReportLeakCount = isUserInputRuntimeSuite() ? countExactSecrets( Buffer.from(report), [ userInputAnswerCanary, userInputAnswerText, ...state.userInput.privateValues, ].filter(isNonEmptyString), ) : 0; const remainingScopedAgentsReportLeakCount = isScopedAgentsSuite() ? countExactSecrets(Buffer.from(report), state.scopedAgents.privateValues) : 0; const remainingProjectSkillReportLeakCount = isProjectSkillSuite() ? countExactSecrets(Buffer.from(report), state.projectSkill.privateValues) : 0; const remainingParallelReadReportLeakCount = isParallelReadSuite() ? countExactSecrets(Buffer.from(report), state.parallelRead.privateValues) : 0; const remainingSupervisorAutonomousPlayableReportLeakCount = isSupervisorAutonomousPlayableLaneDefenseSuite() ? countExactSecrets( Buffer.from(report), state.supervisorAutonomousPlayable.privateValues, ) : 0; const remainingSupervisorSwarmReportLeakCount = isSupervisorSwarmSuite() ? countExactSecrets( Buffer.from(report), state.supervisorSwarm.privateValues, ) : 0; const remainingFormalConfigPathReportLeakCount = isWebSearchSuite() || isContextCompactionSuite() || isUserInputRuntimeSuite() || isScopedAgentsSuite() || isProjectSkillSuite() || isParallelReadSuite() || isSupervisorAutonomousPlayableLaneDefenseSuite() || isSupervisorSwarmSuite() || isSteerRunnerKillSuite() ? countExactSecrets(Buffer.from(report), formalConfigPathVariants()) : 0; if ( remainingProjectPathReportLeakCount > 0 || remainingResponseStreamReportLeakCount > 0 || remainingWebSearchReportLeakCount > 0 || remainingUserInputReportLeakCount > 0 || remainingScopedAgentsReportLeakCount > 0 || remainingProjectSkillReportLeakCount > 0 || remainingParallelReadReportLeakCount > 0 || remainingSupervisorAutonomousPlayableReportLeakCount > 0 || remainingSupervisorSwarmReportLeakCount > 0 || remainingFormalConfigPathReportLeakCount > 0 ) { state.status = 'FAIL'; recordError( remainingProjectPathReportLeakCount > 0 ? 'disposable-project-path-report-redaction-required' : remainingResponseStreamReportLeakCount > 0 ? 'response-stream-report-redaction-required' : remainingUserInputReportLeakCount > 0 ? 'user-input-report-redaction-required' : remainingScopedAgentsReportLeakCount > 0 ? 'scoped-agents-report-redaction-required' : remainingProjectSkillReportLeakCount > 0 ? 'project-skill-report-redaction-required' : remainingParallelReadReportLeakCount > 0 ? 'parallel-read-report-redaction-required' : remainingSupervisorAutonomousPlayableReportLeakCount > 0 ? 'supervisor-autonomous-playable-report-redaction-required' : remainingSupervisorSwarmReportLeakCount > 0 ? 'supervisor-swarm-report-redaction-required' : remainingFormalConfigPathReportLeakCount > 0 ? 'formal-config-path-report-redaction-required' : 'web-search-report-redaction-required', ); const safeSummary = { status: state.status, suite: state.suite, providerUsed: false, blocked: state.blocked, cleanup: { performed: state.cleanupPerformed, kept: Boolean(state.options?.keepProject), }, evidence: { projectPathReportLeakCount: remainingProjectPathReportLeakCount, responseStreamReportLeakCount: remainingResponseStreamReportLeakCount, webSearchReportLeakCount: remainingWebSearchReportLeakCount, userInputReportLeakCount: remainingUserInputReportLeakCount, scopedAgentsReportLeakCount: remainingScopedAgentsReportLeakCount, projectSkillReportLeakCount: remainingProjectSkillReportLeakCount, parallelReadReportLeakCount: remainingParallelReadReportLeakCount, supervisorAutonomousPlayableReportLeakCount: remainingSupervisorAutonomousPlayableReportLeakCount, supervisorSwarmReportLeakCount: remainingSupervisorSwarmReportLeakCount, formalConfigPathReportLeakCount: remainingFormalConfigPathReportLeakCount, ...(isSupervisorSwarmSuite() ? supervisorSwarmChatSessionFailureEvidence() : {}), }, errorCount: state.errors.length, errorHashes: state.errors.map(summarizeRecordedError), }; safeSummary.summaryHash = hashValue(JSON.stringify(safeSummary)); report = JSON.stringify(safeSummary, null, 2); } process.stdout.write(`${report}\n`); process.exitCode = state.shutdownSignal ? state.shutdownSignal === 'SIGINT' ? 130 : 143 : state.status === 'PASS' ? 0 : state.status === 'BLOCKED' ? 2 : 1; } }