1337 lines
50 KiB
JavaScript
1337 lines
50 KiB
JavaScript
import {
|
|
assert,
|
|
hashValue,
|
|
prerequisiteLabel,
|
|
recordError,
|
|
summarizeRecordedError,
|
|
throwIfShutdownRequested,
|
|
} from './assertions/core.mjs';
|
|
import {
|
|
absolutePathVariants,
|
|
countExactSecrets,
|
|
disposableProjectPathVariants,
|
|
formalConfigPathVariants,
|
|
isNonEmptyString,
|
|
} from './assertions/runtime.mjs';
|
|
import {
|
|
closeOwnedRunnerKillHandle,
|
|
closeSourceAppDataDirectoryGuard,
|
|
removeIsolatedSuiteAppData,
|
|
stopOwnedIsolatedRunner,
|
|
} from './harness/app-data.mjs';
|
|
import { loadConfig, parseArguments } from './harness/config.mjs';
|
|
import {
|
|
activeInteractiveCliSessions,
|
|
captureOwnedProcessCleanupSnapshot,
|
|
closeInteractiveCli,
|
|
destroyInteractiveCliOutputStreams,
|
|
interactiveCliOutput,
|
|
verifyOwnedProcessCleanupSnapshot,
|
|
waitForInteractiveCliStdioClose,
|
|
} from './harness/process.mjs';
|
|
import { checkPrerequisites } from './harness/project.mjs';
|
|
import {
|
|
buildSummary,
|
|
emptyEvidence,
|
|
isIsolatedRunnerSuite,
|
|
removeDisposableProject,
|
|
} from './harness/reporting.mjs';
|
|
import { countSecretsInProject } from './harness/runtime.mjs';
|
|
import {
|
|
activeCommandChildren,
|
|
BlockedError,
|
|
commandRootErrorMarker,
|
|
contextCompactionConstraintCanary,
|
|
processStoppedMarker,
|
|
responseStreamThinkingMarkers,
|
|
shutdownWaiters,
|
|
state,
|
|
steerInstruction,
|
|
StreamingSecretScanner,
|
|
userInputAnswerCanary,
|
|
userInputAnswerText,
|
|
} from './runtime-state.mjs';
|
|
import {
|
|
collectPartialContextCompactionEvidence,
|
|
emptyContextCompactionEvidence,
|
|
isContextCompactionSuite,
|
|
runContextCompactionE2e,
|
|
} from './suites/context-compaction.mjs';
|
|
import { runRealE2e } from './suites/full.mjs';
|
|
import {
|
|
collectPartialGoalEvidence,
|
|
emptyGoalEvidence,
|
|
goalPrivateBodyValues,
|
|
isGoalRuntimeSuite,
|
|
runGoalRuntimeE2e,
|
|
} from './suites/goal.mjs';
|
|
import {
|
|
collectPartialParallelReadEvidence,
|
|
emptyParallelReadEvidence,
|
|
isParallelReadSuite,
|
|
runParallelReadE2e,
|
|
} from './suites/parallel-read.mjs';
|
|
import {
|
|
emptyProcessEvidence,
|
|
isProcessSessionSuite,
|
|
runProcessSessionE2e,
|
|
} from './suites/process-session.mjs';
|
|
import {
|
|
collectPartialProjectSkillEvidence,
|
|
emptyProjectSkillEvidence,
|
|
isProjectSkillSuite,
|
|
runProjectSkillE2e,
|
|
} from './suites/project-skill.mjs';
|
|
import {
|
|
collectPartialResponseStreamEvidence,
|
|
emptyResponseStreamEvidence,
|
|
isResponseStreamSuite,
|
|
runResponseStreamE2e,
|
|
} from './suites/response-stream.mjs';
|
|
import {
|
|
collectPartialScopedAgentsEvidence,
|
|
emptyScopedAgentsEvidence,
|
|
isScopedAgentsSuite,
|
|
runScopedAgentsE2e,
|
|
} from './suites/scoped-agents.mjs';
|
|
import { runAgentRuntimeRealE2eSelfTests } from './suites/self-test.mjs';
|
|
import {
|
|
collectPartialSteerRunnerKillEvidence,
|
|
emptySteerRunnerKillEvidence,
|
|
isSteerRunnerKillSuite,
|
|
runSteerRunnerKillE2e,
|
|
} from './suites/steer-runner-kill.mjs';
|
|
import {
|
|
collectPartialSupervisorAutonomousPlayableEvidence,
|
|
emptySupervisorAutonomousPlayableEvidence,
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite,
|
|
runSupervisorAutonomousPlayableLaneDefenseE2e,
|
|
} from './suites/supervisor-autonomous-playable.mjs';
|
|
import {
|
|
collectPartialSupervisorSwarmEvidence,
|
|
emptySupervisorSwarmEvidence,
|
|
isSupervisorSwarmInteractiveChatSuite,
|
|
isSupervisorSwarmSuite,
|
|
isSupervisorSwarmTransientRetrySuite,
|
|
recordSupervisorSwarmChatSessionFailureDiagnostic,
|
|
runSupervisorSwarmE2e,
|
|
supervisorSwarmChatSessionFailureEvidence,
|
|
supervisorSwarmToolPlanHandoffProxyNeedsCleanup,
|
|
} from './suites/supervisor-swarm.mjs';
|
|
import {
|
|
collectPartialUserInputEvidence,
|
|
emptyUserInputEvidence,
|
|
isUserInputRuntimeSuite,
|
|
runUserInputRuntimeE2e,
|
|
} from './suites/user-input.mjs';
|
|
import {
|
|
collectPartialWebSearchEvidence,
|
|
emptyWebSearchEvidence,
|
|
isWebSearchSuite,
|
|
runWebSearchE2e,
|
|
webSearchPrivateLeakValues,
|
|
} from './suites/web-search.mjs';
|
|
|
|
state.evidence = emptyEvidence();
|
|
|
|
export function requestShutdown(signal) {
|
|
if (state.shutdownSignal) return;
|
|
state.shutdownSignal = signal;
|
|
state.status = 'FAIL';
|
|
recordError(`interrupted-${signal.toLowerCase()}`);
|
|
if (state.cleanupInProgress) return;
|
|
for (const waiter of shutdownWaiters) waiter();
|
|
for (const child of activeCommandChildren) {
|
|
if (child.exitCode !== null || child.signalCode !== null) continue;
|
|
child.kill('SIGTERM');
|
|
const forceTimer = setTimeout(() => {
|
|
if (child.exitCode === null && child.signalCode === null) {
|
|
child.kill('SIGKILL');
|
|
}
|
|
}, 1_000);
|
|
forceTimer.unref();
|
|
}
|
|
}
|
|
|
|
for (const signal of ['SIGINT', 'SIGTERM']) {
|
|
process.on(signal, () => requestShutdown(signal));
|
|
}
|
|
|
|
export const selfTestRequested =
|
|
process.argv.length === 3 && process.argv[2] === '--self-test';
|
|
|
|
if (selfTestRequested) {
|
|
const selfTestEvidence = await runAgentRuntimeRealE2eSelfTests();
|
|
process.stdout.write(`${JSON.stringify(selfTestEvidence, null, 2)}\n`);
|
|
} else {
|
|
try {
|
|
state.options = parseArguments(process.argv.slice(2));
|
|
state.suite = state.options.suite;
|
|
state.runtimeConfigDir = state.options.configDir;
|
|
if (isProcessSessionSuite()) state.evidence = emptyProcessEvidence();
|
|
if (isGoalRuntimeSuite()) state.evidence = emptyGoalEvidence();
|
|
if (isResponseStreamSuite()) state.evidence = emptyResponseStreamEvidence();
|
|
if (isWebSearchSuite()) state.evidence = emptyWebSearchEvidence();
|
|
if (isContextCompactionSuite()) {
|
|
state.evidence = emptyContextCompactionEvidence();
|
|
}
|
|
if (isUserInputRuntimeSuite()) state.evidence = emptyUserInputEvidence();
|
|
if (isScopedAgentsSuite()) state.evidence = emptyScopedAgentsEvidence();
|
|
if (isProjectSkillSuite()) state.evidence = emptyProjectSkillEvidence();
|
|
if (isParallelReadSuite()) state.evidence = emptyParallelReadEvidence();
|
|
if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
|
|
state.evidence = emptySupervisorAutonomousPlayableEvidence();
|
|
}
|
|
if (isSupervisorSwarmSuite()) {
|
|
state.evidence = emptySupervisorSwarmEvidence();
|
|
}
|
|
if (isSteerRunnerKillSuite()) {
|
|
state.evidence = emptySteerRunnerKillEvidence();
|
|
}
|
|
const loaded = await loadConfig(state.options.configDir);
|
|
if (
|
|
isWebSearchSuite() ||
|
|
isContextCompactionSuite() ||
|
|
isUserInputRuntimeSuite() ||
|
|
isScopedAgentsSuite() ||
|
|
isProjectSkillSuite() ||
|
|
isParallelReadSuite() ||
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
|
|
isSupervisorSwarmSuite() ||
|
|
isSteerRunnerKillSuite()
|
|
) {
|
|
state.formalConfigPathTranscriptScanner = new StreamingSecretScanner(
|
|
absolutePathVariants(state.options.configDir, loaded.realConfigDir),
|
|
);
|
|
}
|
|
state.secrets = loaded.secrets;
|
|
state.transcriptScanner = new StreamingSecretScanner(state.secrets);
|
|
state.config = await checkPrerequisites(loaded.config);
|
|
|
|
const required = isSupervisorAutonomousPlayableLaneDefenseSuite()
|
|
? ['llmConfigured', 'chromeAvailable']
|
|
: isProcessSessionSuite() || isIsolatedRunnerSuite()
|
|
? ['llmConfigured']
|
|
: ['llmConfigured', 'chromeAvailable'];
|
|
if (state.suite === 'full') {
|
|
required.push('editorApiConfigured');
|
|
}
|
|
state.blocked = required
|
|
.filter((name) => !state.config[name])
|
|
.map((name) => prerequisiteLabel(name));
|
|
if (state.blocked.length > 0) {
|
|
state.status = 'BLOCKED';
|
|
} else {
|
|
if (isSteerRunnerKillSuite()) {
|
|
await runSteerRunnerKillE2e();
|
|
} else if (isGoalRuntimeSuite()) {
|
|
await runGoalRuntimeE2e();
|
|
} else if (isResponseStreamSuite()) {
|
|
await runResponseStreamE2e();
|
|
} else if (isWebSearchSuite()) {
|
|
await runWebSearchE2e();
|
|
} else if (isContextCompactionSuite()) {
|
|
await runContextCompactionE2e();
|
|
} else if (isUserInputRuntimeSuite()) {
|
|
await runUserInputRuntimeE2e();
|
|
} else if (isScopedAgentsSuite()) {
|
|
await runScopedAgentsE2e();
|
|
} else if (isProjectSkillSuite()) {
|
|
await runProjectSkillE2e();
|
|
} else if (isParallelReadSuite()) {
|
|
await runParallelReadE2e();
|
|
} else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
|
|
await runSupervisorAutonomousPlayableLaneDefenseE2e();
|
|
} else if (isSupervisorSwarmSuite()) {
|
|
await runSupervisorSwarmE2e();
|
|
} else if (isProcessSessionSuite()) {
|
|
await runProcessSessionE2e();
|
|
} else {
|
|
await runRealE2e();
|
|
}
|
|
state.status = 'PASS';
|
|
}
|
|
throwIfShutdownRequested();
|
|
} catch (error) {
|
|
if (error instanceof BlockedError) {
|
|
state.status = 'BLOCKED';
|
|
state.blocked = [...new Set([...state.blocked, ...error.components])];
|
|
} else {
|
|
state.status = 'FAIL';
|
|
}
|
|
recordError(error?.code ?? 'unexpected-error', error);
|
|
} finally {
|
|
state.cleanupInProgress = true;
|
|
const stateTrackedInteractiveCliSessions = new Set(
|
|
[
|
|
state.userInputCliSession,
|
|
state.supervisorAutonomousPlayableCliSession,
|
|
state.supervisorSwarmCliSession,
|
|
].filter(Boolean),
|
|
);
|
|
const interactiveCliSessions = [
|
|
...new Set([
|
|
...stateTrackedInteractiveCliSessions,
|
|
...activeInteractiveCliSessions,
|
|
]),
|
|
];
|
|
const supervisorAutonomousPlayableCliSession =
|
|
state.supervisorAutonomousPlayableCliSession;
|
|
if (isUserInputRuntimeSuite() && state.userInputCliSession) {
|
|
try {
|
|
await closeInteractiveCli(state.userInputCliSession);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError('user-input-cli-cleanup-failed', error);
|
|
}
|
|
state.userInputCliSession = null;
|
|
}
|
|
if (
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
|
|
state.supervisorAutonomousPlayableCliSession
|
|
) {
|
|
try {
|
|
await closeInteractiveCli(state.supervisorAutonomousPlayableCliSession);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-autonomous-playable-cli-cleanup-failed', error);
|
|
}
|
|
state.supervisorAutonomousPlayableCliSession = null;
|
|
}
|
|
if (
|
|
isSupervisorSwarmInteractiveChatSuite() &&
|
|
state.supervisorSwarmCliSession
|
|
) {
|
|
try {
|
|
if (
|
|
state.status !== 'PASS' &&
|
|
state.supervisorSwarmCliSession.closed &&
|
|
!state.supervisorSwarm.chatSessionFailureDiagnostic
|
|
) {
|
|
recordSupervisorSwarmChatSessionFailureDiagnostic(
|
|
state.supervisorSwarmCliSession,
|
|
);
|
|
}
|
|
await closeInteractiveCli(state.supervisorSwarmCliSession);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-swarm-autonomous-chat-cli-cleanup-failed',
|
|
error,
|
|
);
|
|
}
|
|
state.supervisorSwarmCliSession = null;
|
|
}
|
|
for (const session of interactiveCliSessions) {
|
|
if (stateTrackedInteractiveCliSessions.has(session)) continue;
|
|
try {
|
|
await closeInteractiveCli(session);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError('interactive-cli-cleanup-failed', error);
|
|
}
|
|
}
|
|
if (
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
|
|
state.isolatedRunner.appDataDir
|
|
) {
|
|
try {
|
|
const runnerPid = state.isolatedRunner.current?.pid ?? null;
|
|
const helperPid =
|
|
state.isolatedRunner.current?.killHandle?.child?.pid ?? null;
|
|
state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot =
|
|
await captureOwnedProcessCleanupSnapshot({
|
|
runnerPid,
|
|
helperPids: Number.isSafeInteger(helperPid) ? [helperPid] : [],
|
|
rootPids: [
|
|
...interactiveCliSessions.map((session) => session.child?.pid),
|
|
...[...activeCommandChildren].map((child) => child.pid),
|
|
].filter((pid) => Number.isSafeInteger(pid) && pid > 0),
|
|
});
|
|
const observed =
|
|
state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot
|
|
.observedCounts;
|
|
assert(
|
|
observed.runner === 1 && observed.helper === 1,
|
|
'supervisor-autonomous-playable-owned-process-snapshot-incomplete',
|
|
);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-autonomous-playable-owned-process-snapshot-failed',
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
if (isIsolatedRunnerSuite() && state.isolatedRunner.appDataDir) {
|
|
try {
|
|
await stopOwnedIsolatedRunner();
|
|
state.isolatedRunner.stopped = true;
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
const safeCleanupErrorCode =
|
|
isNonEmptyString(error?.code) &&
|
|
/^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code)
|
|
? error.code
|
|
: 'isolated-owned-runner-cleanup-failed';
|
|
recordError(safeCleanupErrorCode, error);
|
|
await closeOwnedRunnerKillHandle(
|
|
state.isolatedRunner.current?.killHandle,
|
|
).catch(() => {});
|
|
}
|
|
}
|
|
for (const session of interactiveCliSessions) {
|
|
try {
|
|
await waitForInteractiveCliStdioClose(session, 10_000);
|
|
} catch (error) {
|
|
destroyInteractiveCliOutputStreams(session);
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
error?.code === 'interactive-cli-stdio-close-timeout'
|
|
? error.code
|
|
: 'interactive-cli-stdio-cleanup-failed',
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
if (supervisorAutonomousPlayableCliSession) {
|
|
state.supervisorAutonomousPlayable.cliOutput = interactiveCliOutput(
|
|
supervisorAutonomousPlayableCliSession,
|
|
);
|
|
}
|
|
if (isIsolatedRunnerSuite() && state.isolatedRunner.appDataDir) {
|
|
if (state.isolatedRunner.stopped) {
|
|
try {
|
|
state.isolatedRunner.cleanupPerformed =
|
|
await removeIsolatedSuiteAppData();
|
|
if (!state.isolatedRunner.cleanupPerformed) {
|
|
state.status = 'FAIL';
|
|
recordError('isolated-appdata-cleanup-sentinel-missing');
|
|
}
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
const safeCleanupErrorCode =
|
|
isNonEmptyString(error?.code) &&
|
|
/^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code)
|
|
? error.code
|
|
: 'isolated-appdata-cleanup-failed';
|
|
recordError(safeCleanupErrorCode, error);
|
|
}
|
|
}
|
|
const killMethod =
|
|
state.isolatedRunner.pidfdClaimCount > 0
|
|
? process.platform === 'win32'
|
|
? 'windows-process-handle'
|
|
: 'linux-pidfd'
|
|
: null;
|
|
if (isSteerRunnerKillSuite()) {
|
|
state.evidence.steerRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.steerAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.steerRunnerKillMethod = killMethod;
|
|
state.evidence.steerRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.steerRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigHardlinkCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigLinksVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('steer-runner-kill-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isGoalRuntimeSuite()) {
|
|
state.evidence.goalRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.goalAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.goalRunnerKillMethod = killMethod;
|
|
state.evidence.goalRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.goalRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
} else if (isResponseStreamSuite()) {
|
|
state.evidence.responseStreamRunnerStopped =
|
|
state.isolatedRunner.stopped;
|
|
state.evidence.responseStreamAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.responseStreamRunnerKillMethod = killMethod;
|
|
state.evidence.responseStreamRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.responseStreamRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigHardlinkCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigLinksVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('response-stream-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isWebSearchSuite()) {
|
|
state.evidence.webSearchRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.webSearchAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.webSearchRunnerKillMethod = killMethod;
|
|
state.evidence.webSearchRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.webSearchRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('web-search-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isUserInputRuntimeSuite()) {
|
|
state.evidence.userInputRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.userInputAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.userInputRunnerKillMethod = killMethod;
|
|
state.evidence.userInputRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.userInputRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigHardlinkCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigLinksVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('user-input-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isScopedAgentsSuite()) {
|
|
state.evidence.scopedAgentsRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.scopedAgentsAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.scopedAgentsRunnerKillMethod = killMethod;
|
|
state.evidence.scopedAgentsRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.scopedAgentsRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('scoped-agents-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isProjectSkillSuite()) {
|
|
state.evidence.projectSkillRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.projectSkillAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.projectSkillRunnerKillMethod = killMethod;
|
|
state.evidence.projectSkillRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.projectSkillRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('project-skill-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isParallelReadSuite()) {
|
|
state.evidence.parallelReadRunnerStopped = state.isolatedRunner.stopped;
|
|
state.evidence.parallelReadAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.parallelReadRunnerKillMethod = killMethod;
|
|
state.evidence.parallelReadRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.parallelReadRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('parallel-read-formal-config-cli-call-detected');
|
|
}
|
|
} else if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
|
|
state.evidence.supervisorAutonomousPlayableRunnerStopped =
|
|
state.isolatedRunner.stopped;
|
|
state.evidence.supervisorAutonomousPlayableAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceAppDataDirectoryUntouched =
|
|
state.isolatedRunner.sourceAppDataDirectoryUntouched;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-autonomous-playable-formal-config-cli-call-detected',
|
|
);
|
|
}
|
|
} else if (isSupervisorSwarmSuite()) {
|
|
state.evidence.supervisorSwarmRunnerStopped =
|
|
state.isolatedRunner.stopped;
|
|
state.evidence.supervisorSwarmAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.supervisorSwarmRunnerKillMethod = killMethod;
|
|
state.evidence.supervisorSwarmRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.supervisorSwarmRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceAppDataDirectoryUntouched =
|
|
state.isolatedRunner.sourceAppDataDirectoryUntouched;
|
|
state.evidence.sourceConfigReplicaCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigReplicasVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-swarm-formal-config-cli-call-detected');
|
|
}
|
|
} else {
|
|
assert(
|
|
isContextCompactionSuite(),
|
|
'unknown-isolated-suite-cleanup-profile',
|
|
);
|
|
state.evidence.contextCompactionRunnerStopped =
|
|
state.isolatedRunner.stopped;
|
|
state.evidence.contextCompactionAppDataCleanupPerformed =
|
|
state.isolatedRunner.cleanupPerformed;
|
|
state.evidence.contextCompactionRunnerKillMethod = killMethod;
|
|
state.evidence.contextCompactionRunnerPidfdClaimCount =
|
|
state.isolatedRunner.pidfdClaimCount;
|
|
state.evidence.contextCompactionRunnerPidfdSignalCount =
|
|
state.isolatedRunner.pidfdSignalCount;
|
|
state.evidence.formalConfigCliCallCount =
|
|
state.isolatedRunner.sourceConfigCliCallCount;
|
|
state.evidence.sourceRunnerEndpointUnchanged =
|
|
state.isolatedRunner.sourceRunnerEndpointUnchanged;
|
|
state.evidence.sourceConfigHardlinkCount =
|
|
state.isolatedRunner.configLinks.length;
|
|
state.evidence.sourceConfigLinksVerified =
|
|
state.isolatedRunner.sourceConfigLinksVerified;
|
|
state.evidence.isolatedAppDataUsed = true;
|
|
if (state.isolatedRunner.sourceConfigCliCallCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('context-compaction-formal-config-cli-call-detected');
|
|
}
|
|
}
|
|
}
|
|
if (state.isolatedRunner.sourceAppDataDirectoryWatcher) {
|
|
closeSourceAppDataDirectoryGuard();
|
|
state.status = 'FAIL';
|
|
recordError('source-appdata-directory-guard-not-verified');
|
|
}
|
|
if (
|
|
isSupervisorSwarmTransientRetrySuite() &&
|
|
state.supervisorSwarm.transientFaultProxy
|
|
) {
|
|
try {
|
|
await state.supervisorSwarm.transientFaultProxy.stop();
|
|
const proxyStats = state.supervisorSwarm.transientFaultProxy.getStats();
|
|
state.evidence.transientFaultRequestCount = proxyStats.requestCount;
|
|
state.evidence.transientFaultInjectedCount =
|
|
proxyStats.faultInjectedCount;
|
|
state.evidence.transientFaultHeldRequestCount =
|
|
proxyStats.heldRequestCount;
|
|
state.evidence.transientFaultForwardedRequestCount =
|
|
proxyStats.forwardedRequestCount;
|
|
state.evidence.transientFaultProxyStopped = proxyStats.stopped === true;
|
|
if (!state.evidence.transientFaultProxyStopped) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-swarm-transient-retry-proxy-not-stopped');
|
|
}
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-swarm-transient-retry-proxy-cleanup-failed',
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
if (supervisorSwarmToolPlanHandoffProxyNeedsCleanup()) {
|
|
try {
|
|
await state.supervisorSwarm.toolPlanHandoffProxy.stop();
|
|
const proxyStats =
|
|
state.supervisorSwarm.toolPlanHandoffProxy.getStats();
|
|
state.evidence.toolPlanHandoffProxyRequestCount =
|
|
proxyStats.requestCount;
|
|
state.evidence.toolPlanHandoffProxyForwardedRequestCount =
|
|
proxyStats.forwardedRequestCount;
|
|
state.evidence.toolPlanHandoffProxyStopped =
|
|
proxyStats.stopped === true;
|
|
if (!state.evidence.toolPlanHandoffProxyStopped) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-swarm-tool-plan-handoff-proxy-not-stopped');
|
|
}
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-swarm-tool-plan-handoff-proxy-cleanup-failed',
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
if (
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
|
|
state.isolatedRunner.appDataDir
|
|
) {
|
|
const snapshot =
|
|
state.supervisorAutonomousPlayable.ownedProcessCleanupSnapshot;
|
|
if (snapshot) {
|
|
try {
|
|
const cleanup = await verifyOwnedProcessCleanupSnapshot(snapshot);
|
|
state.evidence.ownedProcessIdentityCaptured = true;
|
|
state.evidence.ownedRunnerObservedCount =
|
|
snapshot.observedCounts.runner;
|
|
state.evidence.ownedHelperObservedCount =
|
|
snapshot.observedCounts.helper;
|
|
state.evidence.ownedNodeDescendantObservedCount =
|
|
snapshot.observedCounts.node;
|
|
state.evidence.ownedBrowserDescendantObservedCount =
|
|
snapshot.observedCounts.browser;
|
|
state.evidence.ownedCommandDescendantObservedCount =
|
|
snapshot.observedCounts.command;
|
|
state.evidence.ownedRunnerResidualCount =
|
|
cleanup.residualCounts.runner;
|
|
state.evidence.ownedHelperResidualCount =
|
|
cleanup.residualCounts.helper;
|
|
state.evidence.ownedNodeDescendantResidualCount =
|
|
cleanup.residualCounts.node;
|
|
state.evidence.ownedBrowserDescendantResidualCount =
|
|
cleanup.residualCounts.browser;
|
|
state.evidence.ownedCommandDescendantResidualCount =
|
|
cleanup.residualCounts.command;
|
|
state.evidence.activeCommandChildrenAfterCleanup =
|
|
cleanup.activeCommandChildCount;
|
|
state.evidence.activeInteractiveCliSessionsAfterCleanup =
|
|
cleanup.activeInteractiveCliSessionCount;
|
|
state.evidence.ownedProcessCleanupPassed = cleanup.clean;
|
|
if (!cleanup.clean) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-autonomous-playable-owned-process-residual-detected',
|
|
);
|
|
}
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-autonomous-playable-owned-process-verification-failed',
|
|
error,
|
|
);
|
|
}
|
|
} else {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
'supervisor-autonomous-playable-owned-process-snapshot-missing',
|
|
);
|
|
}
|
|
}
|
|
if (
|
|
isSteerRunnerKillSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS'
|
|
) {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialSteerRunnerKillEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('steer-runner-kill-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (isGoalRuntimeSuite() && state.projectRoot && state.status !== 'PASS') {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialGoalEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('goal-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (
|
|
isResponseStreamSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS'
|
|
) {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialResponseStreamEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('response-stream-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (isWebSearchSuite() && state.projectRoot && state.status !== 'PASS') {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialWebSearchEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('web-search-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (
|
|
isContextCompactionSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS'
|
|
) {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialContextCompactionEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('context-compaction-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (
|
|
isUserInputRuntimeSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS'
|
|
) {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialUserInputEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('user-input-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (isScopedAgentsSuite() && state.projectRoot && state.status !== 'PASS') {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialScopedAgentsEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('scoped-agents-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (isProjectSkillSuite() && state.projectRoot && state.status !== 'PASS') {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialProjectSkillEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('project-skill-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (isParallelReadSuite() && state.projectRoot && state.status !== 'PASS') {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialParallelReadEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError('parallel-read-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS' &&
|
|
state.evidence.evidenceCompleteness !== 'complete'
|
|
) {
|
|
try {
|
|
state.evidence = {
|
|
...state.evidence,
|
|
...(await collectPartialSupervisorAutonomousPlayableEvidence()),
|
|
};
|
|
} catch (error) {
|
|
recordError(
|
|
'supervisor-autonomous-playable-partial-evidence-read-failed',
|
|
error,
|
|
);
|
|
}
|
|
}
|
|
if (
|
|
isSupervisorSwarmSuite() &&
|
|
state.projectRoot &&
|
|
state.status !== 'PASS'
|
|
) {
|
|
try {
|
|
state.evidence = await collectPartialSupervisorSwarmEvidence(
|
|
state.evidence,
|
|
);
|
|
} catch (error) {
|
|
recordError('supervisor-swarm-partial-evidence-read-failed', error);
|
|
}
|
|
}
|
|
if (state.projectRoot && state.secrets.length > 0) {
|
|
try {
|
|
state.projectLeakCount = await countSecretsInProject(
|
|
state.projectRoot,
|
|
state.secrets,
|
|
);
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError('project-secret-scan-failed', error);
|
|
}
|
|
}
|
|
state.transcriptLeakCount = state.transcriptScanner?.count ?? 0;
|
|
state.projectPathTranscriptLeakCount =
|
|
state.projectPathTranscriptScanner?.count ?? 0;
|
|
state.formalConfigPathTranscriptLeakCount =
|
|
state.formalConfigPathTranscriptScanner?.count ?? 0;
|
|
if (state.transcriptLeakCount + state.projectLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('loaded-key-leak-detected');
|
|
}
|
|
if (state.projectPathTranscriptLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('disposable-project-path-transcript-leak-detected');
|
|
}
|
|
if (state.formalConfigPathTranscriptLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('formal-config-path-transcript-leak-detected');
|
|
}
|
|
const isolatedRunnerAllowsProjectCleanup =
|
|
!isIsolatedRunnerSuite() ||
|
|
!state.isolatedRunner.appDataDir ||
|
|
state.isolatedRunner.stopped;
|
|
if (
|
|
state.projectRoot &&
|
|
!state.options?.keepProject &&
|
|
isolatedRunnerAllowsProjectCleanup
|
|
) {
|
|
try {
|
|
state.cleanupPerformed = await removeDisposableProject();
|
|
if (!state.cleanupPerformed) {
|
|
state.status = 'FAIL';
|
|
recordError('cleanup-sentinel-missing');
|
|
}
|
|
} catch (error) {
|
|
state.status = 'FAIL';
|
|
recordError('cleanup-failed', error);
|
|
}
|
|
}
|
|
|
|
let summary = buildSummary();
|
|
let report = JSON.stringify(summary, null, 2);
|
|
if (isProcessSessionSuite() && state.process.challenge) {
|
|
state.process.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
state.process.challenge,
|
|
state.process.readyLine,
|
|
state.process.echoLine,
|
|
processStoppedMarker,
|
|
].filter(Boolean),
|
|
);
|
|
state.evidence.processReportLeakCount = state.process.reportLeakCount;
|
|
if (state.process.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('process-private-output-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
state.commandMarkerReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[commandRootErrorMarker],
|
|
);
|
|
if (state.commandMarkerReportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('command-output-marker-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
state.steerInstructionReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[steerInstruction],
|
|
);
|
|
state.evidence.steerInstructionReportLeakCount =
|
|
state.steerInstructionReportLeakCount;
|
|
if (state.steerInstructionReportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('steer-instruction-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
if (isGoalRuntimeSuite()) {
|
|
state.goalBodyReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
goalPrivateBodyValues(),
|
|
);
|
|
state.evidence.goalBodyReportLeakCount = state.goalBodyReportLeakCount;
|
|
if (state.goalBodyReportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('goal-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isResponseStreamSuite()) {
|
|
state.responseStream.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
state.responseStream.finalText,
|
|
...responseStreamThinkingMarkers,
|
|
].filter(isNonEmptyString),
|
|
);
|
|
state.evidence.responseStreamReportLeakCount =
|
|
state.responseStream.reportLeakCount;
|
|
if (state.responseStream.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('response-stream-private-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isWebSearchSuite()) {
|
|
state.webSearch.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
webSearchPrivateLeakValues(),
|
|
);
|
|
state.evidence.webSearchReportLeakCount = state.webSearch.reportLeakCount;
|
|
if (state.webSearch.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('web-search-private-context-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isContextCompactionSuite()) {
|
|
state.contextCompaction.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
contextCompactionConstraintCanary,
|
|
...state.contextCompaction.privateSummaries,
|
|
].filter(isNonEmptyString),
|
|
);
|
|
state.evidence.contextCompactionReportLeakCount =
|
|
state.contextCompaction.reportLeakCount;
|
|
if (state.contextCompaction.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('context-compaction-private-context-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isUserInputRuntimeSuite()) {
|
|
state.userInput.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
userInputAnswerCanary,
|
|
userInputAnswerText,
|
|
...state.userInput.privateValues,
|
|
].filter(isNonEmptyString),
|
|
);
|
|
state.evidence.userInputReportLeakCount = state.userInput.reportLeakCount;
|
|
if (state.userInput.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('user-input-private-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isScopedAgentsSuite()) {
|
|
state.scopedAgents.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.scopedAgents.privateValues,
|
|
);
|
|
state.evidence.scopedAgentsReportLeakCount =
|
|
state.scopedAgents.reportLeakCount;
|
|
if (state.scopedAgents.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('scoped-agents-private-instruction-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isProjectSkillSuite()) {
|
|
state.projectSkill.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.projectSkill.privateValues,
|
|
);
|
|
state.evidence.projectSkillReportLeakCount =
|
|
state.projectSkill.reportLeakCount;
|
|
if (state.projectSkill.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('project-skill-private-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isParallelReadSuite()) {
|
|
state.parallelRead.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.parallelRead.privateValues,
|
|
);
|
|
state.evidence.parallelReadReportLeakCount =
|
|
state.parallelRead.reportLeakCount;
|
|
if (state.parallelRead.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('parallel-read-private-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isSupervisorAutonomousPlayableLaneDefenseSuite()) {
|
|
state.supervisorAutonomousPlayable.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.supervisorAutonomousPlayable.privateValues,
|
|
);
|
|
state.evidence.supervisorAutonomousPlayableReportLeakCount =
|
|
state.supervisorAutonomousPlayable.reportLeakCount;
|
|
if (state.supervisorAutonomousPlayable.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-autonomous-playable-report-body-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
if (isSupervisorSwarmSuite()) {
|
|
state.supervisorSwarm.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.supervisorSwarm.privateValues,
|
|
);
|
|
state.evidence.supervisorSwarmReportLeakCount =
|
|
state.supervisorSwarm.reportLeakCount;
|
|
if (state.supervisorSwarm.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('supervisor-swarm-private-body-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
state.projectPathReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
disposableProjectPathVariants(),
|
|
);
|
|
state.evidence.projectPathReportLeakCount =
|
|
state.projectPathReportLeakCount;
|
|
if (state.projectPathReportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('disposable-project-path-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
if (
|
|
isWebSearchSuite() ||
|
|
isContextCompactionSuite() ||
|
|
isUserInputRuntimeSuite() ||
|
|
isScopedAgentsSuite() ||
|
|
isProjectSkillSuite() ||
|
|
isParallelReadSuite() ||
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
|
|
isSupervisorSwarmSuite() ||
|
|
isSteerRunnerKillSuite()
|
|
) {
|
|
state.formalConfigPathReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
formalConfigPathVariants(),
|
|
);
|
|
state.evidence.formalConfigPathReportLeakCount =
|
|
state.formalConfigPathReportLeakCount;
|
|
if (state.formalConfigPathReportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('formal-config-path-report-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
}
|
|
state.reportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
state.secrets,
|
|
);
|
|
if (state.reportLeakCount > 0) {
|
|
state.status = 'FAIL';
|
|
recordError('report-key-leak-detected');
|
|
summary = buildSummary();
|
|
report = JSON.stringify(summary, null, 2);
|
|
}
|
|
const remainingProjectPathReportLeakCount = countExactSecrets(
|
|
Buffer.from(report),
|
|
disposableProjectPathVariants(),
|
|
);
|
|
const remainingResponseStreamReportLeakCount = isResponseStreamSuite()
|
|
? countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
state.responseStream.finalText,
|
|
...responseStreamThinkingMarkers,
|
|
].filter(isNonEmptyString),
|
|
)
|
|
: 0;
|
|
const remainingWebSearchReportLeakCount = isWebSearchSuite()
|
|
? countExactSecrets(Buffer.from(report), webSearchPrivateLeakValues())
|
|
: 0;
|
|
const remainingUserInputReportLeakCount = isUserInputRuntimeSuite()
|
|
? countExactSecrets(
|
|
Buffer.from(report),
|
|
[
|
|
userInputAnswerCanary,
|
|
userInputAnswerText,
|
|
...state.userInput.privateValues,
|
|
].filter(isNonEmptyString),
|
|
)
|
|
: 0;
|
|
const remainingScopedAgentsReportLeakCount = isScopedAgentsSuite()
|
|
? countExactSecrets(Buffer.from(report), state.scopedAgents.privateValues)
|
|
: 0;
|
|
const remainingProjectSkillReportLeakCount = isProjectSkillSuite()
|
|
? countExactSecrets(Buffer.from(report), state.projectSkill.privateValues)
|
|
: 0;
|
|
const remainingParallelReadReportLeakCount = isParallelReadSuite()
|
|
? countExactSecrets(Buffer.from(report), state.parallelRead.privateValues)
|
|
: 0;
|
|
const remainingSupervisorAutonomousPlayableReportLeakCount =
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite()
|
|
? countExactSecrets(
|
|
Buffer.from(report),
|
|
state.supervisorAutonomousPlayable.privateValues,
|
|
)
|
|
: 0;
|
|
const remainingSupervisorSwarmReportLeakCount = isSupervisorSwarmSuite()
|
|
? countExactSecrets(
|
|
Buffer.from(report),
|
|
state.supervisorSwarm.privateValues,
|
|
)
|
|
: 0;
|
|
const remainingFormalConfigPathReportLeakCount =
|
|
isWebSearchSuite() ||
|
|
isContextCompactionSuite() ||
|
|
isUserInputRuntimeSuite() ||
|
|
isScopedAgentsSuite() ||
|
|
isProjectSkillSuite() ||
|
|
isParallelReadSuite() ||
|
|
isSupervisorAutonomousPlayableLaneDefenseSuite() ||
|
|
isSupervisorSwarmSuite() ||
|
|
isSteerRunnerKillSuite()
|
|
? countExactSecrets(Buffer.from(report), formalConfigPathVariants())
|
|
: 0;
|
|
if (
|
|
remainingProjectPathReportLeakCount > 0 ||
|
|
remainingResponseStreamReportLeakCount > 0 ||
|
|
remainingWebSearchReportLeakCount > 0 ||
|
|
remainingUserInputReportLeakCount > 0 ||
|
|
remainingScopedAgentsReportLeakCount > 0 ||
|
|
remainingProjectSkillReportLeakCount > 0 ||
|
|
remainingParallelReadReportLeakCount > 0 ||
|
|
remainingSupervisorAutonomousPlayableReportLeakCount > 0 ||
|
|
remainingSupervisorSwarmReportLeakCount > 0 ||
|
|
remainingFormalConfigPathReportLeakCount > 0
|
|
) {
|
|
state.status = 'FAIL';
|
|
recordError(
|
|
remainingProjectPathReportLeakCount > 0
|
|
? 'disposable-project-path-report-redaction-required'
|
|
: remainingResponseStreamReportLeakCount > 0
|
|
? 'response-stream-report-redaction-required'
|
|
: remainingUserInputReportLeakCount > 0
|
|
? 'user-input-report-redaction-required'
|
|
: remainingScopedAgentsReportLeakCount > 0
|
|
? 'scoped-agents-report-redaction-required'
|
|
: remainingProjectSkillReportLeakCount > 0
|
|
? 'project-skill-report-redaction-required'
|
|
: remainingParallelReadReportLeakCount > 0
|
|
? 'parallel-read-report-redaction-required'
|
|
: remainingSupervisorAutonomousPlayableReportLeakCount > 0
|
|
? 'supervisor-autonomous-playable-report-redaction-required'
|
|
: remainingSupervisorSwarmReportLeakCount > 0
|
|
? 'supervisor-swarm-report-redaction-required'
|
|
: remainingFormalConfigPathReportLeakCount > 0
|
|
? 'formal-config-path-report-redaction-required'
|
|
: 'web-search-report-redaction-required',
|
|
);
|
|
const safeSummary = {
|
|
status: state.status,
|
|
suite: state.suite,
|
|
providerUsed: false,
|
|
blocked: state.blocked,
|
|
cleanup: {
|
|
performed: state.cleanupPerformed,
|
|
kept: Boolean(state.options?.keepProject),
|
|
},
|
|
evidence: {
|
|
projectPathReportLeakCount: remainingProjectPathReportLeakCount,
|
|
responseStreamReportLeakCount: remainingResponseStreamReportLeakCount,
|
|
webSearchReportLeakCount: remainingWebSearchReportLeakCount,
|
|
userInputReportLeakCount: remainingUserInputReportLeakCount,
|
|
scopedAgentsReportLeakCount: remainingScopedAgentsReportLeakCount,
|
|
projectSkillReportLeakCount: remainingProjectSkillReportLeakCount,
|
|
parallelReadReportLeakCount: remainingParallelReadReportLeakCount,
|
|
supervisorAutonomousPlayableReportLeakCount:
|
|
remainingSupervisorAutonomousPlayableReportLeakCount,
|
|
supervisorSwarmReportLeakCount:
|
|
remainingSupervisorSwarmReportLeakCount,
|
|
formalConfigPathReportLeakCount:
|
|
remainingFormalConfigPathReportLeakCount,
|
|
...(isSupervisorSwarmSuite()
|
|
? supervisorSwarmChatSessionFailureEvidence()
|
|
: {}),
|
|
},
|
|
errorCount: state.errors.length,
|
|
errorHashes: state.errors.map(summarizeRecordedError),
|
|
};
|
|
safeSummary.summaryHash = hashValue(JSON.stringify(safeSummary));
|
|
report = JSON.stringify(safeSummary, null, 2);
|
|
}
|
|
process.stdout.write(`${report}\n`);
|
|
process.exitCode = state.shutdownSignal
|
|
? state.shutdownSignal === 'SIGINT'
|
|
? 130
|
|
: 143
|
|
: state.status === 'PASS'
|
|
? 0
|
|
: state.status === 'BLOCKED'
|
|
? 2
|
|
: 1;
|
|
}
|
|
}
|