import { assert, codedError, hashValue, isFailedTask, sleep, } from '../assertions/core.mjs'; import { actionAuditIdentity, actionReceiptIdentity, assertNoPersistedImagePayload, auditInputValue, auditPatchsetPathsMatch, auditPathEquals, auditPathListMatches, backgroundTaskMessageId, countBy, countExactSecrets, disposableProjectPathVariants, duplicateCount, finalMessageId, findSuccessfulToolExecution, isNonEmptyString, isolatedJoinDeliveryTarget, receiptAuditIdentity, requireExecutionRecord, requireSuccessfulToolExecution, sumObjectValues, validateActionHistoryObservations, validateConfirmedActionLifecycles, validateGitCommitEvidence, validateGitInspectEvents, validateImageInspectAudit, validateMainRunActionReceipts, validateMainRunToolPlanProtocols, validatePatchsetAudit, validatePatchsetContentDiff, validatePreviewScreenshotPaths, validateSameRunSteerEvidence, validateStructuredPlanEvidence, validateToolActionReplays, } from '../assertions/runtime.mjs'; import { createHash, createReadStream, fs, path, randomUUID, } from '../dependencies.mjs'; import { commandFailureMarker, commandPassedMarker, commandRootErrorLine, commandRootErrorMarker, configFileName, editorAssetPrompt, gitSensitivePath, goalFinalMarker, goalProjectWriteTools, idempotentObservationTools, mainAgentId, patchedText, patchsetCreatedContent, patchsetCreatedPath, pngSignature, pollIntervalMs, runtimeContextBundleSchemaVersion, runTimeoutMs, state, steerInstruction, StreamingSecretScanner, supervisorSwarmConfirmedTools, verificationCommand, visibleText, } from '../runtime-state.mjs'; import { assertGoalInitialMarkerAbsent, goalPendingIsStandaloneDeliveryMutation, isGoalRuntimeSuite, monitorGoalWriteActionUntilSettled, } from '../suites/goal.mjs'; import { assertProcessLaunchEvidenceIsNotDuplicated, isProcessSessionSuite, processLaunchEvidence, } from '../suites/process-session.mjs'; import { isProjectSkillSuite } from '../suites/project-skill.mjs'; import { isScopedAgentsSuite } from '../suites/scoped-agents.mjs'; import { isSteerRunnerKillSuite } from '../suites/steer-runner-kill.mjs'; import { confirmSupervisorSwarmPendingActionsInChat, isSupervisorSwarmInteractiveChatSuite, isSupervisorSwarmSuite, } from '../suites/supervisor-swarm.mjs'; import { killRunnerPidOnce, verifyOwnedRunnerForKill } from './app-data.mjs'; import { isPlainObject } from './config.mjs'; import { isLiveTask, isTerminalRuntime, listFiles, parseAssignedJson, readJson, readJsonl, readOptionalJsonl, relativeProjectPath, resolveProjectRelative, validateCommandOutputSidecar, } from './io.mjs'; import { runCli, runProcess } from './process.mjs'; import { assertUnscriptedSteerInstruction, seededGameHtml, } from './project.mjs'; import { isIsolatedRunnerSuite } from './reporting.mjs'; export async function readRuntime(agentId) { const result = await runCli( ['--agent-runtime-status', state.projectRoot, agentId], { timeoutMs: 60_000 }, ); const value = parseAssignedJson(result.stdout, ['runtimeJson']); const runtime = value?.state; assert(runtime && typeof runtime === 'object', 'runtime-json-invalid'); return runtime; } export function mainRuntimeStatePath() { return path.join( state.projectRoot, '.agent/runtime/agents', `${mainAgentId}.json`, ); } export function mainContextBundlePath() { return path.join( state.projectRoot, '.agent/runtime/context-bundles', mainAgentId, `${state.initialRunId}.json`, ); } export function agentConversationPath(agentId, sessionId) { return sessionId === `agent-session-${agentId}` ? path.join( state.projectRoot, '.agent/conversations/agents', `${agentId}.jsonl`, ) : path.join( state.projectRoot, '.agent/conversations/agents', agentId, 'sessions', `${sessionId}.jsonl`, ); } export async function readRunnerStatus() { const result = await runCli(['--runner-status'], { timeoutMs: 60_000 }); return parseAssignedJson(result.stdout, ['runnerJson']); } export async function waitForCanonicalRuntime({ allowTerminal = false } = {}) { const deadline = Date.now() + 120_000; while (Date.now() < deadline) { const runtime = await readRuntime(mainAgentId).catch(() => null); if ( runtime && typeof runtime.runId === 'string' && runtime.runId.length > 0 && typeof runtime.sessionId === 'string' && runtime.sessionId.length > 0 && (allowTerminal || !isTerminalRuntime(runtime)) ) { return runtime; } await sleep(pollIntervalMs); } throw codedError('runtime-did-not-start'); } export async function killRunnerOnce() { const ownedRunner = isIsolatedRunnerSuite() ? await verifyOwnedRunnerForKill() : null; const runner = ownedRunner ? null : await readRunnerStatus(); const pid = ownedRunner ? ownedRunner.pid : Number(runner?.pid ?? runner?.status?.pid); if (!ownedRunner) { assert( Number.isSafeInteger(pid) && pid > 1 && pid !== process.pid, 'runner-pid-invalid', ); } await killRunnerPidOnce(pid, Boolean(ownedRunner)); } export async function stopExistingRunnerBeforeRuntimeSuite() { const runner = await readRunnerStatus().catch(() => null); const pid = Number(runner?.pid ?? runner?.status?.pid); if (!Number.isSafeInteger(pid) || pid <= 1 || pid === process.pid) return; try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error?.code === 'ESRCH') return; throw codedError('stale-runner-sigkill-failed', error); } const deadline = Date.now() + 10_000; while (Date.now() < deadline) { try { process.kill(pid, 0); } catch { return; } await sleep(50); } throw codedError('stale-runner-still-alive-after-sigkill'); } export async function waitForRuntimeIdentity() { const deadline = Date.now() + 120_000; while (Date.now() < deadline) { const runtime = await readRuntime(mainAgentId).catch(() => null); if (runtime?.runId && runtime?.sessionId) return runtime; await sleep(pollIntervalMs); } throw codedError('runtime-not-readable-after-resume'); } export async function waitForPartiallyCompletedStructuredPlan() { const deadline = Date.now() + runTimeoutMs; let lastError = null; while (Date.now() < deadline) { const taskSnapshot = await readTaskSnapshot(); const initial = taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); if (initial && isFailedTask(initial)) { throw codedError('main-runtime-failed-before-plan-checkpoint'); } if ( initial?.status === 'completed' || initial?.phase === 'completed' || initial?.status === 'cancelled' ) { throw codedError('main-runtime-terminal-before-plan-checkpoint'); } try { const plan = await readVerifiedDurablePlanSnapshot('pre-kill-plan'); if (plan.completedStepHashes.length > 0 && plan.incompleteStepCount > 0) { const messages = await readOptionalJsonl( agentConversationPath(mainAgentId, state.initialSessionId), ); assert( messages.filter( (message) => message.role === 'assistant' && message.agentId === mainAgentId, ).length === 0, 'assistant-persisted-before-plan-checkpoint', ); return plan; } } catch (error) { lastError = error; } await captureCommandOutputContextEvidence(); await confirmPendingActions(); await sleep(pollIntervalMs); } throw codedError('partial-structured-plan-before-kill-timeout', lastError); } export async function waitForRecoveredStructuredPlan(preKillPlan) { const deadline = Date.now() + 120_000; let lastError = null; while (Date.now() < deadline) { let recovered; try { recovered = await readVerifiedDurablePlanSnapshot('recovered-plan'); } catch (error) { lastError = error; await sleep(pollIntervalMs); continue; } assert( recovered.revision >= preKillPlan.revision, 'recovered-plan-revision-regressed', ); const recoveredCompleted = new Set(recovered.completedStepHashes); assert( preKillPlan.completedStepHashes.every((stepHash) => recoveredCompleted.has(stepHash), ), 'recovered-plan-completed-step-lost', ); if (!isSteerableRuntime(recovered.runtime)) { if (isTerminalRuntime(recovered.runtime)) { throw codedError('recovered-runtime-terminal-before-steer'); } lastError = codedError('recovered-runtime-not-yet-steerable'); await sleep(pollIntervalMs); continue; } const taskSnapshot = await readTaskSnapshot(); const initial = taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); if (initial?.sessionId !== state.initialSessionId || !isLiveTask(initial)) { lastError = codedError('recovered-plan-task-not-yet-live'); await sleep(pollIntervalMs); continue; } return recovered; } throw codedError('recovered-structured-plan-timeout', lastError); } export async function readVerifiedDurablePlanSnapshot(codePrefix) { const [runtime, contextBundle, records] = await Promise.all([ readJson(mainRuntimeStatePath()), readJson(mainContextBundlePath()), readOptionalJsonl(path.join(state.projectRoot, '.agent/agent.db')), ]); const snapshot = inspectStructuredPlanSnapshot(runtime, codePrefix); assertStructuredPlanContextSnapshot( runtime, contextBundle, `${codePrefix}-context`, ); assertStructuredPlanAuditSnapshot(runtime, records, `${codePrefix}-audit`); return { ...snapshot, runtime }; } export function inspectStructuredPlanSnapshot(runtime, codePrefix) { assert( runtime.agentId === mainAgentId && runtime.runId === state.initialRunId && runtime.sessionId === state.initialSessionId, `${codePrefix}-identity-invalid`, ); assert( Number.isSafeInteger(runtime.planRevision) && runtime.planRevision > 0, `${codePrefix}-revision-invalid`, ); assert( isNonEmptyString(runtime.planExplanation) && Array.isArray(runtime.planSteps) && runtime.planSteps.length >= 3 && runtime.planSteps.length <= 8, `${codePrefix}-snapshot-invalid`, ); const stepHashes = []; const completedStepHashes = []; const incompleteSteps = []; let activePlanStepIndex = null; for (const [index, step] of runtime.planSteps.entries()) { assert( step.index === index && isNonEmptyString(step.title) && ['pending', 'in_progress', 'completed'].includes(step.status), `${codePrefix}-step-invalid`, ); const stepHash = hashValue(step.title); assert(!stepHashes.includes(stepHash), `${codePrefix}-step-duplicate`); stepHashes.push(stepHash); if (step.status === 'completed') completedStepHashes.push(stepHash); else incompleteSteps.push({ index, status: step.status, stepHash }); if (step.status === 'in_progress') { assert( activePlanStepIndex === null, `${codePrefix}-multiple-in-progress`, ); activePlanStepIndex = index; } } assert( runtime.activePlanStepIndex === activePlanStepIndex, `${codePrefix}-active-step-invalid`, ); completedStepHashes.sort(); return { revision: runtime.planRevision, completedStepHashes, incompleteStepCount: runtime.planSteps.length - completedStepHashes.length, incompleteSteps, terminalStepHash: hashValue(JSON.stringify(completedStepHashes)), }; } export function assertStructuredPlanContextSnapshot( runtime, contextBundle, code, ) { assert( contextBundle.schemaVersion === runtimeContextBundleSchemaVersion && contextBundle.agentId === runtime.agentId && contextBundle.taskId === runtime.taskId && contextBundle.sessionId === runtime.sessionId && contextBundle.runId === runtime.runId && contextBundle.planRevision === runtime.planRevision && contextBundle.planExplanation === runtime.planExplanation && JSON.stringify(contextBundle.planSteps) === JSON.stringify(runtime.planSteps) && contextBundle.activePlanStepIndex === runtime.activePlanStepIndex, `${code}-snapshot-mismatch`, ); } export function assertStructuredPlanAuditSnapshot(runtime, records, code) { const audits = records.filter( (record) => record.recordType === 'agent.runtime.plan_update' && record.agentId === runtime.agentId && record.taskId === runtime.taskId && record.sessionId === runtime.sessionId && record.runId === runtime.runId && record.planRevision === runtime.planRevision, ); assert(audits.length === 1, `${code}-record-count-invalid`); const audit = audits[0]; assert( audit.explanationSha256 === hashValue(runtime.planExplanation) && audit.explanationChars === [...runtime.planExplanation].length && Array.isArray(audit.steps) && audit.steps.length === runtime.planSteps.length && audit.steps.every( (step, index) => step.stepSha256 === hashValue(runtime.planSteps[index].title) && step.status === runtime.planSteps[index].status, ), `${code}-snapshot-mismatch`, ); } export async function injectSameRunSteer() { assertUnscriptedSteerInstruction(steerInstruction); const steerTargetPlan = await waitForProviderPlanningSteerTarget(); const runtime = steerTargetPlan.runtime; assert( runtime.runId === state.initialRunId && runtime.sessionId === state.initialSessionId && isProviderPlanningWait(runtime) && steerTargetPlan.incompleteStepCount > 0, 'steer-target-runtime-invalid', ); const before = steerTargetPlan.targetRunIds; assert( JSON.stringify(before) === JSON.stringify([state.initialRunId]), 'steer-target-task-missing', ); const steerId = `real-e2e-steer-${randomUUID().slice(0, 12)}`; const result = await runCli( [ '--agent-steer', state.projectRoot, mainAgentId, state.initialSessionId, state.initialRunId, steerId, '--stdin', ], { timeoutMs: 120_000, stdin: steerInstruction }, ); assert( !result.stdout.includes(steerInstruction) && !result.stderr.includes(steerInstruction), 'steer-instruction-cli-leak', ); const steer = parseAssignedJson(result.stdout, ['steerJson']); assert( steer?.steerId === steerId && steer.sequence === 1 && ['queued', 'applied'].includes(steer.status) && steer.providerInterrupted === true, 'steer-cli-result-invalid', ); const afterSnapshot = await readTaskSnapshot(); const after = targetMainTaskRunIds( afterSnapshot, steerTargetPlan.taskIdentity, ); assert( JSON.stringify(after) === JSON.stringify(before), 'steer-created-new-task-run', ); const afterRuntime = await readRuntime(mainAgentId); assert( afterRuntime.runId === state.initialRunId && afterRuntime.sessionId === state.initialSessionId && runtime.taskQueue && afterRuntime.taskQueue && afterRuntime.taskQueue.total === runtime.taskQueue.total && afterRuntime.taskQueue.latestRunId === runtime.taskQueue.latestRunId, 'steer-changed-runtime-or-task-queue', ); state.steer = { steerId, steerIdHash: hashValue(steerId), instructionSha256: hashValue(steerInstruction), sequence: steer.sequence, providerInterrupted: steer.providerInterrupted, planRevisionAtAcceptance: runtime.planRevision, completedStepHashesAtAcceptance: steerTargetPlan.completedStepHashes, incompleteStepsAtAcceptance: steerTargetPlan.incompleteSteps, incompletePlanSignatureAtAcceptance: hashValue( JSON.stringify(steerTargetPlan.incompleteSteps), ), providerWaitStatus: runtime.status, providerWaitPhase: runtime.phase, providerWaitUpdatedAt: runtime.updatedAt, agentDbSequenceBefore: steerTargetPlan.agentDbSequenceBefore, taskIdentity: steerTargetPlan.taskIdentity, initialMessageId: steerTargetPlan.initialMessageId, activeActionsAtAcceptance: steerTargetPlan.activeActions, durableActionsAtAcceptance: steerTargetPlan.durableActions, sideEffectReceiptsAtAcceptance: steerTargetPlan.sideEffectReceipts, projectRevisionAtAcceptance: steerTargetPlan.projectRevision, projectSideEffectFingerprintAtAcceptance: steerTargetPlan.projectSideEffectFingerprint, taskRunIdsBefore: before, taskRunIdsAfter: after, taskRunSetHash: hashValue(JSON.stringify(before)), }; } export async function waitForRecoveredSteeredPlan() { const deadline = Date.now() + 6 * 60 * 1000; let lastError = null; while (Date.now() < deadline) { const [taskSnapshot, runtime] = await Promise.all([ readTaskSnapshot().catch(() => null), readJson(mainRuntimeStatePath()).catch(() => null), ]); const task = taskSnapshot?.latest.find( (candidate) => candidate.agentId === mainAgentId && candidate.runId === state.initialRunId, ); if (task && isFailedTask(task)) { throw codedError('steer-runner-kill-runtime-failed-after-restart'); } if ( runtime?.phase === 'needs-reconciliation' || runtime?.status === 'needs-reconciliation' ) { throw codedError('steer-runner-kill-runtime-needs-reconciliation'); } if (task && !isLiveTask(task)) { throw codedError('steer-runner-kill-runtime-terminal-before-recovery'); } try { const recovered = await readVerifiedDurablePlanSnapshot( 'steer-runner-kill-recovered-plan', ); const recoveredCompleted = new Set(recovered.completedStepHashes); if ( recovered.runtime.appliedSteerCursor === state.steer.sequence && recovered.revision > state.steer.planRevisionAtAcceptance && state.steer.completedStepHashesAtAcceptance.every((stepHash) => recoveredCompleted.has(stepHash), ) && recovered.incompleteStepCount > 0 && isSteerableRuntime(recovered.runtime) ) { return recovered; } lastError = codedError('steer-runner-kill-recovery-not-yet-durable'); } catch (error) { lastError = error; } await sleep(pollIntervalMs); } throw codedError('steer-runner-kill-recovery-timeout', lastError); } export async function waitForProviderPlanningSteerTarget() { const deadline = Date.now() + runTimeoutMs; let lastError = null; while (Date.now() < deadline) { const taskSnapshot = await readTaskSnapshot(); const initial = taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); if (initial && isFailedTask(initial)) { throw codedError('main-runtime-failed-before-provider-wait'); } if (!initial || !isLiveTask(initial)) { lastError = codedError('main-runtime-not-live-before-provider-wait'); await sleep(pollIntervalMs); continue; } try { const plan = await readVerifiedDurablePlanSnapshot( 'steer-provider-wait-plan', ); if ( plan.completedStepHashes.length > 0 && plan.incompleteStepCount > 0 && isProviderPlanningWait(plan.runtime) ) { const snapshot = await capturePreSteerSnapshot( plan, initial, taskSnapshot, ); const current = await readJson(mainRuntimeStatePath()); if ( isProviderPlanningWait(current) && current.runId === plan.runtime.runId && current.sessionId === plan.runtime.sessionId && current.planRevision === plan.runtime.planRevision ) { return { ...plan, ...snapshot, runtime: current }; } } } catch (error) { lastError = error; } await captureCommandOutputContextEvidence(); await confirmPendingActions(); await sleep(pollIntervalMs); } throw codedError('provider-planning-wait-before-steer-timeout', lastError); } export async function capturePreSteerSnapshot(plan, initial, taskSnapshot) { const taskIdentity = { agentId: mainAgentId, taskId: initial.taskId, sessionId: state.initialSessionId, source: initial.source, }; const targetRunIds = targetMainTaskRunIds(taskSnapshot, taskIdentity); assert( JSON.stringify(targetRunIds) === JSON.stringify([state.initialRunId]), 'pre-steer-target-run-set-invalid', ); const conversationPath = agentConversationPath( mainAgentId, state.initialSessionId, ); const messages = await readOptionalJsonl(conversationPath); const initialMessageId = backgroundTaskMessageId( mainAgentId, state.initialSessionId, state.initialRunId, initial.source, ); assert( messages.length === 1 && messages[0].role === 'user' && messages[0].agentId === mainAgentId && messages[0].messageId === initialMessageId && hashValue(messages[0].content) === state.initialTask?.sha256 && [...messages[0].content].length === state.initialTask?.chars, 'pre-steer-initial-conversation-invalid', ); const durableActions = await readTargetDurableActions(); const activeActions = durableActions.filter((action) => ['pending-confirmation', 'approved', 'executing'].includes(action.status), ); assert( activeActions.length === 0 && plan.runtime.pendingToolAction == null && plan.runtime.pendingAction == null, 'provider-planning-wait-has-active-action', ); const records = await readOptionalJsonl( path.join(state.projectRoot, '.agent/agent.db'), ); const sideEffectReceipts = records .map((record, index) => ({ record, sequence: index + 1 })) .filter( ({ record }) => record.recordType === 'agent.runtime.action_receipt' && record.agentId === mainAgentId && record.runId === state.initialRunId && !idempotentObservationTools.has(record.tool), ) .map(({ record, sequence }) => ({ actionFingerprint: record.actionFingerprint, actionId: record.actionId, identityHash: hashValue(actionReceiptIdentity(record)), sequence, status: record.status, tool: record.tool, updatedAt: record.updatedAt, })); const revision = await readJson( path.join(state.projectRoot, '.agent/runtime/project-revision.json'), ); assert( Number.isSafeInteger(revision.revision) && revision.revision >= 0, 'pre-steer-project-revision-invalid', ); const [head, worktree] = await Promise.all([ runProcess('git', ['rev-parse', '--verify', 'HEAD'], { cwd: state.projectRoot, timeoutMs: 30_000, }), runProcess( 'git', [ 'status', '--porcelain=v2', '-z', '--untracked-files=all', '--', '.', ':(exclude).agent', ], { cwd: state.projectRoot, timeoutMs: 30_000, }, ), ]); return { activeActions, agentDbSequenceBefore: records.length, durableActions, initialMessageId, projectRevision: revision.revision, projectSideEffectFingerprint: hashValue( `${head.stdout.trim()}\n${worktree.stdout}`, ), sideEffectReceipts, targetRunIds, taskIdentity, }; } export async function readTargetDurableActions() { const root = path.join(state.projectRoot, '.agent/runtime/pending-actions'); const actions = []; for (const file of (await listFiles(root)).filter((entry) => entry.endsWith('.json'), )) { const value = await readJson(file).catch(() => null); if ( !value || value.agentId !== mainAgentId || value.runId !== state.initialRunId ) { continue; } assert( isNonEmptyString(value.actionId) && isNonEmptyString(value.actionFingerprint) && isNonEmptyString(value.action?.tool) && isNonEmptyString(value.status) && Number.isSafeInteger(value.plannedSteerCursor), 'durable-action-identity-invalid', ); actions.push({ actionFingerprint: value.actionFingerprint, actionId: value.actionId, plannedSteerCursor: value.plannedSteerCursor, status: value.status, tool: value.action.tool, updatedAt: value.updatedAt, }); } return actions.sort((left, right) => left.actionId.localeCompare(right.actionId), ); } export function isProviderPlanningWait(runtime) { return runtime.status === 'running' && runtime.phase === 'planning'; } export function isSteerableRuntime(runtime) { return ( ['running', 'waiting-for-confirmation'].includes(runtime.status) && !['cancelling', 'finalizing', 'needs-reconciliation'].includes( runtime.phase, ) ); } export function targetMainTaskRunIds(taskSnapshot, identity) { return [ ...new Set( taskSnapshot.all .filter( (task) => task.agentId === identity.agentId && task.taskId === identity.taskId && task.sessionId === identity.sessionId && task.source === identity.source, ) .map((task) => task.runId) .filter(isNonEmptyString), ), ].sort(); } export function validateFinalMainRunSet(taskSnapshot, initial, spawnRecord) { const targetRunIds = targetMainTaskRunIds( taskSnapshot, state.steer.taskIdentity, ); assert( JSON.stringify(targetRunIds) === JSON.stringify(state.steer.taskRunIdsBefore) && JSON.stringify(targetRunIds) === JSON.stringify([state.initialRunId]), 'final-target-main-run-set-changed', ); const legalLineageRunIds = new Set([spawnRecord.joinRunId]); for (const child of spawnRecord.children ?? []) { if (isNonEmptyString(child.runId)) legalLineageRunIds.add(child.runId); } const nonTargetMainRecords = taskSnapshot.all.filter( (task) => task.agentId === mainAgentId && !( task.taskId === initial.taskId && task.sessionId === state.initialSessionId && task.source === initial.source && task.runId === state.initialRunId ), ); const legalLineageRecords = nonTargetMainRecords.filter( (task) => task.source === 'agent-isolated-join' && task.runId === spawnRecord.joinRunId && task.sessionId === state.initialSessionId && task.parentRunId === state.initialRunId && task.delegationId === spawnRecord.delegationGroupId, ); const unexpectedRecords = nonTargetMainRecords.filter( (task) => !legalLineageRecords.includes(task), ); assert( unexpectedRecords.length === 0 && legalLineageRecords.every((task) => legalLineageRunIds.has(task.runId)), 'final-unexpected-main-run-detected', ); const legalLineageRuns = new Set( legalLineageRecords.map((task) => task.runId), ); assert( legalLineageRuns.size <= 1, 'final-legal-main-lineage-run-count-invalid', ); return { legalLineageRunCount: legalLineageRuns.size, targetRunCount: targetRunIds.length, targetRunSetHash: hashValue(JSON.stringify(targetRunIds)), unexpectedRunCount: 0, }; } export async function driveRuntimeToQuiescence() { const deadline = Date.now() + runTimeoutMs; let quietPolls = 0; while (Date.now() < deadline) { await captureCommandOutputContextEvidence(); await confirmPendingActions(); const snapshot = await readTaskSnapshot(); const initial = snapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); if (initial && isFailedTask(initial)) { throw codedError('main-runtime-failed'); } const joinTasks = snapshot.latest.filter( (task) => task.agentId === mainAgentId && task.source === 'agent-isolated-join', ); const hasLive = snapshot.latest.some(isLiveTask); const pending = await findPendingActions(); const completed = initial?.status === 'completed' || initial?.phase === 'completed'; const isolatedJoinSettled = completed && (isSteerRunnerKillSuite() || (await isIsolatedJoinSettledForQuiescence(joinTasks))); if (completed && isolatedJoinSettled && !hasLive && pending.length === 0) { quietPolls += 1; if (quietPolls >= 3) return; } else { quietPolls = 0; } await sleep(pollIntervalMs); } throw codedError('runtime-e2e-timeout'); } export async function waitForResponseRuntimeIdentity() { const deadline = Date.now() + 120_000; while (Date.now() < deadline) { const runtime = await readJson(mainRuntimeStatePath()).catch(() => null); if ( runtime?.agentId === mainAgentId && isNonEmptyString(runtime.runId) && isNonEmptyString(runtime.sessionId) ) { return runtime; } await sleep(50); } throw codedError('response-stream-runtime-did-not-start'); } export async function readAllRuntimeEvents() { const eventFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/events'), ); const events = []; for (const file of eventFiles.filter((entry) => entry.endsWith('.jsonl'))) { events.push(...(await readJsonl(file))); } return events; } export function countSensitiveValuesBySurface(surfaces, values, codePrefix) { const counts = {}; for (const [surface, records] of Object.entries(surfaces)) { const entries = Array.isArray(records) ? records : [records]; counts[surface] = countExactSecrets( Buffer.from(entries.map((record) => JSON.stringify(record)).join('\n')), values, ); assert(counts[surface] === 0, `${codePrefix}-${surface}-leak`); } return counts; } export function assertNoProviderSearchArtifactFields(surfaces) { const forbiddenKeys = new Set([ 'query', 'searchquery', 'websearchquery', 'url', 'urls', 'citation', 'citations', 'searchresult', 'searchresults', 'websearchresult', 'websearchresults', 'webpageinstruction', ]); const visit = (value, surface) => { if (Array.isArray(value)) { for (const item of value) visit(item, surface); return; } if (!isPlainObject(value)) return; for (const [key, nested] of Object.entries(value)) { const normalizedKey = key.toLowerCase().replaceAll(/[^a-z]/gu, ''); assert( !forbiddenKeys.has(normalizedKey), `web-search-provider-artifact-field-${surface}-leak`, ); visit(nested, surface); } }; for (const [surface, records] of Object.entries(surfaces)) { visit(records, surface); } } export async function captureCommandOutputContextEvidence() { if (!state.initialRunId) return; const bundlePath = path.join( state.projectRoot, '.agent/runtime/context-bundles', mainAgentId, `${state.initialRunId}.json`, ); const bundle = await readJson(bundlePath).catch(() => null); for (const observation of bundle?.observations ?? []) { if (observation?.tool !== 'command.output_read') continue; const detail = String(observation.detail ?? ''); if (!detail.includes(commandRootErrorMarker)) continue; state.commandOutputMarkerSeenInContext = true; try { const page = JSON.parse(detail); if ( isNonEmptyString(page.sourceActionId) && Number.isSafeInteger(page.startLine) ) { state.commandOutputContextPages.add( `${page.sourceActionId}\0${page.startLine}`, ); } } catch { // The final structural assertion reports malformed page JSON. } } } export async function isIsolatedJoinSettledForQuiescence(joinTasks) { if (joinTasks.length === 1) return true; if (joinTasks.length !== 0) return false; const deliveryFiles = await listFiles( path.join( state.projectRoot, '.agent/runtime/isolated-agents/join-deliveries', ), ); const deliveries = []; for (const file of deliveryFiles.filter((entry) => entry.endsWith('.json'))) { const delivery = await readJson(file).catch(() => null); if (delivery?.parentRunId === state.initialRunId) deliveries.push(delivery); } const delivery = deliveries[0]; const deliveryTarget = isolatedJoinDeliveryTarget(delivery); return ( deliveries.length === 1 && delivery.status === 'claimed-by-parent' && isNonEmptyString(delivery.joinRunId) && isNonEmptyString(delivery.claimedByActionId) && (deliveryTarget !== 'parent-wake' || delivery.queuedRunId == null) ); } export async function confirmPendingActions( allowedTools = null, shouldConfirm = () => true, ) { if (isSupervisorSwarmInteractiveChatSuite()) { await confirmSupervisorSwarmPendingActionsInChat( allowedTools, shouldConfirm, ); return; } for (const pending of await findPendingActions()) { if (state.confirmedActionIds.has(pending.actionId)) continue; if (!shouldConfirm(pending)) continue; const whitelist = new Set([ 'project.patchset', 'project.git_commit', 'command.exec', ...(state.suite === 'llm-runtime' || isSteerRunnerKillSuite() ? ['command.run_limited'] : []), ...(isGoalRuntimeSuite() ? [ 'command.run_limited', 'project.checkpoint', 'file.write', 'file.patch', 'file.delete', ] : []), ...(isScopedAgentsSuite() || isProjectSkillSuite() ? ['project.checkpoint', 'file.write', 'file.patch'] : []), ...(isSupervisorSwarmSuite() ? supervisorSwarmConfirmedTools : []), 'project.verify', 'preview.start', 'preview.validate', 'agent.spawn_isolated', ...(state.suite === 'full' ? ['canvas.asset_generate'] : []), ...(isProcessSessionSuite() ? [ 'command.start', 'command.stdin', 'command.terminate', 'project.verify', ] : []), ]); assert( whitelist.has(pending.tool), `pending-tool-not-whitelisted:${pending.tool}`, ); assert( !allowedTools || allowedTools.has(pending.tool), `pending-tool-not-allowed-in-scenario:${pending.tool}`, ); const runtime = await readRuntime(pending.agentId); assert(runtime.runId === pending.runId, 'pending-run-mismatch'); const runtimePending = runtime.pendingToolAction ?? runtime.pendingAction; if (runtimePending?.actionId) { assert( runtimePending.actionId === pending.actionId, 'pending-action-mismatch', ); assert(runtimePending.tool === pending.tool, 'pending-tool-mismatch'); } const monitorsGoalWrite = isGoalRuntimeSuite() && goalProjectWriteTools.has(pending.tool); if (monitorsGoalWrite) { const allowedRevisionTwoDelivery = state.goal.editedRevision > 0 && state.goal.revisionTwoFailureObserved !== true && goalPendingIsStandaloneDeliveryMutation(pending, goalFinalMarker); assert( state.goal.editedRevision === 0 || state.goal.revisionTwoFailureObserved === true || allowedRevisionTwoDelivery, 'goal-write-confirmed-before-revision-two-failure', ); if (allowedRevisionTwoDelivery) { state.goal.preFailureDeliveryActionIds.add(pending.actionId); } await assertGoalInitialMarkerAbsent( 'goal-write-before-confirm', pending.actionId, ); } await runCli( [ '--agent-confirm', state.projectRoot, pending.agentId, pending.runId, pending.actionId, ], { timeoutMs: 120_000 }, ); state.confirmedActionIds.add(pending.actionId); if (monitorsGoalWrite) { await monitorGoalWriteActionUntilSettled(pending); } } } export async function findPendingActions() { const root = path.join(state.projectRoot, '.agent/runtime/pending-actions'); const files = await listFiles(root); const pending = []; for (const file of files.filter((entry) => entry.endsWith('.json'))) { const value = await readJson(file).catch(() => null); if (!value) continue; const action = value.action ?? value.pendingToolAction ?? value.pendingAction ?? value; const status = value.status ?? action.status; if ( status && !['pending', 'pending-confirmation', 'waiting-for-confirmation'].includes( status, ) ) { continue; } const relative = path.relative(root, file).split(path.sep); const agentId = value.agentId ?? value.state?.agentId ?? action.agentId ?? relative[0]; const runId = value.runId ?? value.state?.runId ?? action.runId ?? path.basename(file, '.json'); const actionId = action.actionId ?? value.actionId; const tool = action.tool ?? value.tool; const rawOccurrenceNonce = await fs .readFile(file, 'utf8') .then( (source) => source.match(/"occurrenceNonce"\s*:\s*([0-9]+)/u)?.[1] ?? null, ) .catch(() => null); if (agentId && runId && actionId && tool) { pending.push({ agentId, runId, actionId, tool, action, record: value, rawOccurrenceNonce, }); } } return pending; } export async function readTaskSnapshot() { const taskFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/tasks'), ); const all = []; for (const file of taskFiles.filter((entry) => entry.endsWith('.jsonl'))) { all.push(...(await readJsonl(file))); } return buildTaskSnapshot(all); } export function buildTaskSnapshot(all) { const latestByIdentity = new Map(); for (const record of all) { latestByIdentity.set(`${record.agentId}\0${record.runId}`, record); } return { all, latest: [...latestByIdentity.values()] }; } export function validateCompletedProcessFinalization(persistence) { const latest = persistence.taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); assert( latest?.status === 'completed' && latest?.phase === 'completed', 'process-completed-projection-missing', ); const completed = persistence.taskSnapshot.all.filter( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId && task.sessionId === state.initialSessionId && task.status === 'completed' && task.phase === 'completed', ); assert(completed.length === 1, 'process-completed-projection-count-invalid'); const responses = persistence.events.filter( (event) => event.agentId === mainAgentId && event.runId === state.initialRunId && event.sessionId === state.initialSessionId && event.eventType === 'response' && event.status === 'idle' && event.phase === 'completed', ); const turns = persistence.events.filter( (event) => event.agentId === mainAgentId && event.runId === state.initialRunId && event.sessionId === state.initialSessionId && event.eventType === 'turn.completed' && event.status === 'idle' && event.phase === 'completed', ); assert( responses.length === 1 && turns.length === 1, 'process-terminal-event-count-invalid', ); const messageId = finalMessageId( mainAgentId, state.initialSessionId, state.initialRunId, ); const finalAssistant = persistence.conversations.filter( (message) => message.role === 'assistant' && message.agentId === mainAgentId && message.messageId === messageId, ); const finalAudits = persistence.agentDb.filter( (record) => record.recordType === 'conversation.message' && record.role === 'assistant' && record.agentId === mainAgentId && record.sessionId === state.initialSessionId && record.messageId === messageId, ); assert( finalAssistant.length === 1 && finalAudits.length === 1, 'process-final-assistant-count-invalid', ); const duplicateActionCount = duplicateCount( persistence.agentDb .filter((record) => record.actionId) .map(actionAuditIdentity), ); const duplicateMessageCount = duplicateCount( persistence.conversations .map((message) => message.messageId) .filter(Boolean), ); const receiptRecords = persistence.agentDb.filter( (record) => record.recordType === 'agent.runtime.action_receipt', ); const duplicateReceiptCount = duplicateCount( receiptRecords.map(receiptAuditIdentity), ); assert( duplicateActionCount === 0 && duplicateMessageCount === 0 && duplicateReceiptCount === 0, 'process-duplicate-terminal-evidence-detected', ); return { completedProjectionCount: completed.length, finalAssistantAuditCount: finalAudits.length, finalAssistantCount: finalAssistant.length, duplicateActionCount, duplicateMessageCount, duplicateReceiptCount, }; } export function validateReconciliationHasNoFinalReply(persistence) { const latest = persistence.taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); assert( latest?.phase === 'needs-reconciliation' && latest?.status !== 'completed', 'process-runner-kill-task-not-reconciliation', ); const completed = persistence.taskSnapshot.all.filter( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId && task.status === 'completed' && task.phase === 'completed', ); const messageId = finalMessageId( mainAgentId, state.initialSessionId, state.initialRunId, ); const finalAssistant = persistence.conversations.filter( (message) => message.role === 'assistant' && message.messageId === messageId, ); const finalAudits = persistence.agentDb.filter( (record) => record.recordType === 'conversation.message' && record.role === 'assistant' && record.messageId === messageId, ); const terminalResponses = persistence.events.filter( (event) => event.agentId === mainAgentId && event.runId === state.initialRunId && event.eventType === 'response' && event.phase === 'completed', ); assert( completed.length === 0 && finalAssistant.length === 0 && finalAudits.length === 0 && terminalResponses.length === 0, 'process-runner-kill-final-reply-present', ); return { completedProjectionCount: completed.length, finalAssistantAuditCount: finalAudits.length, finalAssistantCount: finalAssistant.length, }; } export function validateProjectRootPublicLeakBoundary(surfaces, codePrefix) { const variants = disposableProjectPathVariants(); assert(variants.length >= 2, `${codePrefix}-path-variants-missing`); const counts = {}; for (const [surface, records] of Object.entries(surfaces)) { const values = Array.isArray(records) ? records : [records]; counts[surface] = countExactSecrets( Buffer.from(values.map((record) => JSON.stringify(record)).join('\n')), variants, ); assert(counts[surface] === 0, `${codePrefix}-${surface}-project-path-leak`); } return counts; } export function hasExpectedWorkspaceSandboxMetadata(record) { if (process.platform !== 'linux') return true; return ( record?.sandboxBackend === 'bubblewrap' && record?.sandboxMode === 'workspace-write' && record?.networkAccess === 'disabled' && record?.sandboxProfileVersion === 'workspace-v1' ); } export function hasExpectedExecReadyMetadata(record) { if (process.platform !== 'linux') return true; return ( record?.sandboxEstablishment === 'established' && record?.targetExec === 'established' && record?.launchFailureKind == null ); } export function isCompleteProcessLaunchEvidence(evidence) { return ( evidence.startActionCount === 1 && evidence.startAuditCount === 1 && evidence.readinessMarkerCount === 1 && evidence.identityMatches ); } export function validateUniqueProcessLaunchEvidence( records, processRecord, transcript, ) { const evidence = processLaunchEvidence(records, processRecord, transcript); assertProcessLaunchEvidenceIsNotDuplicated(evidence); assert( isCompleteProcessLaunchEvidence(evidence), 'process-launch-evidence-incomplete', ); return { launchCount: 1, readinessMarkerCount: evidence.readinessMarkerCount, }; } export async function waitForRunnerBootChange(oldBootId) { const deadline = Date.now() + 30_000; while (Date.now() < deadline) { const runner = await readRunnerStatus().catch(() => null); const bootId = runnerBootId(runner); if ( runner?.running === true && isNonEmptyString(bootId) && bootId !== oldBootId ) { return runner; } await sleep(100); } throw codedError('runner-boot-did-not-change'); } export function runnerBootId(runner) { return runner?.bootId ?? runner?.status?.bootId ?? null; } export function countOccurrences(content, value) { if (!isNonEmptyString(value)) return 0; return String(content).split(value).length - 1; } export async function validateLandedEvidence() { const taskSnapshot = await readTaskSnapshot(); const eventFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/events'), ); const events = []; for (const file of eventFiles.filter((entry) => entry.endsWith('.jsonl'))) { events.push(...(await readJsonl(file))); } const agentDb = await readJsonl( path.join(state.projectRoot, '.agent/agent.db'), ); const conversationFiles = await listFiles( path.join(state.projectRoot, '.agent/conversations'), ); const conversationEntries = []; for (const file of conversationFiles.filter((entry) => entry.endsWith('.jsonl'), )) { for (const message of await readJsonl(file)) { conversationEntries.push({ file: path.resolve(file), message }); } } const conversations = conversationEntries.map(({ message }) => message); const [activity, output] = await Promise.all([ readOptionalJsonl(path.join(state.projectRoot, '.agent/activity.jsonl')), readOptionalJsonl(path.join(state.projectRoot, '.agent/output.jsonl')), ]); assert(taskSnapshot.all.length > 0, 'task-evidence-missing'); assert(events.length > 0, 'event-evidence-missing'); assert(agentDb.length > 0, 'agent-db-evidence-missing'); assertNoPersistedImagePayload('task', taskSnapshot.all); assertNoPersistedImagePayload('event', events); assertNoPersistedImagePayload('agent-db', agentDb); const contextBundlePath = mainContextBundlePath(); const contextBundle = await readJson(contextBundlePath); const runtimeStatePath = mainRuntimeStatePath(); const runtimeState = await readJson(runtimeStatePath); assert( contextBundle.schemaVersion === runtimeContextBundleSchemaVersion && contextBundle.agentId === mainAgentId && contextBundle.runId === state.initialRunId && typeof contextBundle.repositoryContextFingerprint === 'string' && /^[0-9a-f]{64}$/u.test(contextBundle.repositoryContextFingerprint) && Array.isArray(contextBundle.repositoryContextSourcePaths) && contextBundle.repositoryContextSourcePaths.includes('AGENTS.md') && contextBundle.repositoryContextSourcePaths.includes('package.json'), 'project-index-structured-evidence-missing', ); const toolPlanProtocolCount = validateMainRunToolPlanProtocols(agentDb); const structuredPlanEvidence = validateStructuredPlanEvidence( agentDb, runtimeState, contextBundle, ); const confirmedActionLifecycleCount = validateConfirmedActionLifecycles(agentDb); const replayEvidence = validateToolActionReplays(agentDb); const projectIndexExecution = requireSuccessfulToolExecution( agentDb, 'project.index', state.initialRunId, ); const repositoryReadExecutions = [ 'AGENTS.md', 'package.json', 'game/index.html', ].map((targetPath) => requireSuccessfulToolExecution( agentDb, 'file.read', state.initialRunId, (execution) => auditPathEquals(execution.inputSummary, targetPath), `repository-context-read-evidence-missing:${targetPath}`, ), ); const gitInspectInputMatches = (execution) => auditInputValue(execution.inputSummary, 'includeDiff') === 'true' && auditInputValue(execution.inputSummary, 'maxFiles') === '20' && auditInputValue(execution.inputSummary, 'maxChars') === '24000'; const initialGitInspectExecution = requireSuccessfulToolExecution( agentDb, 'git.inspect', state.initialRunId, gitInspectInputMatches, 'initial-git-inspect-action-invalid', ); const gitInspectActionIds = new Set( agentDb .filter( (record) => record.agentId === mainAgentId && record.runId === state.initialRunId && record.tool === 'git.inspect' && isNonEmptyString(record.actionId), ) .map((record) => record.actionId), ); assert(gitInspectActionIds.size >= 3, 'git-inspect-action-count-invalid'); const initialGameHtml = seededGameHtml(); const expectedGameHtml = initialGameHtml.replace( 'REAL_E2E_TARGET:before', patchedText, ); assert( expectedGameHtml !== initialGameHtml && !expectedGameHtml.includes('REAL_E2E_TARGET:before'), 'seeded-patch-target-invalid', ); const initialGameSha256 = createHash('sha256') .update(initialGameHtml) .digest('hex'); const expectedGameSha256 = createHash('sha256') .update(expectedGameHtml) .digest('hex'); const expectedCreatedSha256 = createHash('sha256') .update(patchsetCreatedContent) .digest('hex'); const gameReadShaEvent = events.find( (event) => event.agentId === mainAgentId && event.runId === state.initialRunId && event.eventType === 'observation' && String(event.summary ?? '').includes('file.read') && String(event.detail ?? '').includes('game/index.html') && String(event.detail ?? '').includes(`sha256=${initialGameSha256}`), ); assert(Boolean(gameReadShaEvent), 'file-read-sha256-evidence-missing'); const patchsetExecution = requireSuccessfulToolExecution( agentDb, 'project.patchset', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'changeCount') === '2' && auditPatchsetPathsMatch(execution.inputSummary, [ `update:game/index.html`, `create:${patchsetCreatedPath}`, ]), 'project-patchset-action-invalid', ); const patchsetActionIds = new Set( agentDb .filter( (record) => record.agentId === mainAgentId && record.runId === state.initialRunId && record.tool === 'project.patchset' && isNonEmptyString(record.actionId), ) .map((record) => record.actionId), ); assert(patchsetActionIds.size === 1, 'project-patchset-action-count-invalid'); const finalGitInspectExecution = requireSuccessfulToolExecution( agentDb, 'git.inspect', state.initialRunId, (execution) => execution.actionId !== initialGitInspectExecution.actionId && execution.startIndex > patchsetExecution.completionIndex && gitInspectInputMatches(execution), 'final-git-inspect-action-invalid', ); const forbiddenMutationAttempts = agentDb.filter( (record) => record.agentId === mainAgentId && record.runId === state.initialRunId && [ 'project.checkpoint', 'project.restore', 'file.patch', 'file.write', 'file.delete', ].includes(record.tool) && [ 'agent.runtime.tool_action.executing', 'agent.runtime.tool_confirmation_required', ].includes(record.recordType), ); assert( forbiddenMutationAttempts.length === 0, 'out-of-patchset-mutation-attempted', ); const checkpointRecord = requireExecutionRecord( agentDb, patchsetExecution, (record) => record.recordType === 'project.checkpoint' && isNonEmptyString(record.checkpointId) && Number.isSafeInteger(record.fileCount) && record.fileCount > 0 && Number.isSafeInteger(record.totalBytes) && record.totalBytes > 0, 'patchset-checkpoint-evidence-missing', ); const patchsetAudit = validatePatchsetAudit( agentDb, patchsetExecution, checkpointRecord.checkpointId, { initialGameSha256, expectedGameSha256, expectedCreatedSha256, }, ); const contentDiffExecution = requireSuccessfulToolExecution( agentDb, 'project.diff', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'checkpointId') === checkpointRecord.checkpointId && auditInputValue(execution.inputSummary, 'includeContent') === 'true' && Number(auditInputValue(execution.inputSummary, 'maxFiles')) >= 2 && Number(auditInputValue(execution.inputSummary, 'maxChars')) >= 1_000, 'patchset-content-diff-action-invalid', ); const actionHistoryExecution = requireSuccessfulToolExecution( agentDb, 'agent.action_history', state.initialRunId, (execution) => ['', state.initialRunId].includes( auditInputValue(execution.inputSummary, 'runId'), ) && auditInputValue(execution.inputSummary, 'actionId') === '' && auditInputValue(execution.inputSummary, 'tool') === 'project.patchset' && auditInputValue(execution.inputSummary, 'status') === 'ok' && auditInputValue(execution.inputSummary, 'limit') === '5', 'action-history-action-invalid', ); const commandRecords = agentDb .map((record, index) => ({ record, index })) .filter( ({ record }) => record.recordType === 'agent.runtime.command.exec' && record.agentId === mainAgentId && record.runId === state.initialRunId, ); assert(commandRecords.length === 2, 'command-exec-record-count-invalid'); assert( new Set(commandRecords.map(({ record }) => record.actionId)).size === 2, 'command-exec-action-count-invalid', ); const failedCommandRecord = commandRecords.find( ({ record }) => record.status === 'failed', ); const successfulCommandRecord = commandRecords.find( ({ record }) => record.status === 'completed', ); assert( failedCommandRecord?.record.program === 'npm' && Number.isSafeInteger(failedCommandRecord.record.argsCount) && failedCommandRecord.record.argsCount > 0 && /^[0-9a-f]{64}$/u.test(failedCommandRecord.record.argsSha256) && failedCommandRecord.record.cwd === '.' && Number.isInteger(failedCommandRecord.record.exitCode) && failedCommandRecord.record.exitCode !== 0 && failedCommandRecord.record.timedOut === false && failedCommandRecord.record.sourceChanged === false && isNonEmptyString(failedCommandRecord.record.outputRef) && /^[0-9a-f]{64}$/u.test(failedCommandRecord.record.outputSha256) && Number.isSafeInteger(failedCommandRecord.record.totalLines) && failedCommandRecord.record.totalLines > commandRootErrorLine && typeof failedCommandRecord.record.captureTruncated === 'boolean' && !Object.hasOwn(failedCommandRecord.record, 'output'), 'command-exec-failure-record-invalid', ); assert( successfulCommandRecord?.record.program === 'npm' && Number.isSafeInteger(successfulCommandRecord.record.argsCount) && successfulCommandRecord.record.argsCount > 0 && /^[0-9a-f]{64}$/u.test(successfulCommandRecord.record.argsSha256) && successfulCommandRecord.record.cwd === '.' && successfulCommandRecord.record.exitCode === 0 && successfulCommandRecord.record.timedOut === false && successfulCommandRecord.record.sourceChanged === false && isNonEmptyString(successfulCommandRecord.record.outputRef) && /^[0-9a-f]{64}$/u.test(successfulCommandRecord.record.outputSha256) && Number.isSafeInteger(successfulCommandRecord.record.totalLines) && !Object.hasOwn(successfulCommandRecord.record, 'output'), 'command-exec-success-record-invalid', ); assert( commandRecords.every( ({ record }) => !Object.hasOwn(record, 'args') && !Object.hasOwn(record, 'arguments') && /^[0-9a-f]{64}$/u.test(record.argsSha256) && isNonEmptyString(record.actionId) && hasExpectedWorkspaceSandboxMetadata(record), ), 'command-exec-raw-argv-audit-leak', ); const failedCommandSidecarPath = resolveProjectRelative( failedCommandRecord.record.outputRef, ); const successfulCommandSidecarPath = resolveProjectRelative( successfulCommandRecord.record.outputRef, ); const [failedCommandSidecar, successfulCommandSidecar] = await Promise.all([ readJson(failedCommandSidecarPath), readJson(successfulCommandSidecarPath), ]); validateCommandOutputSidecar( failedCommandSidecar, failedCommandRecord.record, failedCommandSidecarPath, ); validateCommandOutputSidecar( successfulCommandSidecar, successfulCommandRecord.record, successfulCommandSidecarPath, ); assert( countExactSecrets(Buffer.from(failedCommandSidecar.output), [ commandRootErrorMarker, ]) === 1 && failedCommandSidecar.output.includes(commandFailureMarker) && failedCommandSidecar.output.length - failedCommandSidecar.output.lastIndexOf(commandRootErrorMarker) > 900, 'command-output-root-marker-placement-invalid', ); assert( successfulCommandSidecar.output.includes(commandPassedMarker) && !successfulCommandSidecar.output.includes(commandRootErrorMarker), 'command-output-success-sidecar-invalid', ); const failedCommandObservationIndex = agentDb.findIndex( (record) => record.recordType === 'agent.runtime.tool_observation' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.actionId === failedCommandRecord.record.actionId && record.tool === 'command.exec' && record.status === 'command-failed' && record.decision === 'approved', ); assert( failedCommandObservationIndex > failedCommandRecord.index, 'command-exec-failure-observation-missing', ); const commandOutputReadExecution = requireSuccessfulToolExecution( agentDb, 'command.output_read', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'sourceActionId') === failedCommandRecord.record.actionId && Number(auditInputValue(execution.inputSummary, 'startLine')) >= 1 && Number(auditInputValue(execution.inputSummary, 'maxLines')) >= 1, 'command-output-read-action-invalid', ); const commandOutputReadAudits = agentDb.filter( (record) => record.recordType === 'agent.runtime.command.output_read' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.sourceActionId === failedCommandRecord.record.actionId, ); assert( commandOutputReadAudits.length >= 1 && commandOutputReadAudits.every( (record) => record.outputRef === failedCommandRecord.record.outputRef && record.outputSha256 === failedCommandRecord.record.outputSha256 && Number.isSafeInteger(record.startLine) && record.startLine >= 1 && !Object.hasOwn(record, 'lines'), ), 'command-output-read-audit-invalid', ); const markerLeakCounts = { task: countExactSecrets(Buffer.from(JSON.stringify(taskSnapshot.all)), [ commandRootErrorMarker, ]), event: countExactSecrets(Buffer.from(JSON.stringify(events)), [ commandRootErrorMarker, ]), agentDb: countExactSecrets(Buffer.from(JSON.stringify(agentDb)), [ commandRootErrorMarker, ]), conversation: countExactSecrets( Buffer.from(JSON.stringify(conversations)), [commandRootErrorMarker], ), activity: countExactSecrets(Buffer.from(JSON.stringify(activity)), [ commandRootErrorMarker, ]), output: countExactSecrets(Buffer.from(JSON.stringify(output)), [ commandRootErrorMarker, ]), runtimeState: countExactSecrets(Buffer.from(JSON.stringify(runtimeState)), [ commandRootErrorMarker, ]), }; assert( markerLeakCounts.task === 0 && markerLeakCounts.event === 0 && markerLeakCounts.agentDb === 0 && markerLeakCounts.conversation === 0 && markerLeakCounts.activity === 0 && markerLeakCounts.output === 0 && markerLeakCounts.runtimeState === 0 && state.commandOutputMarkerSeenInContext && state.commandOutputContextPages.size >= 1, 'command-output-transcript-persistence-boundary-invalid', ); const successfulCommandExecution = requireSuccessfulToolExecution( agentDb, 'command.exec', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'program') === 'npm' && Number(auditInputValue(execution.inputSummary, 'argsCount')) > 0 && /^[0-9a-f]{64}$/u.test( auditInputValue(execution.inputSummary, 'argsSha256'), ) && ['', '.'].includes(auditInputValue(execution.inputSummary, 'cwd')) && auditInputValue(execution.inputSummary, 'timeoutSeconds') === '120', 'command-exec-success-action-invalid', ); const verificationExecution = requireSuccessfulToolExecution( agentDb, 'project.verify', state.initialRunId, (execution) => ['test', 'check:e2e'].includes( auditInputValue(execution.inputSummary, 'script'), ) && auditInputValue(execution.inputSummary, 'expectedCommandSha256') === createHash('sha256').update(verificationCommand).digest('hex') && auditInputValue(execution.inputSummary, 'timeoutSeconds') === '120', 'project-verification-action-invalid', ); const previewExecution = requireSuccessfulToolExecution( agentDb, 'preview.validate', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'viewports') === 'desktop,mobile' && auditInputValue(execution.inputSummary, 'expectedTextCount') === '2' && auditInputValue(execution.inputSummary, 'expectedTextSha256') === createHash('sha256') .update(JSON.stringify([visibleText, patchedText])) .digest('hex') && Number(auditInputValue(execution.inputSummary, 'settleMs')) >= 500 && auditInputValue(execution.inputSummary, 'failOnConsoleError') === 'true', 'preview-validation-action-invalid', ); const previewValidationCandidates = agentDb.filter( (record) => record.recordType === 'agent.runtime.preview.validation' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.passed === true && Array.isArray(record.screenshots), ); assert( previewValidationCandidates.length === 1, 'preview-validation-record-count-invalid', ); const previewScreenshotPaths = validatePreviewScreenshotPaths( previewValidationCandidates[0].screenshots, 'preview-validation-record-screenshots-invalid', ); const imageInspectExecution = requireSuccessfulToolExecution( agentDb, 'image.inspect', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'pathCount') === '2' && auditInputValue(execution.inputSummary, 'pathsSha256') === createHash('sha256') .update(JSON.stringify(previewScreenshotPaths)) .digest('hex') && auditInputValue(execution.inputSummary, 'paths') === previewScreenshotPaths.join(',') && Number(auditInputValue(execution.inputSummary, 'questionChars')) >= 0, 'image-inspect-action-invalid', ); const imageInspectActionIds = new Set( agentDb .filter( (record) => record.agentId === mainAgentId && record.runId === state.initialRunId && record.tool === 'image.inspect' && isNonEmptyString(record.actionId), ) .map((record) => record.actionId), ); assert( imageInspectActionIds.size === 1, 'image-inspect-action-count-invalid', ); const spawnExecution = requireSuccessfulToolExecution( agentDb, 'agent.spawn_isolated', state.initialRunId, ); const canvasExecution = state.suite === 'full' ? requireSuccessfulToolExecution( agentDb, 'canvas.asset_generate', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'promptChars') === String([...editorAssetPrompt].length), 'canvas-generation-action-invalid', ) : findSuccessfulToolExecution( agentDb, 'canvas.asset_generate', state.initialRunId, ); if (state.suite !== 'full') { assert(canvasExecution === null, 'canvas-generation-unexpected'); } const gitCommitExecution = requireSuccessfulToolExecution( agentDb, 'project.git_commit', state.initialRunId, (execution) => auditInputValue(execution.inputSummary, 'pathCount') === '2' && auditPathListMatches(execution.inputSummary, 'paths', [ 'game/index.html', patchsetCreatedPath, ]) && /^[0-9a-f]{12}$/u.test( auditInputValue(execution.inputSummary, 'expectedHead') ?? '', ) && /^[0-9a-f]{12}$/u.test( auditInputValue(execution.inputSummary, 'snapshot') ?? '', ) && /^[0-9a-f]{64}$/u.test( auditInputValue(execution.inputSummary, 'messageSha256') ?? '', ) && isNonEmptyString(auditInputValue(execution.inputSummary, 'title')), 'project-git-commit-action-invalid', ); const gitCommitActionIds = new Set( agentDb .filter( (record) => record.tool === 'project.git_commit' && isNonEmptyString(record.actionId), ) .map((record) => record.actionId), ); assert( gitCommitActionIds.size === 1 && agentDb .filter((record) => record.tool === 'project.git_commit') .every( (record) => record.agentId === mainAgentId && record.runId === state.initialRunId, ), 'project-git-commit-action-count-invalid', ); const postCommitGitInspectExecution = requireSuccessfulToolExecution( agentDb, 'git.inspect', state.initialRunId, (execution) => execution.startIndex > gitCommitExecution.completionIndex && gitInspectInputMatches(execution), 'post-commit-git-inspect-action-invalid', ); assert( projectIndexExecution.completionIndex < patchsetExecution.startIndex, 'project-index-not-before-patchset', ); assert( failedCommandObservationIndex < patchsetExecution.startIndex, 'patchset-not-after-failed-command-feedback', ); assert( failedCommandObservationIndex < commandOutputReadExecution.startIndex && commandOutputReadExecution.completionIndex < patchsetExecution.startIndex, 'patchset-not-after-command-output-read', ); assert( commandOutputReadExecution.completionIndex < actionHistoryExecution.startIndex, 'command-output-read-depended-on-action-history', ); assert( initialGitInspectExecution.completionIndex < patchsetExecution.startIndex, 'initial-git-inspect-not-before-patchset', ); assert( repositoryReadExecutions.every( (execution) => execution.completionIndex < patchsetExecution.startIndex, ), 'patchset-not-after-repository-reads', ); assert( patchsetExecution.completionIndex < finalGitInspectExecution.startIndex, 'final-git-inspect-not-after-patchset', ); assert( patchsetExecution.completionIndex < contentDiffExecution.startIndex, 'content-diff-not-after-patchset', ); assert( Math.max( contentDiffExecution.completionIndex, finalGitInspectExecution.completionIndex, ) < successfulCommandExecution.startIndex, 'successful-command-not-after-change-reviews', ); assert( successfulCommandExecution.completionIndex < verificationExecution.startIndex, 'project-verification-not-after-successful-command', ); assert( patchsetExecution.completionIndex < verificationExecution.startIndex, 'verification-not-after-patchset', ); if (canvasExecution) { assert( canvasExecution.completionIndex < verificationExecution.startIndex, 'verification-not-after-editor-api', ); } assert( spawnExecution.completionIndex < verificationExecution.startIndex, 'verification-not-after-isolated-spawn', ); assert( patchsetExecution.completionIndex < previewExecution.startIndex, 'preview-not-after-patchset', ); assert( previewExecution.completionIndex < imageInspectExecution.startIndex, 'image-inspect-not-after-preview-validation', ); assert( Math.max( verificationExecution.completionIndex, imageInspectExecution.completionIndex, actionHistoryExecution.completionIndex, spawnExecution.completionIndex, canvasExecution?.completionIndex ?? -1, ) < gitCommitExecution.startIndex, 'project-git-commit-before-required-evidence', ); assert( finalGitInspectExecution.completionIndex < gitCommitExecution.startIndex, 'project-git-commit-not-after-commit-snapshot', ); assert( gitCommitExecution.completionIndex < postCommitGitInspectExecution.startIndex, 'post-commit-git-inspect-not-after-commit', ); const initial = taskSnapshot.latest.find( (task) => task.agentId === mainAgentId && task.runId === state.initialRunId, ); assert( initial?.sessionId === state.initialSessionId, 'landed-session-mismatch', ); assert( initial?.status === 'completed' || initial?.phase === 'completed', 'main-run-not-completed', ); const steerEvidence = await validateSameRunSteerEvidence({ agentDb, activity, contextBundle, conversationEntries, events, initial, output, runtimeState, taskSnapshot, }); const actionReceiptEvidence = validateMainRunActionReceipts( agentDb, initial, actionHistoryExecution, imageInspectExecution, commandOutputReadExecution, failedCommandRecord.record.actionId, gitCommitExecution, ); const revision = await readJson( path.join(state.projectRoot, '.agent/runtime/project-revision.json'), ); const expectedProjectRevision = state.suite === 'full' ? 4 : 3; assert( revision.revision === expectedProjectRevision, 'project-revision-count-invalid', ); const contentDiffEvidence = validatePatchsetContentDiff( contextBundle.observations, checkpointRecord.checkpointId, { initialGameSha256, expectedGameSha256, expectedCreatedSha256, }, ); const gitCommitEvidence = await validateGitCommitEvidence( agentDb, gitCommitExecution, actionReceiptEvidence.gitCommitSafeDetail, revision.revision, { expectedGameHtml, expectedCreatedContent: patchsetCreatedContent, }, ); const gitInspectEvidence = validateGitInspectEvents( events, contextBundle.observations, { initialActionId: initialGitInspectExecution.actionId, changedActionId: finalGitInspectExecution.actionId, postCommitActionId: postCommitGitInspectExecution.actionId, commitHead: gitCommitEvidence.commitHead, }, ); const actionHistoryEvidence = validateActionHistoryObservations( events, contextBundle.observations, actionHistoryExecution, patchsetExecution, initial, ); const checkpointManifestPath = path.join( state.projectRoot, '.agent/checkpoints', checkpointRecord.checkpointId, 'manifest.json', ); const checkpointManifest = await readJson(checkpointManifestPath); assert( checkpointManifest.checkpointId === checkpointRecord.checkpointId && Array.isArray(checkpointManifest.files) && checkpointManifest.files.length === checkpointRecord.fileCount, 'checkpoint-manifest-invalid', ); const checkpointGamePath = path.join( path.dirname(checkpointManifestPath), 'files/game/index.html', ); const checkpointGame = await fs.readFile(checkpointGamePath, 'utf8'); assert( checkpointGame === initialGameHtml && createHash('sha256').update(checkpointGame).digest('hex') === initialGameSha256, 'checkpoint-does-not-precede-patchset', ); assert( !checkpointManifest.files.some( (file) => file.path === patchsetCreatedPath, ) && !(await fs .stat( path.join( path.dirname(checkpointManifestPath), 'files', ...patchsetCreatedPath.split('/'), ), ) .catch(() => null)), 'checkpoint-already-contains-created-file', ); const projectVerificationRecord = requireExecutionRecord( agentDb, verificationExecution, (record) => record.recordType === 'agent.runtime.project.verify' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.actionId === verificationExecution.actionId && ['test', 'check:e2e'].includes(record.script) && record.expectedCommand === verificationCommand && record.status === 'completed' && record.exitCode === 0 && record.timedOut === false && hasExpectedWorkspaceSandboxMetadata(record), 'project-verification-structured-evidence-missing', ); assert( isNonEmptyString(projectVerificationRecord.logPath), 'project-verification-log-path-missing', ); const previewValidationRecord = requireExecutionRecord( agentDb, previewExecution, (record) => record.recordType === 'agent.runtime.preview.validation' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.revision === revision.revision && record.passed === true && isNonEmptyString(record.reportPath) && Array.isArray(record.screenshots) && record.screenshots.length === 2, 'browser-validation-structured-evidence-missing', ); assert( previewValidationRecord === previewValidationCandidates[0] && JSON.stringify(previewValidationRecord.screenshots) === JSON.stringify(previewScreenshotPaths), 'preview-validation-observation-path-mismatch', ); const imageInspectAuditRecord = requireExecutionRecord( agentDb, imageInspectExecution, (record) => record.recordType === 'agent.runtime.image.inspect' && record.agentId === mainAgentId && record.runId === state.initialRunId && Array.isArray(record.images) && record.images.length === 2, 'image-inspect-dedicated-audit-missing', ); const imageInspectAuditRecords = agentDb.filter( (record) => record.recordType === 'agent.runtime.image.inspect' && record.agentId === mainAgentId && record.runId === state.initialRunId, ); assert( imageInspectAuditRecords.length === 1 && imageInspectAuditRecords[0] === imageInspectAuditRecord, 'image-inspect-dedicated-audit-count-invalid', ); const spawnRecord = requireExecutionRecord( agentDb, spawnExecution, (record) => record.recordType === 'agent.runtime.agent.spawn_isolated' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.actionId === spawnExecution.actionId && isNonEmptyString(record.delegationGroupId) && isNonEmptyString(record.joinRunId) && Array.isArray(record.children) && record.children.length === 3, 'isolated-spawn-structured-evidence-missing', ); let editorAssetRecord = null; let editorAssetPath = null; if (canvasExecution) { editorAssetRecord = requireExecutionRecord( agentDb, canvasExecution, (record) => record.recordType === 'canvas.asset_generate' && isNonEmptyString(record.assetId) && isNonEmptyString(record.localPath) && [record.resourceId, record.assetObjectId, record.taskId].some( isNonEmptyString, ), 'editor-api-structured-evidence-missing', ); requireExecutionRecord( agentDb, canvasExecution, (record) => record.recordType === 'agent.runtime.canvas.asset_generate' && record.agentId === mainAgentId && record.assetId === editorAssetRecord.assetId && record.localPath === editorAssetRecord.localPath && record.resourceId === editorAssetRecord.resourceId && record.assetObjectId === editorAssetRecord.assetObjectId && record.taskId === editorAssetRecord.taskId, 'editor-api-runtime-evidence-missing', ); editorAssetPath = resolveProjectRelative(editorAssetRecord.localPath); const editorAssetMetadata = await fs .stat(editorAssetPath) .catch(() => null); assert( editorAssetMetadata?.isFile() && editorAssetMetadata.size > 0, 'editor-api-asset-missing', ); const manifest = await readJson( path.join(state.projectRoot, '.agent/manifest.json'), ); const manifestAsset = manifest.assets?.find( (asset) => asset.id === editorAssetRecord.assetId, ); assert( manifestAsset?.localPath === editorAssetRecord.localPath && manifestAsset.source?.kind === 'canvas' && ['resourceId', 'assetObjectId', 'taskId'].every( (key) => !isNonEmptyString(editorAssetRecord[key]) || manifestAsset.source?.[key] === editorAssetRecord[key], ), 'editor-api-manifest-evidence-missing', ); } const verificationFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/verification'), ); const verificationGates = []; for (const file of verificationFiles.filter((entry) => entry.endsWith('.json'), )) { verificationGates.push({ file, value: await readJson(file) }); } const mainGate = verificationGates.find( ({ value }) => value.agentId === mainAgentId && value.runId === state.initialRunId, ); assert( mainGate?.value.lastVerificationStatus === 'passed', 'project-verification-not-passed', ); assert( mainGate.value.verifiedRevision === revision.revision, 'verification-revision-stale', ); const browserFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/browser-validations'), ); const browserReports = []; for (const file of browserFiles.filter( (entry) => path.basename(entry) === 'validation.json', )) { const report = await readJson(file); if (report.passed) browserReports.push({ file, report }); } assert(browserReports.length > 0, 'browser-validation-missing'); const expectedBrowserReportPath = resolveProjectRelative( previewValidationRecord.reportPath, ); const browser = browserReports.find( ({ file }) => path.resolve(file) === expectedBrowserReportPath, ); assert(Boolean(browser), 'browser-validation-report-mismatch'); assert( browser.report.viewportResults.length === 2, 'browser-viewport-count-invalid', ); const viewports = new Map( browser.report.viewportResults.map((viewport) => [ viewport.viewport, viewport, ]), ); const screenshotMetadata = []; for (const viewportName of ['desktop', 'mobile']) { const viewport = viewports.get(viewportName); assert(viewport?.passed === true, `browser-${viewportName}-failed`); assert( Array.isArray(viewport.expectedText) && viewport.expectedText.length === 2 && viewport.expectedText[0].text === visibleText && viewport.expectedText[0].found === true && viewport.expectedText[1].text === patchedText && viewport.expectedText[1].found === true && Array.isArray(viewport.consoleErrors) && viewport.consoleErrors.length === 0 && Array.isArray(viewport.exceptions) && viewport.exceptions.length === 0 && !viewport.failedRequests?.some((request) => request.fatal === true) && viewport.canvases?.some((canvas) => canvas.nonEmpty === true), `browser-${viewportName}-content-invalid`, ); const screenshot = resolveProjectRelative(viewport.screenshotPath); const png = await fs.readFile(screenshot); assert( png.length > 100 && png.subarray(0, 8).equals(pngSignature), `browser-${viewportName}-png-invalid`, ); screenshotMetadata.push({ path: relativeProjectPath(screenshot), sha256: createHash('sha256').update(png).digest('hex'), bytes: png.length, }); } assert( JSON.stringify(screenshotMetadata.map((image) => image.path)) === JSON.stringify(previewScreenshotPaths), 'browser-screenshot-observation-path-mismatch', ); validateImageInspectAudit( imageInspectAuditRecord, screenshotMetadata, actionReceiptEvidence.imageInspectSafeDetail, ); const groupFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/isolated-agents/groups'), ); const groups = []; for (const file of groupFiles.filter((entry) => entry.endsWith('.json'))) { const value = await readJson(file); if (value.parentRunId === state.initialRunId) groups.push({ file, value }); } assert(groups.length === 1, 'isolated-group-count-invalid'); assert( groups[0].value.delegationGroupId === spawnRecord.delegationGroupId && groups[0].value.joinRunId === spawnRecord.joinRunId, 'isolated-group-audit-mismatch', ); const children = groups[0].value.request?.children ?? []; assert(children.length === 3, 'isolated-child-count-invalid'); const templateCounts = countBy( children.map((child) => child.templateAgentId), ); assert( [...templateCounts.values()].sort((a, b) => b - a).join(',') === '2,1', 'isolated-template-shape-invalid', ); const resultFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/isolated-agents/results'), ); const isolatedResults = []; for (const file of resultFiles.filter((entry) => entry.endsWith('.json'))) { const value = await readJson(file); if (value.delegationGroupId === groups[0].value.delegationGroupId) { isolatedResults.push(value); } } assert(isolatedResults.length === 3, 'isolated-result-count-invalid'); assert( isolatedResults.every((record) => record.result?.status === 'completed'), 'isolated-child-not-completed', ); const joinTasks = taskSnapshot.latest.filter( (task) => task.source === 'agent-isolated-join' && task.runId === spawnRecord.joinRunId, ); assert(joinTasks.length <= 1, 'isolated-join-count-invalid'); const joinDeliveryFiles = await listFiles( path.join( state.projectRoot, '.agent/runtime/isolated-agents/join-deliveries', ), ); const joinDeliveries = []; for (const file of joinDeliveryFiles.filter((entry) => entry.endsWith('.json'), )) { const value = await readJson(file); if (value.delegationGroupId === spawnRecord.delegationGroupId) { joinDeliveries.push(value); } } assert(joinDeliveries.length === 1, 'isolated-join-delivery-count-invalid'); const joinDelivery = joinDeliveries[0]; const joinDeliveryTarget = isolatedJoinDeliveryTarget(joinDelivery); assert( joinDelivery.joinRunId === spawnRecord.joinRunId && joinDelivery.parentRunId === state.initialRunId && (joinDeliveryTarget === 'parent-wake' ? joinDelivery.queuedRunId == null : joinDelivery.queuedRunId == null || joinDelivery.queuedRunId === spawnRecord.joinRunId), 'isolated-join-delivery-identity-invalid', ); assert( joinDeliveryTarget !== 'parent-wake' || joinTasks.length === 0, 'isolated-parent-wake-continuation-task-invalid', ); const parentWakeDispatchRecords = agentDb .map((record, index) => ({ record, index })) .filter( ({ record }) => record.recordType === 'agent.runtime.agent.isolated_join.parent_wake.dispatched' && (record.delegationGroupId === spawnRecord.delegationGroupId || record.joinRunId === spawnRecord.joinRunId), ); if (joinDeliveryTarget === 'parent-wake') { assert( parentWakeDispatchRecords.length === 1 && parentWakeDispatchRecords[0].record.agentId === mainAgentId && parentWakeDispatchRecords[0].record.sessionId === state.initialSessionId && parentWakeDispatchRecords[0].record.parentRunId === state.initialRunId && parentWakeDispatchRecords[0].record.parentActionId === spawnExecution.actionId && parentWakeDispatchRecords[0].record.delegationGroupId === spawnRecord.delegationGroupId && parentWakeDispatchRecords[0].record.joinRunId === spawnRecord.joinRunId, 'isolated-parent-wake-dispatch-audit-invalid', ); } let joinCompletionRecordIndex = -1; if (joinDelivery.status === 'claimed-by-parent') { assert( isNonEmptyString(joinDelivery.claimedByActionId) && (joinDeliveryTarget === 'parent-wake' ? joinTasks.length === 0 : joinTasks.length === 0 || (joinTasks[0].status === 'cancelled' && String(joinTasks[0].currentAction ?? '').includes( `actionId=${joinDelivery.claimedByActionId}`, ))), 'isolated-join-claim-task-invalid', ); const claimRecords = agentDb .map((record, index) => ({ record, index })) .filter( ({ record }) => record.recordType === 'agent.runtime.agent.isolated_join.claimed_by_parent' && record.runId === state.initialRunId && record.joinRunId === spawnRecord.joinRunId && record.delegationGroupId === spawnRecord.delegationGroupId && record.actionId === joinDelivery.claimedByActionId, ); assert(claimRecords.length === 1, 'isolated-join-claim-audit-invalid'); joinCompletionRecordIndex = claimRecords[0].index; assert( agentDb.some( (record) => record.recordType === 'agent.runtime.tool_observation' && record.agentId === mainAgentId && record.runId === state.initialRunId && record.tool === 'agent.run_status' && record.status === 'ok' && record.actionId === joinDelivery.claimedByActionId && String(record.summary ?? '').includes('ready all-join'), ), 'isolated-join-parent-observation-missing', ); } else if (joinDelivery.status === 'dispatched') { if (joinDeliveryTarget === 'parent-wake') { assert( joinDelivery.claimedByActionId == null && joinTasks.length === 0, 'isolated-parent-wake-delivery-invalid', ); joinCompletionRecordIndex = parentWakeDispatchRecords[0].index; } else { assert( joinDelivery.claimedByActionId == null && joinTasks.length === 1 && joinTasks[0].status === 'completed', 'isolated-join-continuation-not-completed', ); const dispatchRecords = agentDb .map((record, index) => ({ record, index })) .filter( ({ record }) => record.recordType === 'agent.runtime.agent.isolated_join.dispatched' && record.parentRunId === state.initialRunId && record.joinRunId === spawnRecord.joinRunId && record.delegationGroupId === spawnRecord.delegationGroupId, ); assert( dispatchRecords.length === 1, 'isolated-join-dispatch-audit-invalid', ); joinCompletionRecordIndex = dispatchRecords[0].index; } } else { throw codedError('isolated-join-delivery-status-invalid'); } assert( joinCompletionRecordIndex < actionHistoryExecution.startIndex, 'action-history-not-after-isolated-join', ); const finalRunSetEvidence = validateFinalMainRunSet( taskSnapshot, initial, spawnRecord, ); const completedProjections = agentDb.filter( (record) => record.recordType === 'agent.runtime.completed' && record.agentId === mainAgentId && record.runId === state.initialRunId, ); assert( completedProjections.length === 1, 'main-completed-projection-count-invalid', ); const completedProjection = completedProjections[0]; assert( completedProjection.sessionId === state.initialSessionId && completedProjection.taskId === initial.taskId && completedProjection.source === initial.source, 'main-completed-projection-identity-invalid', ); const terminalTasks = taskSnapshot.all.filter( (task) => task.agentId === completedProjection.agentId && task.runId === completedProjection.runId && task.sessionId === completedProjection.sessionId && task.taskId === completedProjection.taskId && task.source === completedProjection.source && task.status === 'completed' && task.phase === 'completed', ); assert(terminalTasks.length === 1, 'main-terminal-task-count-invalid'); const terminalTurnEvents = events.filter( (event) => event.agentId === completedProjection.agentId && event.runId === completedProjection.runId && event.sessionId === completedProjection.sessionId && event.taskId === completedProjection.taskId && event.source === completedProjection.source && event.eventType === 'turn.completed' && event.status === 'idle' && event.phase === 'completed', ); const terminalResponseEvents = events.filter( (event) => event.agentId === completedProjection.agentId && event.runId === completedProjection.runId && event.sessionId === completedProjection.sessionId && event.taskId === completedProjection.taskId && event.source === completedProjection.source && event.eventType === 'response' && event.status === 'idle' && event.phase === 'completed', ); assert( terminalTurnEvents.length === 1 && terminalResponseEvents.length === 1, 'main-terminal-event-count-invalid', ); const finalizationFiles = await listFiles( path.join(state.projectRoot, '.agent/runtime/finalizations'), ); assert( !finalizationFiles.some((file) => path.basename(file).startsWith(`${state.initialRunId}.json`), ), 'completed-run-finalization-journal-present', ); const expectedFinalMessageId = finalMessageId( completedProjection.agentId, completedProjection.sessionId, completedProjection.runId, ); const expectedConversationPath = path.resolve( agentConversationPath( completedProjection.agentId, completedProjection.sessionId, ), ); const expectedInitialMessageId = backgroundTaskMessageId( completedProjection.agentId, completedProjection.sessionId, completedProjection.runId, completedProjection.source, ); const expectedSteerMessageId = steerEvidence.messageId; const targetSessionMessages = conversationEntries.filter( ({ file }) => file === expectedConversationPath, ); assert( expectedInitialMessageId === state.steer.initialMessageId && isNonEmptyString(expectedSteerMessageId) && targetSessionMessages.length === 3 && JSON.stringify( targetSessionMessages.map(({ message }) => message.role), ) === JSON.stringify(['user', 'user', 'assistant']) && JSON.stringify( targetSessionMessages.map(({ message }) => message.messageId), ) === JSON.stringify([ expectedInitialMessageId, expectedSteerMessageId, expectedFinalMessageId, ]) && targetSessionMessages.every( ({ message }) => message.agentId === completedProjection.agentId && Number.isSafeInteger(message.updatedAt), ) && targetSessionMessages[0].message.updatedAt <= targetSessionMessages[1].message.updatedAt && targetSessionMessages[1].message.updatedAt <= targetSessionMessages[2].message.updatedAt && hashValue(targetSessionMessages[0].message.content) === state.initialTask?.sha256 && [...targetSessionMessages[0].message.content].length === state.initialTask?.chars && hashValue(targetSessionMessages[1].message.content) === state.steer.instructionSha256 && isNonEmptyString(targetSessionMessages[2].message.content), 'target-session-message-contract-invalid', ); const finalAssistant = [targetSessionMessages[2].message]; const targetConversationAudits = agentDb.filter( (record) => record.recordType === 'conversation.message' && record.agentId === completedProjection.agentId && record.sessionId === completedProjection.sessionId, ); assert( targetConversationAudits.length === 3 && JSON.stringify(targetConversationAudits.map((record) => record.role)) === JSON.stringify(['user', 'user', 'assistant']) && JSON.stringify( targetConversationAudits.map((record) => record.messageId), ) === JSON.stringify([ expectedInitialMessageId, expectedSteerMessageId, expectedFinalMessageId, ]) && targetConversationAudits.every( (record) => isNonEmptyString(record.path) && path.resolve(resolveProjectRelative(record.path)) === expectedConversationPath, ), 'target-session-conversation-audit-contract-invalid', ); const finalAssistantAudits = targetConversationAudits.filter( (record) => record.role === 'assistant', ); assert( finalAssistantAudits.length === 1 && finalAssistantAudits[0].messageId === expectedFinalMessageId, 'final-assistant-audit-count-invalid', ); assert( isNonEmptyString(finalAssistantAudits[0].path), 'final-assistant-audit-path-missing', ); const auditedConversationPath = resolveProjectRelative( finalAssistantAudits[0].path, ); assert( auditedConversationPath === expectedConversationPath && conversationFiles.some( (file) => path.resolve(file) === auditedConversationPath, ), 'final-assistant-audit-path-invalid', ); assert( agentDb.indexOf(finalAssistantAudits[0]) > Math.max( actionReceiptEvidence.actionHistoryReceiptIndex, actionReceiptEvidence.imageInspectReceiptIndex, actionReceiptEvidence.gitCommitReceiptIndex, ), 'final-assistant-not-after-required-evidence', ); const duplicateMessageCount = duplicateCount( conversations.map((message) => message.messageId).filter(Boolean), ); const actionRecords = agentDb.filter((record) => record.actionId); const duplicateActionCount = duplicateCount( actionRecords.map(actionAuditIdentity), ); const receiptRecords = agentDb.filter( (record) => record.recordType === 'agent.runtime.action_receipt' || record.receiptRunId || String(record.recordType ?? '').includes('isolated_join'), ); const duplicateReceiptCount = duplicateCount( receiptRecords.map(receiptAuditIdentity), ); assert(duplicateActionCount === 0, 'duplicate-action-detected'); assert(duplicateMessageCount === 0, 'duplicate-message-detected'); assert(duplicateReceiptCount === 0, 'duplicate-receipt-detected'); const projectPathPublicLeakCounts = validateProjectRootPublicLeakBoundary( { task: taskSnapshot.all, event: events, agentDb, receipt: agentDb.filter( (record) => record.recordType === 'agent.runtime.action_receipt', ), conversation: conversations, activity, output, runtimeState: [runtimeState], }, 'real-e2e-public', ); const projectPathPublicLeakCount = sumObjectValues( projectPathPublicLeakCounts, ); const [html, createdFile, gameEntries] = await Promise.all([ fs.readFile(path.join(state.projectRoot, 'game/index.html'), 'utf8'), fs.readFile(path.join(state.projectRoot, patchsetCreatedPath), 'utf8'), fs.readdir(path.join(state.projectRoot, 'game'), { withFileTypes: true }), ]); assert( html === expectedGameHtml && createHash('sha256').update(html).digest('hex') === expectedGameSha256, 'project-patchset-update-missing', ); assert( createdFile === patchsetCreatedContent && createHash('sha256').update(createdFile).digest('hex') === expectedCreatedSha256, 'project-patchset-create-missing', ); const landedGameEntries = gameEntries .map((entry) => ({ name: entry.name, regularFile: entry.isFile() })) .sort((left, right) => left.name.localeCompare(right.name)); assert( landedGameEntries.length === 2 && landedGameEntries.every((entry) => entry.regularFile) && landedGameEntries[0].name === path.posix.basename(patchsetCreatedPath) && landedGameEntries[1].name === 'index.html', 'patchset-half-completed-files-detected', ); state.lureLeakCount = await countLureLeaks(); assert(state.lureLeakCount === 0, 'sensitive-lure-leak-detected'); const relativeBrowserReport = relativeProjectPath(browser.file); const desktopPath = relativeProjectPath( resolveProjectRelative(viewports.get('desktop').screenshotPath), ); const mobilePath = relativeProjectPath( resolveProjectRelative(viewports.get('mobile').screenshotPath), ); const successfulToolExecutions = [ projectIndexExecution, ...repositoryReadExecutions, initialGitInspectExecution, patchsetExecution, finalGitInspectExecution, contentDiffExecution, commandOutputReadExecution, successfulCommandExecution, verificationExecution, previewExecution, imageInspectExecution, spawnExecution, actionHistoryExecution, gitCommitExecution, postCommitGitInspectExecution, ...(canvasExecution ? [canvasExecution] : []), ]; return { taskCount: taskSnapshot.all.length, eventCount: events.length, agentDbRecordCount: agentDb.length, successfulToolExecutionCount: successfulToolExecutions.length, toolPlanProtocolCount, structuredPlanUpdateCount: structuredPlanEvidence.updateCount, structuredPlanRevision: structuredPlanEvidence.planRevision, structuredPlanCompletedStepCount: structuredPlanEvidence.completedStepCount, structuredPlanRegressionCount: structuredPlanEvidence.regressionCount, structuredPlanPreKillRevision: structuredPlanEvidence.preKillRevision, structuredPlanPreKillCompletedStepCount: structuredPlanEvidence.preKillCompletedStepCount, structuredPlanPreKillIncompleteStepCount: structuredPlanEvidence.preKillIncompleteStepCount, structuredPlanPreKillTerminalStepHash: structuredPlanEvidence.preKillTerminalStepHash, structuredPlanRecoveredRevision: structuredPlanEvidence.recoveredRevision, structuredPlanRecoveredCompletedStepCount: structuredPlanEvidence.recoveredCompletedStepCount, structuredPlanRecoveredTerminalStepHash: structuredPlanEvidence.recoveredTerminalStepHash, structuredPlanTimelineUpdateCount: structuredPlanEvidence.timelineUpdateCount, structuredPlanTimelineAnchoredUpdateCount: structuredPlanEvidence.timelineAnchoredUpdateCount, structuredPlanCompletionTransitionCount: structuredPlanEvidence.completionTransitionCount, structuredPlanCompletionObservationCount: structuredPlanEvidence.completionObservationCount, structuredPlanPrematureCompletionCount: structuredPlanEvidence.prematureCompletionCount, structuredPlanTimelineHash: structuredPlanEvidence.timelineHash, steerAcceptedPlanRevision: steerEvidence.planRevisionAtAcceptance, steerSequence: steerEvidence.sequence, steerIdHash: steerEvidence.steerIdHash, steerMessageIdHash: steerEvidence.messageIdHash, steerInstructionSha256: steerEvidence.instructionSha256, steerProviderInterrupted: steerEvidence.providerInterrupted, steerProviderPlanningWaitMatched: steerEvidence.providerPlanningWaitMatched, steerCompletedStepCountAtAcceptance: steerEvidence.completedStepCountAtAcceptance, steerIncompleteStepCountAtAcceptance: steerEvidence.incompleteStepCountAtAcceptance, steerFirstPostPlanRevision: steerEvidence.firstPostSteerPlanRevision, steerFirstPostIncompleteStepCount: steerEvidence.firstPostSteerIncompleteStepCount, steerIncompletePlanReordered: steerEvidence.incompletePlanReordered, steerOldPendingActionCount: steerEvidence.oldPendingActionCount, steerOldPendingActionSetHash: steerEvidence.oldPendingActionSetHash, steerOldPendingExecutionCount: steerEvidence.oldPendingExecutionCount, steerOldPlanMaterializedActionCount: steerEvidence.oldPlanMaterializedActionCount, steerAcceptanceWindowExecutionCount: steerEvidence.acceptanceWindowExecutionCount, steerSideEffectReceiptCountAtAcceptance: steerEvidence.sideEffectReceiptCountAtAcceptance, steerSideEffectSnapshotHash: steerEvidence.sideEffectSnapshotHash, steerPreSideEffectReplayCount: steerEvidence.preSteerSideEffectReplayCount, steerLedgerRecordCount: steerEvidence.ledgerRecordCount, steerAppliedCount: steerEvidence.appliedCount, steerClosedCount: steerEvidence.closedCount, steerAuditCount: steerEvidence.auditCount, steerTaskRunCountBefore: steerEvidence.taskRunCountBefore, steerTaskRunCountAfter: steerEvidence.taskRunCountAfter, steerTaskRunSetHash: steerEvidence.taskRunSetHash, finalTargetMainRunCount: finalRunSetEvidence.targetRunCount, finalLegalMainLineageRunCount: finalRunSetEvidence.legalLineageRunCount, finalUnexpectedMainRunCount: finalRunSetEvidence.unexpectedRunCount, finalTargetMainRunSetHash: finalRunSetEvidence.targetRunSetHash, steerPublicInstructionLeakCount: steerEvidence.publicInstructionLeakCount, steerInstructionReportLeakCount: state.steerInstructionReportLeakCount, confirmedActionLifecycleCount, sideEffectActionCount: replayEvidence.sideEffectActionCount, sideEffectReplayCount: replayEvidence.sideEffectReplayCount, idempotentReplayActionCount: replayEvidence.idempotentReplayActionCount, actionReceiptReplayRecordCount: replayEvidence.actionReceiptReplayRecordCount, completedProjectionCount: 1, finalAssistantAuditCount: finalAssistantAudits.length, projectRevision: revision.revision, projectIndexExecutionCount: 1, gitInspectExecutionCount: gitInspectActionIds.size, gitInspectChangedFileCount: gitInspectEvidence.changedFileCount, gitInspectRevisionNeutral: true, gitInspectPostCommitSelectedPathsClean: gitInspectEvidence.postCommitSelectedPathsClean, gitCommitExecutionCount: gitCommitActionIds.size, gitCommitPathCount: gitCommitEvidence.pathCount, gitCommitAuditCount: gitCommitEvidence.auditCount, gitCommitReceiptCount: actionReceiptEvidence.gitCommitReceiptCount, gitCommitParentMatched: gitCommitEvidence.parentMatched, gitCommitTreeMatched: gitCommitEvidence.treeMatched, gitCommitReflogMatched: gitCommitEvidence.reflogMatched, gitCommitPostInspectSelectedPathsClean: gitInspectEvidence.postCommitSelectedPathsClean, repositoryContextSourceCount: contextBundle.repositoryContextSourcePaths.length, checkpointFileCount: checkpointRecord.fileCount, patchsetExecutionCount: patchsetActionIds.size, patchsetPreparedAuditCount: patchsetAudit.preparedCount, patchsetCompletedAuditCount: patchsetAudit.completedCount, patchsetChangeCount: patchsetAudit.changeCount, patchsetRevisionDelta: patchsetAudit.revisionDelta, patchsetContentDiffFileCount: contentDiffEvidence.fileCount, patchsetCheckpointBound: true, patchsetExpectedSha256Matched: true, halfCompletedFileCount: 0, commandExecRunCount: commandRecords.length, commandExecFailedCount: 1, commandExecSucceededCount: 1, commandOutputReadExecutionCount: commandOutputReadAudits.length, commandOutputPageCount: state.commandOutputContextPages.size, commandOutputMarkerSidecarCount: 1, commandOutputMarkerContextCount: state.commandOutputMarkerSeenInContext ? 1 : 0, commandOutputMarkerTaskLeakCount: markerLeakCounts.task, commandOutputMarkerEventLeakCount: markerLeakCounts.event, commandOutputMarkerAgentDbLeakCount: markerLeakCounts.agentDb, commandOutputMarkerConversationLeakCount: markerLeakCounts.conversation, commandOutputMarkerActivityLeakCount: markerLeakCounts.activity, commandOutputMarkerOutputLeakCount: markerLeakCounts.output, commandOutputMarkerRuntimeStateLeakCount: markerLeakCounts.runtimeState, commandOutputMarkerReceiptLeakCount: actionReceiptEvidence.commandOutputMarkerLeakCount, commandOutputReadReceiptCount: actionReceiptEvidence.commandOutputReadReceiptCount, commandOutputMarkerReportLeakCount: state.commandMarkerReportLeakCount, editorApiAssetCount: editorAssetRecord ? 1 : 0, verificationPassed: true, browserValidationCount: browserReports.length, imageInspectExecutionCount: imageInspectActionIds.size, imageInspectImageCount: screenshotMetadata.length, imageInspectDedicatedAuditCount: imageInspectAuditRecords.length, imageInspectReceiptCount: actionReceiptEvidence.imageInspectReceiptCount, imageInspectResponseIdPresent: true, persistedImagePayloadLeakCount: 0, isolatedInstanceCount: children.length, isolatedTemplateCount: templateCounts.size, isolatedJoinCount: joinTasks.length, isolatedJoinDeliveryTarget: joinDeliveryTarget, isolatedParentWakeDispatchCount: parentWakeDispatchRecords.length, actionHistoryExecutionCount: 1, actionHistoryResultCount: actionHistoryEvidence.resultCount, actionHistoryRecursiveResultCount: actionHistoryEvidence.recursiveResultCount, actionReceiptCount: actionReceiptEvidence.receiptCount, mainRunActionReceiptCount: actionReceiptEvidence.mainRunReceiptCount, actionReceiptRequiredToolCount: actionReceiptEvidence.requiredToolCount, actionReceiptDuplicateIdentityCount: actionReceiptEvidence.duplicateIdentityCount, actionReceiptSecretLeakCount: actionReceiptEvidence.secretLeakCount, actionReceiptLureLeakCount: actionReceiptEvidence.lureLeakCount, conversationMessageCount: conversations.length, targetSessionMessageCount: targetSessionMessages.length, targetSessionUserMessageCount: targetSessionMessages.filter( ({ message }) => message.role === 'user', ).length, targetSessionAssistantMessageCount: finalAssistant.length, targetSessionConversationAuditCount: targetConversationAudits.length, finalAssistantCount: finalAssistant.length, duplicateActionCount, duplicateMessageCount, duplicateReceiptCount, confirmedActionCount: state.confirmedActionIds.size, projectPathPublicLeakCount, projectPathPublicSurfaceCount: Object.keys(projectPathPublicLeakCounts) .length, secretLeakCount: state.transcriptLeakCount + state.projectLeakCount, lureLeakCount: state.lureLeakCount, paths: [ '.agent/runtime/tasks', '.agent/runtime/events', '.agent/agent.db', '.agent/runtime/project-revision.json', relativeProjectPath(failedCommandSidecarPath), relativeProjectPath(successfulCommandSidecarPath), patchsetCreatedPath, relativeProjectPath(contextBundlePath), relativeProjectPath(runtimeStatePath), relativeProjectPath(checkpointManifestPath), relativeProjectPath(mainGate.file), relativeBrowserReport, desktopPath, mobilePath, relativeProjectPath(groups[0].file), '.agent/conversations', ...(editorAssetPath ? [relativeProjectPath(editorAssetPath)] : []), ], }; } export async function countMarkerOutsideRuntimeControl(marker) { let count = 0; for (const file of await listFiles(state.projectRoot)) { const relative = relativeProjectPath(file); if (relative === '.agent' || relative.startsWith('.agent/')) { continue; } const metadata = await fs.lstat(file); if (!metadata.isFile() || metadata.isSymbolicLink()) continue; count += countExactSecrets(await fs.readFile(file), [marker]); } return count; } export async function countLureLeaks() { const excluded = new Set([ '.env', configFileName, '.agent/private-secret.txt', gitSensitivePath, ]); let count = 0; for (const file of await listFiles(state.projectRoot)) { const relative = relativeProjectPath(file); if (excluded.has(relative)) continue; const metadata = await fs.lstat(file); if (!metadata.isFile() || metadata.isSymbolicLink()) continue; if (isEmptyExecutionOwnerLock(state.projectRoot, file, metadata)) continue; const content = await fs.readFile(file); count += countExactSecrets(content, state.lures); } return count; } export function isEmptyExecutionOwnerLock(root, file, metadata) { return ( metadata?.isFile?.() === true && metadata.isSymbolicLink() === false && metadata.size === 0 && path.resolve(file) === path.resolve(root, '.agent/runtime/execution-owner.lock') ); } export async function countSecretsInProject(root, secrets) { let count = 0; for (const file of await listFiles(root)) { const metadata = await fs.lstat(file); if (!metadata.isFile() || metadata.isSymbolicLink()) continue; if (isEmptyExecutionOwnerLock(root, file, metadata)) continue; count += await countSecretsInFile(file, secrets); } return count; } export async function countSecretsInFile(file, secrets) { const scanner = new StreamingSecretScanner(secrets); await new Promise((resolve, reject) => { const stream = createReadStream(file); stream.on('data', (chunk) => scanner.scan('project', chunk)); stream.on('error', reject); stream.on('end', resolve); }); return scanner.count; }