diff --git a/apps/ai-game-creator-shell/package.json b/apps/ai-game-creator-shell/package.json index d475dd99e..e00bc7ee5 100644 --- a/apps/ai-game-creator-shell/package.json +++ b/apps/ai-game-creator-shell/package.json @@ -19,6 +19,7 @@ "agent-runtime:mixed-swarm-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-static-isolated-autonomous-chat", "agent-runtime:supervisor-swarm-autonomous-chat-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-autonomous-chat", "agent-runtime:supervisor-swarm-transient-retry-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-transient-retry", + "agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-final-reply-transient-retry", "agent-runtime:steer-real-e2e": "node scripts/agent-runtime-steer-real-e2e.mjs", "agent-runtime:steer-runner-kill-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite steer-runner-kill", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --noEmit && node scripts/check-config.mjs" diff --git a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs index 1076b73ba..ba9ff87e2 100644 --- a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs +++ b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs @@ -71,6 +71,10 @@ const supervisorSwarmTransientRetryAppDataSentinelFileName = '.agent-runtime-real-e2e-supervisor-swarm-transient-retry-appdata.json'; const supervisorSwarmTransientRetryAppDataSentinelSchema = 'genarrative-agent-runtime-real-e2e-supervisor-swarm-transient-retry-appdata.v1'; +const supervisorSwarmFinalReplyTransientRetryAppDataSentinelFileName = + '.agent-runtime-real-e2e-supervisor-swarm-final-reply-transient-retry-appdata.json'; +const supervisorSwarmFinalReplyTransientRetryAppDataSentinelSchema = + 'genarrative-agent-runtime-real-e2e-supervisor-swarm-final-reply-transient-retry-appdata.v1'; const supervisorSwarmAutonomousChatAppDataSentinelFileName = '.agent-runtime-real-e2e-supervisor-swarm-autonomous-chat-appdata.json'; const supervisorSwarmAutonomousChatAppDataSentinelSchema = @@ -113,6 +117,8 @@ const steerRunnerKillSuite = 'steer-runner-kill'; const parallelReadSuite = 'parallel-read'; const supervisorSwarmSuite = 'supervisor-swarm'; const supervisorSwarmTransientRetrySuite = 'supervisor-swarm-transient-retry'; +const supervisorSwarmFinalReplyTransientRetrySuite = + 'supervisor-swarm-final-reply-transient-retry'; const supervisorSwarmTransientRetryTargetAgentId = projectSupervisorAgentId; const supervisorSwarmTransientRetryBackoffMs = 30_000; const supervisorSwarmTransientRetryPreDueGuardMs = 1_500; @@ -418,6 +424,7 @@ const processEchoPrefix = 'GENARRATIVE_PROCESS_ECHO'; const processStoppedMarker = 'GENARRATIVE_PROCESS_STOPPED'; const pollIntervalMs = 750; const runTimeoutMs = 30 * 60 * 1000; +const supervisorSwarmTerminalSidecarCleanupTimeoutMs = 10_000; const processRunnerKillStartTimeoutMs = 5 * 60 * 1000; const commandOutputLimit = 4 * 1024 * 1024; const supportedToolPlanProtocols = new Set([ @@ -781,10 +788,16 @@ const state = { newRunnerBootId: null, transientFaultProxy: null, transientFaultCheckpoint: null, + transientFaultSelection: null, + transientFaultSelectorError: false, + transientFaultRequestSequence: 0, + transientFaultPreFaultHostVerificationPassed: false, + transientFaultPreFaultHostVerificationCompletedAtMs: 0, transientRetryPreKillSidecar: null, transientRetryPostRestartSidecar: null, transientRetryPreKillRunnerBootId: null, transientRetryPostRestartRunnerBootId: null, + transientRetryRunnerKilledAtMs: 0, transientRetryRequestCountBeforeKill: 0, transientRetryRequestCountAfterRestart: 0, transientRetryRequestCountBeforeDue: 0, @@ -797,6 +810,9 @@ const state = { transientRetryIncidentalProviderFailureCount: 0, transientRetryIncidentalProviderRetryCount: 0, transientRetryRecoveredFromPreviousBoot: false, + transientRetryToolPlanCountAtFault: 0, + transientRetryToolPlanCountAfterRecovery: 0, + transientRetryFinalReplyStreamCommitted: false, autonomousTaskRecipeFree: false, autonomousRepositoryRecipeFree: false, interactiveCliUsed: false, @@ -10019,12 +10035,14 @@ async function driveSupervisorSwarmToRepairKillBoundary() { state.supervisorSwarm.repairPendingIdentity = supervisorSwarmPendingIdentity(repairPending); state.supervisorSwarm.repairPendingTool = repairPending.tool; + if (isSupervisorSwarmInitialTransientRetrySuite()) { assert( state.supervisorSwarm.runnerKillBoundaryObserved && state.identityStable && state.supervisorSwarm.transientRetryRecoveredFromPreviousBoot, 'supervisor-swarm-transient-retry-recovery-not-observed-before-repair', ); + } return; } if (isSupervisorSwarmMixedHarnessSuite()) { @@ -10098,6 +10116,7 @@ async function driveSupervisorSwarmToRepairKillBoundary() { async function driveSupervisorSwarmRuntimeToCompletion() { const deadline = Date.now() + runTimeoutMs; let quietPolls = 0; + let terminalSidecarCleanupStartedAt = 0; while (Date.now() < deadline) { const persistence = await readSupervisorSwarmPersistence(); assertSupervisorSwarmProviderFailureRecoverable(persistence); @@ -10154,7 +10173,7 @@ async function driveSupervisorSwarmRuntimeToCompletion() { (delivery) => delivery.status === 'claimed-by-parent', ) && validateSupervisorSwarmMixedIsolatedPersistence(persistence)); - const completed = + const terminalStateObserved = deliveries.length === 3 && deliveries.every((delivery) => delivery.status === 'claimed-by-parent') && childTasks.length === 3 && @@ -10172,11 +10191,25 @@ async function driveSupervisorSwarmRuntimeToCompletion() { (message) => message.role === 'assistant', ).length === 1 && (await findPendingActions()).length === 0; - if (completed) { + if (terminalStateObserved) { + const residualSidecars = await readSupervisorSwarmResidualSidecarCounts(); + if (!supervisorSwarmResidualSidecarsEmpty(residualSidecars)) { + terminalSidecarCleanupStartedAt ||= Date.now(); + if ( + Date.now() - terminalSidecarCleanupStartedAt >= + supervisorSwarmTerminalSidecarCleanupTimeoutMs + ) { + throw codedError('supervisor-swarm-terminal-sidecar-present'); + } + quietPolls = 0; + await sleep(25); + continue; + } quietPolls += 1; if (quietPolls >= 3) return; } else { quietPolls = 0; + terminalSidecarCleanupStartedAt = 0; } await sleep(150); } @@ -10245,6 +10278,9 @@ async function prepareSupervisorSwarmRuntimeAppData() { upstreamBaseUrl: targetConfig.baseUrl, faultCount: 1, holdAfterFault: true, + ...(isSupervisorSwarmFinalReplyTransientRetrySuite() + ? { shouldInjectFault: shouldInjectSupervisorSwarmFinalReplyFault } + : {}), }); state.supervisorSwarm.transientFaultProxy = proxy; assert( @@ -10303,6 +10339,278 @@ async function readSupervisorSwarmProjectRevision() { }); } +function supervisorSwarmOpenProviderStarts(agentDb) { + if (!isNonEmptyString(state.initialRunId)) return []; + const terminalRequestIds = new Set( + agentDb + .filter( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + ['completed', 'failed', 'interrupted'].includes(record.status), + ) + .map((record) => record.requestId), + ); + return agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.status === 'started' && + record.agentId === projectSupervisorAgentId && + record.sessionId === supervisorSwarmSessionId && + record.runId === state.initialRunId && + !terminalRequestIds.has(record.requestId), + ); +} + +function supervisorSwarmFinalReplyFaultPrerequisites( + persistence, + finalReplyStarted, + metadata, +) { + const deliveries = supervisorSwarmParentDeliveries(persistence.deliveries); + const repairs = deliveries.filter( + (delivery) => delivery.repairOfDelegationId != null, + ); + const deliveryIds = new Set( + deliveries.map((delivery) => delivery.delegationId), + ); + const relevantClaims = persistence.claims.filter((claim) => + (claim.receipts ?? []).some((receipt) => + deliveryIds.has(receipt.delegationId), + ), + ); + const claimedReceiptIds = relevantClaims.flatMap((claim) => + (claim.receipts ?? []).map((receipt) => receipt.delegationId), + ); + const verificationAudits = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.project.verify' && + record.agentId === projectSupervisorAgentId && + record.runId === state.initialRunId && + record.status === 'completed' && + record.exitCode === 0 && + record.timedOut === false && + isNonEmptyString(record.actionId), + ); + const verificationAudit = verificationAudits.at(-1); + const verificationReceipts = verificationAudit + ? persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.action_receipt' && + record.agentId === projectSupervisorAgentId && + record.runId === state.initialRunId && + record.actionId === verificationAudit.actionId && + record.tool === 'project.verify' && + record.status === 'ok', + ) + : []; + const verificationObservations = verificationAudit + ? persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.tool_observation' && + record.agentId === projectSupervisorAgentId && + record.runId === state.initialRunId && + record.actionId === verificationAudit.actionId && + record.tool === 'project.verify' && + record.status === 'ok', + ) + : []; + const finalReplyStartedIndex = persistence.agentDb.indexOf(finalReplyStarted); + const verificationAuditIndex = verificationAudit + ? persistence.agentDb.indexOf(verificationAudit) + : -1; + const verificationObservationIndex = verificationObservations[0] + ? persistence.agentDb.indexOf(verificationObservations[0]) + : -1; + const runtimeVerificationOrderValid = + verificationAuditIndex >= 0 && + verificationAuditIndex < verificationObservationIndex && + verificationObservationIndex < finalReplyStartedIndex; + const assistantCount = persistence.supervisorConversation.filter( + (message) => message.role === 'assistant', + ).length; + const parentToolPlanStartedCount = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.status === 'started' && + record.agentId === projectSupervisorAgentId && + record.runId === state.initialRunId && + record.requestKind === 'tool-plan', + ).length; + const repair = repairs[0]; + const prerequisiteChecks = { + deliveryCount: deliveries.length === 3, + deliveryTerminal: deliveries.every( + (delivery) => + delivery.status === 'claimed-by-parent' && + delivery.terminalStatus === 'completed', + ), + repairDelivery: + repairs.length === 1 && + repair.status === 'claimed-by-parent' && + repair.terminalStatus === 'completed', + observedClaims: + relevantClaims.length === 2 && + relevantClaims.every((claim) => claim.status === 'observed'), + observedClaimReceipts: + claimedReceiptIds.length === 3 && + new Set(claimedReceiptIds).size === 3 && + claimedReceiptIds.every((delegationId) => + deliveryIds.has(delegationId), + ), + finalReplyTaskIdentity: finalReplyStarted.taskId != null, + assistantAbsent: assistantCount === 0, + projectRevision: + Number.isSafeInteger(persistence.projectRevision?.revision) && + persistence.projectRevision.revision > 0, + }; + const invalidPrerequisiteFields = Object.entries(prerequisiteChecks) + .filter(([, valid]) => !valid) + .map(([field]) => field); + return { + sequence: metadata.sequence, + acceptedAtMs: metadata.acceptedAtMs, + requestId: finalReplyStarted.requestId, + requestKind: finalReplyStarted.requestKind, + requestSlot: finalReplyStarted.requestSlot, + deliveryCount: deliveries.length, + claimedDeliveryCount: deliveries.filter( + (delivery) => delivery.status === 'claimed-by-parent', + ).length, + repairDeliveryCount: repairs.length, + observedClaimCount: relevantClaims.filter( + (claim) => claim.status === 'observed', + ).length, + observedClaimReceiptCount: claimedReceiptIds.length, + runtimeVerificationCount: verificationAudits.length, + runtimeVerificationReceiptCount: verificationReceipts.length, + runtimeVerificationObservationCount: verificationObservations.length, + runtimeVerificationOrderValid, + assistantCount, + projectRevision: persistence.projectRevision?.revision ?? 0, + parentToolPlanStartedCount, + invalidPrerequisiteFields, + preconditionsValid: invalidPrerequisiteFields.length === 0, + }; +} + +function supervisorSwarmHostVerificationPassed(result) { + return ( + String(result?.stdout ?? '').includes(commandPassedMarker) && + !String(result?.stdout ?? '').includes(commandFailureMarker) && + !String(result?.stderr ?? '').includes(commandFailureMarker) + ); +} + +function supervisorSwarmFinalReplyFaultInjectionAllowed(selection) { + return selection?.preconditionsValid === true; +} + +async function shouldInjectSupervisorSwarmFinalReplyFault(metadata) { + if (!isNonEmptyString(state.initialRunId)) return false; + const deadline = Date.now() + 1_000; + let candidates = []; + while (Date.now() < deadline) { + const agentDb = await readOptionalJsonl( + path.join(state.projectRoot, '.agent/agent.db'), + ); + const openRequests = supervisorSwarmOpenProviderStarts(agentDb); + candidates = openRequests.filter( + (record) => + record.requestKind === 'final-reply' && + !String(record.requestSlot ?? '').includes('-transient-'), + ); + if (candidates.length > 0) break; + if (openRequests.length > 0) return false; + await sleep(10); + } + if (candidates.length === 0) return false; + if (candidates.length !== 1) { + state.supervisorSwarm.transientFaultSelectorError = true; + state.supervisorSwarm.transientFaultSelection = { + sequence: metadata.sequence, + acceptedAtMs: metadata.acceptedAtMs, + candidateCount: candidates.length, + invalidPrerequisiteFields: ['finalReplyCandidateCount'], + preconditionsValid: false, + }; + return false; + } + try { + const [persistence, projectRevision] = await Promise.all([ + readSupervisorSwarmPersistence(), + readSupervisorSwarmProjectRevision(), + ]); + const persistedCandidate = persistence.agentDb.find( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.status === 'started' && + record.requestId === candidates[0].requestId && + record.requestKind === 'final-reply', + ); + let selection = supervisorSwarmFinalReplyFaultPrerequisites( + { ...persistence, projectRevision }, + persistedCandidate ?? candidates[0], + metadata, + ); + if (selection.preconditionsValid) { + let hostVerification = null; + try { + hostVerification = await runProcess( + process.execPath, + ['verify-e2e.mjs'], + { + cwd: state.projectRoot, + timeoutMs: 120_000, + }, + ); + } catch { + hostVerification = null; + } + const completedAtMs = Date.now(); + const hostVerificationPassed = + supervisorSwarmHostVerificationPassed(hostVerification); + const hostVerificationValid = + hostVerificationPassed && completedAtMs >= metadata.acceptedAtMs; + selection = { + ...selection, + preFaultHostVerificationCompletedAtMs: completedAtMs, + preFaultHostVerificationPassed: hostVerificationValid, + invalidPrerequisiteFields: hostVerificationValid + ? selection.invalidPrerequisiteFields + : [...selection.invalidPrerequisiteFields, 'hostVerification'], + preconditionsValid: hostVerificationValid, + }; + } else { + selection = { + ...selection, + preFaultHostVerificationCompletedAtMs: 0, + preFaultHostVerificationPassed: false, + }; + } + state.supervisorSwarm.transientFaultSelection = selection; + state.supervisorSwarm.transientFaultSelectorError = + !selection.preconditionsValid; + state.supervisorSwarm.transientFaultRequestSequence = metadata.sequence; + state.supervisorSwarm.transientFaultPreFaultHostVerificationPassed = + selection.preFaultHostVerificationPassed; + state.supervisorSwarm.transientFaultPreFaultHostVerificationCompletedAtMs = + selection.preFaultHostVerificationCompletedAtMs; + state.supervisorSwarm.transientRetryToolPlanCountAtFault = + selection.parentToolPlanStartedCount; + return supervisorSwarmFinalReplyFaultInjectionAllowed(selection); + } catch { + state.supervisorSwarm.transientFaultSelectorError = true; + state.supervisorSwarm.transientFaultSelection = { + sequence: metadata.sequence, + acceptedAtMs: metadata.acceptedAtMs, + candidateCount: 1, + invalidPrerequisiteFields: ['prerequisiteRead'], + preconditionsValid: false, + }; + return false; + } +} + async function readSupervisorSwarmProviderRetrySidecar(agentId, runId) { const directory = path.join( state.projectRoot, @@ -10340,6 +10648,43 @@ async function readSupervisorSwarmProviderRetrySidecar(agentId, runId) { return matches[0] ?? null; } +async function driveSupervisorSwarmUntilFinalReplyFault() { + assert( + isSupervisorSwarmFinalReplyTransientRetrySuite(), + 'supervisor-swarm-final-reply-fault-driver-outside-suite', + ); + const proxy = state.supervisorSwarm.transientFaultProxy; + assert(proxy, 'supervisor-swarm-final-reply-fault-proxy-missing'); + const deadline = Date.now() + runTimeoutMs; + while (Date.now() < deadline) { + if (proxy.getStats().faultInjectedCount === 1) return; + const persistence = await readSupervisorSwarmPersistence(); + assertSupervisorSwarmProviderFailureRecoverable(persistence); + assertSupervisorSwarmRuntimeHealthy(persistence); + const deliveries = supervisorSwarmParentDeliveries(persistence.deliveries); + const confirmRunKeys = new Set([ + `${projectSupervisorAgentId}\0${state.initialRunId}`, + ...deliveries.map( + (delivery) => `${delivery.targetAgentId}\0${delivery.targetRunId}`, + ), + ]); + await confirmSupervisorSwarmPendingActions(confirmRunKeys); + const parentTask = persistence.taskSnapshot.latest.find( + (task) => + task.agentId === projectSupervisorAgentId && + task.runId === state.initialRunId, + ); + if ( + parentTask?.status === 'completed' || + parentTask?.phase === 'completed' + ) { + throw codedError('supervisor-swarm-final-reply-completed-before-fault'); + } + await sleep(50); + } + throw codedError('supervisor-swarm-final-reply-fault-timeout'); +} + function supervisorSwarmTransientRetrySideEffects( persistence, pending, @@ -10421,7 +10766,24 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { if (!isSupervisorSwarmTransientRetrySuite()) return; const proxy = state.supervisorSwarm.transientFaultProxy; assert(proxy, 'supervisor-swarm-transient-retry-proxy-missing'); + if (isSupervisorSwarmFinalReplyTransientRetrySuite()) { + await driveSupervisorSwarmUntilFinalReplyFault(); + } else { await proxy.waitForFault(60_000); + } + const finalReplyMode = isSupervisorSwarmFinalReplyTransientRetrySuite(); + const expectedRequestKind = finalReplyMode ? 'final-reply' : 'tool-plan'; + const faultSelection = state.supervisorSwarm.transientFaultSelection; + const faultSequence = finalReplyMode ? faultSelection?.sequence : 1; + assert( + Number.isSafeInteger(faultSequence) && + faultSequence > 0 && + (!finalReplyMode || + (faultSelection?.preconditionsValid === true && + faultSelection.requestKind === 'final-reply' && + state.supervisorSwarm.transientFaultSelectorError === false)), + 'supervisor-swarm-transient-retry-fault-selection-invalid', + ); const target = { agentId: supervisorSwarmTransientRetryTargetAgentId, runId: state.initialRunId, @@ -10458,7 +10820,9 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { const failedStarted = lifecycle.find( (record) => record.status === 'started' && - !record.requestSlot?.includes('-transient-'), + record.requestKind === expectedRequestKind && + !record.requestSlot?.includes('-transient-') && + (!finalReplyMode || record.requestId === faultSelection.requestId), ); const failedTerminal = failedStarted ? lifecycle.find( @@ -10467,7 +10831,10 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { record.status === 'failed', ) : null; - const retryAudit = retries[0]; + const matchingRetries = failedStarted + ? retries.filter((record) => record.requestId === failedStarted.requestId) + : []; + const retryAudit = matchingRetries[0]; if (!failedStarted || !failedTerminal || !retryAudit) { await sleep(25); continue; @@ -10477,6 +10844,9 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { record.status === 'started' && record.requestSlot === retryAudit.nextRequestSlot, ); + const requestLifecycle = lifecycle.filter( + (record) => record.requestId === failedStarted.requestId, + ); const sidecar = await readSupervisorSwarmProviderRetrySidecar( target.agentId, target.runId, @@ -10517,10 +10887,10 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { const proxyStats = proxy.getStats(); const retryRecord = sidecar.record; assert( - lifecycle.length === 2 && - retries.length === 1 && + requestLifecycle.length === 2 && + matchingRetries.length === 1 && retryStarted.length === 0 && - failedStarted.requestKind === 'tool-plan' && + failedStarted.requestKind === expectedRequestKind && failedStarted.requestId === failedTerminal.requestId && failedStarted.requestSlot === failedTerminal.requestSlot && failedStarted.requestId === retryAudit.requestId && @@ -10564,17 +10934,29 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { targetRuntime.phase === 'waiting-for-provider-retry' && targetTask.status === 'running' && targetTask.phase === 'waiting-for-provider-retry' && - proxyStats.requestCount === 1 && + proxyStats.requestCount === faultSequence && proxyStats.faultInjectedCount === 1 && proxyStats.heldRequestCount === 0 && - proxyStats.forwardedRequestCount === 0 && + proxyStats.forwardedRequestCount === faultSequence - 1 && proxyStats.forwardingReleased === false, 'supervisor-swarm-transient-retry-waiting-boundary-invalid', ); + if (finalReplyMode) { + assert( + sideEffects.childDeliveryCount === 3 && + sideEffects.claimCount === 2 && + sideEffects.assistantCount === 0 && + sideEffects.pendingCount === 0 && + sideEffects.projectRevision > 0 && + sideEffects.designArtifactPresent === true, + 'supervisor-swarm-final-reply-transient-retry-pre-kill-prerequisite-invalid', + ); + } else { assertSupervisorSwarmTransientRetrySideEffectsZero( sideEffects, 'supervisor-swarm-transient-retry-pre-kill-side-effect', ); + } waitingBoundary = { failedStarted, failedTerminal, @@ -10582,6 +10964,24 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { target, sidecar, sideEffects, + faultSelection, + faultSequence, + finalReplyMode, + deliveryIdentities: supervisorSwarmParentDeliveries( + persistence.deliveries, + ) + .map(supervisorSwarmDeliveryIdentity) + .sort((left, right) => + left.delegationId.localeCompare(right.delegationId), + ), + claimIdentities: persistence.claims + .filter( + (claim) => + claim.parentAgentId === target.agentId && + claim.parentRunId === target.runId, + ) + .map(supervisorSwarmClaimIdentity) + .sort((left, right) => left.actionId.localeCompare(right.actionId)), taskIdentity: supervisorSwarmTaskIdentity(targetTask), }; break; @@ -10605,6 +11005,7 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { state.supervisorSwarm.oldRunnerBootId = currentRunner.bootId; await killRunnerOnce(); + state.supervisorSwarm.transientRetryRunnerKilledAtMs = Date.now(); state.supervisorSwarm.runnerKillBoundaryObserved = true; await runCli(['--agent-resume', state.projectRoot], { timeoutMs: 120_000 }); state.resumed = true; @@ -10655,6 +11056,21 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { postRestartDesignMetadata, waitingBoundary.target, ); + const postRestartDeliveryIdentities = supervisorSwarmParentDeliveries( + postRestartPersistence.deliveries, + ) + .map(supervisorSwarmDeliveryIdentity) + .sort((left, right) => + left.delegationId.localeCompare(right.delegationId), + ); + const postRestartClaimIdentities = postRestartPersistence.claims + .filter( + (claim) => + claim.parentAgentId === waitingBoundary.target.agentId && + claim.parentRunId === waitingBoundary.target.runId, + ) + .map(supervisorSwarmClaimIdentity) + .sort((left, right) => left.actionId.localeCompare(right.actionId)); state.supervisorSwarm.transientRetryPostRestartSidecar = postRestartSidecar; state.supervisorSwarm.transientRetryRequestCountAfterRestart = proxy.getStats().requestCount; @@ -10684,17 +11100,35 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { JSON.stringify(supervisorSwarmTaskIdentity(postRestartTask)) === JSON.stringify(waitingBoundary.taskIdentity), ); + if (waitingBoundary.finalReplyMode) { + assert( + JSON.stringify(postRestartSideEffects) === + JSON.stringify(waitingBoundary.sideEffects), + 'supervisor-swarm-final-reply-transient-retry-post-restart-prerequisite-drift', + ); + } else { assertSupervisorSwarmTransientRetrySideEffectsZero( postRestartSideEffects, 'supervisor-swarm-transient-retry-post-restart-side-effect', ); + } assert( state.supervisorSwarm.transientRetrySidecarIdentityStable && state.supervisorSwarm.transientRetryAttemptStable && state.supervisorSwarm.transientRetryAtStable && state.supervisorSwarm.transientRetryRuntimeIdentityStable && - state.supervisorSwarm.transientRetryRequestCountBeforeKill === 1 && - state.supervisorSwarm.transientRetryRequestCountAfterRestart === 1 && + JSON.stringify(postRestartDeliveryIdentities) === + JSON.stringify(waitingBoundary.deliveryIdentities) && + JSON.stringify(postRestartClaimIdentities) === + JSON.stringify(waitingBoundary.claimIdentities) && + state.supervisorSwarm.transientRetryRunnerKilledAtMs >= + waitingBoundary.sidecar.record.updatedAtMs && + state.supervisorSwarm.transientRetryRunnerKilledAtMs < + waitingBoundary.sidecar.record.retryAtMs && + state.supervisorSwarm.transientRetryRequestCountBeforeKill === + waitingBoundary.faultSequence && + state.supervisorSwarm.transientRetryRequestCountAfterRestart === + waitingBoundary.faultSequence && Date.now() + supervisorSwarmTransientRetryPreDueGuardMs < waitingBoundary.sidecar.record.retryAtMs, 'supervisor-swarm-transient-retry-post-restart-invalid', @@ -10715,28 +11149,32 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { preDueStats.requestCount; state.supervisorSwarm.transientRetryEarlyRequestCount = Math.max( 0, - preDueStats.requestCount - 1, + preDueStats.requestCount - waitingBoundary.faultSequence, ); assert( Date.now() < waitingBoundary.sidecar.record.retryAtMs && - preDueStats.requestCount === 1 && + preDueStats.requestCount === waitingBoundary.faultSequence && preDueStats.heldRequestCount === 0 && - preDueStats.forwardedRequestCount === 0 && + preDueStats.forwardedRequestCount === waitingBoundary.faultSequence - 1 && state.supervisorSwarm.transientRetryEarlyRequestCount === 0, 'supervisor-swarm-transient-retry-request-sent-before-due', ); await proxy.waitForHeldRequest(60_000); const heldRequestLog = proxy.getRequestLog(); + const faultedRequestLog = heldRequestLog.find( + (entry) => entry.sequence === waitingBoundary.faultSequence, + ); + const retryRequestLog = heldRequestLog.find( + (entry) => entry.sequence === waitingBoundary.faultSequence + 1, + ); state.supervisorSwarm.transientRetryAcceptedAtOrAfterDue = Boolean( - heldRequestLog.length === 2 && - heldRequestLog[0].sequence === 1 && - heldRequestLog[0].faultInjectedAtMs >= heldRequestLog[0].acceptedAtMs && - heldRequestLog[1].sequence === 2 && - heldRequestLog[1].acceptedAtMs >= + heldRequestLog.length === waitingBoundary.faultSequence + 1 && + faultedRequestLog?.faultInjectedAtMs >= faultedRequestLog?.acceptedAtMs && + retryRequestLog?.acceptedAtMs >= waitingBoundary.sidecar.record.retryAtMs && - heldRequestLog[1].heldAtMs >= heldRequestLog[1].acceptedAtMs && - heldRequestLog[1].forwardingStartedAtMs == null, + retryRequestLog?.heldAtMs >= retryRequestLog?.acceptedAtMs && + retryRequestLog?.forwardingStartedAtMs == null, ); assert( state.supervisorSwarm.transientRetryAcceptedAtOrAfterDue, @@ -10767,6 +11205,9 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { record.agentId === waitingBoundary.target.agentId && record.runId === waitingBoundary.target.runId, ); + const matchingRetries = retries.filter( + (record) => record.requestId === waitingBoundary.failedStarted.requestId, + ); const retryStarted = lifecycle.filter( (record) => record.status === 'started' && @@ -10810,13 +11251,26 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { record.requestId === waitingBoundary.failedStarted.requestId, ); const retryStartedIndex = persistence.agentDb.indexOf(retryStarted[0]); + if (waitingBoundary.finalReplyMode) { + assert( + JSON.stringify(sideEffects) === + JSON.stringify(waitingBoundary.sideEffects), + 'supervisor-swarm-final-reply-transient-retry-pre-forward-prerequisite-drift', + ); + } else { assertSupervisorSwarmTransientRetrySideEffectsZero( sideEffects, 'supervisor-swarm-transient-retry-pre-forward-side-effect', ); + } + const requestLifecycle = lifecycle.filter( + (record) => + record.requestId === waitingBoundary.failedStarted.requestId || + record.requestId === retryStarted[0].requestId, + ); assert( - lifecycle.length === 3 && - retries.length === 1 && + requestLifecycle.length === 3 && + matchingRetries.length === 1 && retryStarted[0].requestId !== waitingBoundary.failedStarted.requestId && retryStarted[0].requestSlot === waitingBoundary.retryAudit.nextRequestSlot && @@ -10828,10 +11282,11 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { failedStartedIndex < failedTerminalIndex && failedTerminalIndex < retryAuditIndex && retryAuditIndex < retryStartedIndex && - proxyStats.requestCount === 2 && + proxyStats.requestCount === waitingBoundary.faultSequence + 1 && proxyStats.faultInjectedCount === 1 && proxyStats.heldRequestCount === 1 && - proxyStats.forwardedRequestCount === 0 && + proxyStats.forwardedRequestCount === + waitingBoundary.faultSequence - 1 && proxyStats.forwardingReleased === false, 'supervisor-swarm-transient-retry-held-attempt-invalid', ); @@ -10847,6 +11302,28 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { source: waitingBoundary.failedStarted.source, requestKind: waitingBoundary.failedStarted.requestKind, ...sideEffects, + faultSequence: waitingBoundary.faultSequence, + parentToolPlanStartedCount: + waitingBoundary.faultSelection?.parentToolPlanStartedCount ?? 0, + finalReplyPrerequisitesValid: + waitingBoundary.faultSelection?.preconditionsValid === true, + preFaultHostVerificationPassed: + waitingBoundary.faultSelection?.preFaultHostVerificationPassed === + true, + preFaultHostVerificationCompletedAtMs: + waitingBoundary.faultSelection + ?.preFaultHostVerificationCompletedAtMs ?? 0, + repairDeliveryCount: + waitingBoundary.faultSelection?.repairDeliveryCount ?? 0, + observedClaimReceiptCount: + waitingBoundary.faultSelection?.observedClaimReceiptCount ?? 0, + runtimeVerificationCount: + waitingBoundary.faultSelection?.runtimeVerificationCount ?? 0, + runtimeVerificationReceiptCount: + waitingBoundary.faultSelection?.runtimeVerificationReceiptCount ?? 0, + runtimeVerificationObservationCount: + waitingBoundary.faultSelection?.runtimeVerificationObservationCount ?? + 0, upstreamRequestCountBeforeRelease: proxyStats.forwardedRequestCount, forwardingReleased: false, }; @@ -10919,11 +11396,16 @@ async function runSupervisorSwarmE2e() { 'supervisor-swarm-parent-runtime-identity-invalid', ); - await captureSupervisorSwarmTransientRetryCheckpoint(); + if (isSupervisorSwarmInitialTransientRetrySuite()) { + await captureSupervisorSwarmTransientRetryCheckpoint(); + } await captureSupervisorSwarmInitialProviderBatch(); await captureSupervisorSwarmCollaborationPolicySnapshotAndDrift(); await restartSupervisorSwarmRunnerAtInitialBatchBoundary(); await driveSupervisorSwarmToRepairKillBoundary(); + if (isSupervisorSwarmFinalReplyTransientRetrySuite()) { + await captureSupervisorSwarmTransientRetryCheckpoint(); + } await driveSupervisorSwarmRuntimeToCompletion(); if (isSupervisorSwarmInteractiveChatSuite()) { await captureSupervisorSwarmAutonomousTurnReport(); @@ -11206,11 +11688,18 @@ function validateSupervisorSwarmProviderLifecycle( record.runId === checkpoint.runId, ) : []; + const targetToolPlanStartedCount = targetStarted.filter( + (record) => record.requestKind === 'tool-plan', + ).length; + state.supervisorSwarm.transientRetryToolPlanCountAfterRecovery = + targetToolPlanStartedCount; const proxyStats = state.supervisorSwarm.transientFaultProxy?.getStats(); state.supervisorSwarm.transientRetryIncidentalProviderFailureCount = Math.max(0, failed.length - injectedFailed.length); - state.supervisorSwarm.transientRetryIncidentalProviderRetryCount = - Math.max(0, retries.length - injectedRetries.length); + state.supervisorSwarm.transientRetryIncidentalProviderRetryCount = Math.max( + 0, + retries.length - injectedRetries.length, + ); assert( checkpoint && injectedFailed.length === 1 && @@ -11236,6 +11725,17 @@ function validateSupervisorSwarmProviderLifecycle( proxyStats.faultInjectedCount === 1 && proxyStats.forwardedRequestCount === targetStarted.length - 1 && targetCompleted.length + targetFailed.length === targetStarted.length && + (!isSupervisorSwarmFinalReplyTransientRetrySuite() || + (checkpoint.requestKind === 'final-reply' && + checkpoint.finalReplyPrerequisitesValid === true && + checkpoint.preFaultHostVerificationPassed === true && + checkpoint.preFaultHostVerificationCompletedAtMs > 0 && + checkpoint.repairDeliveryCount === 1 && + checkpoint.observedClaimReceiptCount === 3 && + checkpoint.parentToolPlanStartedCount === + targetToolPlanStartedCount && + state.supervisorSwarm.transientRetryToolPlanCountAtFault === + targetToolPlanStartedCount)) && state.supervisorSwarm.transientRetryIncidentalProviderFailureCount === state.supervisorSwarm.transientRetryIncidentalProviderRetryCount, 'supervisor-swarm-forced-transient-retry-invalid', @@ -11948,9 +12448,9 @@ function validateSupervisorSwarmFinalization(agentDb, identity, assistant) { ); for (const [index, record] of records.entries()) { assert( - record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && + record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && record.journalSchemaVersion === - 'game-creator-runtime-finalization.v3' && + 'game-creator-runtime-finalization.v4' && record.finalizationId === finalizationId && record.messageId === assistant.messageId && record.agentId === identity.agentId && @@ -12235,14 +12735,18 @@ async function readSupervisorSwarmResidualSidecarCounts() { parallelReadBatches: '.agent/runtime/parallel-read-batches', pendingActions: '.agent/runtime/pending-actions', providerActionBatches: '.agent/runtime/provider-action-batches', + providerHandoffs: '.agent/runtime/provider-handoffs', providerRetries: '.agent/runtime/provider-retries', userInput: '.agent/runtime/user-input', }; const counts = {}; for (const [name, relative] of Object.entries(roots)) { const files = await listFiles(path.join(state.projectRoot, relative)); - counts[name] = - name === 'providerRetries' + counts[name] = [ + 'finalizations', + 'providerHandoffs', + 'providerRetries', + ].includes(name) ? files.length : files.filter((file) => file.endsWith('.json')).length; } @@ -12267,6 +12771,58 @@ async function readSupervisorSwarmResidualSidecarCounts() { return counts; } +function supervisorSwarmResidualSidecarsEmpty(counts) { + return Object.values(counts).every((count) => count === 0); +} + +function supervisorSwarmParentResponseStreamPath() { + return path.join( + state.projectRoot, + '.agent/runtime/response-streams', + hashValue(projectSupervisorAgentId).slice(0, 32), + `${hashValue(state.initialRunId).slice(0, 32)}.json`, + ); +} + +async function validateSupervisorSwarmFinalReplyResponseStream( + parentRuntime, + assistant, +) { + if (!isSupervisorSwarmFinalReplyTransientRetrySuite()) return null; + const checkpoint = state.supervisorSwarm.transientFaultCheckpoint; + const streamPath = supervisorSwarmParentResponseStreamPath(); + const [stream, parentStreamFiles] = await Promise.all([ + readJson(streamPath), + listFiles(path.dirname(streamPath)), + ]); + const assistantText = String(assistant.content ?? ''); + assert( + checkpoint?.requestKind === 'final-reply' && + parentStreamFiles.length === 1 && + path.resolve(parentStreamFiles[0]) === path.resolve(streamPath) && + stream.schemaVersion === 'game-creator-runtime-response-stream.v1' && + stream.agentId === projectSupervisorAgentId && + stream.taskId === parentRuntime.taskId && + stream.sessionId === supervisorSwarmSessionId && + stream.runId === state.initialRunId && + stream.requestKind === 'final-reply' && + stream.requestSlot === checkpoint.failedRequestSlot && + stream.status === 'committed' && + Number.isSafeInteger(stream.sequence) && + stream.sequence > 0 && + stream.finishReason !== 'fallback' && + stream.accumulatedText === assistantText, + 'supervisor-swarm-final-reply-transient-retry-stream-invalid', + ); + state.supervisorSwarm.transientRetryFinalReplyStreamCommitted = true; + return { + sequence: stream.sequence, + chars: [...assistantText].length, + fingerprint: hashValue(assistantText), + requestSlotHash: hashValue(stream.requestSlot), + }; +} + async function countSupervisorSwarmSteerRecords() { const files = ( await listFiles(path.join(state.projectRoot, '.agent/runtime/steers')) @@ -12296,6 +12852,7 @@ function supervisorSwarmTransientRetrySnapshotEvidence( ) { const enabled = isSupervisorSwarmTransientRetrySuite(); const checkpoint = state.supervisorSwarm.transientFaultCheckpoint; + const selection = state.supervisorSwarm.transientFaultSelection; const proxy = state.supervisorSwarm.transientFaultProxy; const stats = proxy?.getStats() ?? { requestCount: 0, @@ -12322,7 +12879,13 @@ function supervisorSwarmTransientRetrySnapshotEvidence( ), transientRetryRunnerKilledDuringBackoff: Boolean( state.supervisorSwarm.runnerKillBoundaryObserved && - state.supervisorSwarm.transientRetryPreKillRunnerBootId, + state.supervisorSwarm.transientRetryPreKillRunnerBootId && + state.supervisorSwarm.transientRetryPreKillSidecar && + state.supervisorSwarm.transientRetryRunnerKilledAtMs >= + state.supervisorSwarm.transientRetryPreKillSidecar.record + .updatedAtMs && + state.supervisorSwarm.transientRetryRunnerKilledAtMs < + state.supervisorSwarm.transientRetryPreKillSidecar.record.retryAtMs, ), transientRetryRunnerBootChanged: Boolean( state.supervisorSwarm.transientRetryPreKillRunnerBootId && @@ -12362,6 +12925,52 @@ function supervisorSwarmTransientRetrySnapshotEvidence( transientFaultPreRetryAssistantCount: checkpoint?.assistantCount ?? 0, transientFaultPreRetryPendingCount: checkpoint?.pendingCount ?? 0, transientFaultPreRetryProjectRevision: checkpoint?.projectRevision ?? 0, + transientFaultPreRetryDesignArtifactPresent: + checkpoint?.designArtifactPresent === true, + transientFaultRequestKind: + checkpoint?.requestKind ?? selection?.requestKind ?? '', + transientFaultRequestSequence: + checkpoint?.faultSequence ?? selection?.sequence ?? 0, + transientFaultFinalReplyPrerequisitesValid: + checkpoint?.finalReplyPrerequisitesValid === true || + selection?.preconditionsValid === true, + transientFaultInvalidPrerequisiteFields: + selection?.invalidPrerequisiteFields ?? [], + transientFaultInvalidPrerequisiteCount: + selection?.invalidPrerequisiteFields?.length ?? 0, + transientFaultPreFaultHostVerificationPassed: + checkpoint?.preFaultHostVerificationPassed === true || + selection?.preFaultHostVerificationPassed === true, + transientFaultPreFaultHostVerificationCompletedAfterAccepted: + Number.isSafeInteger( + checkpoint?.preFaultHostVerificationCompletedAtMs ?? + selection?.preFaultHostVerificationCompletedAtMs, + ) && + (checkpoint?.preFaultHostVerificationCompletedAtMs ?? + selection?.preFaultHostVerificationCompletedAtMs) >= + (selection?.acceptedAtMs ?? Number.MAX_SAFE_INTEGER), + transientFaultRepairDeliveryCount: + checkpoint?.repairDeliveryCount ?? selection?.repairDeliveryCount ?? 0, + transientFaultObservedClaimReceiptCount: + checkpoint?.observedClaimReceiptCount ?? + selection?.observedClaimReceiptCount ?? + 0, + transientFaultRuntimeVerificationCount: + checkpoint?.runtimeVerificationCount ?? + selection?.runtimeVerificationCount ?? + 0, + transientFaultRuntimeVerificationReceiptCount: + checkpoint?.runtimeVerificationReceiptCount ?? + selection?.runtimeVerificationReceiptCount ?? + 0, + transientFaultRuntimeVerificationObservationCount: + checkpoint?.runtimeVerificationObservationCount ?? + selection?.runtimeVerificationObservationCount ?? + 0, + transientRetryToolPlanCountAtFault: + state.supervisorSwarm.transientRetryToolPlanCountAtFault, + transientRetryToolPlanCountAfterRecovery: + state.supervisorSwarm.transientRetryToolPlanCountAfterRecovery, transientFaultUpstreamRequestCountBeforeRelease: checkpoint?.upstreamRequestCountBeforeRelease ?? 0, transientFaultRequestIdentityChanged: Boolean( @@ -12399,6 +13008,7 @@ function supervisorSwarmTransientRetryEvidence(provider) { provider.forcedTransientRetryVerified, ); if (evidence.transientFaultModeEnabled) { + const finalReplyMode = isSupervisorSwarmFinalReplyTransientRetrySuite(); assert( provider.forcedTransientRetryVerified === true && evidence.transientFaultRequestCount >= 2 && @@ -12411,9 +13021,13 @@ function supervisorSwarmTransientRetryEvidence(provider) { evidence.transientRetryRunnerKilledDuringBackoff && evidence.transientRetryRunnerBootChanged && evidence.transientRetryRecoveredFromPreviousBoot && - evidence.transientRetryRequestCountBeforeKill === 1 && - evidence.transientRetryRequestCountAfterRestart === 1 && - evidence.transientRetryRequestCountBeforeDue === 1 && + evidence.transientFaultRequestSequence > 0 && + evidence.transientRetryRequestCountBeforeKill === + evidence.transientFaultRequestSequence && + evidence.transientRetryRequestCountAfterRestart === + evidence.transientFaultRequestSequence && + evidence.transientRetryRequestCountBeforeDue === + evidence.transientFaultRequestSequence && evidence.transientRetryEarlyRequestCount === 0 && evidence.transientRetrySidecarIdentityStable && evidence.transientRetryAttemptStable && @@ -12423,14 +13037,36 @@ function supervisorSwarmTransientRetryEvidence(provider) { evidence.transientRetryIncidentalProviderFailureCount === evidence.transientRetryIncidentalProviderRetryCount && evidence.transientFaultPreRetryCheckpointCaptured && - evidence.transientFaultPreRetryActionCount === 0 && - evidence.transientFaultPreRetryReceiptCount === 0 && - evidence.transientFaultPreRetryChildDeliveryCount === 0 && - evidence.transientFaultPreRetryClaimCount === 0 && - evidence.transientFaultPreRetryAssistantCount === 0 && - evidence.transientFaultPreRetryPendingCount === 0 && - evidence.transientFaultPreRetryProjectRevision === 0 && - evidence.transientFaultUpstreamRequestCountBeforeRelease === 0 && + (finalReplyMode + ? evidence.transientFaultRequestKind === 'final-reply' && + evidence.transientFaultFinalReplyPrerequisitesValid && + evidence.transientFaultInvalidPrerequisiteCount === 0 && + evidence.transientFaultPreFaultHostVerificationPassed && + evidence.transientFaultPreFaultHostVerificationCompletedAfterAccepted && + evidence.transientFaultPreRetryActionCount > 0 && + evidence.transientFaultPreRetryReceiptCount > 0 && + evidence.transientFaultPreRetryChildDeliveryCount === 3 && + evidence.transientFaultPreRetryClaimCount === 2 && + evidence.transientFaultPreRetryAssistantCount === 0 && + evidence.transientFaultPreRetryPendingCount === 0 && + evidence.transientFaultPreRetryProjectRevision > 0 && + evidence.transientFaultPreRetryDesignArtifactPresent && + evidence.transientFaultRepairDeliveryCount === 1 && + evidence.transientFaultObservedClaimReceiptCount === 3 && + evidence.transientRetryToolPlanCountAtFault > 0 && + evidence.transientRetryToolPlanCountAtFault === + evidence.transientRetryToolPlanCountAfterRecovery + : evidence.transientFaultRequestKind === 'tool-plan' && + evidence.transientFaultPreRetryActionCount === 0 && + evidence.transientFaultPreRetryReceiptCount === 0 && + evidence.transientFaultPreRetryChildDeliveryCount === 0 && + evidence.transientFaultPreRetryClaimCount === 0 && + evidence.transientFaultPreRetryAssistantCount === 0 && + evidence.transientFaultPreRetryPendingCount === 0 && + evidence.transientFaultPreRetryProjectRevision === 0 && + !evidence.transientFaultPreRetryDesignArtifactPresent) && + evidence.transientFaultUpstreamRequestCountBeforeRelease === + evidence.transientFaultRequestSequence - 1 && evidence.transientFaultRequestIdentityChanged && evidence.transientFaultRetrySlotValid && evidence.transientFaultStableLogicalIdentity && @@ -13176,6 +13812,11 @@ async function validateSupervisorSwarmEvidence() { }, supervisorAssistants[0], ); + const finalReplyResponseStream = + await validateSupervisorSwarmFinalReplyResponseStream( + parentRuntime, + supervisorAssistants[0], + ); const staticClaimObservationIndexes = [ initialClaims[0].actionId, repairClaims[0].actionId, @@ -13302,7 +13943,7 @@ async function validateSupervisorSwarmEvidence() { const residualSidecars = await readSupervisorSwarmResidualSidecarCounts(); assert( - Object.values(residualSidecars).every((count) => count === 0), + supervisorSwarmResidualSidecarsEmpty(residualSidecars), 'supervisor-swarm-terminal-sidecar-present', ); const publicLeakEvidence = collectSupervisorSwarmPublicLeakEvidence( @@ -13361,6 +14002,8 @@ async function validateSupervisorSwarmEvidence() { { scenario: isSupervisorSwarmCollaborationPolicyMixedRecoverySuite() ? 'project-supervisor-collaboration-policy-mixed-initial-batch-recovery' + : isSupervisorSwarmFinalReplyTransientRetrySuite() + ? 'project-supervisor-final-reply-transient-retry-runner-recovery' : isSupervisorSwarmMixedHarnessSuite() ? 'project-supervisor-autonomous-chat-static-multi-isolated-single-repair-runner-recovery' : autonomousModeEnabled @@ -13750,6 +14393,14 @@ async function validateSupervisorSwarmEvidence() { finalReplyProviderRequestCount: provider.finalReplyCount, contextCompactionProviderRequestCount: provider.contextCompactionCount, parentFinalReplyProviderRequestCount: provider.parentFinalReplyCount, + finalReplyResponseStreamCommitted: + state.supervisorSwarm.transientRetryFinalReplyStreamCommitted, + finalReplyResponseStreamSequence: finalReplyResponseStream?.sequence ?? 0, + finalReplyResponseStreamChars: finalReplyResponseStream?.chars ?? 0, + finalReplyResponseStreamFingerprint: + finalReplyResponseStream?.fingerprint ?? 'not-required', + finalReplyResponseStreamRequestSlotHash: + finalReplyResponseStream?.requestSlotHash ?? 'not-required', finalAssistantCount: supervisorAssistants.length, professionalAssistantCount: professionalAssistants.length, isolatedAssistantCount: isolatedAssistants.length, @@ -13829,6 +14480,7 @@ async function validateSupervisorSwarmEvidence() { targetedContractReadCount: actions.targetedContractReadCount, pendingActionCount: residualSidecars.pendingActions, providerActionBatchSidecarCount: residualSidecars.providerActionBatches, + providerHandoffSidecarCount: residualSidecars.providerHandoffs, providerRetrySidecarCount: residualSidecars.providerRetries, parallelReadBatchSidecarCount: residualSidecars.parallelReadBatches, finalizationJournalCount: residualSidecars.finalizations, @@ -13858,6 +14510,8 @@ async function validateSupervisorSwarmEvidence() { '.agent/runtime/collaboration-policy-snapshots', '.agent/runtime/collaboration-policy-snapshot-bindings', '.agent/runtime/provider-action-batches', + '.agent/runtime/provider-handoffs', + '.agent/runtime/response-streams', '.agent/runtime/tasks', '.agent/runtime/events', '.agent/agent.db', @@ -14063,6 +14717,7 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { collaborationPolicySnapshotFile, collaborationPolicySnapshotBindingFile, collaborationPolicySidecar, + finalReplyResponseStream, ] = await Promise.all([ toleratePartialRead( 'design-artifact', @@ -14094,6 +14749,8 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { parallelReadBatches: 0, pendingActions: 0, providerActionBatches: 0, + providerHandoffs: 0, + providerRetries: 0, userInput: 0, collaborationPolicySnapshotArtifacts: 0, collaborationPolicySnapshotBindingArtifacts: 0, @@ -14153,6 +14810,16 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { null, { ignoreMissing: true }, ), + toleratePartialRead( + 'final-reply-response-stream', + () => + isSupervisorSwarmFinalReplyTransientRetrySuite() && + isNonEmptyString(state.initialRunId) + ? readJson(supervisorSwarmParentResponseStreamPath()) + : null, + null, + { ignoreMissing: true }, + ), ]); const parentTasks = persistence.taskSnapshot.all.filter( (task) => task.agentId === projectSupervisorAgentId, @@ -14877,6 +15544,28 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { instance.runId === record.runId, ), ).length, + finalReplyResponseStreamCommitted: + finalReplyResponseStream?.status === 'committed', + finalReplyResponseStreamSequence: Number.isSafeInteger( + finalReplyResponseStream?.sequence, + ) + ? finalReplyResponseStream.sequence + : 0, + finalReplyResponseStreamChars: isNonEmptyString( + finalReplyResponseStream?.accumulatedText, + ) + ? [...finalReplyResponseStream.accumulatedText].length + : 0, + finalReplyResponseStreamFingerprint: isNonEmptyString( + finalReplyResponseStream?.accumulatedText, + ) + ? hashValue(finalReplyResponseStream.accumulatedText) + : 'not-observed', + finalReplyResponseStreamRequestSlotHash: isNonEmptyString( + finalReplyResponseStream?.requestSlot, + ) + ? hashValue(finalReplyResponseStream.requestSlot) + : 'not-observed', duplicateIsolatedGroupCount: duplicateCount( isolatedRecords.groups.map((group) => group.delegationGroupId), ), @@ -14899,6 +15588,7 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { legacyConversationMessageCount: persistence.legacyConversation.length, pendingActionCount: pending.length, providerActionBatchSidecarCount: residualSidecars.providerActionBatches, + providerHandoffSidecarCount: residualSidecars.providerHandoffs, providerRetrySidecarCount: residualSidecars.providerRetries, parallelReadBatchSidecarCount: residualSidecars.parallelReadBatches, finalizationJournalCount: residualSidecars.finalizations, @@ -14921,7 +15611,9 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { '.agent/runtime/collaboration-policy-snapshots', '.agent/runtime/collaboration-policy-snapshot-bindings', '.agent/runtime/provider-action-batches', + '.agent/runtime/provider-handoffs', '.agent/runtime/provider-retries', + '.agent/runtime/response-streams', '.agent/runtime/tasks', '.agent/runtime/events', '.agent/agent.db', @@ -15000,6 +15692,7 @@ function parseArguments(args) { suite === parallelReadSuite || suite === supervisorSwarmSuite || suite === supervisorSwarmTransientRetrySuite || + suite === supervisorSwarmFinalReplyTransientRetrySuite || suite === supervisorSwarmAutonomousChatSuite || suite === supervisorSwarmStaticIsolatedAutonomousChatSuite || suite === supervisorSwarmCollaborationPolicyMixedRecoverySuite || @@ -15229,7 +15922,18 @@ function isolatedSuiteAppDataProfile() { codePrefix: 'parallel-read-appdata', }; } - if (isSupervisorSwarmTransientRetrySuite()) { + if (isSupervisorSwarmFinalReplyTransientRetrySuite()) { + return { + prefix: + '.agent-runtime-real-e2e-supervisor-swarm-final-reply-transient-retry-', + sentinelName: + supervisorSwarmFinalReplyTransientRetryAppDataSentinelFileName, + sentinelSchema: + supervisorSwarmFinalReplyTransientRetryAppDataSentinelSchema, + codePrefix: 'supervisor-swarm-final-reply-transient-retry-appdata', + }; + } + if (isSupervisorSwarmInitialTransientRetrySuite()) { return { prefix: '.agent-runtime-real-e2e-supervisor-swarm-transient-retry-', sentinelName: supervisorSwarmTransientRetryAppDataSentinelFileName, @@ -19667,9 +20371,9 @@ function validateResponseStreamFinalization( ); for (const [index, record] of records.entries()) { assert( - record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && + record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && record.journalSchemaVersion === - 'game-creator-runtime-finalization.v3' && + 'game-creator-runtime-finalization.v4' && record.finalizationId === finalizationId && record.messageId === messageId && record.taskId === runtimeState.taskId && @@ -26290,9 +26994,9 @@ function validateGoalFinalizationLifecycle(agentDb, goal, runtimeState) { const expectedGoalSnapshotFingerprint = goalSnapshotFingerprint(goal); for (const [index, record] of records.entries()) { assert( - record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && + record.auditSchemaVersion === 'game-creator-finalization-lifecycle.v1' && record.journalSchemaVersion === - 'game-creator-runtime-finalization.v3' && + 'game-creator-runtime-finalization.v4' && record.finalizationId === finalizationId && record.messageId === messageId && record.taskId === runtimeState.taskId && @@ -27385,6 +28089,21 @@ function supervisorSwarmEvidenceFieldTemplate() { transientFaultPreRetryAssistantCount: 0, transientFaultPreRetryPendingCount: 0, transientFaultPreRetryProjectRevision: 0, + transientFaultPreRetryDesignArtifactPresent: false, + transientFaultRequestKind: '', + transientFaultRequestSequence: 0, + transientFaultFinalReplyPrerequisitesValid: false, + transientFaultInvalidPrerequisiteFields: [], + transientFaultInvalidPrerequisiteCount: 0, + transientFaultPreFaultHostVerificationPassed: false, + transientFaultPreFaultHostVerificationCompletedAfterAccepted: false, + transientFaultRepairDeliveryCount: 0, + transientFaultObservedClaimReceiptCount: 0, + transientFaultRuntimeVerificationCount: 0, + transientFaultRuntimeVerificationReceiptCount: 0, + transientFaultRuntimeVerificationObservationCount: 0, + transientRetryToolPlanCountAtFault: 0, + transientRetryToolPlanCountAfterRecovery: 0, transientFaultUpstreamRequestCountBeforeRelease: 0, transientFaultRequestIdentityChanged: false, transientFaultRetrySlotValid: false, @@ -27396,6 +28115,11 @@ function supervisorSwarmEvidenceFieldTemplate() { finalReplyProviderRequestCount: 0, contextCompactionProviderRequestCount: 0, parentFinalReplyProviderRequestCount: 0, + finalReplyResponseStreamCommitted: false, + finalReplyResponseStreamSequence: 0, + finalReplyResponseStreamChars: 0, + finalReplyResponseStreamFingerprint: 'not-required', + finalReplyResponseStreamRequestSlotHash: 'not-required', finalAssistantCount: 0, professionalAssistantCount: 0, isolatedAssistantCount: 0, @@ -27427,6 +28151,7 @@ function supervisorSwarmEvidenceFieldTemplate() { targetedContractReadCount: 0, pendingActionCount: 0, providerActionBatchSidecarCount: 0, + providerHandoffSidecarCount: 0, providerRetrySidecarCount: 0, parallelReadBatchSidecarCount: 0, finalizationJournalCount: 0, @@ -28474,9 +29199,20 @@ function isSupervisorSwarmSuite() { } function isSupervisorSwarmTransientRetrySuite() { + return ( + isSupervisorSwarmInitialTransientRetrySuite() || + isSupervisorSwarmFinalReplyTransientRetrySuite() + ); +} + +function isSupervisorSwarmInitialTransientRetrySuite() { return state.suite === supervisorSwarmTransientRetrySuite; } +function isSupervisorSwarmFinalReplyTransientRetrySuite() { + return state.suite === supervisorSwarmFinalReplyTransientRetrySuite; +} + function isSupervisorSwarmAutonomousChatSuite() { return state.suite === supervisorSwarmAutonomousChatSuite; } @@ -31285,6 +32021,161 @@ function runAgentRuntimeRealE2eSelfTests() { 'agent-runtime-real-e2e-self-test-source-endpoint-lifecycle-guard-invalid', ); + const previousInitialRunId = state.initialRunId; + const syntheticFinalReplyRunId = 'synthetic-final-reply-run'; + state.initialRunId = syntheticFinalReplyRunId; + const syntheticToolPlanStarted = { + recordType: 'agent.runtime.provider_request.lifecycle', + status: 'started', + agentId: projectSupervisorAgentId, + runId: syntheticFinalReplyRunId, + requestKind: 'tool-plan', + requestId: 'synthetic-tool-plan-request', + }; + const syntheticVerificationAudit = { + recordType: 'agent.runtime.project.verify', + agentId: projectSupervisorAgentId, + runId: syntheticFinalReplyRunId, + actionId: 'synthetic-verification-action', + status: 'completed', + exitCode: 0, + timedOut: false, + }; + const syntheticVerificationReceipt = { + recordType: 'agent.runtime.action_receipt', + agentId: projectSupervisorAgentId, + runId: syntheticFinalReplyRunId, + actionId: syntheticVerificationAudit.actionId, + tool: 'project.verify', + status: 'ok', + }; + const syntheticVerificationObservation = { + recordType: 'agent.runtime.tool_observation', + agentId: projectSupervisorAgentId, + runId: syntheticFinalReplyRunId, + actionId: syntheticVerificationAudit.actionId, + tool: 'project.verify', + status: 'ok', + }; + const syntheticFinalReplyStarted = { + recordType: 'agent.runtime.provider_request.lifecycle', + status: 'started', + agentId: projectSupervisorAgentId, + taskId: 'synthetic-final-reply-task', + sessionId: supervisorSwarmSessionId, + runId: syntheticFinalReplyRunId, + requestKind: 'final-reply', + requestId: 'synthetic-final-reply-request', + requestSlot: 'synthetic-final-reply-slot', + }; + const syntheticDeliveries = [ + ['synthetic-design', null], + ['synthetic-quality', null], + ['synthetic-repair', 'synthetic-quality'], + ].map(([delegationId, repairOfDelegationId]) => ({ + parentAgentId: projectSupervisorAgentId, + parentSessionId: supervisorSwarmSessionId, + parentRunId: syntheticFinalReplyRunId, + delegationId, + repairOfDelegationId, + status: 'claimed-by-parent', + terminalStatus: 'completed', + })); + const syntheticFinalReplyPrerequisites = + supervisorSwarmFinalReplyFaultPrerequisites( + { + deliveries: syntheticDeliveries, + claims: [ + { + status: 'observed', + receipts: [ + { delegationId: 'synthetic-design' }, + { delegationId: 'synthetic-quality' }, + ], + }, + { + status: 'observed', + receipts: [{ delegationId: 'synthetic-repair' }], + }, + ], + agentDb: [ + syntheticToolPlanStarted, + syntheticVerificationAudit, + syntheticVerificationReceipt, + syntheticVerificationObservation, + syntheticFinalReplyStarted, + ], + supervisorConversation: [], + projectRevision: { revision: 2 }, + }, + syntheticFinalReplyStarted, + { sequence: 7, acceptedAtMs: 1234 }, + ); + const syntheticInvalidFinalReplyPrerequisites = + supervisorSwarmFinalReplyFaultPrerequisites( + { + deliveries: syntheticDeliveries, + claims: [ + { + status: 'observed', + receipts: [ + { delegationId: 'synthetic-design' }, + { delegationId: 'synthetic-quality' }, + ], + }, + { + status: 'observed', + receipts: [{ delegationId: 'synthetic-repair' }], + }, + ], + agentDb: [ + syntheticToolPlanStarted, + syntheticFinalReplyStarted, + syntheticVerificationAudit, + syntheticVerificationReceipt, + syntheticVerificationObservation, + ], + supervisorConversation: [{ role: 'assistant' }], + projectRevision: { revision: 2 }, + }, + syntheticFinalReplyStarted, + { sequence: 7, acceptedAtMs: 1234 }, + ); + state.initialRunId = previousInitialRunId; + assert( + syntheticFinalReplyPrerequisites.preconditionsValid && + syntheticFinalReplyPrerequisites.requestKind === 'final-reply' && + syntheticFinalReplyPrerequisites.deliveryCount === 3 && + syntheticFinalReplyPrerequisites.repairDeliveryCount === 1 && + syntheticFinalReplyPrerequisites.observedClaimReceiptCount === 3 && + syntheticFinalReplyPrerequisites.runtimeVerificationCount === 1 && + syntheticFinalReplyPrerequisites.parentToolPlanStartedCount === 1 && + !syntheticInvalidFinalReplyPrerequisites.preconditionsValid && + JSON.stringify( + syntheticInvalidFinalReplyPrerequisites.invalidPrerequisiteFields, + ) === JSON.stringify(['assistantAbsent']) && + supervisorSwarmHostVerificationPassed({ + stdout: commandPassedMarker, + stderr: '', + }) && + !supervisorSwarmHostVerificationPassed({ + stdout: commandFailureMarker, + stderr: '', + }) && + supervisorSwarmFinalReplyFaultInjectionAllowed({ + preconditionsValid: true, + }) && + !supervisorSwarmFinalReplyFaultInjectionAllowed({ + preconditionsValid: false, + }) && + supervisorSwarmResidualSidecarsEmpty({ finalizations: 0, retries: 0 }) && + !supervisorSwarmResidualSidecarsEmpty({ + finalizations: 1, + retries: 0, + }), + 'agent-runtime-real-e2e-self-test-final-reply-fault-prerequisite-invalid', + ); + const newlineValue = 'private first line\nprivate second line'; const quotedValue = 'private value says "quoted"'; const backslashValue = 'private\\nested\\value'; @@ -31971,6 +32862,7 @@ function runAgentRuntimeRealE2eSelfTests() { collaborationPolicySnapshotBindingStable: true, durableSnapshotEligibilityAndContractBindingValidated: true, sourceEndpointAbsentLifecycleGuardValidated, + finalReplyFaultPrerequisiteValidated: true, collaborationPolicyDriftFixtureValidated: true, collaborationPolicyDriftStatusObserved: true, duplicateCollaborationPolicySnapshotCount: syntheticSnapshotDuplicateCount, diff --git a/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs index f538def54..bd742b01a 100644 --- a/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs +++ b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs @@ -84,6 +84,12 @@ function normalizeOptions(upstreamBaseUrlOrOptions, faultCount, extraOptions) { if (options.listen !== undefined && typeof options.listen !== 'function') { throw proxyError('listen must be a function'); } + if ( + options.shouldInjectFault !== undefined && + typeof options.shouldInjectFault !== 'function' + ) { + throw proxyError('shouldInjectFault must be a function'); + } const fallbackPorts = options.fallbackPorts ?? DEFAULT_FALLBACK_PORTS; if ( @@ -103,6 +109,7 @@ function normalizeOptions(upstreamBaseUrlOrOptions, faultCount, extraOptions) { holdAfterFault: options.holdAfterFault ?? false, fallbackPorts: [...new Set(fallbackPorts)], listen: options.listen ?? listenOnLoopback, + shouldInjectFault: options.shouldInjectFault ?? null, }; } @@ -466,6 +473,25 @@ export async function startLlmTransientFaultProxy( return; } + let shouldInjectFault = faultInjectedCount < options.faultCount; + if (shouldInjectFault && options.shouldInjectFault) { + const decision = await options.shouldInjectFault( + Object.freeze({ + sequence: requestMetadata.sequence, + acceptedAtMs: requestMetadata.acceptedAtMs, + }), + ); + if (typeof decision !== 'boolean') { + throw proxyError('shouldInjectFault must resolve to a boolean'); + } + shouldInjectFault = decision; + } + + if (stopping || request.destroyed || response.destroyed) { + resetSocket(request.socket); + return; + } + if ( options.holdAfterFault && faultInjectedCount > 0 && @@ -481,7 +507,7 @@ export async function startLlmTransientFaultProxy( } } - if (faultInjectedCount < options.faultCount) { + if (shouldInjectFault && faultInjectedCount < options.faultCount) { faultInjectedCount += 1; requestMetadata.faultInjectedAtMs = Date.now(); notifyCounterWaiters('fault'); diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent.rs b/apps/ai-game-creator-shell/src-tauri/src/agent.rs index d421458ae..dceefc4e4 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent.rs @@ -27114,17 +27114,18 @@ fn observe_agent_runtime_file_write( }; } }; - let _lock = match acquire_project_write_lock(root, "file.write") { - Ok(lock) => lock, - Err(error) => { - return AgentRuntimeToolObservation { - tool: "file.write".to_string(), - status: "failed".to_string(), - summary: sanitize_agent_runtime_text(&error, 240), - detail: None, - }; - } - }; + let _lock = + match acquire_game_creator_agent_runtime_project_write_lock_with_wait(root, "file.write") { + Ok(lock) => lock, + Err(error) => { + return AgentRuntimeToolObservation { + tool: "file.write".to_string(), + status: "failed".to_string(), + summary: redact_agent_runtime_project_paths(root, &error, 240), + detail: None, + }; + } + }; if let Err(error) = prepare_agent_runtime_project_mutation_locked(root, agent_id, run_id, "file.write") { @@ -27155,7 +27156,7 @@ fn observe_agent_runtime_file_write( Err(error) => AgentRuntimeToolObservation { tool: "file.write".to_string(), status: "failed".to_string(), - summary: sanitize_agent_runtime_text(&error, 240), + summary: redact_agent_runtime_project_paths(root, &error, 240), detail: None, }, } @@ -27177,7 +27178,10 @@ fn observe_agent_runtime_file_delete( detail: None, }; } - let _lock = match acquire_project_write_lock(root, "file.delete") { + let _lock = match acquire_game_creator_agent_runtime_project_write_lock_with_wait( + root, + "file.delete", + ) { Ok(lock) => lock, Err(error) => { return AgentRuntimeToolObservation { @@ -27346,17 +27350,18 @@ fn observe_agent_runtime_file_patch( } }; - let _lock = match acquire_project_write_lock(root, "file.patch") { - Ok(lock) => lock, - Err(error) => { - return AgentRuntimeToolObservation { - tool: "file.patch".to_string(), - status: "failed".to_string(), - summary: redact_agent_runtime_project_paths(root, &error, 240), - detail: None, - }; - } - }; + let _lock = + match acquire_game_creator_agent_runtime_project_write_lock_with_wait(root, "file.patch") { + Ok(lock) => lock, + Err(error) => { + return AgentRuntimeToolObservation { + tool: "file.patch".to_string(), + status: "failed".to_string(), + summary: redact_agent_runtime_project_paths(root, &error, 240), + detail: None, + }; + } + }; if let Err(error) = prepare_agent_runtime_project_mutation_locked(root, agent_id, run_id, "file.patch") { diff --git a/apps/ai-game-creator-shell/src-tauri/src/tests.rs b/apps/ai-game-creator-shell/src-tauri/src/tests.rs index 7994fc8c6..563130913 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/tests.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/tests.rs @@ -46306,6 +46306,97 @@ fn project_write_lock_rejects_parallel_writer_and_releases_on_drop() { fs::remove_dir_all(root).ok(); } +#[tokio::test] +async fn agent_runtime_file_write_waits_for_short_parallel_project_writer() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "并行文件写入等待").expect("project init"); + write_project_permission_policy_at( + &root, + ProjectPermissionPolicy { + denied_commands: Vec::new(), + confirm_commands: Vec::new(), + agent_policies: BTreeMap::new(), + }, + ) + .expect("allow direct file write"); + let barrier = Arc::new(Barrier::new(2)); + let holder_barrier = Arc::clone(&barrier); + let holder_root = root.clone(); + let holder = std::thread::spawn(move || { + let lock = acquire_project_write_lock(&holder_root, "concurrent-writer") + .expect("acquire short-lived project writer"); + holder_barrier.wait(); + std::thread::sleep(Duration::from_millis(50)); + drop(lock); + }); + barrier.wait(); + + let observation = execute_game_creator_agent_runtime_tool_action( + &root, + "design-director", + "design-parallel-file-write-run", + "等待并行写锁后写入文件", + &AgentRuntimeToolAction { + tool: "file.write".to_string(), + reason: Some("验证 Runtime 文件写入会等待短暂锁竞争".to_string()), + input: serde_json::json!({ + "path": "game/parallel-write.txt", + "content": "parallel write completed\n" + }), + }, + ) + .await; + + holder.join().expect("join short-lived project writer"); + assert_eq!(observation.status, "ok"); + assert_eq!( + fs::read_to_string(root.join("game/parallel-write.txt")).expect("read written file"), + "parallel write completed\n" + ); + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn agent_runtime_file_write_lock_failure_redacts_project_path() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "并行文件写入脱敏").expect("project init"); + write_project_permission_policy_at( + &root, + ProjectPermissionPolicy { + denied_commands: Vec::new(), + confirm_commands: Vec::new(), + agent_policies: BTreeMap::new(), + }, + ) + .expect("allow direct file write"); + let lock = acquire_project_write_lock(&root, "persistent-writer") + .expect("acquire persistent project writer"); + + let observation = execute_game_creator_agent_runtime_tool_action( + &root, + "design-director", + "design-file-write-lock-failure-run", + "验证写锁失败观察可以安全持久化", + &AgentRuntimeToolAction { + tool: "file.write".to_string(), + reason: Some("验证项目绝对路径不会进入失败观察".to_string()), + input: serde_json::json!({ + "path": "game/blocked-write.txt", + "content": "must not be written\n" + }), + }, + ) + .await; + + drop(lock); + assert_eq!(observation.status, "failed"); + assert!(!observation + .summary + .contains(root.to_string_lossy().as_ref())); + assert!(!root.join("game/blocked-write.txt").exists()); + fs::remove_dir_all(root).ok(); +} + #[cfg(unix)] #[test] fn project_write_lock_reclaims_dead_process_owner() { diff --git a/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts index 79a0c33b9..88ecbf441 100644 --- a/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts +++ b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts @@ -313,6 +313,123 @@ describe('LLM transient fault proxy', () => { }); }); + it('uses only frozen timing metadata to select a later fault', async () => { + const capturedBodies: string[] = []; + const selectorCalls: Array<{ + metadata: { sequence: number; acceptedAtMs: number }; + frozen: boolean; + }> = []; + const upstream = await startServer(async (incoming, response) => { + capturedBodies.push(await readBody(incoming)); + response.end('forwarded'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 1, + async shouldInjectFault(metadata: { + sequence: number; + acceptedAtMs: number; + }) { + selectorCalls.push({ + metadata: { ...metadata }, + frozen: Object.isFrozen(metadata), + }); + await Promise.resolve(); + return metadata.sequence === 2; + }, + }); + + await expect( + request(proxy.url, { + headers: { authorization: 'Bearer a' }, + body: 'first-sensitive-body', + }), + ).resolves.toMatchObject({ statusCode: 200, body: 'forwarded' }); + await expect( + request(proxy.url, { + headers: { authorization: 'Bearer b' }, + body: 'selected-sensitive-body', + }), + ).rejects.toBeInstanceOf(Error); + await expect(proxy.waitForFault()).resolves.toMatchObject({ + faultInjectedCount: 1, + }); + await expect( + request(proxy.url, { body: 'after-fault-body' }), + ).resolves.toMatchObject({ statusCode: 200, body: 'forwarded' }); + + expect(selectorCalls).toEqual([ + { + metadata: { + sequence: 1, + acceptedAtMs: expect.any(Number), + }, + frozen: true, + }, + { + metadata: { + sequence: 2, + acceptedAtMs: expect.any(Number), + }, + frozen: true, + }, + ]); + expect(Object.keys(selectorCalls[0].metadata).sort()).toEqual([ + 'acceptedAtMs', + 'sequence', + ]); + expect(JSON.stringify(selectorCalls)).not.toMatch( + /secret|sensitive|authorization|body/iu, + ); + expect(capturedBodies).toEqual([ + 'first-sensitive-body', + 'after-fault-body', + ]); + expect(proxy.stats).toEqual({ + requestCount: 3, + faultInjectedCount: 1, + heldRequestCount: 0, + forwardedRequestCount: 2, + forwardingReleased: true, + stopped: false, + }); + }); + + it('does not count a delayed selector decision after the proxy stops', async () => { + let upstreamRequestCount = 0; + const selectorStarted = deferred(); + const selectorDecision = deferred(); + const upstream = await startServer(async (incoming, response) => { + upstreamRequestCount += 1; + await readBody(incoming); + response.end('unexpected'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 1, + async shouldInjectFault() { + selectorStarted.resolve(); + return selectorDecision.promise; + }, + }); + + const pendingRequest = request(proxy.url, { body: 'delayed-selector' }); + await selectorStarted.promise; + const stopPromise = proxy.stop(); + selectorDecision.resolve(true); + + await expect(pendingRequest).rejects.toBeInstanceOf(Error); + await expect(stopPromise).resolves.toBeUndefined(); + expect(upstreamRequestCount).toBe(0); + expect(proxy.stats).toMatchObject({ + requestCount: 1, + faultInjectedCount: 0, + heldRequestCount: 0, + forwardedRequestCount: 0, + stopped: true, + }); + }); + it('holds the post-fault request before upstream until forwarding is released', async () => { const captured: CapturedRequest[] = []; const upstream = await startServer(async (incoming, response) => { diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 67a08b007..0e3a87c23 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -4920,3 +4920,16 @@ - 回复流事务：finalization journal 升级为 v4，并固定保存 `responseRequestSlot / responseSteerCursor / responseRevision / response`。assistant 与 Runtime/Goal 已幂等完成后，journal 仍必须保留到匹配 response stream 写成 `committed` 且立即回读身份、状态和正文完全一致；之后才可删除 journal 和剩余 handoff。提交使用 journal 的固定 Agent/task/Session/run/request slot/steer cursor/revision，禁止调用面向 UI 的可见性过滤或用当前全局 project revision 静默跳过既定 run 的 stream。 - 回复流恢复：stream 缺失或仍为 `streaming` 时，可用 journal 固定身份和正文重建 `ready` 后提交；已 `committed` 且正文一致时按幂等成功继续清理。既有 stream 身份冲突、ready/committed 正文冲突、写入失败或回读失败都必须保留 journal 并保持可恢复 finalization，不能删除证据、覆盖冲突正文或把 finalization 当成已经清理。 - Runner 与边界：primary、`.previous` 或损坏的 handoff 都使所属 root 保持 busy，并阻止 `runner.shutdown_if_idle`。handoff 原子提交并回读前强杀 Runner，仍可能留下 Provider 已成功但本地只有未闭合 `started` 的未知窗口；Runtime 只能失败关闭，V1.41 不因此承诺端到端 exactly-once。`tool-plan` 和 function arguments 明确不在本协议内，真实外部 Provider 的 final-reply 退避期 Runner 强杀仍需独立 E2E 后才能记 PASS。 + +## 2026-07-20 AI 游戏创作 Agent Runtime V1.42 Project Supervisor final-reply 瞬时重试 Runner 强杀真实门禁 + +- 决策：不修改生产 Runtime、retry、handoff、finalization 或 response stream 协议，只扩展一次性 loopback fault proxy 与真实 E2E harness。新 suite 为 `supervisor-swarm-final-reply-transient-retry`；旧 `supervisor-swarm-transient-retry` 保持首次 tool-plan 故障语义，只证明该边界，不能替代新 suite。 +- 选择器隐私：proxy 的异步 selector 只能收到冻结的 `sequence / acceptedAtMs`，不得收到或保存 URL、header、body、API Key 或凭据。harness 只能选择当前 `project-supervisor` 同一父 Session/run 的唯一 base `final-reply` started lifecycle；不得命中 transient 后继、专业 Agent/child、tool-plan 或 compaction。 +- 命中前门禁：持久 delivery 必须精确为 `2` 条初始加 `1` 条 repair，三者均已由父 Agent claim；两次 `observed` claim 必须完整覆盖 `3` 条 receipt，父 assistant 仍为 `0`。随后 selector 必须在 proxy reset/forward 目标请求前，以可信宿主 Node 在 disposable project cwd 同步运行固定的 `node verify-e2e.mjs`；只有 `real-e2e-command=passed` marker 成功且无失败 marker 才允许注入。失败、超时、非零退出或 marker 无效均不注入，捕获的 stdout/stderr 不得进入 selector state、checkpoint、report 或公共日志。父 run 的 `project.verify` audit/receipt/observation 计数与顺序仅作诊断，不是注入前提。 +- 强杀与恢复：base final-reply 形成唯一 failed lifecycle、retry audit、持久 retry sidecar 和 `running / waiting-for-provider-retry` 后，在 `30s` backoff 内对 suite 自有 Runner 执行 pidfd `SIGKILL`。新 boot 保持 Session/run/task/request fingerprint/attempt/next slot/sidecar 字节/retryAt；重启后及到期前零新增请求，到期后只出现唯一 `-transient-1`，网络 `acceptedAtMs` 不得早于 `retryAtMs`。 +- 终态等待：task/Runtime 到达终态不等于 durable 清理已经完成。验收器必须在终态后继续显式等待相关 sidecar 全部清零，并设置 `10s` 硬超时；超时或仍有残留即判该轮失败，不能把早期采样到的 journal 与后续轮次拼接。 +- 写锁修正：并行 Agent 的 `file.write / file.patch / file.delete` 统一使用 Runtime 短等待项目写锁。`file.write` 的锁竞争错误在写入 observation 和 pending 前脱敏，禁止携带绝对锁路径；该路径曾导致 pending 持久化拒绝并把 run 推入 `needs-reconciliation`，现以 `2` 条 Rust 回归测试固定短等待与错误脱敏边界。 +- 终局：父 tool-plan 数在故障前后相等；唯一 `-transient-1` 成功后，只能有唯一成功 parent final-reply、唯一 Supervisor assistant 和唯一 committed response stream。retry/handoff/finalization artifacts、重复 delivery/claim/receipt/action/message/lifecycle，以及公共正文、API Key、项目/发布配置绝对路径命中全部为 `0`。复验命令为 `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径>`。 +- 确定性证据：fault proxy `14/14`、E2E self-test **PASS**、前端 `308/308`，以及 shell typecheck、`platform-llm 41/41`、`platform-agent 17/17`、`shared-contracts 7/7` 均已完成。 +- 六轮记录：真实外部 Provider suite 共执行六轮，前五轮均为 **FAIL** 且不得拼接。第一、二轮沿用既有失败记录；第三轮已走通故障、重试和唯一回复，但过早观察到 `1` 个 finalization journal；第四轮在 quality-review 普通 tool-plan 连续 transport/connectivity 失败并耗尽重试，未进入目标故障；第五轮命中上述项目写锁竞争与绝对锁路径泄漏问题。第六轮在同一轮内完整 **PASS**。 +- 第六轮证据：正式路由为 `gpt-5.5 / openai_chat`，`2` 条初始加 `1` 条 repair delivery、`3` 条专业 Agent assistant，目标为 Project Supervisor base final-reply，可信宿主 verify marker 门禁通过；受控 Provider `failed=1 / retry=1`，incidental `failure=0 / retry=0`，`30s` backoff，pidfd `claim=2 / signal=2`，Runner `resumed=true / identityStable=true`。父 tool-plan 故障前后均为 `13`，parent final-reply 与最终 assistant 唯一，response stream `sequence=2 / committed`；pending、retry、handoff、finalization、confirmation sidecar、全部重复计数及 API Key、私有正文、项目路径、正式配置路径和公共报告泄漏扫描命中均为 `0`。本决策不关闭 V1.41 handoff 原子落盘并回读前的 unknown-result 窗口，也不覆盖 tool-plan 成功响应/function arguments 的 durable handoff。 diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index 5e4e2131f..0605c0e7a 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -135,6 +135,26 @@ cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml durab V1.41 只覆盖无 tool call 的 `context-compaction / final-reply-context-compaction / final-reply`。Runner 在 Provider 成功后、handoff 原子提交并回读前被硬杀时，仍只能把未闭合 `started` 视为结果未知并失败关闭；该窗口不是 exactly-once。`tool-plan` 及其 function arguments 不进入 handoff，真实外部 Provider 的 final-reply 退避期 Runner 强杀仍需独立 E2E，不能用上述确定性测试或 V1.39 PASS 代替。 +### AI 游戏创作 Runtime V1.42 Supervisor final-reply 强杀真实复验 + +V1.42 不改变生产 Runtime 协议，只补一次性 fault proxy selector 和独立真实 suite。先完成确定性门禁，再显式传入发布 AppData 的绝对路径运行长链路： + +```bash +npm run test -- apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts +node apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs --self-test +npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径> +``` + +1. proxy selector 只允许读取冻结的 `sequence / acceptedAtMs`，不得接触 URL、header、body 或 Key。harness 从持久层识别同一父 Session/run 的唯一 base final-reply，先核对 `2` 初始加 `1` repair delivery 均已 claim、两次 observed claim 完整覆盖 `3` receipts 且 assistant 为 `0`；selector 再在 proxy reset/forward 该请求前，以可信宿主 Node 在 disposable project cwd 同步运行固定的 `node verify-e2e.mjs`。仅 `real-e2e-command=passed` marker 成功且无失败 marker 才允许注入；失败、超时或 marker 无效时不注入，stdout/stderr 不得写入 state、checkpoint、report 或公共日志。父 run 的 `project.verify` audit/receipt/observation 计数仅用于诊断，不是注入门禁。 +2. base final-reply failed lifecycle、retry audit、sidecar 和 task/Runtime `running / waiting-for-provider-retry` 全部成立后，才在 `30s` backoff 内用 pidfd `SIGKILL` 强杀 suite 自有 Runner。新 boot 必须保持 Session/run/request fingerprint/attempt/next slot/sidecar 字节/retryAt；重启后和到期前零新请求，到期后只允许唯一 `-transient-1`，并证明 `acceptedAtMs >= retryAtMs`。 +3. forwarding gate 放行前再次比较 delivery/claim/receipt、assistant、pending、project revision、可信宿主 marker 结果和父 tool-plan 数；后者在恢复前后必须相等，父 `project.verify` 仍只保留诊断计数。终局要求唯一成功 parent final-reply、唯一 Supervisor assistant、唯一 committed response stream，retry/handoff/finalization artifacts、重复和公共正文/Key/绝对路径泄漏均为 `0`。 +4. task/Runtime 到达终态后仍要显式等待 pending、retry、handoff、finalization、confirmation sidecar 全部清零，并设置 `10s` 硬超时。终态采样与 durable 清理之间允许存在短窗口，但超时仍有残留必须判该轮 **FAIL**，不得复用后续轮次的清理结果。 +5. 并行 Agent 执行 `file.write / file.patch / file.delete` 时统一走 Runtime 短等待项目写锁。锁竞争失败只返回脱敏错误，尤其不得让 `file.write` observation 携带绝对锁路径后再进入 pending 持久化；定向 Rust 回归至少覆盖短等待写锁和 `file.write` 错误脱敏两条边界。 + +旧 `supervisor-swarm-transient-retry` 继续只复验首次 tool-plan，不能替代新 suite。确定性门禁已完成 fault proxy `14/14`、E2E self-test **PASS**、前端 `308/308`，以及 shell typecheck、`platform-llm 41/41`、`platform-agent 17/17`、`shared-contracts 7/7`。真实外部 Provider suite 共执行六轮，前五轮均为 **FAIL** 且不得拼接：第一、二轮沿用既有失败记录；第三轮因终态后过早观察到 `1` 个 finalization journal 失败；第四轮因 quality-review 普通 tool-plan 连续 transport/connectivity 失败并耗尽重试、未进入目标故障而失败；第五轮因并行 `file.write` 与项目写锁竞争，绝对锁路径进入失败 observation 后触发 pending 持久化拒绝和 `needs-reconciliation` 而失败。完成终态 sidecar 等待和写锁/脱敏修正后，第六轮在同一轮内完整 **PASS**。 + +第六轮使用 `gpt-5.5 / openai_chat`，形成 `2` 条初始加 `1` 条 repair delivery 和 `3` 条专业 Agent assistant，精确命中 Project Supervisor base final-reply，可信宿主 verify marker 门禁通过；受控 Provider `failed=1 / retry=1`、incidental `failure=0 / retry=0`，`30s` backoff，pidfd `claim=2 / signal=2`，Runner `resumed=true / identityStable=true`。父 tool-plan 在故障前后均为 `13`，parent final-reply 与最终 assistant 唯一，response stream `sequence=2 / committed`；pending、retry、handoff、finalization、confirmation sidecar、全部重复计数及 API Key、私有正文、项目路径、正式配置路径和公共报告泄漏扫描命中均为 `0`。V1.41 handoff 落盘前 unknown-result 边界和 tool-plan handoff 未覆盖状态保持不变。 + suite 只能读正式 AppData，在其同级目录写入 sentinel 管理的 `0600` 私有副本和 overlay；启动 CLI/Runner 时须把 loopback 合并进大小写两套 no-proxy 环境，防止系统 HTTP 代理绕过本地故障门禁；source-dir guard 必须证明本 suite 前缀未进入源目录，源配置和 endpoint 身份保持不变，报告不得保存 Provider URL、headers、正文、凭据或绝对配置路径。sidecar 先于 task/state 投影是合法提交窗口，验收器应等待完整等待态后再强杀；若后续协作或终局失败，partial report 仍应保留已取得的 retry checkpoint，但失败轮不得与后续成功轮拼接。 ### AI 游戏创作自主 Swarm 终端复验 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 6ad1e8bf0..0e79621e5 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3378,3 +3378,14 @@ - 排障与验证：先核对 handoff 的 `providerRequestId / requestSlot / attempt` 与 Agent DB lifecycle，再看 retry/handoff/finalization/response-stream sidecar，最后才看 Runtime/UI 投影。用关闭 mock Provider 后恢复证明 handoff 回放零网络；分别覆盖 compaction 与 final-reply 消费窗口、handoff/retry 冲突、stream 缺失/streaming、commit 写失败、committed 后清理前退出、全局 revision 漂移和 Runner busy。日志与断言只公开指纹、字符数、状态和差异字段，不能打印 handoff 正文、请求体、凭据、URL 或绝对路径。 - 保留边界：Runner 若在 Provider 成功后、handoff 原子提交并回读前被硬杀，本地仍只有结果未知的 `started`，不能安全补发或宣称 exactly-once。handoff 只覆盖无 tool call 的 `context-compaction / final-reply-context-compaction / final-reply`；`tool-plan` 及其 function arguments 不在内，真实外部 Provider 的 final-reply Runner 强杀门禁也需单独完成。 - 关联：`apps/ai-game-creator-shell/src-tauri/src/provider_handoff.rs`、`provider_retry.rs`、`agent.rs`、`project.rs`、`runner.rs`、`tests.rs`、`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`。 + +## final-reply 强杀门禁不能按请求正文选目标，也不能在协作证据未闭合时动手 + +- 现象：复用 fail-first 代理后，suite 仍只命中第一个 tool-plan；或者为了定位 final-reply，让代理 selector 读取 URL/header/body；又或者把父 run 的 `project.verify` 误当成注入许可，没有在目标请求 reset/forward 前运行可信宿主检查。最终即使出现 failed/retry 和唯一 assistant，也不能证明 Project Supervisor 收尾边界真实可恢复。 +- 原因：网络 sequence 本身不表达请求种类，Provider payload 又包含正文、工具上下文和凭据，不能成为故障选择 API。final-reply 前的 `2` 初始 delivery、`1` repair delivery、`3` receipts 和零 assistant 用于证明“父 run 已完成协作、正要唯一收尾”；真正允许故障注入的项目正确性 oracle 是 selector 在 disposable project cwd 同步运行的可信宿主 `node verify-e2e.mjs` 成功 marker。父 `project.verify` audit/receipt/observation 可能不存在，只能作为诊断计数。 +- 处理：fault proxy 只向异步 selector 暴露冻结的 `sequence / acceptedAtMs`；harness 自己从 Agent DB、delivery/claim sidecar 和 conversation 中选择同一父 Session/run 的唯一 base final-reply，并先验证 `2+1` delivery 已 claim、两次 observed claim 覆盖 `3` receipts、assistant 为 `0`。selector 随后必须在 proxy reset/forward 目标请求前，以可信宿主 Node 同步运行固定的 `node verify-e2e.mjs`；只有 stdout 含 `real-e2e-command=passed` 且 stdout/stderr 无失败 marker 才返回允许注入。失败、超时、非零退出或 marker 无效时不得注入，捕获的 stdout/stderr 只能用于内存判定与敏感扫描，不得写入 state、checkpoint、report 或公共日志；父 `project.verify` 只记录诊断。候选重复或前提不完整必须失败，禁止回退到首请求或解析正文。 +- 恢复门禁：等 base final-reply 形成 failed lifecycle、retry audit、持久 sidecar 和完整 `running / waiting-for-provider-retry` 后，才在 `30s` backoff 内 pidfd `SIGKILL` Runner。重启保持 Session/run/request fingerprint/attempt/slot/retryAt，`retryAt` 前零请求，到期后只允许唯一 `-transient-1`；父 tool-plan 数不得增加。task/Runtime 进入终态后还必须显式等待 pending、retry、handoff、finalization、confirmation sidecar 全部清零，并设置 `10s` 硬超时；看到唯一回复后立即采样到残留 journal 只能判该轮 **FAIL**，不能与后续清理或其它轮次拼接。 +- 写锁竞争：并行 Agent 的 `file.write` 可能与项目写锁竞争。若失败 observation 原样携带绝对锁路径，后续 pending 持久化会因安全门禁拒绝并进入 `needs-reconciliation`，把原本可恢复的锁竞争扩大为持久状态故障。`file.write / file.patch / file.delete` 应统一使用 Runtime 短等待项目写锁，`file.write` 错误在进入 observation/pending 前脱敏，并以 `2` 条 Rust 回归测试固定短等待和脱敏边界。 +- 验证：新命令为 `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径>`；旧 `supervisor-swarm-transient-retry` 只保留首次 tool-plan 证明。fault proxy `14/14`、E2E self-test **PASS**、前端 `308/308` 及 shell typecheck、`platform-llm 41/41`、`platform-agent 17/17`、`shared-contracts 7/7` 等确定性门禁已完成。真实外部 Provider suite 共执行六轮，前五轮均为 **FAIL** 且不得拼接：第一、二轮沿用既有失败记录，第三轮为终态后过早观察到 `1` 个 finalization journal，第四轮为 quality-review 普通 tool-plan 连续 transport/connectivity 失败耗尽重试且未进入目标故障，第五轮为上述写锁竞争与绝对锁路径问题；修复后第六轮在同一轮内完整 **PASS**。 +- 第六轮证据：`gpt-5.5 / openai_chat`，`2` 条初始加 `1` 条 repair delivery、`3` 条专业 Agent assistant，Project Supervisor base final-reply 目标与可信宿主 verify marker 门禁均成立；受控 Provider `failed=1 / retry=1`、incidental `failure=0 / retry=0`，`30s` backoff，pidfd `claim=2 / signal=2`，Runner `resumed=true / identityStable=true`。父 tool-plan 故障前后均为 `13`，parent final-reply 与最终 assistant 唯一，response stream `sequence=2 / committed`；pending、retry、handoff、finalization、confirmation sidecar、全部重复计数及 API Key、私有正文、项目路径、正式配置路径和公共报告泄漏扫描命中均为 `0`。V1.41 handoff 原子提交前的 unknown-result 和 tool-plan handoff 未覆盖边界继续保留。 +- 关联：`apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs`、`apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs`、`apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts`、`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`。 diff --git a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md index 8ea12db4a..380565121 100644 --- a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md +++ b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md @@ -1439,6 +1439,38 @@ V1.41 为 V1.40 明确留下的成功响应交接窗口增加 `.agent/runtime/pr 上表是 V1.41 的确定性门禁，不代表真实外部 Provider E2E 结论。除表内窗口外，还必须继续扫描 task/event/Agent DB/CLI/report，确认 Provider 响应、compaction summary、API Key、Provider URL 和项目/配置绝对路径公共泄漏均为 0。 +## V1.42 Project Supervisor final-reply 瞬时重试 Runner 强杀真实门禁 + +V1.42 不改变生产 Runtime、Provider lifecycle、retry sidecar、handoff、finalization 或 response stream 协议，只扩展一次性 loopback fault proxy 和真实 E2E harness。新 suite 固定为 `supervisor-swarm-final-reply-transient-retry`，发布 AppData 复验入口为： + +```bash +npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径> +``` + +### 受控故障选择与命中前事实 + +- fault proxy 的异步 `shouldInjectFault` selector 只接收 `Object.freeze({ sequence, acceptedAtMs })`。selector 不得看到或保存 URL、header、请求/响应 body、API Key 或其它凭据，也不得靠解析 Provider payload 判断请求种类；代理既有 metadata-only 请求日志边界保持不变。 +- harness 只能选择当前 `project-supervisor` 父 Agent、同一父 Session/run 的唯一 base `final-reply` `started` lifecycle；带 `-transient-N` 的后继、专业 Agent/child 请求、tool-plan 和 context-compaction 都不得成为注入目标。候选缺失、重复或身份不符必须让 suite 失败，不能退回“首个请求”语义。 +- 命中 base final-reply 前，持久选择上下文必须已经证明：恰好 `2` 条初始 delivery 加 `1` 条 repair delivery 均为 `claimed-by-parent / completed`；两次 `observed` claim 完整观察这 `3` 条 receipt；父 conversation assistant 仍为 `0`，且项目 revision 与 final-reply task identity 有效。候选或这些事实不唯一时不能注入，也不能退回“首个请求”语义。 +- selector 必须在 proxy 对目标请求执行 reset 或正常 forward **之前**，以可信宿主 Node 在 disposable project cwd 同步运行固定命令 `node verify-e2e.mjs`。只有命令正常结束、stdout 含 `real-e2e-command=passed`，且 stdout/stderr 均不含 `real-e2e-command=failed` 时才返回允许注入；spawn 失败、超时、非零退出、marker 缺失或冲突一律返回不注入。捕获的 stdout/stderr 只用于本次内存判定与敏感扫描，失败内容不得进入 selector state、checkpoint、report 或公共日志。 +- 父 run 的 `project.verify` audit/receipt/observation 数量与顺序继续写入有界诊断字段，但它们不是 fault injection prerequisite；计数为 `0` 也不能单独阻止注入，更不能替代上述可信宿主 marker oracle。 + +### 30 秒退避、Runner 强杀与恢复 + +- base final-reply 必须先形成唯一 `started -> failed` lifecycle、唯一 retry audit 和已原子落盘的 provider retry sidecar，并把父 task/Runtime 都投影为 `running / waiting-for-provider-retry`。只有完整等待态成立且 `retryAt` 仍留有保护窗口时，才允许对本 suite 持有的 Runner 执行 pidfd `SIGKILL`。 +- 新 boot 接管后必须保持原父 Session/run、task、request fingerprint、retry identity、attempt、next request slot、sidecar 字节和 `retryAt` 不变，并明确证明从旧 boot 恢复。重启后及 `retryAt` 前新增代理请求数都必须为 `0`；到期后只允许出现一个 `-transient-1`，且其 `acceptedAtMs >= retryAtMs`。 +- `-transient-1` 在 forwarding gate 内仍不得进入 upstream；放行前再次核对 delivery、claim、receipt、assistant、pending、project revision、可信宿主 marker 结果和父 tool-plan 计数均未漂移。父 `project.verify` 只保留诊断计数，不提升为放行前置条件。放行后只能由该唯一后继成功完成，受控 failed/retry 与其它 incidental Provider failure/retry 必须按 request identity 分开计数。 + +### 终局门禁与六轮证据 + +- 恢复前后的父 tool-plan `started` 数必须完全相等，不得为收尾新增 tool-plan。终局只允许唯一成功的 parent final-reply、唯一 Project Supervisor assistant 和唯一 `committed` response stream，stream 身份绑定原 base final-reply slot 且正文与 assistant 完全一致。 +- retry、handoff、finalization artifacts 必须全部为 `0`；重复 delivery/claim/receipt/action/message/lifecycle 必须为 `0`。公共 task/event/Agent DB/CLI/report 中的 Provider/assistant 正文、API Key 和项目/发布配置绝对路径命中必须为 `0`。 +- 旧 `supervisor-swarm-transient-retry` 继续只证明 Project Supervisor 首次 tool-plan 的持久退避与 Runner 强杀，不能替代本 suite，也不能把它的历史 PASS 外推为 V1.42 final-reply PASS。 +- 2026-07-20 确定性与静态门禁已完成：fault proxy `14/14`、E2E self-test **PASS**、前端 `308/308`，以及 shell typecheck、`platform-llm 41/41`、`platform-agent 17/17`、`shared-contracts 7/7` 均通过。 +- 真实外部 Provider suite 总计执行六轮，逐轮独立裁决且严禁拼接：第一、二轮沿用既有失败记录，均为 **FAIL**；第三轮已走通故障、持久重试和唯一回复，但验收过早观察到 `1` 个 finalization journal，仍为 **FAIL**，随后改为终态后显式等待 sidecar 全部清零并设置 `10s` 硬超时；第四轮在 quality-review 的普通 tool-plan 连续发生 transport/connectivity 失败并耗尽重试，未进入目标 final-reply 故障，仍为 **FAIL**；第五轮暴露并修复并行 Agent 的 `file.write` 与项目写锁竞争，失败 observation 携带绝对锁路径，继而触发 pending 持久化拒绝并进入 `needs-reconciliation`，仍为 **FAIL**。修复后 `file.write / file.patch / file.delete` 统一使用 Runtime 短等待项目写锁，`file.write` 错误在持久化前脱敏，并新增 `2` 条 Rust 回归测试。 +- 第六轮在同一轮内完整 **PASS**：正式路由为 `gpt-5.5 / openai_chat`；形成 `2` 条初始 delivery、`1` 条 repair delivery 和 `3` 条专业 Agent assistant；受控目标精确命中 Project Supervisor base final-reply，可信宿主 verify marker 门禁通过。受控 Provider `failed=1 / retry=1`，incidental `failure=0 / retry=0`；`30s` backoff 成立，pidfd `claim=2 / signal=2`，Runner `resumed=true / identityStable=true`；故障前后父 tool-plan 均为 `13`，只产生唯一 parent final-reply 和唯一最终 assistant，response stream 以 `sequence=2` 提交为 `committed`。pending、retry、handoff、finalization、confirmation sidecar 全部为 `0`，全部重复计数为 `0`，API Key、私有正文、项目路径、正式配置路径及公共报告泄漏扫描命中均为 `0`。 +- 第六轮 PASS 只关闭“final-reply 瞬态失败进入持久退避后强杀 Runner”的真实证据缺口。V1.41 中“Provider 已成功返回、但 handoff 尚未原子落盘并回读”仍是 unknown-result 边界；tool-plan 成功响应及 function arguments 的 durable handoff 仍未覆盖。 + ## 验收命令 - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml structured_plan_ -- --nocapture` @@ -1482,6 +1514,7 @@ V1.41 为 V1.40 明确留下的成功响应交接窗口增加 `.agent/runtime/pr - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite parallel-read` - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite supervisor-swarm` - `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e -- --config-dir ` +- `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径>` - `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-autonomous-chat-real-e2e -- --config-dir ` - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite full` - `npm run check:encoding` diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 846ab24d9..1b77ff836 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -611,3 +611,10 @@ game-project/ - V1.41 对支持范围执行 handoff-first：Provider 成功后先提交并回读 handoff，再闭合同一实际 requestId 的 `completed` lifecycle；恢复先消费 handoff，匹配 retry 时复用原 attempt/slot 且零新网络请求，retry identity/attempt/slot 冲突进入 reconciliation，稳定身份漂移先闭合旧 lifecycle 再作废并同 run 重规划。自动 compaction 必须在规范 sidecar 写入并回读一致后才清理 handoff；final-reply handoff 则保留到 finalization 和 stream commit 全部完成。任一 primary、`.previous` 或损坏 handoff 都让 Runner 保持 busy，阻止 `runner.shutdown_if_idle`。 - finalization schema 已正式升为 `game-creator-runtime-finalization.v4`：`responseRequestSlot` 与 response revision/fingerprint、steer cursor、计划及 Goal 快照共同进入 `finalizationId` 指纹，篡改 slot 必须失败关闭；`project.rs` 的 Agent DB finalization lifecycle 白名单兼容 `journalSchemaVersion` v1-v4，缺少 slot 的 v3 journal 继续按旧指纹和 legacy slot 派生规则读取。assistant、Runtime 和 Goal 提交后，缺失或仍为 streaming/failed/discarded 的回复流由 journal 固定身份重建为 ready，再推进 committed；committed 写后必须回读身份、状态和正文，成功后才一起删除 finalization journal/handoff，失败则保留 `finalizing` 供 Runner 幂等恢复。确定性测试矩阵以 Runtime V1.41 章节中的 handoff、compaction、四 checkpoint、stream 重建、schema 兼容和 Runner busy 用例为准。2026-07-20 最新验证为 `provider_handoff_` 11/11、`provider_retry_` 21/21、`response_stream_` 23/23、`finalization_resume_` 12/12；Tauri/Rust 串行全量共 989 tests，`985 passed / 4 ignored / 0 failed`，不再沿用 V1.40 的旧局部计数。 - V1.41 的承诺严格限定为 **handoff 已落盘后的 exactly-once 消费**。外部 Provider 已返回完整响应、但 Runtime 尚未写 handoff 的硬杀窗口仍不能承诺 Provider 调用 exactly-once，后续只能进入 reconciliation 并可能需要人工决定是否重发；tool-plan 成功响应仍不在本轮 handoff。真实外部 Provider 的 final-reply 退避期 Runner 强杀继续是独立 E2E，不能用 V1.39 首次 tool-plan PASS 或确定性 mock 结果替代。 +- 2026-07-20 起，同一 Runtime 文档的“V1.42 Project Supervisor final-reply 瞬时重试 Runner 强杀真实门禁”作为该独立 E2E 的当前事实源。本切片不改变生产 Runtime 协议，只为一次性 loopback fault proxy 增加安全异步 selector，并新增 `supervisor-swarm-final-reply-transient-retry` suite；selector 只可见冻结的 `sequence / acceptedAtMs`，不得看到 URL、header、body 或 API Key。 +- 新 suite 只命中同一父 Session/run 的 Project Supervisor base final-reply。注入前的持久选择上下文必须证明 `2` 条初始 delivery 加 `1` 条 repair delivery 已全部 claim、两次 observed claim 已完整观察 `3` 条 receipt、assistant 仍为 `0`；随后 selector 在 proxy reset/forward 该请求前，以可信宿主 Node 在 disposable project cwd 同步执行固定的 `node verify-e2e.mjs`，仅 `real-e2e-command=passed` marker 成功且无失败 marker 才允许注入。失败、超时或 marker 无效时不得注入，捕获的 stdout/stderr 不得写入 state、checkpoint、report 或公共日志。父 run 的 `project.verify` audit/receipt/observation 计数仅作诊断，不是注入前提。进入 `30s` retry backoff 后才用 pidfd `SIGKILL` 强杀 Runner；新 boot 必须保持 Session/run/request fingerprint/attempt/slot/retryAt，重启后和到期前零新请求，到期后只有唯一 `-transient-1`。 +- 终局不得增加父 tool-plan，只允许唯一成功 parent final-reply、唯一 Supervisor assistant 和唯一 committed response stream；retry/handoff/finalization artifacts、重复和公共正文/API Key/项目及发布配置绝对路径泄漏均为 `0`。旧 `supervisor-swarm-transient-retry` 继续只证明首次 tool-plan，不能替代新 suite。规范命令为 `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e -- --config-dir <发布AppData绝对路径>`。 +- V1.42 已完成 fault proxy `14/14`、E2E self-test **PASS**、前端 `308/308`，以及 shell typecheck、`platform-llm 41/41`、`platform-agent 17/17`、`shared-contracts 7/7` 等确定性门禁。真实外部 Provider suite 总计执行六轮，前五轮均为 **FAIL**，各轮证据不得拼接。第一、二轮沿用既有失败记录；第三轮虽已走通故障、重试和唯一回复，但验收过早观察到 `1` 个 finalization journal，现已改为终态后显式等待 sidecar 清零并设置 `10s` 硬超时；第四轮在 quality-review 普通 tool-plan 连续发生 transport/connectivity 失败并耗尽重试，未进入目标故障。 +- 第五轮暴露并修复并行 Agent 的 `file.write` 与项目写锁竞争：失败 observation 携带绝对锁路径，导致 pending 持久化拒绝并进入 `needs-reconciliation`。现 `file.write / file.patch / file.delete` 统一使用 Runtime 短等待项目写锁，`file.write` 错误在持久化前脱敏，并新增 `2` 条 Rust 回归测试。 +- 第六轮在同一轮内完整 **PASS**：`gpt-5.5 / openai_chat` 路由形成 `2` 条初始加 `1` 条 repair delivery、`3` 条专业 Agent assistant；故障精确命中 Project Supervisor base final-reply，可信宿主 verify marker 门禁通过。受控 Provider `failed=1 / retry=1`、incidental `failure=0 / retry=0`，`30s` backoff，pidfd `claim=2 / signal=2`，Runner `resumed=true / identityStable=true`；父 tool-plan 在故障前后均为 `13`，parent final-reply 与最终 assistant 唯一，response stream `sequence=2 / committed`。pending、retry、handoff、finalization、confirmation sidecar、全部重复计数，以及 API Key、私有正文、项目路径、正式配置路径和公共报告泄漏扫描命中均为 `0`。 +- 第六轮 PASS 不改变 V1.41 handoff 原子落盘并回读前的 unknown-result 边界，tool-plan 成功响应/function arguments 的 durable handoff 仍未覆盖。 diff --git a/package.json b/package.json index 6818c8de8..dd3a30f4d 100644 --- a/package.json +++ b/package.json @@ -150,6 +150,7 @@ "ai-game-creator-shell:agent-runtime:real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:real-e2e --", "ai-game-creator-shell:agent-runtime:supervisor-swarm-autonomous-chat-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:supervisor-swarm-autonomous-chat-real-e2e --", "ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:supervisor-swarm-transient-retry-real-e2e --", + "ai-game-creator-shell:agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:supervisor-swarm-final-reply-transient-retry-real-e2e --", "ai-game-creator-shell:agent-runtime:steer-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:steer-real-e2e --", "ai-game-creator-shell:agent-runtime:steer-runner-kill-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:steer-runner-kill-real-e2e --", "ai-game-creator-shell:typecheck": "npm --prefix apps/ai-game-creator-shell run typecheck",