From 160fe3a8b4fb64ea65c0ca8403fd7b2b7615221c Mon Sep 17 00:00:00 2001 From: AIGameCreator App Date: Sun, 19 Jul 2026 23:24:51 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84=20Agent=20Provider=20?= =?UTF-8?q?=E6=8C=81=E4=B9=85=E9=87=8D=E8=AF=95=E7=9C=9F=E5=AE=9E=E9=AA=8C?= =?UTF-8?q?=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将瞬态故障注入迁移到 Project Supervisor 首次工具规划并验证持久退避恢复 为故障代理补充仅含序号和时间的元数据请求审计与对应测试 强化真实 E2E 对并行委派、重启恢复、重试时序、分账和终局清理的验收 修正瞬态重试目标的 maxRetries 配置门禁并同步 Runtime 文档与项目记忆 --- .../scripts/agent-runtime-real-e2e.mjs | 862 +++++++++++++++--- .../scripts/llm-transient-fault-proxy.mjs | 19 +- .../tests/llmTransientFaultProxy.test.ts | 34 + .../shared-memory/decision-log.md | 1 + .../shared-memory/development-workflow.md | 6 +- docs/project-memory/shared-memory/pitfalls.md | 3 +- ...案】AI游戏创作Agent Runtime V1.1-2026-07-12.md | 6 +- ...案】AI游戏创作智能体App实施计划-2026-06-24.md | 2 +- 8 files changed, 792 insertions(+), 141 deletions(-) diff --git a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs index 09595474a..1076b73ba 100644 --- a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs +++ b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs @@ -113,6 +113,9 @@ const steerRunnerKillSuite = 'steer-runner-kill'; const parallelReadSuite = 'parallel-read'; const supervisorSwarmSuite = 'supervisor-swarm'; const supervisorSwarmTransientRetrySuite = 'supervisor-swarm-transient-retry'; +const supervisorSwarmTransientRetryTargetAgentId = projectSupervisorAgentId; +const supervisorSwarmTransientRetryBackoffMs = 30_000; +const supervisorSwarmTransientRetryPreDueGuardMs = 1_500; const supervisorSwarmAutonomousChatSuite = 'supervisor-swarm-autonomous-chat'; const supervisorSwarmStaticIsolatedAutonomousChatSuite = 'supervisor-swarm-static-isolated-autonomous-chat'; @@ -778,6 +781,22 @@ const state = { newRunnerBootId: null, transientFaultProxy: null, transientFaultCheckpoint: null, + transientRetryPreKillSidecar: null, + transientRetryPostRestartSidecar: null, + transientRetryPreKillRunnerBootId: null, + transientRetryPostRestartRunnerBootId: null, + transientRetryRequestCountBeforeKill: 0, + transientRetryRequestCountAfterRestart: 0, + transientRetryRequestCountBeforeDue: 0, + transientRetryEarlyRequestCount: 0, + transientRetrySidecarIdentityStable: false, + transientRetryAttemptStable: false, + transientRetryAtStable: false, + transientRetryRuntimeIdentityStable: false, + transientRetryAcceptedAtOrAfterDue: false, + transientRetryIncidentalProviderFailureCount: 0, + transientRetryIncidentalProviderRetryCount: 0, + transientRetryRecoveredFromPreviousBoot: false, autonomousTaskRecipeFree: false, autonomousRepositoryRecipeFree: false, interactiveCliUsed: false, @@ -1022,10 +1041,7 @@ try { /^(?:isolated|source)-[a-z0-9-]+$/u.test(error.code) ? error.code : 'isolated-owned-runner-cleanup-failed'; - recordError( - safeCleanupErrorCode, - error, - ); + recordError(safeCleanupErrorCode, error); await closeOwnedRunnerKillHandle( state.isolatedRunner.current?.killHandle, ).catch(() => {}); @@ -6133,6 +6149,19 @@ async function writeSupervisorSwarmProjectPolicy(denyQualityMutations) { } function expectedSupervisorSwarmCollaborationPolicy() { + if (isSupervisorSwarmTransientRetrySuite()) { + return { + schemaVersion: supervisorCollaborationPolicySchemaVersion, + requiredInitialWave: 'static', + minStaticDelegates: 2, + requiredStaticAgentIds: [ + supervisorSwarmDesignAgentId, + supervisorSwarmQualityAgentId, + ], + minIsolatedChildren: 0, + orchestratorOnlyAfterDelegation: true, + }; + } if (isSupervisorSwarmMixedHarnessSuite()) { return { schemaVersion: supervisorCollaborationPolicySchemaVersion, @@ -6206,7 +6235,8 @@ async function replaceSupervisorSwarmCollaborationPolicy(policy, failureCode) { async function writeSupervisorSwarmCollaborationPolicy() { assert( - isSupervisorSwarmMixedHarnessSuite(), + isSupervisorSwarmMixedHarnessSuite() || + isSupervisorSwarmTransientRetrySuite(), 'supervisor-swarm-collaboration-policy-write-outside-suite', ); const policy = expectedSupervisorSwarmCollaborationPolicy(); @@ -6265,7 +6295,8 @@ async function seedSupervisorSwarmDisposableProject() { supervisorSwarmWeakQualityContent, ), writeSupervisorSwarmProjectPolicy(true), - ...(isSupervisorSwarmMixedHarnessSuite() + ...(isSupervisorSwarmMixedHarnessSuite() || + isSupervisorSwarmTransientRetrySuite() ? [writeSupervisorSwarmCollaborationPolicy()] : []), ]); @@ -7698,6 +7729,7 @@ function validateSupervisorSwarmInitialProviderBatch(batch) { sessionId: batch.sessionId, runId: batch.runId, status: batch.status, + nextActionIndex: batch.nextActionIndex, actionIds: [...actionIds], delegateActionIds: [...delegateActionIds], actions: batch.actions.map((pending) => ({ @@ -7801,7 +7833,8 @@ function supervisorSwarmInitialBatchBindsPolicySnapshot( initialBatch.runId === parentRunId && initialBatch.status !== 'aborted' && contract?.schemaVersion === supervisorCollaborationContractSchemaVersion && - contract.policySchemaVersion === supervisorCollaborationPolicySchemaVersion && + contract.policySchemaVersion === + supervisorCollaborationPolicySchemaVersion && isPlainObject(contract.policySnapshot) && /^[0-9a-f]{64}$/u.test(contract.policyFingerprint ?? '') && /^[0-9a-f]{64}$/u.test(contract.contractFingerprint ?? '') @@ -9976,6 +10009,24 @@ async function driveSupervisorSwarmToRepairKillBoundary() { supervisorSwarmConfirmedTools.includes(repairPending.tool), `supervisor-swarm-repair-pending-tool-invalid:${repairPending.tool}`, ); + if (isSupervisorSwarmTransientRetrySuite()) { + const provider = supervisorSwarmRepairProviderLifecycle( + persistence.agentDb, + repair, + ); + state.supervisorSwarm.repairProviderRequestId = provider.requestId; + state.supervisorSwarm.repairPendingActionId = repairPending.actionId; + state.supervisorSwarm.repairPendingIdentity = + supervisorSwarmPendingIdentity(repairPending); + state.supervisorSwarm.repairPendingTool = repairPending.tool; + assert( + state.supervisorSwarm.runnerKillBoundaryObserved && + state.identityStable && + state.supervisorSwarm.transientRetryRecoveredFromPreviousBoot, + 'supervisor-swarm-transient-retry-recovery-not-observed-before-repair', + ); + return; + } if (isSupervisorSwarmMixedHarnessSuite()) { if (!supervisorSwarmMixedIsolatedClaimsReady(persistence)) { await sleep(50); @@ -10172,7 +10223,7 @@ async function prepareSupervisorSwarmRuntimeAppData() { const loaded = await loadConfig(state.options.configDir); const targetConfig = effectiveAgentLlmConfig( loaded.config, - supervisorSwarmDesignAgentId, + supervisorSwarmTransientRetryTargetAgentId, ); const upstream = new URL(targetConfig.baseUrl); assert( @@ -10202,10 +10253,10 @@ async function prepareSupervisorSwarmRuntimeAppData() { ); const configOverlay = { agentLlm: { - [supervisorSwarmDesignAgentId]: { + [supervisorSwarmTransientRetryTargetAgentId]: { baseUrl: proxy.baseUrl, maxRetries: 1, - retryBackoffMs: 100, + retryBackoffMs: supervisorSwarmTransientRetryBackoffMs, }, }, }; @@ -10213,9 +10264,11 @@ async function prepareSupervisorSwarmRuntimeAppData() { JSON.stringify(Object.keys(configOverlay)) === JSON.stringify(['agentLlm']) && JSON.stringify(Object.keys(configOverlay.agentLlm)) === - JSON.stringify([supervisorSwarmDesignAgentId]) && + JSON.stringify([supervisorSwarmTransientRetryTargetAgentId]) && JSON.stringify( - Object.keys(configOverlay.agentLlm[supervisorSwarmDesignAgentId]), + Object.keys( + configOverlay.agentLlm[supervisorSwarmTransientRetryTargetAgentId], + ), ) === JSON.stringify(['baseUrl', 'maxRetries', 'retryBackoffMs']), 'supervisor-swarm-transient-retry-overlay-shape-invalid', ); @@ -10224,7 +10277,7 @@ async function prepareSupervisorSwarmRuntimeAppData() { const isolated = await loadConfig(state.isolatedRunner.appDataDir); const isolatedTarget = effectiveAgentLlmConfig( isolated.config, - supervisorSwarmDesignAgentId, + supervisorSwarmTransientRetryTargetAgentId, ); assert( isolatedTarget.baseUrl === proxy.baseUrl && @@ -10232,7 +10285,8 @@ async function prepareSupervisorSwarmRuntimeAppData() { isolatedTarget.model === targetConfig.model && isolatedTarget.apiKind === targetConfig.apiKind && isolatedTarget.maxRetries === 1 && - isolatedTarget.retryBackoffMs === 100 && + isolatedTarget.retryBackoffMs === + supervisorSwarmTransientRetryBackoffMs && state.isolatedRunner.configOverlayCreated, 'supervisor-swarm-transient-retry-effective-overlay-invalid', ); @@ -10249,14 +10303,134 @@ async function readSupervisorSwarmProjectRevision() { }); } +async function readSupervisorSwarmProviderRetrySidecar(agentId, runId) { + const directory = path.join( + state.projectRoot, + '.agent/runtime/provider-retries', + ); + const matches = []; + for (const file of (await listFiles(directory)).filter((candidate) => + candidate.endsWith('.json'), + )) { + const [bytes, metadata] = await Promise.all([ + fs.readFile(file), + fs.lstat(file), + ]); + const record = JSON.parse(bytes.toString('utf8')); + if ( + record?.identity?.agentId === agentId && + record.identity.runId === runId + ) { + assert( + metadata.isFile() && + !metadata.isSymbolicLink() && + (process.platform === 'win32' || (metadata.mode & 0o077) === 0), + 'supervisor-swarm-provider-retry-sidecar-permissions-invalid', + ); + matches.push({ + record, + bytesSha256: hashValue(bytes), + }); + } + } + assert( + matches.length <= 1, + 'supervisor-swarm-provider-retry-sidecar-duplicate', + ); + return matches[0] ?? null; +} + +function supervisorSwarmTransientRetrySideEffects( + persistence, + pending, + revision, + designMetadata, + target, +) { + const actionRecords = persistence.agentDb.filter( + (record) => + record.agentId === target.agentId && + record.runId === target.runId && + [ + 'agent.runtime.tool_action.executing', + 'agent.runtime.tool_action.observed', + 'agent.runtime.tool_observation', + 'agent.runtime.action_receipt', + 'agent.runtime.tool_confirmation_required', + 'agent.runtime.tool_confirmation.approved', + ].includes(record.recordType), + ); + const childDeliveries = persistence.deliveries.filter( + (delivery) => + delivery.parentAgentId === target.agentId && + delivery.parentRunId === target.runId, + ); + const claims = persistence.claims.filter((claim) => { + if (target.delegationId) { + return (claim.receipts ?? []).some( + (receipt) => receipt.delegationId === target.delegationId, + ); + } + return ( + claim.parentAgentId === target.agentId && + claim.parentRunId === target.runId + ); + }); + const messages = + target.agentId === projectSupervisorAgentId + ? persistence.supervisorConversation + : (persistence.professionalConversations.find( + (entry) => + entry.agentId === target.agentId && + entry.sessionId === target.sessionId, + )?.messages ?? []); + return { + actionCount: actionRecords.length, + receiptCount: actionRecords.filter( + (record) => record.recordType === 'agent.runtime.action_receipt', + ).length, + childDeliveryCount: childDeliveries.length, + claimCount: claims.length, + assistantCount: messages.filter((message) => message.role === 'assistant') + .length, + pendingCount: pending.filter( + (candidate) => + candidate.agentId === target.agentId && + candidate.runId === target.runId, + ).length, + projectRevision: revision.revision, + designArtifactPresent: designMetadata !== null, + }; +} + +function assertSupervisorSwarmTransientRetrySideEffectsZero(sideEffects, code) { + assert( + sideEffects.actionCount === 0 && + sideEffects.receiptCount === 0 && + sideEffects.childDeliveryCount === 0 && + sideEffects.claimCount === 0 && + sideEffects.assistantCount === 0 && + sideEffects.pendingCount === 0 && + sideEffects.projectRevision === 0 && + sideEffects.designArtifactPresent === false, + code, + ); +} + async function captureSupervisorSwarmTransientRetryCheckpoint() { if (!isSupervisorSwarmTransientRetrySuite()) return; const proxy = state.supervisorSwarm.transientFaultProxy; assert(proxy, 'supervisor-swarm-transient-retry-proxy-missing'); await proxy.waitForFault(60_000); - await proxy.waitForHeldRequest(60_000); + const target = { + agentId: supervisorSwarmTransientRetryTargetAgentId, + runId: state.initialRunId, + sessionId: supervisorSwarmSessionId, + delegationId: null, + }; const deadline = Date.now() + 60_000; + let waitingBoundary = null; while (Date.now() < deadline) { const [persistence, pending, revision, designMetadata] = await Promise.all([ readSupervisorSwarmPersistence(), @@ -10269,28 +10443,17 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { throw error; }), ]); - const initial = supervisorSwarmParentDeliveries( - persistence.deliveries, - ).filter((delivery) => delivery.repairOfDelegationId == null); - const designDelivery = initial.find( - (delivery) => delivery.targetAgentId === supervisorSwarmDesignAgentId, - ); - if (!designDelivery || initial.length !== 2) { - await sleep(25); - continue; - } - const lifecycle = persistence.agentDb.filter( (record) => record.recordType === 'agent.runtime.provider_request.lifecycle' && - record.agentId === designDelivery.targetAgentId && - record.runId === designDelivery.targetRunId, + record.agentId === target.agentId && + record.runId === target.runId, ); const retries = persistence.agentDb.filter( (record) => record.recordType === 'agent.runtime.provider_request.retry' && - record.agentId === designDelivery.targetAgentId && - record.runId === designDelivery.targetRunId, + record.agentId === target.agentId && + record.runId === target.runId, ); const failedStarted = lifecycle.find( (record) => @@ -10305,14 +10468,33 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { ) : null; const retryAudit = retries[0]; - const retryStarted = retryAudit - ? lifecycle.find( + if (!failedStarted || !failedTerminal || !retryAudit) { + await sleep(25); + continue; + } + const retryStarted = lifecycle.filter( (record) => record.status === 'started' && record.requestSlot === retryAudit.nextRequestSlot, - ) - : null; - if (!failedStarted || !failedTerminal || !retryAudit || !retryStarted) { + ); + const sidecar = await readSupervisorSwarmProviderRetrySidecar( + target.agentId, + target.runId, + ); + const targetRuntime = persistence.runtimeStates.find( + (runtime) => + runtime.agentId === target.agentId && runtime.runId === target.runId, + ); + const targetTask = persistence.taskSnapshot.latest.find( + (task) => task.agentId === target.agentId && task.runId === target.runId, + ); + if ( + !sidecar || + targetRuntime?.status !== 'running' || + targetRuntime.phase !== 'waiting-for-provider-retry' || + targetTask?.status !== 'running' || + targetTask.phase !== 'waiting-for-provider-retry' + ) { await sleep(25); continue; } @@ -10325,104 +10507,346 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { 'source', 'requestKind', ]; - const actionRecords = persistence.agentDb.filter( - (record) => - record.agentId === designDelivery.targetAgentId && - record.runId === designDelivery.targetRunId && - [ - 'agent.runtime.tool_action.executing', - 'agent.runtime.tool_action.observed', - 'agent.runtime.tool_observation', - 'agent.runtime.action_receipt', - 'agent.runtime.tool_confirmation_required', - 'agent.runtime.tool_confirmation.approved', - ].includes(record.recordType), + const sideEffects = supervisorSwarmTransientRetrySideEffects( + persistence, + pending, + revision, + designMetadata, + target, ); - const receipts = actionRecords.filter( - (record) => record.recordType === 'agent.runtime.action_receipt', - ); - const childDeliveries = persistence.deliveries.filter( - (delivery) => - delivery.parentAgentId === designDelivery.targetAgentId && - delivery.parentRunId === designDelivery.targetRunId, - ); - const claims = persistence.claims.filter((claim) => - (claim.receipts ?? []).some( - (receipt) => receipt.delegationId === designDelivery.delegationId, - ), - ); - const designConversation = persistence.professionalConversations.find( - (entry) => - entry.agentId === designDelivery.targetAgentId && - entry.sessionId === designDelivery.targetSessionId, - ); - const assistantCount = (designConversation?.messages ?? []).filter( - (message) => message.role === 'assistant', - ).length; - const pendingCount = pending.filter( - (candidate) => - candidate.agentId === designDelivery.targetAgentId && - candidate.runId === designDelivery.targetRunId, - ).length; const proxyStats = proxy.getStats(); + const retryRecord = sidecar.record; assert( - lifecycle.length === 3 && + lifecycle.length === 2 && retries.length === 1 && + retryStarted.length === 0 && failedStarted.requestKind === 'tool-plan' && failedStarted.requestId === failedTerminal.requestId && failedStarted.requestSlot === failedTerminal.requestSlot && failedStarted.requestId === retryAudit.requestId && retryAudit.retryAttempt === 1 && retryAudit.maxRetries === 1 && + retryAudit.backoffMs === supervisorSwarmTransientRetryBackoffMs && retryAudit.nextRequestSlot === `${failedStarted.requestSlot}-transient-1` && - retryStarted.requestSlot === retryAudit.nextRequestSlot && - retryStarted.requestId !== failedStarted.requestId && stableIdentityFields.every( (field) => failedStarted[field] === failedTerminal[field] && - failedStarted[field] === retryAudit[field] && - failedStarted[field] === retryStarted[field], + failedStarted[field] === retryAudit[field], ) && persistence.agentDb.indexOf(failedStarted) < persistence.agentDb.indexOf(failedTerminal) && persistence.agentDb.indexOf(failedTerminal) < persistence.agentDb.indexOf(retryAudit) && - persistence.agentDb.indexOf(retryAudit) < - persistence.agentDb.indexOf(retryStarted) && - actionRecords.length === 0 && - receipts.length === 0 && - childDeliveries.length === 0 && - claims.length === 0 && - assistantCount === 0 && - pendingCount === 0 && - revision.revision === 0 && - designMetadata === null && + retryRecord.schemaVersion === 'game-creator-provider-retry.v1' && + retryRecord.identity.agentId === failedStarted.agentId && + retryRecord.identity.taskId === failedStarted.taskId && + retryRecord.identity.sessionId === failedStarted.sessionId && + retryRecord.identity.runId === failedStarted.runId && + retryRecord.identity.source === failedStarted.source && + retryRecord.identity.requestKind === failedStarted.requestKind && + retryRecord.identity.baseRequestSlot === failedStarted.requestSlot && + /^[a-f0-9]{64}$/u.test(retryRecord.identity.requestFingerprint) && + /^[a-f0-9]{64}$/u.test( + retryRecord.identity.providerConfigFingerprint, + ) && + retryRecord.nextRequestSlot === retryAudit.nextRequestSlot && + retryRecord.nextAttempt === 1 && + retryRecord.maxRetries === 1 && + retryRecord.backoffMs === supervisorSwarmTransientRetryBackoffMs && + retryRecord.retryAtMs >= + retryRecord.updatedAtMs + supervisorSwarmTransientRetryBackoffMs && + retryRecord.retryAtMs > + Date.now() + supervisorSwarmTransientRetryPreDueGuardMs && + retryRecord.errorKind === retryAudit.errorKind && + retryRecord.errorFingerprint === retryAudit.errorFingerprint && + targetRuntime.status === 'running' && + targetRuntime.phase === 'waiting-for-provider-retry' && + targetTask.status === 'running' && + targetTask.phase === 'waiting-for-provider-retry' && + proxyStats.requestCount === 1 && + proxyStats.faultInjectedCount === 1 && + proxyStats.heldRequestCount === 0 && + proxyStats.forwardedRequestCount === 0 && + proxyStats.forwardingReleased === false, + 'supervisor-swarm-transient-retry-waiting-boundary-invalid', + ); + assertSupervisorSwarmTransientRetrySideEffectsZero( + sideEffects, + 'supervisor-swarm-transient-retry-pre-kill-side-effect', + ); + waitingBoundary = { + failedStarted, + failedTerminal, + retryAudit, + target, + sidecar, + sideEffects, + taskIdentity: supervisorSwarmTaskIdentity(targetTask), + }; + break; + } + if (!waitingBoundary) { + throw codedError('supervisor-swarm-transient-retry-waiting-timeout'); + } + + const ownerBefore = await readSupervisorSwarmExecutionOwner(); + const currentRunner = await verifyOwnedRunnerForKill(); + assert( + ownerBefore.bootId === currentRunner.bootId && + ownerBefore.pid === currentRunner.pid, + 'supervisor-swarm-transient-retry-owner-before-kill-invalid', + ); + state.supervisorSwarm.transientRetryPreKillSidecar = waitingBoundary.sidecar; + state.supervisorSwarm.transientRetryPreKillRunnerBootId = + currentRunner.bootId; + state.supervisorSwarm.transientRetryRequestCountBeforeKill = + proxy.getStats().requestCount; + state.supervisorSwarm.oldRunnerBootId = currentRunner.bootId; + + await killRunnerOnce(); + state.supervisorSwarm.runnerKillBoundaryObserved = true; + await runCli(['--agent-resume', state.projectRoot], { timeoutMs: 120_000 }); + state.resumed = true; + const restarted = await waitForRunnerBootChange(currentRunner.bootId); + const claimed = await claimOwnedRunner(restarted); + const ownerAfter = await readSupervisorSwarmExecutionOwner(claimed.bootId); + state.supervisorSwarm.transientRetryPostRestartRunnerBootId = claimed.bootId; + state.supervisorSwarm.newRunnerBootId = claimed.bootId; + state.supervisorSwarm.transientRetryRecoveredFromPreviousBoot = + ownerAfter.bootId === claimed.bootId && + ownerAfter.recoveredFromBootId === currentRunner.bootId; + assert( + claimed.bootId !== currentRunner.bootId && + state.supervisorSwarm.transientRetryRecoveredFromPreviousBoot, + 'supervisor-swarm-transient-retry-owner-recovery-invalid', + ); + + const [postRestartSidecar, postRestartPersistence, postRestartPending] = + await Promise.all([ + readSupervisorSwarmProviderRetrySidecar( + waitingBoundary.target.agentId, + waitingBoundary.target.runId, + ), + readSupervisorSwarmPersistence(), + findPendingActions(), + ]); + const postRestartRevision = await readSupervisorSwarmProjectRevision(); + const postRestartDesignMetadata = await fs + .lstat(path.join(state.projectRoot, supervisorSwarmDesignPath)) + .catch((error) => { + if (error?.code === 'ENOENT') return null; + throw error; + }); + const postRestartRuntime = postRestartPersistence.runtimeStates.find( + (runtime) => + runtime.agentId === waitingBoundary.target.agentId && + runtime.runId === waitingBoundary.target.runId, + ); + const postRestartTask = postRestartPersistence.taskSnapshot.latest.find( + (task) => + task.agentId === waitingBoundary.target.agentId && + task.runId === waitingBoundary.target.runId, + ); + const postRestartSideEffects = supervisorSwarmTransientRetrySideEffects( + postRestartPersistence, + postRestartPending, + postRestartRevision, + postRestartDesignMetadata, + waitingBoundary.target, + ); + state.supervisorSwarm.transientRetryPostRestartSidecar = postRestartSidecar; + state.supervisorSwarm.transientRetryRequestCountAfterRestart = + proxy.getStats().requestCount; + state.supervisorSwarm.transientRetrySidecarIdentityStable = Boolean( + postRestartSidecar && + JSON.stringify(postRestartSidecar.record.identity) === + JSON.stringify(waitingBoundary.sidecar.record.identity), + ); + state.supervisorSwarm.transientRetryAttemptStable = Boolean( + postRestartSidecar && + postRestartSidecar.record.nextAttempt === + waitingBoundary.sidecar.record.nextAttempt && + postRestartSidecar.record.nextRequestSlot === + waitingBoundary.sidecar.record.nextRequestSlot && + postRestartSidecar.bytesSha256 === waitingBoundary.sidecar.bytesSha256, + ); + state.supervisorSwarm.transientRetryAtStable = Boolean( + postRestartSidecar && + postRestartSidecar.record.retryAtMs === + waitingBoundary.sidecar.record.retryAtMs, + ); + state.supervisorSwarm.transientRetryRuntimeIdentityStable = Boolean( + postRestartRuntime?.status === 'running' && + postRestartRuntime.phase === 'waiting-for-provider-retry' && + postRestartTask?.status === 'running' && + postRestartTask.phase === 'waiting-for-provider-retry' && + JSON.stringify(supervisorSwarmTaskIdentity(postRestartTask)) === + JSON.stringify(waitingBoundary.taskIdentity), + ); + assertSupervisorSwarmTransientRetrySideEffectsZero( + postRestartSideEffects, + 'supervisor-swarm-transient-retry-post-restart-side-effect', + ); + assert( + state.supervisorSwarm.transientRetrySidecarIdentityStable && + state.supervisorSwarm.transientRetryAttemptStable && + state.supervisorSwarm.transientRetryAtStable && + state.supervisorSwarm.transientRetryRuntimeIdentityStable && + state.supervisorSwarm.transientRetryRequestCountBeforeKill === 1 && + state.supervisorSwarm.transientRetryRequestCountAfterRestart === 1 && + Date.now() + supervisorSwarmTransientRetryPreDueGuardMs < + waitingBoundary.sidecar.record.retryAtMs, + 'supervisor-swarm-transient-retry-post-restart-invalid', + ); + state.identityStable = true; + + const preDueDelayMs = + waitingBoundary.sidecar.record.retryAtMs - + supervisorSwarmTransientRetryPreDueGuardMs - + Date.now(); + assert( + preDueDelayMs > 0, + 'supervisor-swarm-transient-retry-pre-due-window-missed', + ); + await sleep(preDueDelayMs); + const preDueStats = proxy.getStats(); + state.supervisorSwarm.transientRetryRequestCountBeforeDue = + preDueStats.requestCount; + state.supervisorSwarm.transientRetryEarlyRequestCount = Math.max( + 0, + preDueStats.requestCount - 1, + ); + assert( + Date.now() < waitingBoundary.sidecar.record.retryAtMs && + preDueStats.requestCount === 1 && + preDueStats.heldRequestCount === 0 && + preDueStats.forwardedRequestCount === 0 && + state.supervisorSwarm.transientRetryEarlyRequestCount === 0, + 'supervisor-swarm-transient-retry-request-sent-before-due', + ); + + await proxy.waitForHeldRequest(60_000); + const heldRequestLog = proxy.getRequestLog(); + state.supervisorSwarm.transientRetryAcceptedAtOrAfterDue = Boolean( + heldRequestLog.length === 2 && + heldRequestLog[0].sequence === 1 && + heldRequestLog[0].faultInjectedAtMs >= heldRequestLog[0].acceptedAtMs && + heldRequestLog[1].sequence === 2 && + heldRequestLog[1].acceptedAtMs >= + waitingBoundary.sidecar.record.retryAtMs && + heldRequestLog[1].heldAtMs >= heldRequestLog[1].acceptedAtMs && + heldRequestLog[1].forwardingStartedAtMs == null, + ); + assert( + state.supervisorSwarm.transientRetryAcceptedAtOrAfterDue, + 'supervisor-swarm-transient-retry-network-arrived-before-due', + ); + const heldDeadline = Date.now() + 60_000; + while (Date.now() < heldDeadline) { + const [persistence, pending, revision, designMetadata] = await Promise.all([ + readSupervisorSwarmPersistence(), + findPendingActions(), + readSupervisorSwarmProjectRevision(), + fs + .lstat(path.join(state.projectRoot, supervisorSwarmDesignPath)) + .catch((error) => { + if (error?.code === 'ENOENT') return null; + throw error; + }), + ]); + const lifecycle = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.agentId === waitingBoundary.target.agentId && + record.runId === waitingBoundary.target.runId, + ); + const retries = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.retry' && + record.agentId === waitingBoundary.target.agentId && + record.runId === waitingBoundary.target.runId, + ); + const retryStarted = lifecycle.filter( + (record) => + record.status === 'started' && + record.requestSlot === waitingBoundary.retryAudit.nextRequestSlot, + ); + if (retryStarted.length !== 1) { + await sleep(25); + continue; + } + const stableIdentityFields = [ + 'agentId', + 'taskId', + 'sessionId', + 'runId', + 'source', + 'requestKind', + ]; + const sideEffects = supervisorSwarmTransientRetrySideEffects( + persistence, + pending, + revision, + designMetadata, + waitingBoundary.target, + ); + const proxyStats = proxy.getStats(); + const failedStartedIndex = persistence.agentDb.findIndex( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.status === 'started' && + record.requestId === waitingBoundary.failedStarted.requestId, + ); + const failedTerminalIndex = persistence.agentDb.findIndex( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.status === 'failed' && + record.requestId === waitingBoundary.failedStarted.requestId, + ); + const retryAuditIndex = persistence.agentDb.findIndex( + (record) => + record.recordType === 'agent.runtime.provider_request.retry' && + record.requestId === waitingBoundary.failedStarted.requestId, + ); + const retryStartedIndex = persistence.agentDb.indexOf(retryStarted[0]); + assertSupervisorSwarmTransientRetrySideEffectsZero( + sideEffects, + 'supervisor-swarm-transient-retry-pre-forward-side-effect', + ); + assert( + lifecycle.length === 3 && + retries.length === 1 && + retryStarted[0].requestId !== waitingBoundary.failedStarted.requestId && + retryStarted[0].requestSlot === + waitingBoundary.retryAudit.nextRequestSlot && + stableIdentityFields.every( + (field) => + retryStarted[0][field] === waitingBoundary.failedStarted[field], + ) && + failedStartedIndex >= 0 && + failedStartedIndex < failedTerminalIndex && + failedTerminalIndex < retryAuditIndex && + retryAuditIndex < retryStartedIndex && proxyStats.requestCount === 2 && proxyStats.faultInjectedCount === 1 && proxyStats.heldRequestCount === 1 && proxyStats.forwardedRequestCount === 0 && proxyStats.forwardingReleased === false, - 'supervisor-swarm-transient-retry-pre-forward-side-effect', + 'supervisor-swarm-transient-retry-held-attempt-invalid', ); state.supervisorSwarm.transientFaultCheckpoint = { - failedRequestId: failedStarted.requestId, - retryRequestId: retryStarted.requestId, - failedRequestSlot: failedStarted.requestSlot, - retryRequestSlot: retryStarted.requestSlot, - agentId: failedStarted.agentId, - taskId: failedStarted.taskId, - sessionId: failedStarted.sessionId, - runId: failedStarted.runId, - source: failedStarted.source, - requestKind: failedStarted.requestKind, - actionCount: actionRecords.length, - receiptCount: receipts.length, - childDeliveryCount: childDeliveries.length, - claimCount: claims.length, - assistantCount, - pendingCount, - projectRevision: revision.revision, + failedRequestId: waitingBoundary.failedStarted.requestId, + retryRequestId: retryStarted[0].requestId, + failedRequestSlot: waitingBoundary.failedStarted.requestSlot, + retryRequestSlot: retryStarted[0].requestSlot, + agentId: waitingBoundary.failedStarted.agentId, + taskId: waitingBoundary.failedStarted.taskId, + sessionId: waitingBoundary.failedStarted.sessionId, + runId: waitingBoundary.failedStarted.runId, + source: waitingBoundary.failedStarted.source, + requestKind: waitingBoundary.failedStarted.requestKind, + ...sideEffects, upstreamRequestCountBeforeRelease: proxyStats.forwardedRequestCount, forwardingReleased: false, }; @@ -10434,7 +10858,7 @@ async function captureSupervisorSwarmTransientRetryCheckpoint() { state.supervisorSwarm.transientFaultCheckpoint.forwardingReleased = true; return; } - throw codedError('supervisor-swarm-transient-retry-checkpoint-timeout'); + throw codedError('supervisor-swarm-transient-retry-held-attempt-timeout'); } async function runSupervisorSwarmE2e() { @@ -10495,10 +10919,10 @@ async function runSupervisorSwarmE2e() { 'supervisor-swarm-parent-runtime-identity-invalid', ); + await captureSupervisorSwarmTransientRetryCheckpoint(); await captureSupervisorSwarmInitialProviderBatch(); await captureSupervisorSwarmCollaborationPolicySnapshotAndDrift(); await restartSupervisorSwarmRunnerAtInitialBatchBoundary(); - await captureSupervisorSwarmTransientRetryCheckpoint(); await driveSupervisorSwarmToRepairKillBoundary(); await driveSupervisorSwarmRuntimeToCompletion(); if (isSupervisorSwarmInteractiveChatSuite()) { @@ -10743,8 +11167,18 @@ function validateSupervisorSwarmProviderLifecycle( let forcedTransientRetryVerified = false; if (isSupervisorSwarmTransientRetrySuite()) { const checkpoint = state.supervisorSwarm.transientFaultCheckpoint; - const failedTerminal = failed[0]; - const retryAudit = retries[0]; + const injectedFailed = checkpoint + ? failed.filter( + (record) => record.requestId === checkpoint.failedRequestId, + ) + : []; + const injectedRetries = checkpoint + ? retries.filter( + (record) => record.requestId === checkpoint.failedRequestId, + ) + : []; + const failedTerminal = injectedFailed[0]; + const retryAudit = injectedRetries[0]; const retryGroup = checkpoint ? byRequest.get(checkpoint.retryRequestId) : undefined; @@ -10764,11 +11198,23 @@ function validateSupervisorSwarmProviderLifecycle( record.runId === checkpoint.runId, ) : []; + const targetFailed = checkpoint + ? lifecycle.filter( + (record) => + record.status === 'failed' && + record.agentId === checkpoint.agentId && + record.runId === checkpoint.runId, + ) + : []; const proxyStats = state.supervisorSwarm.transientFaultProxy?.getStats(); + state.supervisorSwarm.transientRetryIncidentalProviderFailureCount = + Math.max(0, failed.length - injectedFailed.length); + state.supervisorSwarm.transientRetryIncidentalProviderRetryCount = + Math.max(0, retries.length - injectedRetries.length); assert( checkpoint && - failed.length === 1 && - retries.length === 1 && + injectedFailed.length === 1 && + injectedRetries.length === 1 && failedTerminal.requestId === checkpoint.failedRequestId && retryAudit.requestId === checkpoint.failedRequestId && retryAudit.nextRequestSlot === checkpoint.retryRequestSlot && @@ -10788,7 +11234,10 @@ function validateSupervisorSwarmProviderLifecycle( ].every((field) => retryGroup[0][field] === checkpoint[field]) && proxyStats?.requestCount === targetStarted.length && proxyStats.faultInjectedCount === 1 && - proxyStats.forwardedRequestCount === targetCompleted.length, + proxyStats.forwardedRequestCount === targetStarted.length - 1 && + targetCompleted.length + targetFailed.length === targetStarted.length && + state.supervisorSwarm.transientRetryIncidentalProviderFailureCount === + state.supervisorSwarm.transientRetryIncidentalProviderRetryCount, 'supervisor-swarm-forced-transient-retry-invalid', ); forcedTransientRetryVerified = true; @@ -11786,19 +12235,21 @@ async function readSupervisorSwarmResidualSidecarCounts() { parallelReadBatches: '.agent/runtime/parallel-read-batches', pendingActions: '.agent/runtime/pending-actions', providerActionBatches: '.agent/runtime/provider-action-batches', + providerRetries: '.agent/runtime/provider-retries', userInput: '.agent/runtime/user-input', }; const counts = {}; for (const [name, relative] of Object.entries(roots)) { - counts[name] = ( - await listFiles(path.join(state.projectRoot, relative)) - ).filter((file) => file.endsWith('.json')).length; + const files = await listFiles(path.join(state.projectRoot, relative)); + counts[name] = + name === 'providerRetries' + ? files.length + : files.filter((file) => file.endsWith('.json')).length; } const snapshotFiles = await listFiles( supervisorSwarmCollaborationPolicySnapshotDirectory(), ); - const expectedSnapshotPath = - supervisorSwarmInitialBatchBindsPolicySnapshot() + const expectedSnapshotPath = supervisorSwarmInitialBatchBindsPolicySnapshot() ? supervisorSwarmCollaborationPolicySnapshotPath() : null; counts.collaborationPolicySnapshotArtifacts = snapshotFiles.filter( @@ -11807,8 +12258,7 @@ async function readSupervisorSwarmResidualSidecarCounts() { const bindingFiles = await listFiles( supervisorSwarmCollaborationPolicySnapshotBindingDirectory(), ); - const expectedBindingPath = - supervisorSwarmInitialBatchBindsPolicySnapshot() + const expectedBindingPath = supervisorSwarmInitialBatchBindsPolicySnapshot() ? supervisorSwarmCollaborationPolicySnapshotBindingPath() : null; counts.collaborationPolicySnapshotBindingArtifacts = bindingFiles.filter( @@ -11857,11 +12307,52 @@ function supervisorSwarmTransientRetrySnapshotEvidence( }; return { transientFaultModeEnabled: enabled, - transientFaultTargetAgentId: enabled ? supervisorSwarmDesignAgentId : '', + transientFaultTargetAgentId: enabled + ? supervisorSwarmTransientRetryTargetAgentId + : '', transientFaultRequestCount: stats.requestCount, transientFaultInjectedCount: stats.faultInjectedCount, transientFaultHeldRequestCount: stats.heldRequestCount, transientFaultForwardedRequestCount: stats.forwardedRequestCount, + transientRetryBackoffMs: enabled + ? supervisorSwarmTransientRetryBackoffMs + : 0, + transientRetryWaitingBoundaryCaptured: Boolean( + state.supervisorSwarm.transientRetryPreKillSidecar, + ), + transientRetryRunnerKilledDuringBackoff: Boolean( + state.supervisorSwarm.runnerKillBoundaryObserved && + state.supervisorSwarm.transientRetryPreKillRunnerBootId, + ), + transientRetryRunnerBootChanged: Boolean( + state.supervisorSwarm.transientRetryPreKillRunnerBootId && + state.supervisorSwarm.transientRetryPostRestartRunnerBootId && + state.supervisorSwarm.transientRetryPreKillRunnerBootId !== + state.supervisorSwarm.transientRetryPostRestartRunnerBootId, + ), + transientRetryRecoveredFromPreviousBoot: + state.supervisorSwarm.transientRetryRecoveredFromPreviousBoot, + transientRetryRequestCountBeforeKill: + state.supervisorSwarm.transientRetryRequestCountBeforeKill, + transientRetryRequestCountAfterRestart: + state.supervisorSwarm.transientRetryRequestCountAfterRestart, + transientRetryRequestCountBeforeDue: + state.supervisorSwarm.transientRetryRequestCountBeforeDue, + transientRetryEarlyRequestCount: + state.supervisorSwarm.transientRetryEarlyRequestCount, + transientRetrySidecarIdentityStable: + state.supervisorSwarm.transientRetrySidecarIdentityStable, + transientRetryAttemptStable: + state.supervisorSwarm.transientRetryAttemptStable, + transientRetryAtStable: state.supervisorSwarm.transientRetryAtStable, + transientRetryRuntimeIdentityStable: + state.supervisorSwarm.transientRetryRuntimeIdentityStable, + transientRetryAcceptedAtOrAfterDue: + state.supervisorSwarm.transientRetryAcceptedAtOrAfterDue, + transientRetryIncidentalProviderFailureCount: + state.supervisorSwarm.transientRetryIncidentalProviderFailureCount, + transientRetryIncidentalProviderRetryCount: + state.supervisorSwarm.transientRetryIncidentalProviderRetryCount, transientFaultPreRetryCheckpointCaptured: Boolean(checkpoint), transientFaultPreRetryActionCount: checkpoint?.actionCount ?? 0, transientFaultPreRetryReceiptCount: checkpoint?.receiptCount ?? 0, @@ -11874,7 +12365,10 @@ function supervisorSwarmTransientRetrySnapshotEvidence( transientFaultUpstreamRequestCountBeforeRelease: checkpoint?.upstreamRequestCountBeforeRelease ?? 0, transientFaultRequestIdentityChanged: Boolean( - checkpoint && checkpoint.failedRequestId !== checkpoint.retryRequestId, + checkpoint && + isNonEmptyString(checkpoint.failedRequestId) && + isNonEmptyString(checkpoint.retryRequestId) && + checkpoint.failedRequestId !== checkpoint.retryRequestId, ), transientFaultRetrySlotValid: Boolean( checkpoint && @@ -11911,6 +12405,23 @@ function supervisorSwarmTransientRetryEvidence(provider) { evidence.transientFaultInjectedCount === 1 && evidence.transientFaultHeldRequestCount === 1 && evidence.transientFaultForwardedRequestCount >= 1 && + evidence.transientRetryBackoffMs === + supervisorSwarmTransientRetryBackoffMs && + evidence.transientRetryWaitingBoundaryCaptured && + evidence.transientRetryRunnerKilledDuringBackoff && + evidence.transientRetryRunnerBootChanged && + evidence.transientRetryRecoveredFromPreviousBoot && + evidence.transientRetryRequestCountBeforeKill === 1 && + evidence.transientRetryRequestCountAfterRestart === 1 && + evidence.transientRetryRequestCountBeforeDue === 1 && + evidence.transientRetryEarlyRequestCount === 0 && + evidence.transientRetrySidecarIdentityStable && + evidence.transientRetryAttemptStable && + evidence.transientRetryAtStable && + evidence.transientRetryRuntimeIdentityStable && + evidence.transientRetryAcceptedAtOrAfterDue && + evidence.transientRetryIncidentalProviderFailureCount === + evidence.transientRetryIncidentalProviderRetryCount && evidence.transientFaultPreRetryCheckpointCaptured && evidence.transientFaultPreRetryActionCount === 0 && evidence.transientFaultPreRetryReceiptCount === 0 && @@ -12066,8 +12577,7 @@ async function validateSupervisorSwarmEvidence() { state.supervisorSwarm.collaborationPolicySnapshotInitialBytesSha256; collaborationPolicySnapshotBindingBytesStable = isNonEmptyString( - state.supervisorSwarm - .collaborationPolicySnapshotBindingInitialBytes, + state.supervisorSwarm.collaborationPolicySnapshotBindingInitialBytes, ) && finalBindingFile.bytes === state.supervisorSwarm @@ -12107,7 +12617,10 @@ async function validateSupervisorSwarmEvidence() { } } let finalCollaborationPolicySidecar = null; - if (isSupervisorSwarmMixedHarnessSuite()) { + if ( + isSupervisorSwarmMixedHarnessSuite() || + isSupervisorSwarmTransientRetrySuite() + ) { finalCollaborationPolicySidecar = await readJson( supervisorSwarmCollaborationPolicyPath(), ); @@ -12925,6 +13438,24 @@ async function validateSupervisorSwarmEvidence() { initialProviderBatchCaptured: true, initialProviderBatchSchemaVersion: initialBatch.schemaVersion, initialProviderBatchActionCount: initialBatch.actionIds.length, + initialProviderBatchObservedStatus: initialBatch.status, + initialProviderBatchObservedPlanActionCount: + initialBatch.actionIds.length, + initialProviderBatchObservedDelegateActionCount: + initialBatch.actions.filter( + (action) => action.tool === 'agent.delegate', + ).length, + initialProviderBatchObservedSpawnIsolatedActionCount: + initialBatch.actions.filter( + (action) => action.tool === 'agent.spawn_isolated', + ).length, + initialProviderBatchObservedOtherActionCount: initialBatch.actions.filter( + (action) => + action.tool !== 'agent.delegate' && + action.tool !== 'agent.spawn_isolated', + ).length, + initialProviderBatchObservedCollaborationContractPresent: true, + initialProviderBatchObservedNextActionIndex: initialBatch.nextActionIndex, initialProviderBatchCompletedEventCount: batchEvents.length, initialCollaborationContractPresent: Boolean(collaborationContract), initialCollaborationContractSchemaVersion: @@ -13298,6 +13829,7 @@ async function validateSupervisorSwarmEvidence() { targetedContractReadCount: actions.targetedContractReadCount, pendingActionCount: residualSidecars.pendingActions, providerActionBatchSidecarCount: residualSidecars.providerActionBatches, + providerRetrySidecarCount: residualSidecars.providerRetries, parallelReadBatchSidecarCount: residualSidecars.parallelReadBatches, finalizationJournalCount: residualSidecars.finalizations, confirmationSidecarCount: residualSidecars.confirmations, @@ -13681,6 +14213,17 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { gitSensitivePath, ].includes(changedPath), ); + const observedInitialBatch = await readJson( + supervisorSwarmInitialProviderBatchPath(), + ).catch(() => null); + const observedInitialBatchActions = Array.isArray( + observedInitialBatch?.actions, + ) + ? observedInitialBatch.actions + : []; + const observedInitialBatchTools = observedInitialBatchActions.map( + (pendingAction) => pendingAction?.action?.tool, + ); const partialInitialBatch = state.supervisorSwarm.initialProviderBatch; const partialCollaborationContract = partialInitialBatch?.collaborationContract; @@ -13854,9 +14397,37 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { state.supervisorSwarm.initialProviderBatch, ), initialProviderBatchSchemaVersion: - partialInitialBatch?.schemaVersion ?? null, + partialInitialBatch?.schemaVersion ?? + observedInitialBatch?.schemaVersion ?? + null, initialProviderBatchActionCount: - state.supervisorSwarm.initialProviderBatch?.actionIds?.length ?? 0, + state.supervisorSwarm.initialProviderBatch?.actionIds?.length ?? + observedInitialBatchActions.length, + initialProviderBatchObservedStatus: observedInitialBatch?.status ?? null, + initialProviderBatchObservedPlanActionCount: Array.isArray( + observedInitialBatch?.plan?.actions, + ) + ? observedInitialBatch.plan.actions.length + : 0, + initialProviderBatchObservedDelegateActionCount: + observedInitialBatchTools.filter((tool) => tool === 'agent.delegate') + .length, + initialProviderBatchObservedSpawnIsolatedActionCount: + observedInitialBatchTools.filter( + (tool) => tool === 'agent.spawn_isolated', + ).length, + initialProviderBatchObservedOtherActionCount: + observedInitialBatchTools.filter( + (tool) => tool !== 'agent.delegate' && tool !== 'agent.spawn_isolated', + ).length, + initialProviderBatchObservedCollaborationContractPresent: isPlainObject( + observedInitialBatch?.collaborationContract, + ), + initialProviderBatchObservedNextActionIndex: Number.isSafeInteger( + observedInitialBatch?.nextActionIndex, + ) + ? observedInitialBatch.nextActionIndex + : null, initialProviderBatchCompletedEventCount: persistence.events.filter( (event) => event.agentId === projectSupervisorAgentId && @@ -14328,6 +14899,7 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { legacyConversationMessageCount: persistence.legacyConversation.length, pendingActionCount: pending.length, providerActionBatchSidecarCount: residualSidecars.providerActionBatches, + providerRetrySidecarCount: residualSidecars.providerRetries, parallelReadBatchSidecarCount: residualSidecars.parallelReadBatches, finalizationJournalCount: residualSidecars.finalizations, confirmationSidecarCount: residualSidecars.confirmations, @@ -14349,6 +14921,7 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { '.agent/runtime/collaboration-policy-snapshots', '.agent/runtime/collaboration-policy-snapshot-bindings', '.agent/runtime/provider-action-batches', + '.agent/runtime/provider-retries', '.agent/runtime/tasks', '.agent/runtime/events', '.agent/agent.db', @@ -15295,7 +15868,7 @@ async function prepareIsolatedSuiteAppData({ effective.maxRetries >= (isSupervisorSwarmInteractiveChatSuite() || (isSupervisorSwarmTransientRetrySuite() && - agentId !== supervisorSwarmDesignAgentId) + agentId !== supervisorSwarmTransientRetryTargetAgentId) ? 0 : 1) && effective.maxRetries <= 3 && @@ -26608,6 +27181,13 @@ function supervisorSwarmEvidenceFieldTemplate() { initialProviderBatchCaptured: false, initialProviderBatchSchemaVersion: null, initialProviderBatchActionCount: 0, + initialProviderBatchObservedStatus: null, + initialProviderBatchObservedPlanActionCount: 0, + initialProviderBatchObservedDelegateActionCount: 0, + initialProviderBatchObservedSpawnIsolatedActionCount: 0, + initialProviderBatchObservedOtherActionCount: 0, + initialProviderBatchObservedCollaborationContractPresent: false, + initialProviderBatchObservedNextActionIndex: null, initialProviderBatchCompletedEventCount: 0, initialCollaborationContractPresent: false, initialCollaborationContractSchemaVersion: null, @@ -26781,6 +27361,22 @@ function supervisorSwarmEvidenceFieldTemplate() { transientFaultInjectedCount: 0, transientFaultHeldRequestCount: 0, transientFaultForwardedRequestCount: 0, + transientRetryBackoffMs: 0, + transientRetryWaitingBoundaryCaptured: false, + transientRetryRunnerKilledDuringBackoff: false, + transientRetryRunnerBootChanged: false, + transientRetryRecoveredFromPreviousBoot: false, + transientRetryRequestCountBeforeKill: 0, + transientRetryRequestCountAfterRestart: 0, + transientRetryRequestCountBeforeDue: 0, + transientRetryEarlyRequestCount: 0, + transientRetrySidecarIdentityStable: false, + transientRetryAttemptStable: false, + transientRetryAtStable: false, + transientRetryRuntimeIdentityStable: false, + transientRetryAcceptedAtOrAfterDue: false, + transientRetryIncidentalProviderFailureCount: 0, + transientRetryIncidentalProviderRetryCount: 0, transientFaultPreRetryCheckpointCaptured: false, transientFaultPreRetryActionCount: 0, transientFaultPreRetryReceiptCount: 0, @@ -26831,6 +27427,7 @@ function supervisorSwarmEvidenceFieldTemplate() { targetedContractReadCount: 0, pendingActionCount: 0, providerActionBatchSidecarCount: 0, + providerRetrySidecarCount: 0, parallelReadBatchSidecarCount: 0, finalizationJournalCount: 0, confirmationSidecarCount: 0, @@ -30773,12 +31370,10 @@ function runAgentRuntimeRealE2eSelfTests() { syntheticRecoveredRepairSessionId, syntheticRecoveredRepairIdentity.runId, ); - const syntheticModernRecoveredRepairRecords = - syntheticRecoveredRepairRecords( + const syntheticModernRecoveredRepairRecords = syntheticRecoveredRepairRecords( 'agent.runtime.provider_action_batch.confirmation_required', ); - const syntheticLegacyRecoveredRepairRecords = - syntheticRecoveredRepairRecords( + const syntheticLegacyRecoveredRepairRecords = syntheticRecoveredRepairRecords( 'agent.runtime.tool_confirmation_required', ); const syntheticModernRecoveredRepairLifecycle = @@ -31191,8 +31786,7 @@ function runAgentRuntimeRealE2eSelfTests() { /^[0-9a-f]{64}$/u.test(syntheticEnospcDiagnostic.stderrSha256) && syntheticChatSessionFailureEvidence.chatSessionUnexpectedlyClosed === true && - syntheticChatSessionFailureEvidence.chatSessionFailureKind === - 'enospc', + syntheticChatSessionFailureEvidence.chatSessionFailureKind === 'enospc', 'agent-runtime-real-e2e-self-test-collaboration-policy-snapshot-binding-invalid', ); assert( diff --git a/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs index 9c53b4265..f538def54 100644 --- a/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs +++ b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs @@ -265,6 +265,7 @@ export async function startLlmTransientFaultProxy( const upstreamSockets = new Set(); const heldForwardingWaiters = new Set(); const counterWaiters = new Set(); + const requestLog = []; let requestCount = 0; let faultInjectedCount = 0; @@ -284,6 +285,8 @@ export async function startLlmTransientFaultProxy( forwardingReleased, stopped, }); + const requestLogSnapshot = () => + Object.freeze(requestLog.map((entry) => Object.freeze({ ...entry }))); const notifyCounterWaiters = (kind) => { for (const waiter of [...counterWaiters]) { @@ -356,7 +359,8 @@ export async function startLlmTransientFaultProxy( failClosed(request, response, 502); }; - const forwardRequest = (request, response) => { + const forwardRequest = (request, response, requestMetadata) => { + requestMetadata.forwardingStartedAtMs = Date.now(); const transport = options.upstream.protocol === 'https:' ? https : http; let upstreamRequest; try { @@ -439,6 +443,14 @@ export async function startLlmTransientFaultProxy( const handleRequest = async (request, response, expectsContinue) => { requestCount += 1; + const requestMetadata = { + sequence: requestCount, + acceptedAtMs: Date.now(), + faultInjectedAtMs: null, + heldAtMs: null, + forwardingStartedAtMs: null, + }; + requestLog.push(requestMetadata); request.on('error', () => {}); response.on('error', () => {}); @@ -460,6 +472,7 @@ export async function startLlmTransientFaultProxy( !forwardingReleased ) { heldRequestCount += 1; + requestMetadata.heldAtMs = Date.now(); notifyCounterWaiters('held'); const shouldForward = await waitForForwardingRelease(request, response); if (!shouldForward) { @@ -470,6 +483,7 @@ export async function startLlmTransientFaultProxy( if (faultInjectedCount < options.faultCount) { faultInjectedCount += 1; + requestMetadata.faultInjectedAtMs = Date.now(); notifyCounterWaiters('fault'); resetSocket(request.socket); return; @@ -480,7 +494,7 @@ export async function startLlmTransientFaultProxy( return; } if (expectsContinue) response.writeContinue(); - forwardRequest(request, response); + forwardRequest(request, response, requestMetadata); }; const dispatchRequest = (request, response, expectsContinue = false) => { @@ -568,6 +582,7 @@ export async function startLlmTransientFaultProxy( return stats(); }, getStats: stats, + getRequestLog: requestLogSnapshot, waitForFault(timeoutMs = 5_000) { return waitForCounter('fault', timeoutMs); }, diff --git a/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts index c2b05fa67..79a0c33b9 100644 --- a/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts +++ b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts @@ -27,6 +27,13 @@ interface ProxyHandle { port: number; stats: ProxyStats; getStats(): ProxyStats; + getRequestLog(): ReadonlyArray<{ + sequence: number; + acceptedAtMs: number; + faultInjectedAtMs: number | null; + heldAtMs: number | null; + forwardingStartedAtMs: number | null; + }>; waitForFault(timeoutMs?: number): Promise; waitForHeldRequest(timeoutMs?: number): Promise; releaseForwarding(): ProxyStats; @@ -341,6 +348,30 @@ describe('LLM transient fault proxy', () => { heldRequestCount: 1, forwardedRequestCount: 0, }); + const heldLog = proxy.getRequestLog(); + expect(heldLog).toEqual([ + { + sequence: 1, + acceptedAtMs: expect.any(Number), + faultInjectedAtMs: expect.any(Number), + heldAtMs: null, + forwardingStartedAtMs: null, + }, + { + sequence: 2, + acceptedAtMs: expect.any(Number), + faultInjectedAtMs: null, + heldAtMs: expect.any(Number), + forwardingStartedAtMs: null, + }, + ]); + expect(heldLog[1].acceptedAtMs).toBeGreaterThanOrEqual( + heldLog[0].acceptedAtMs, + ); + expect(heldLog[1].heldAtMs).toBeGreaterThanOrEqual(heldLog[1].acceptedAtMs); + expect(JSON.stringify(heldLog)).not.toMatch( + /held-secret|request-body|responses|authorization/iu, + ); expect(captured).toHaveLength(0); expect(proxy.releaseForwarding()).toMatchObject({ forwardingReleased: true, @@ -363,6 +394,9 @@ describe('LLM transient fault proxy', () => { }), ]); expect(proxy.stats.forwardedRequestCount).toBe(1); + expect( + proxy.getRequestLog()[1].forwardingStartedAtMs, + ).toBeGreaterThanOrEqual(heldLog[1].heldAtMs); }); it('holds before injecting a remaining configured fault', async () => { diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 9b0db8e8e..76a9f5d09 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -4901,3 +4901,4 @@ - 调度：等待态释放执行 lane，允许不同 Agent 并行；同 Agent 当前 running 等待任务继续阻挡后续 pending task，保持 FIFO。活跃 sidecar 阻断 finalization 与 `shutdown_if_idle`，cancel、steer、终态和耗尽负责清理。 - 稳定身份：Provider 请求指纹不得包含工具策略 `updatedAt` 等非语义刷新时间。Goal pause 保留 sidecar，resume 恢复原等待态；跨秒恢复必须仍命中相同 request fingerprint 和 `-transient-N` slot。final-reply、手动压缩与 tool-plan `repair-N` 继续使用进程内重试，待具备可无歧义重建的持久请求上下文后再单独升级。 - 证据边界：`provider_retry_` 19/19、`provider_transient_retry_` 6/6，Tauri/Rust 串行全量 968 passed/4 ignored，`cargo check`、rustfmt、编码与 diff 检查通过。本轮确定性验证与 V1.38 真实 Provider E2E 分开记账；任何旧失败轮、旧瞬态重试 PASS 或最小探针都不能拼接成 V1.39 真实 PASS。 +- 真实验收：最终代码的独立 `gpt-5.5 / openai_chat / high` suite 以 Project Supervisor 首次 tool-plan 为受控故障目标，在子 Agent 请求产生前进入 `30s` 持久退避并执行一次 pidfd Runner 强杀。新 boot 恢复后 sidecar identity/字节/attempt/slot/retryAt 不变，强杀前、重启后、到期前请求数均为 1，metadata-only 代理证明第二个请求网络接收时间不早于 retryAt；随后同一父 run 由 static collaboration policy 强制同批两个指定专业 Agent，完整完成真重叠、2+1 delivery、唯一 repair、宿主验证和唯一最终回复。最终 `37 started / 37 terminal / 36 completed / 1 injected failed / 1 retry`，incidental failure/retry 均为 0；重复、残留 sidecar、正文、Key、项目/正式配置路径泄漏均为 0，隔离现场完整清理，单轮耗时 `891.1s`。验收器按 request identity 分开统计受控注入与额外真实瞬态失败，额外失败仍须逐条通过原 lifecycle/retry/后继终态门禁且 failure/retry 计数相等；此前各失败样本不得与该 PASS 拼接。 diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index a38d24dd7..2b63491b8 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -99,10 +99,14 @@ npm run ai-game-creator-shell:typecheck ```bash npm run test -- apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts npm run ai-game-creator-shell:typecheck +cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml provider_retry_ -- --nocapture --test-threads=1 +cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml provider_transient_retry_ -- --nocapture --test-threads=1 npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e -- --config-dir ``` -真实 PASS 必须恰好包含 1 个 failed lifecycle、1 条 retry audit 和 1 个 `-transient-1` 后继 identity；forwarding gate 放行前 action、receipt、目标 Agent 子委派、claim、assistant、pending、project revision 与 upstream forwarding 全为 0。放行后仍须完成双专业 Agent 重叠、唯一 repair、Runner 强杀恢复、唯一 Supervisor assistant 和零重复/残留/泄漏。suite 只能读正式 AppData，在其同级目录写入 sentinel 管理的 `0600` 私有副本和 overlay；启动 CLI/Runner 时须把 loopback 合并进大小写两套 no-proxy 环境，防止系统 HTTP 代理绕过本地故障门禁；source-dir guard 必须证明本 suite 前缀未进入源目录，源配置和 endpoint 身份保持不变，报告不得保存 Provider URL、headers、正文、凭据或绝对配置路径。若后续 repair/恢复/终局失败，partial report 仍应保留已经取得的 retry checkpoint。 +V1.39 真实门禁把一次故障注入到 Project Supervisor 的首次 tool-plan，使退避期强杀发生在子 Agent Provider 请求产生之前；恢复后仍须在同一父 Session/run 完成双专业 Agent 真重叠与唯一 repair。suite 使用 `30s` 退避，必须先观察 sidecar 已落盘且 task/state 均为 `running / waiting-for-provider-retry`，再强杀 Runner；新 boot 接管后 sidecar identity、字节、attempt、slot 与 `retryAt` 不变，重启后和到期前代理请求数都只能为 1。代理的 metadata-only 请求日志只允许保存序号与毫秒时间，第二个请求的 `acceptedAtMs` 必须不早于 `retryAtMs`，且不得包含 URL、method、headers 或正文。forwarding gate 放行前 action、receipt、子委派、claim、assistant、pending、project revision 与 upstream forwarding 全为 0；受控 request identity 必须恰好包含 1 个 failed lifecycle、1 条 retry audit 和 1 个 `-transient-1` 后继 identity。其它真实瞬态失败按 incidental failure/retry 分开计数，每条仍须通过既有 lifecycle、retry audit、唯一后继和终局门禁且两类计数相等；不能把它们混入受控注入链，也不能跳过唯一 Supervisor assistant 和零重复/残留/泄漏要求。首批双专业 Agent 使用正式 collaboration policy 固定为同批两个指定 static delegate，不能只靠提示碰运气；该 suite 已在 Provider 退避边界完成唯一一次 Runner 强杀，后续 repair 只验证持久 pending/确认链，不重复制造第二个 kill 边界。 + +suite 只能读正式 AppData，在其同级目录写入 sentinel 管理的 `0600` 私有副本和 overlay；启动 CLI/Runner 时须把 loopback 合并进大小写两套 no-proxy 环境，防止系统 HTTP 代理绕过本地故障门禁；source-dir guard 必须证明本 suite 前缀未进入源目录，源配置和 endpoint 身份保持不变，报告不得保存 Provider URL、headers、正文、凭据或绝对配置路径。sidecar 先于 task/state 投影是合法提交窗口，验收器应等待完整等待态后再强杀；若后续协作或终局失败，partial report 仍应保留已取得的 retry checkpoint，但失败轮不得与后续成功轮拼接。 ### AI 游戏创作自主 Swarm 终端复验 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index 97de300a4..8c1bb5e7d 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3363,5 +3363,6 @@ - 现象：Provider 首次规划请求发生 timeout/connectivity/transport 后，Runner 在 backoff 期间退出会丢失 retry，或重启后清零 attempt、提前补发；另一种偶发现象是 Goal pause/resume 看似保留 sidecar，但只要等待跨过一秒，恢复请求就被判为 context drift，旧 `-transient-N` attempt 被删除并改成新 loop 请求。 - 原因：退避 attempt 和到期时间只存在于进程内；或虽然已有 sidecar，请求 prompt 却直接序列化 Runtime 工具策略快照，把每次刷新都会变化的 `updatedAt` 带进 request fingerprint。同一权限内容因此仅因时间变化产生不同请求身份。 - 处理：先闭合物理请求 lifecycle，再原子持久 retry sidecar/.previous，最后投影 `waiting-for-provider-retry` 并释放 lane；Runner 启动扫描并按绝对到期时间恢复。请求指纹只绑定实际 Provider 请求的稳定语义，UI/审计时间戳、剩余等待毫秒和等待态文案不得进入 prompt。Goal pause 保留 sidecar，resume 必须校验同一 Goal/steer/request/config 身份后恢复原 attempt。 -- 验证：测试必须故意让 pause/resume 跨秒，捕获失败请求与恢复请求的 HTTP body 并比较 SHA-256，同时断言 lifecycle 使用原 `-transient-N` slot而不是新 loop；另覆盖 `.previous` 扫描、Runner idle blocker、同 Agent FIFO、跨 Agent 并行、cancel/steer/耗尽清理和重启未到期零请求。 +- 验证：测试必须故意让 pause/resume 跨秒，捕获失败请求与恢复请求的 HTTP body 并比较 SHA-256，同时断言 lifecycle 使用原 `-transient-N` slot而不是新 loop；另覆盖 `.previous` 扫描、Runner idle blocker、同 Agent FIFO、跨 Agent 并行、cancel/steer/耗尽清理和重启未到期零请求。真实网络门禁不能只在 `retryAt` 前留一个静默窗口后等待请求，代理还要用不含 URL/header/body 的毫秒 metadata 证明第二个请求 `acceptedAtMs >= retryAtMs`。 +- 真实验收陷阱：sidecar 按设计早于 task/state 等待投影落盘，验收器看到 sidecar 后必须继续等完整 `running / waiting-for-provider-retry`，不能把合法提交窗口误判为 torn projection。共享 Runner 强杀会同时中断其它 Agent 的 in-flight Provider 请求；要隔离验证单个持久 retry，应在子请求产生前对父 Agent 首次规划注入故障，恢复后再完成同一 run 的并行协作。首批同批双委派若只依赖自然语言提示会受模型波动影响，真实 suite 应使用正式 collaboration policy/preflight 固定两个指定 static Agent，并保留无正文的 batch 数量诊断。长链路还可能发生额外真实瞬态失败，不能用“全局 failed/retry 必须等于 1”把已正确恢复的网络抖动误判为注入失败；应按 request identity 锁定唯一受控链，额外 failure/retry 独立计数并继续执行全部 lifecycle、后继终态和零残留门禁。 - 关联：`apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs`、`agent.rs`、`runner.rs`、`tests.rs`、`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`。 diff --git a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md index b4fa33fce..0a440530e 100644 --- a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md +++ b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md @@ -1369,9 +1369,11 @@ V1.39 把 V1.28 的显式瞬态重试从单纯进程内退避扩展为可跨 Run 确定性回归必须覆盖：未到期零请求、到期唯一请求、耗尽清理、cancel、steer、Goal pause/resume 跨秒保持正文与 attempt、自动 context-compaction 先恢复再进入 tool-plan、sidecar 先于 task/state 的 torn projection、Runner 重启扫描、同 Agent FIFO、跨 Agent 并行、finalization/idle blocker、`.previous` 恢复/去重/排序和危险路径身份。原有 final-reply/repair 进程内重试回归必须继续通过。 -V1.39 当前只完成确定性实现与回归，不把 V1.38 的失败轮、旧瞬态重试真实样本或最小 HTTP 探针拼接成新的真实 Provider PASS。需要真实验收时必须在最终代码上以独立 disposable 项目和隔离 AppData 完整运行，并证明 Runner 在退避期强杀后仍只发送同一 `-transient-N` attempt、其它 Agent 真并行、同 Agent FIFO、唯一最终回复、零重复/残留/正文/密钥/路径泄漏。 +V1.39 的真实 suite 把受控故障放在 Project Supervisor 首次 tool-plan，此时尚无子 Agent in-flight Provider 请求；否则强杀共享 Runner 会让无可信终态的旁路请求按既有 orphan 规则进入失败关闭，不能用它证明单一 retry sidecar 的恢复。隔离 collaboration policy 要求首批同批产生 `design-director + quality-review` 两个 static delegate，单委派由正式 preflight/协议修复拒绝，不能只依赖提示。故障代理使用 `30s` 退避与 metadata-only 时间日志：只记录请求序号及 `acceptedAtMs / faultInjectedAtMs / heldAtMs / forwardingStartedAtMs`，不记录 URL、method、headers、正文或凭据；退避期强杀后，sidecar 完整身份、字节、attempt、slot 和 `retryAt` 必须保持不变，重启后与到期前请求数均为 1，第二个请求的网络接收时间不得早于 `retryAt`。 -2026-07-19 最终代码的确定性门禁已通过：`provider_retry_` 19/19、`provider_transient_retry_` 6/6；Tauri/Rust 串行全量为 968 passed、4 个环境依赖用例按设计 ignored，`cargo check`、rustfmt、编码与 diff 检查通过。以上结果不替代尚未执行的 V1.39 独立真实 Provider E2E。 +2026-07-19 最终代码的确定性门禁已通过：`provider_retry_` 19/19、`provider_transient_retry_` 6/6；Tauri/Rust 串行全量为 968 passed、4 个环境依赖用例按设计 ignored，`cargo check`、rustfmt、编码与 diff 检查通过。 + +同日最终代码的独立 `openai_chat / gpt-5.5 / high` 真实 Provider suite **PASS**，最终样本单轮耗时 `891.1s`。37 个 request identity 全部形成唯一终态：`37 started / 37 terminal / 36 completed / 1 injected failed / 1 retry`，incidental Provider failure/retry 均为 0；目标父 Agent 的首次请求在正文转发前断线，retry sidecar 与 task/state 完整进入等待态后执行一次 pidfd Runner 强杀，新 boot 从旧 boot 恢复同一父 Session/run/loop/attempt。强杀前、重启后、到期前代理请求数均为 1，early request 为 0，第二个请求 `acceptedAtMs >= retryAtMs`，后继仅有一个稳定 `-transient-1` identity。放行后完成同批两个指定 static delegate、专业 Provider 真重叠、2 条初始 delivery + 1 条继承原合同的 repair、2 次 Observed claim、宿主验证、5 个父计划步骤、唯一 Supervisor assistant 与 3 条内部专业 assistant；重复 lifecycle/action/receipt/delivery/message 均为 0，pending/confirmation/finalization/provider-action-batch/provider-retry sidecar 全为 0，Provider 正文、私有正文、API Key、项目路径和正式配置路径泄漏均为 0，代理、Runner、隔离项目与 AppData 全部清理。验收器按 request identity 区分受控注入与真实网络抖动：额外瞬态失败只有逐条通过既有 lifecycle/retry/后继终态门禁并以相等 incidental failure/retry 计数公开时才允许继续，不能混入注入链。此前首批合同不完整、在专业 Agent in-flight 时强杀、sidecar-first 投影窗口误判、单委派和额外已恢复 Provider 抖动样本均只保留为独立失败证据，不与本次 PASS 拼接；V1.38 的真实 E2E 结论仍独立记账。 ## 验收命令 diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 8249b21dd..6a7fca784 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -605,4 +605,4 @@ game-project/ - snapshot v1 固定且完整包含 `schemaVersion / projectId / parentAgentId / parentRunId / boundFrom / policy / policyFingerprint / snapshotFingerprint / boundAt`；snapshot fingerprint 绑定除 `snapshotFingerprint / boundAt` 外的全部稳定字段。binding v1 固定包含 `schemaVersion / projectId / parentAgentId / parentRunId / boundFrom / policyFingerprint / snapshotFingerprint / boundAt`，必须与 snapshot 逐字段一致。安全 ID 可原样作 key；不安全 Agent/run ID 必须使用有界安全前缀加原始 ID 稳定 SHA-256，锁 key 对完整父 Agent/run 身份计算稳定指纹，禁止 lossy 规范化碰撞。 - 恢复优先级为 existing valid snapshot > 完整验真的 v2 batch contract > 符合严格状态门禁的 legacy 当前有效 policy。snapshot 存在但 binding 缺失时可从 snapshot 补写；binding 存在但 snapshot 丢失时只允许可信 v2 contract 按首次身份恢复，没有可信 v2 contract 时禁止按 live policy 重绑。contractless/v1 collaboration batch 必须先失败关闭，不能伪装 fresh run。`legacy-current-project-policy` 仅允许无 snapshot/binding、无可信 v2 contract，且不存在上述旧 batch，并由 durable 身份和状态明确证明属于 `pending / running / waiting-for-confirmation / waiting-for-user-input` 的旧父 run；terminal、`needs-reconciliation` 或身份/状态未知 run 的状态读取不得新建 snapshot。 - 已有 durable/未观察 claim 与 legacy claimed delivery 继续按原 action/group 身份恢复，不要求先创建新绑定；新 claim 必须先成功解析 effective snapshot 并核对 binding，再进入 V1.35-V1.37 的全锁、预算、完整 observation 和 group 数量门禁。snapshot 绑定后 global policy 的 `matched / drifted / unreadable` 只进入有界 status/诊断，不能改变后续执行；新 policy 只由后续新父 run 采用。2026-07-19 self-test、52/52 collaboration 定向回归和 949 passed/4 ignored Rust 全量已完成，终态快照保留也有独立回归；真实 mixed-swarm 功能样本已闭合但受正式 endpoint 外部重启污染，私有配置源的后续独立运行又连续耗尽 transient Provider retry，不能拼接证据，当前仍**不得声称 V1.38 真实 E2E 已 PASS**。详细报告以 Runtime 技术方案 V1.38 节为准。 -- 2026-07-19 起，同一 Runtime 文档的“V1.39 首次规划 Provider 瞬态重试持久等待态”作为后台首次规划重试的恢复事实源。tool-plan `repair-0` 及其自动 context-compaction 在瞬态失败后先写 per-Agent/run retry sidecar，再投影 `waiting-for-provider-retry` 并释放 lane；Runner 重启按到期时间恢复同一 Session/run/loop/attempt，同 Agent 后续任务保持 FIFO，其它 Agent 可并行。final-reply、手动压缩和 tool-plan `repair-N` 暂不扩展为持久重试；工具策略刷新时间不得进入请求指纹。当前只记录确定性回归，不改变 V1.38 真实 E2E 尚未 PASS 的结论。 +- 2026-07-19 起，同一 Runtime 文档的“V1.39 首次规划 Provider 瞬态重试持久等待态”作为后台首次规划重试的恢复事实源。tool-plan `repair-0` 及其自动 context-compaction 在瞬态失败后先写 per-Agent/run retry sidecar，再投影 `waiting-for-provider-retry` 并释放 lane；Runner 重启按到期时间恢复同一 Session/run/loop/attempt，同 Agent 后续任务保持 FIFO，其它 Agent 可并行。final-reply、手动压缩和 tool-plan `repair-N` 暂不扩展为持久重试；工具策略刷新时间不得进入请求指纹。最终代码已完成独立真实 `gpt-5.5 / openai_chat / high` PASS：父 Agent 首次规划在 `30s` 退避期执行一次 pidfd Runner 强杀，重启后 sidecar 身份/attempt/retryAt 稳定，到期前零早发且第二个请求网络接收时间不早于 retryAt；随后同一父 run 完成双专业 Agent 真重叠、2+1 delivery、唯一 repair、唯一最终回复和零重复/残留/正文/Key/路径泄漏，`37/37` lifecycle 闭合为 `36 completed + 1 injected failed + 1 retry`，incidental failure/retry 均为 0，隔离现场完整清理。额外真实瞬态失败按 request identity 单独计数且仍须完整恢复，不能混入受控注入链。该证据不改变 V1.38 真实 E2E 尚未 PASS 的独立结论。