diff --git a/apps/ai-game-creator-shell/package.json b/apps/ai-game-creator-shell/package.json index f2a16f265..578876e64 100644 --- a/apps/ai-game-creator-shell/package.json +++ b/apps/ai-game-creator-shell/package.json @@ -15,6 +15,7 @@ "agent-run": "node scripts/run-cli-with-config.mjs --agent-run", "agent-run:smoke": "node scripts/smoke-agent-run-local-provider.mjs", "agent-runtime:real-e2e": "node scripts/agent-runtime-real-e2e.mjs", + "agent-runtime:supervisor-swarm-transient-retry-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite supervisor-swarm-transient-retry", "agent-runtime:steer-real-e2e": "node scripts/agent-runtime-steer-real-e2e.mjs", "agent-runtime:steer-runner-kill-real-e2e": "node scripts/agent-runtime-real-e2e.mjs --suite steer-runner-kill", "typecheck": "node ../../node_modules/typescript/bin/tsc -p tsconfig.json --noEmit && node scripts/check-config.mjs" diff --git a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs index 438da42a6..1a9e3cf72 100644 --- a/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs +++ b/apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs @@ -1,12 +1,20 @@ import { spawn } from 'node:child_process'; import { createHash, randomUUID } from 'node:crypto'; -import { constants as fsConstants, createReadStream } from 'node:fs'; +import { + constants as fsConstants, + createReadStream, + watch as watchFileSystem, +} from 'node:fs'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { TextDecoder } from 'node:util'; +import { + startLlmTransientFaultProxy, + withLoopbackNoProxy, +} from './llm-transient-fault-proxy.mjs'; import { buildProcessSessionFixtureSource } from './process-session-real-e2e-fixture.mjs'; const appRoot = path.resolve(fileURLToPath(new URL('..', import.meta.url))); @@ -59,6 +67,10 @@ const supervisorSwarmAppDataSentinelFileName = '.agent-runtime-real-e2e-supervisor-swarm-appdata.json'; const supervisorSwarmAppDataSentinelSchema = 'genarrative-agent-runtime-real-e2e-supervisor-swarm-appdata.v1'; +const supervisorSwarmTransientRetryAppDataSentinelFileName = + '.agent-runtime-real-e2e-supervisor-swarm-transient-retry-appdata.json'; +const supervisorSwarmTransientRetryAppDataSentinelSchema = + 'genarrative-agent-runtime-real-e2e-supervisor-swarm-transient-retry-appdata.v1'; const mainAgentId = 'code-prototype'; const projectSupervisorAgentId = 'project-supervisor'; const requestedRunId = `real-e2e-${Date.now()}-${randomUUID().slice(0, 8)}`; @@ -88,6 +100,7 @@ const projectSkillSuite = 'project-skill'; const steerRunnerKillSuite = 'steer-runner-kill'; const parallelReadSuite = 'parallel-read'; const supervisorSwarmSuite = 'supervisor-swarm'; +const supervisorSwarmTransientRetrySuite = 'supervisor-swarm-transient-retry'; const runtimeContextBundleSchemaVersion = 'game-creator-runtime-context-bundle.v5'; const providerRequestLifecycleSchemaVersion = @@ -390,7 +403,11 @@ const isolatedRunnerState = { streamOverrideCreated: false, webSearchOverrideCreated: false, mcpOverrideCreated: false, + configOverlayCreated: false, sourceConfigCliCallCount: 0, + sourceAppDataDirectoryWatcher: null, + sourceAppDataDirectoryViolationCount: 0, + sourceAppDataDirectoryUntouched: false, sourceEndpointSnapshot: null, sourceRunnerEndpointUnchanged: false, sourceConfigLinksVerified: false, @@ -597,6 +614,8 @@ const state = { hostVerificationPassed: false, oldRunnerBootId: null, newRunnerBootId: null, + transientFaultProxy: null, + transientFaultCheckpoint: null, }, confirmedActionIds: new Set(), cleanupPerformed: false, @@ -973,6 +992,8 @@ try { state.isolatedRunner.sourceConfigCliCallCount; state.evidence.sourceRunnerEndpointUnchanged = state.isolatedRunner.sourceRunnerEndpointUnchanged; + state.evidence.sourceAppDataDirectoryUntouched = + state.isolatedRunner.sourceAppDataDirectoryUntouched; state.evidence.sourceConfigReplicaCount = state.isolatedRunner.configLinks.length; state.evidence.sourceConfigReplicasVerified = @@ -1011,6 +1032,38 @@ try { } } } + if (state.isolatedRunner.sourceAppDataDirectoryWatcher) { + closeSourceAppDataDirectoryGuard(); + state.status = 'FAIL'; + recordError('source-appdata-directory-guard-not-verified'); + } + if ( + isSupervisorSwarmTransientRetrySuite() && + state.supervisorSwarm.transientFaultProxy + ) { + try { + await state.supervisorSwarm.transientFaultProxy.stop(); + const proxyStats = state.supervisorSwarm.transientFaultProxy.getStats(); + state.evidence.transientFaultRequestCount = proxyStats.requestCount; + state.evidence.transientFaultInjectedCount = + proxyStats.faultInjectedCount; + state.evidence.transientFaultHeldRequestCount = + proxyStats.heldRequestCount; + state.evidence.transientFaultForwardedRequestCount = + proxyStats.forwardedRequestCount; + state.evidence.transientFaultProxyStopped = proxyStats.stopped === true; + if (!state.evidence.transientFaultProxyStopped) { + state.status = 'FAIL'; + recordError('supervisor-swarm-transient-retry-proxy-not-stopped'); + } + } catch (error) { + state.status = 'FAIL'; + recordError( + 'supervisor-swarm-transient-retry-proxy-cleanup-failed', + error, + ); + } + } if ( isSteerRunnerKillSuite() && state.projectRoot && @@ -7170,11 +7223,297 @@ async function driveSupervisorSwarmRuntimeToCompletion() { throw codedError('supervisor-swarm-runtime-timeout'); } +function registerSupervisorSwarmPrivateTransportValues(values) { + const normalized = [ + ...new Set(values.filter(isNonEmptyString).map((value) => value.trim())), + ]; + assert(normalized.length > 0, 'supervisor-swarm-private-transport-missing'); + state.supervisorSwarm.privateValues.push(...normalized); + const previousLeakCount = state.transcriptScanner?.count ?? 0; + state.secrets = [...new Set([...state.secrets, ...normalized])]; + state.transcriptScanner = new StreamingSecretScanner(state.secrets); + state.transcriptScanner.count = previousLeakCount; +} + +async function prepareSupervisorSwarmRuntimeAppData() { + if (!isSupervisorSwarmTransientRetrySuite()) { + await prepareIsolatedSuiteAppData(); + return; + } + + const loaded = await loadConfig(state.options.configDir); + const targetConfig = effectiveAgentLlmConfig( + loaded.config, + supervisorSwarmDesignAgentId, + ); + const upstream = new URL(targetConfig.baseUrl); + assert( + targetConfig.apiKind === 'openai_chat' && + targetConfig.model === 'gpt-5.5' && + ['http:', 'https:'].includes(upstream.protocol) && + upstream.username === '' && + upstream.password === '' && + upstream.hash === '', + 'supervisor-swarm-transient-retry-upstream-invalid', + ); + registerSupervisorSwarmPrivateTransportValues([ + targetConfig.baseUrl, + upstream.origin, + upstream.host, + upstream.hostname, + ]); + const proxy = await startLlmTransientFaultProxy({ + upstreamBaseUrl: targetConfig.baseUrl, + faultCount: 1, + holdAfterFault: true, + }); + state.supervisorSwarm.transientFaultProxy = proxy; + assert( + isNonEmptyString(proxy.baseUrl), + 'supervisor-swarm-transient-retry-proxy-base-url-missing', + ); + const configOverlay = { + agentLlm: { + [supervisorSwarmDesignAgentId]: { + baseUrl: proxy.baseUrl, + maxRetries: 1, + retryBackoffMs: 100, + }, + }, + }; + assert( + JSON.stringify(Object.keys(configOverlay)) === + JSON.stringify(['agentLlm']) && + JSON.stringify(Object.keys(configOverlay.agentLlm)) === + JSON.stringify([supervisorSwarmDesignAgentId]) && + JSON.stringify( + Object.keys(configOverlay.agentLlm[supervisorSwarmDesignAgentId]), + ) === JSON.stringify(['baseUrl', 'maxRetries', 'retryBackoffMs']), + 'supervisor-swarm-transient-retry-overlay-shape-invalid', + ); + await prepareIsolatedSuiteAppData({ configOverlay }); + + const isolated = await loadConfig(state.isolatedRunner.appDataDir); + const isolatedTarget = effectiveAgentLlmConfig( + isolated.config, + supervisorSwarmDesignAgentId, + ); + assert( + isolatedTarget.baseUrl === proxy.baseUrl && + isolatedTarget.apiKey === targetConfig.apiKey && + isolatedTarget.model === targetConfig.model && + isolatedTarget.apiKind === targetConfig.apiKind && + isolatedTarget.maxRetries === 1 && + isolatedTarget.retryBackoffMs === 100 && + state.isolatedRunner.configOverlayCreated, + 'supervisor-swarm-transient-retry-effective-overlay-invalid', + ); +} + +async function readSupervisorSwarmProjectRevision() { + return readJson( + path.join(state.projectRoot, '.agent/runtime/project-revision.json'), + ).catch((error) => { + if (error?.code === 'ENOENT') { + return { schemaVersion: null, revision: 0, updatedAt: 0 }; + } + throw error; + }); +} + +async function captureSupervisorSwarmTransientRetryCheckpoint() { + if (!isSupervisorSwarmTransientRetrySuite()) return; + const proxy = state.supervisorSwarm.transientFaultProxy; + assert(proxy, 'supervisor-swarm-transient-retry-proxy-missing'); + await proxy.waitForFault(60_000); + await proxy.waitForHeldRequest(60_000); + + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const [persistence, pending, revision, designMetadata] = await Promise.all([ + readSupervisorSwarmPersistence(), + findPendingActions(), + readSupervisorSwarmProjectRevision(), + fs + .lstat(path.join(state.projectRoot, supervisorSwarmDesignPath)) + .catch((error) => { + if (error?.code === 'ENOENT') return null; + throw error; + }), + ]); + const initial = supervisorSwarmParentDeliveries( + persistence.deliveries, + ).filter((delivery) => delivery.repairOfDelegationId == null); + const designDelivery = initial.find( + (delivery) => delivery.targetAgentId === supervisorSwarmDesignAgentId, + ); + if (!designDelivery || initial.length !== 2) { + await sleep(25); + continue; + } + + const lifecycle = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.lifecycle' && + record.agentId === designDelivery.targetAgentId && + record.runId === designDelivery.targetRunId, + ); + const retries = persistence.agentDb.filter( + (record) => + record.recordType === 'agent.runtime.provider_request.retry' && + record.agentId === designDelivery.targetAgentId && + record.runId === designDelivery.targetRunId, + ); + const failedStarted = lifecycle.find( + (record) => + record.status === 'started' && + !record.requestSlot?.includes('-transient-'), + ); + const failedTerminal = failedStarted + ? lifecycle.find( + (record) => + record.requestId === failedStarted.requestId && + record.status === 'failed', + ) + : null; + const retryAudit = retries[0]; + const retryStarted = retryAudit + ? lifecycle.find( + (record) => + record.status === 'started' && + record.requestSlot === retryAudit.nextRequestSlot, + ) + : null; + if (!failedStarted || !failedTerminal || !retryAudit || !retryStarted) { + await sleep(25); + continue; + } + + const stableIdentityFields = [ + 'agentId', + 'taskId', + 'sessionId', + 'runId', + 'source', + 'requestKind', + ]; + const actionRecords = persistence.agentDb.filter( + (record) => + record.agentId === designDelivery.targetAgentId && + record.runId === designDelivery.targetRunId && + [ + 'agent.runtime.tool_action.executing', + 'agent.runtime.tool_action.observed', + 'agent.runtime.tool_observation', + 'agent.runtime.action_receipt', + 'agent.runtime.tool_confirmation_required', + 'agent.runtime.tool_confirmation.approved', + ].includes(record.recordType), + ); + const receipts = actionRecords.filter( + (record) => record.recordType === 'agent.runtime.action_receipt', + ); + const childDeliveries = persistence.deliveries.filter( + (delivery) => + delivery.parentAgentId === designDelivery.targetAgentId && + delivery.parentRunId === designDelivery.targetRunId, + ); + const claims = persistence.claims.filter((claim) => + (claim.receipts ?? []).some( + (receipt) => receipt.delegationId === designDelivery.delegationId, + ), + ); + const designConversation = persistence.professionalConversations.find( + (entry) => + entry.agentId === designDelivery.targetAgentId && + entry.sessionId === designDelivery.targetSessionId, + ); + const assistantCount = (designConversation?.messages ?? []).filter( + (message) => message.role === 'assistant', + ).length; + const pendingCount = pending.filter( + (candidate) => + candidate.agentId === designDelivery.targetAgentId && + candidate.runId === designDelivery.targetRunId, + ).length; + const proxyStats = proxy.getStats(); + assert( + lifecycle.length === 3 && + retries.length === 1 && + failedStarted.requestKind === 'tool-plan' && + failedStarted.requestId === failedTerminal.requestId && + failedStarted.requestSlot === failedTerminal.requestSlot && + failedStarted.requestId === retryAudit.requestId && + retryAudit.retryAttempt === 1 && + retryAudit.maxRetries === 1 && + retryAudit.nextRequestSlot === + `${failedStarted.requestSlot}-transient-1` && + retryStarted.requestSlot === retryAudit.nextRequestSlot && + retryStarted.requestId !== failedStarted.requestId && + stableIdentityFields.every( + (field) => + failedStarted[field] === failedTerminal[field] && + failedStarted[field] === retryAudit[field] && + failedStarted[field] === retryStarted[field], + ) && + persistence.agentDb.indexOf(failedStarted) < + persistence.agentDb.indexOf(failedTerminal) && + persistence.agentDb.indexOf(failedTerminal) < + persistence.agentDb.indexOf(retryAudit) && + persistence.agentDb.indexOf(retryAudit) < + persistence.agentDb.indexOf(retryStarted) && + actionRecords.length === 0 && + receipts.length === 0 && + childDeliveries.length === 0 && + claims.length === 0 && + assistantCount === 0 && + pendingCount === 0 && + revision.revision === 0 && + designMetadata === null && + proxyStats.requestCount === 2 && + proxyStats.faultInjectedCount === 1 && + proxyStats.heldRequestCount === 1 && + proxyStats.forwardedRequestCount === 0 && + proxyStats.forwardingReleased === false, + 'supervisor-swarm-transient-retry-pre-forward-side-effect', + ); + state.supervisorSwarm.transientFaultCheckpoint = { + failedRequestId: failedStarted.requestId, + retryRequestId: retryStarted.requestId, + failedRequestSlot: failedStarted.requestSlot, + retryRequestSlot: retryStarted.requestSlot, + agentId: failedStarted.agentId, + taskId: failedStarted.taskId, + sessionId: failedStarted.sessionId, + runId: failedStarted.runId, + source: failedStarted.source, + requestKind: failedStarted.requestKind, + actionCount: actionRecords.length, + receiptCount: receipts.length, + childDeliveryCount: childDeliveries.length, + claimCount: claims.length, + assistantCount, + pendingCount, + projectRevision: revision.revision, + upstreamRequestCountBeforeRelease: proxyStats.forwardedRequestCount, + forwardingReleased: false, + }; + const released = proxy.releaseForwarding(); + assert( + released.forwardingReleased === true, + 'supervisor-swarm-transient-retry-release-failed', + ); + state.supervisorSwarm.transientFaultCheckpoint.forwardingReleased = true; + return; + } + throw codedError('supervisor-swarm-transient-retry-checkpoint-timeout'); +} + async function runSupervisorSwarmE2e() { await ensureOwnedRunnerStableKillSupport(); await seedSupervisorSwarmDisposableProject(); state.cliBinary = await prepareCliBinary(); - await prepareIsolatedSuiteAppData(); + await prepareSupervisorSwarmRuntimeAppData(); const task = buildSupervisorSwarmTaskPrompt(); assertSupervisorSwarmTaskPrompt(task); @@ -7208,6 +7547,7 @@ async function runSupervisorSwarmE2e() { ); await captureSupervisorSwarmInitialProviderBatch(); + await captureSupervisorSwarmTransientRetryCheckpoint(); await driveSupervisorSwarmToRepairKillBoundary(); await driveSupervisorSwarmRuntimeToCompletion(); state.evidence = await validateSupervisorSwarmEvidence(); @@ -7367,6 +7707,59 @@ function validateSupervisorSwarmProviderLifecycle(agentDb, deliveries) { duplicateCount(retries.map((record) => record.requestId)) === 0, 'supervisor-swarm-provider-failed-retry-cardinality-invalid', ); + let forcedTransientRetryVerified = false; + if (isSupervisorSwarmTransientRetrySuite()) { + const checkpoint = state.supervisorSwarm.transientFaultCheckpoint; + const failedTerminal = failed[0]; + const retryAudit = retries[0]; + const retryGroup = checkpoint + ? byRequest.get(checkpoint.retryRequestId) + : undefined; + const targetStarted = checkpoint + ? lifecycle.filter( + (record) => + record.status === 'started' && + record.agentId === checkpoint.agentId && + record.runId === checkpoint.runId, + ) + : []; + const targetCompleted = checkpoint + ? lifecycle.filter( + (record) => + record.status === 'completed' && + record.agentId === checkpoint.agentId && + record.runId === checkpoint.runId, + ) + : []; + const proxyStats = state.supervisorSwarm.transientFaultProxy?.getStats(); + assert( + checkpoint && + failed.length === 1 && + retries.length === 1 && + failedTerminal.requestId === checkpoint.failedRequestId && + retryAudit.requestId === checkpoint.failedRequestId && + retryAudit.nextRequestSlot === checkpoint.retryRequestSlot && + retryGroup?.length === 2 && + retryGroup[0].status === 'started' && + retryGroup[1].status === 'completed' && + retryGroup[0].requestId === checkpoint.retryRequestId && + retryGroup[0].requestSlot === checkpoint.retryRequestSlot && + retryGroup[0].requestId !== checkpoint.failedRequestId && + [ + 'agentId', + 'taskId', + 'sessionId', + 'runId', + 'source', + 'requestKind', + ].every((field) => retryGroup[0][field] === checkpoint[field]) && + proxyStats?.requestCount === targetStarted.length && + proxyStats.faultInjectedCount === 1 && + proxyStats.forwardedRequestCount === targetCompleted.length, + 'supervisor-swarm-forced-transient-retry-invalid', + ); + forcedTransientRetryVerified = true; + } for (const key of relevantRuns) { const [agentId, runId] = key.split('\0'); assert( @@ -7415,6 +7808,7 @@ function validateSupervisorSwarmProviderLifecycle(agentDb, deliveries) { finalReplyCount: completedKindCounts.finalReply, contextCompactionCount: completedKindCounts.contextCompaction, parentFinalReplyCount: parentFinalReplies.length, + forcedTransientRetryVerified, duplicateCount: duplicateCount( lifecycle.map((record) => `${record.requestId}:${record.status}`), ), @@ -7775,6 +8169,96 @@ async function countSupervisorSwarmSteerRecords() { return count; } +function supervisorSwarmTransientRetrySnapshotEvidence( + forcedTransientRetryVerified = false, +) { + const enabled = isSupervisorSwarmTransientRetrySuite(); + const checkpoint = state.supervisorSwarm.transientFaultCheckpoint; + const proxy = state.supervisorSwarm.transientFaultProxy; + const stats = proxy?.getStats() ?? { + requestCount: 0, + faultInjectedCount: 0, + heldRequestCount: 0, + forwardedRequestCount: 0, + forwardingReleased: false, + stopped: false, + }; + return { + transientFaultModeEnabled: enabled, + transientFaultTargetAgentId: enabled ? supervisorSwarmDesignAgentId : '', + transientFaultRequestCount: stats.requestCount, + transientFaultInjectedCount: stats.faultInjectedCount, + transientFaultHeldRequestCount: stats.heldRequestCount, + transientFaultForwardedRequestCount: stats.forwardedRequestCount, + transientFaultPreRetryCheckpointCaptured: Boolean(checkpoint), + transientFaultPreRetryActionCount: checkpoint?.actionCount ?? 0, + transientFaultPreRetryReceiptCount: checkpoint?.receiptCount ?? 0, + transientFaultPreRetryChildDeliveryCount: + checkpoint?.childDeliveryCount ?? 0, + transientFaultPreRetryClaimCount: checkpoint?.claimCount ?? 0, + transientFaultPreRetryAssistantCount: checkpoint?.assistantCount ?? 0, + transientFaultPreRetryPendingCount: checkpoint?.pendingCount ?? 0, + transientFaultPreRetryProjectRevision: checkpoint?.projectRevision ?? 0, + transientFaultUpstreamRequestCountBeforeRelease: + checkpoint?.upstreamRequestCountBeforeRelease ?? 0, + transientFaultRequestIdentityChanged: Boolean( + checkpoint && checkpoint.failedRequestId !== checkpoint.retryRequestId, + ), + transientFaultRetrySlotValid: Boolean( + checkpoint && + checkpoint.retryRequestSlot === + `${checkpoint.failedRequestSlot}-transient-1`, + ), + transientFaultStableLogicalIdentity: Boolean( + checkpoint && + [ + checkpoint.agentId, + checkpoint.taskId, + checkpoint.sessionId, + checkpoint.runId, + checkpoint.source, + checkpoint.requestKind, + ].every(isNonEmptyString), + ), + transientFaultForwardingReleased: + checkpoint?.forwardingReleased === true && + stats.forwardingReleased === true, + transientFaultRetryVerified: forcedTransientRetryVerified === true, + transientFaultProxyStopped: stats.stopped === true, + }; +} + +function supervisorSwarmTransientRetryEvidence(provider) { + const evidence = supervisorSwarmTransientRetrySnapshotEvidence( + provider.forcedTransientRetryVerified, + ); + if (evidence.transientFaultModeEnabled) { + assert( + provider.forcedTransientRetryVerified === true && + evidence.transientFaultRequestCount >= 2 && + evidence.transientFaultInjectedCount === 1 && + evidence.transientFaultHeldRequestCount === 1 && + evidence.transientFaultForwardedRequestCount >= 1 && + evidence.transientFaultPreRetryCheckpointCaptured && + evidence.transientFaultPreRetryActionCount === 0 && + evidence.transientFaultPreRetryReceiptCount === 0 && + evidence.transientFaultPreRetryChildDeliveryCount === 0 && + evidence.transientFaultPreRetryClaimCount === 0 && + evidence.transientFaultPreRetryAssistantCount === 0 && + evidence.transientFaultPreRetryPendingCount === 0 && + evidence.transientFaultPreRetryProjectRevision === 0 && + evidence.transientFaultUpstreamRequestCountBeforeRelease === 0 && + evidence.transientFaultRequestIdentityChanged && + evidence.transientFaultRetrySlotValid && + evidence.transientFaultStableLogicalIdentity && + evidence.transientFaultForwardingReleased && + evidence.transientFaultRetryVerified, + 'supervisor-swarm-transient-retry-evidence-invalid', + ); + } + return evidence; +} + async function validateSupervisorSwarmEvidence() { const persistence = await readSupervisorSwarmPersistence(); assertSupervisorSwarmRuntimeHealthy(persistence); @@ -7962,6 +8446,7 @@ async function validateSupervisorSwarmEvidence() { persistence.agentDb, deliveries, ); + const transientRetry = supervisorSwarmTransientRetryEvidence(provider); const actions = validateSupervisorSwarmActionPersistence( persistence.agentDb, deliveries, @@ -8300,6 +8785,7 @@ async function validateSupervisorSwarmEvidence() { isolatedAppDataUsed: true, formalConfigCliCallCount: state.isolatedRunner.sourceConfigCliCallCount, sourceRunnerEndpointUnchanged: false, + sourceAppDataDirectoryUntouched: false, sourceConfigReplicaCount: state.isolatedRunner.configLinks.length, sourceConfigReplicasVerified: false, taskCount: persistence.taskSnapshot.all.length, @@ -8357,6 +8843,7 @@ async function validateSupervisorSwarmEvidence() { providerLifecycleFailedCount: provider.failedCount, providerRetryCount: provider.retryCount, providerRetryPathTriggered: provider.retryCount > 0, + ...transientRetry, toolPlanProviderRequestCount: provider.toolPlanCount, finalReplyProviderRequestCount: provider.finalReplyCount, contextCompactionProviderRequestCount: provider.contextCompactionCount, @@ -8623,6 +9110,7 @@ async function collectPartialSupervisorSwarmEvidence(baseEvidence) { confirmedActionCount: state.supervisorSwarm.confirmedActionCount, failedRepairActionCount: state.supervisorSwarm.failedRepairActionCount, targetedContractReadCount: state.supervisorSwarm.targetedContractReadCount, + ...supervisorSwarmTransientRetrySnapshotEvidence(false), }); } @@ -8695,6 +9183,7 @@ function parseArguments(args) { suite === projectSkillSuite || suite === parallelReadSuite || suite === supervisorSwarmSuite || + suite === supervisorSwarmTransientRetrySuite || suite === steerRunnerKillSuite || processSessionSuites.has(suite), 'unsupported-suite', @@ -8824,6 +9313,10 @@ function sameEffectiveAgentLlm(left, right) { ); } +function isolatedSuiteUsesSiblingAppData() { + return isWebSearchSuite() || isSupervisorSwarmTransientRetrySuite(); +} + function isolatedSuiteAppDataProfile() { if (isSteerRunnerKillSuite()) { return { @@ -8897,6 +9390,14 @@ function isolatedSuiteAppDataProfile() { codePrefix: 'parallel-read-appdata', }; } + if (isSupervisorSwarmTransientRetrySuite()) { + return { + prefix: '.agent-runtime-real-e2e-supervisor-swarm-transient-retry-', + sentinelName: supervisorSwarmTransientRetryAppDataSentinelFileName, + sentinelSchema: supervisorSwarmTransientRetryAppDataSentinelSchema, + codePrefix: 'supervisor-swarm-transient-retry-appdata', + }; + } if (isSupervisorSwarmSuite()) { return { prefix: '.agent-runtime-real-e2e-supervisor-swarm-', @@ -8976,20 +9477,74 @@ async function verifySourceRunnerEndpointUnchanged() { state.isolatedRunner.sourceRunnerEndpointUnchanged = true; } +function closeSourceAppDataDirectoryGuard() { + const watcher = state.isolatedRunner.sourceAppDataDirectoryWatcher; + if (!watcher) return; + watcher.close(); + state.isolatedRunner.sourceAppDataDirectoryWatcher = null; +} + +function startSourceAppDataDirectoryGuard(sourceConfigDir, profile) { + if (!isSupervisorSwarmTransientRetrySuite()) return; + assert( + !state.isolatedRunner.sourceAppDataDirectoryWatcher, + 'source-appdata-directory-guard-already-started', + ); + const watcher = watchFileSystem( + sourceConfigDir, + { persistent: false }, + (_eventType, fileName) => { + const name = Buffer.isBuffer(fileName) + ? fileName.toString('utf8') + : String(fileName ?? ''); + if (name.startsWith(profile.prefix)) { + state.isolatedRunner.sourceAppDataDirectoryViolationCount += 1; + } + }, + ); + watcher.on('error', () => { + state.isolatedRunner.sourceAppDataDirectoryViolationCount += 1; + }); + state.isolatedRunner.sourceAppDataDirectoryWatcher = watcher; +} + +async function verifySourceAppDataDirectoryUntouched() { + if (!isSupervisorSwarmTransientRetrySuite()) return; + closeSourceAppDataDirectoryGuard(); + const sourceConfigDir = await fs.realpath(state.options.configDir); + const profile = isolatedSuiteAppDataProfile(); + const entries = await fs.readdir(sourceConfigDir); + assert( + state.isolatedRunner.sourceAppDataDirectoryViolationCount === 0 && + !entries.some((name) => name.startsWith(profile.prefix)), + 'source-appdata-directory-touched-by-suite', + ); + state.isolatedRunner.sourceAppDataDirectoryUntouched = true; +} + async function prepareIsolatedSuiteAppData({ streamAgentId = null, webSearchAgentId = null, mcpConfigFactory = null, + configOverlay = null, } = {}) { assert( isIsolatedRunnerSuite(), 'isolated-appdata-used-outside-isolated-suite', ); assert( - [streamAgentId, webSearchAgentId, mcpConfigFactory].filter(Boolean) - .length <= 1, + [streamAgentId, webSearchAgentId, mcpConfigFactory, configOverlay].filter( + Boolean, + ).length <= 1, 'isolated-appdata-multiple-overlays-forbidden', ); + if (configOverlay) { + assert( + isPlainObject(configOverlay) && + collectApiKeys(configOverlay).length === 0, + 'isolated-appdata-config-overlay-invalid', + ); + } const profile = isolatedSuiteAppDataProfile(); const suiteSecrets = new Set(state.secrets); const sourceConfigDir = await fs.realpath(state.options.configDir); @@ -8997,7 +9552,7 @@ async function prepareIsolatedSuiteAppData({ await captureSourceRunnerEndpointSnapshot(sourceConfigDir); const ownerToken = randomUUID(); const createdAt = Date.now(); - const appDataParent = isWebSearchSuite() + const appDataParent = isolatedSuiteUsesSiblingAppData() ? path.dirname(sourceConfigDir) : sourceConfigDir; const appDataDir = await createSentinelOwnedTempDirectory({ @@ -9014,6 +9569,15 @@ async function prepareIsolatedSuiteAppData({ state.isolatedRunner.appDataDir = appDataDir; state.isolatedRunner.ownerToken = ownerToken; state.isolatedRunner.createdAt = createdAt; + if (isSupervisorSwarmTransientRetrySuite()) { + const realAppDataDir = await fs.realpath(appDataDir); + assert( + !isPathInside(sourceConfigDir, realAppDataDir) && + path.dirname(realAppDataDir) === path.dirname(sourceConfigDir), + `${profile.codePrefix}-source-appdata-write-boundary-invalid`, + ); + startSourceAppDataDirectoryGuard(sourceConfigDir, profile); + } const mcpOverlay = mcpConfigFactory ? await mcpConfigFactory(appDataDir) : null; @@ -9121,11 +9685,13 @@ async function prepareIsolatedSuiteAppData({ } for (const source of sourceConfigs) { - const linkedName = activeConfigSource - ? source === activeConfigSource - ? configFileName - : `.source-${source.name}` - : source.name; + const linkedName = configOverlay + ? `.source-${source.name}` + : activeConfigSource + ? source === activeConfigSource + ? configFileName + : `.source-${source.name}` + : source.name; const linkedPath = path.join(appDataDir, linkedName); const storageMode = isWebSearchSuite() || @@ -9185,12 +9751,40 @@ async function prepareIsolatedSuiteAppData({ sha256: createHash('sha256').update(source.sourceContent).digest('hex'), }); } - assert( - state.isolatedRunner.configLinks.some( - (link) => link.linkedName === configFileName, - ), - `${profile.codePrefix}-primary-config-link-missing`, - ); + if (configOverlay) { + const primaryConfigPath = path.join(appDataDir, configFileName); + const localOverlayPath = path.join(appDataDir, localConfigFileName); + await fs.writeFile( + primaryConfigPath, + `${JSON.stringify(mergedSourceConfig)}\n`, + { flag: 'wx', mode: 0o600 }, + ); + await fs.writeFile(localOverlayPath, `${JSON.stringify(configOverlay)}\n`, { + flag: 'wx', + mode: 0o600, + }); + const [primaryMetadata, overlayMetadata] = await Promise.all([ + fs.lstat(primaryConfigPath), + fs.lstat(localOverlayPath), + ]); + assert( + primaryMetadata.isFile() && + !primaryMetadata.isSymbolicLink() && + overlayMetadata.isFile() && + !overlayMetadata.isSymbolicLink() && + (primaryMetadata.mode & 0o077) === 0 && + (overlayMetadata.mode & 0o077) === 0, + `${profile.codePrefix}-materialized-config-invalid`, + ); + state.isolatedRunner.configOverlayCreated = true; + } else { + assert( + state.isolatedRunner.configLinks.some( + (link) => link.linkedName === configFileName, + ), + `${profile.codePrefix}-primary-config-link-missing`, + ); + } if (activeConfigSource) { const overrideKey = webSearchAgentId ? 'webSearchEnabled' : 'stream'; @@ -9383,14 +9977,18 @@ async function prepareIsolatedSuiteAppData({ ]); assert( effectiveConfigs.every( - ([, effective]) => + ([agentId, effective]) => effective.model === 'gpt-5.5' && effective.apiKind === 'openai_chat' && isNonEmptyString(effective.reasoningEffort) && Number.isSafeInteger(effective.requestTimeoutMs) && effective.requestTimeoutMs > 0 && Number.isSafeInteger(effective.maxRetries) && - effective.maxRetries >= 1 && + effective.maxRetries >= + (isSupervisorSwarmTransientRetrySuite() && + agentId !== supervisorSwarmDesignAgentId + ? 0 + : 1) && effective.maxRetries <= 3 && Number.isSafeInteger(effective.retryBackoffMs) && effective.retryBackoffMs > 0 && @@ -9976,7 +10574,7 @@ async function removeIsolatedSuiteAppData() { fs.realpath(state.options.configDir), fs.realpath(state.isolatedRunner.appDataDir), ]); - const cleanupPathValid = isWebSearchSuite() + const cleanupPathValid = isolatedSuiteUsesSiblingAppData() ? !isPathInside(sourceConfigDir, appDataDir) && path.dirname(appDataDir) === path.dirname(sourceConfigDir) : isPathInside(sourceConfigDir, appDataDir); @@ -9986,6 +10584,7 @@ async function removeIsolatedSuiteAppData() { ); let ownershipError = null; try { + await verifySourceAppDataDirectoryUntouched(); await verifyIsolatedSuiteConfigLinksUnchanged(); await verifySourceRunnerEndpointUnchanged(); } catch (error) { @@ -10729,10 +11328,22 @@ async function runCli(args, options = {}) { timeoutMs: options.timeoutMs ?? 60_000, stdin: options.stdin, allowNonZero: options.allowNonZero ?? false, + env: buildCliChildEnvironment(), }, ); } +function buildCliChildEnvironment() { + const environment = { + ...process.env, + NO_COLOR: '1', + RUST_BACKTRACE: '0', + }; + return isSupervisorSwarmTransientRetrySuite() + ? withLoopbackNoProxy(environment) + : environment; +} + function startInteractiveCli(args) { assert(Boolean(state.cliBinary), 'interactive-cli-binary-not-ready'); assert(Boolean(state.runtimeConfigDir), 'interactive-config-dir-not-ready'); @@ -10749,7 +11360,7 @@ function startInteractiveCli(args) { [...args, '--config-dir', state.runtimeConfigDir], { cwd: appRoot, - env: { ...process.env, NO_COLOR: '1', RUST_BACKTRACE: '0' }, + env: buildCliChildEnvironment(), stdio: ['pipe', 'pipe', 'pipe'], }, ); @@ -20464,6 +21075,7 @@ function supervisorSwarmEvidenceFieldTemplate() { isolatedAppDataUsed: false, formalConfigCliCallCount: 0, sourceRunnerEndpointUnchanged: false, + sourceAppDataDirectoryUntouched: false, sourceConfigReplicaCount: 0, sourceConfigReplicasVerified: false, taskCount: 0, @@ -20513,6 +21125,27 @@ function supervisorSwarmEvidenceFieldTemplate() { providerLifecycleFailedCount: 0, providerRetryCount: 0, providerRetryPathTriggered: false, + transientFaultModeEnabled: false, + transientFaultTargetAgentId: '', + transientFaultRequestCount: 0, + transientFaultInjectedCount: 0, + transientFaultHeldRequestCount: 0, + transientFaultForwardedRequestCount: 0, + transientFaultPreRetryCheckpointCaptured: false, + transientFaultPreRetryActionCount: 0, + transientFaultPreRetryReceiptCount: 0, + transientFaultPreRetryChildDeliveryCount: 0, + transientFaultPreRetryClaimCount: 0, + transientFaultPreRetryAssistantCount: 0, + transientFaultPreRetryPendingCount: 0, + transientFaultPreRetryProjectRevision: 0, + transientFaultUpstreamRequestCountBeforeRelease: 0, + transientFaultRequestIdentityChanged: false, + transientFaultRetrySlotValid: false, + transientFaultStableLogicalIdentity: false, + transientFaultForwardingReleased: false, + transientFaultRetryVerified: false, + transientFaultProxyStopped: false, toolPlanProviderRequestCount: 0, finalReplyProviderRequestCount: 0, contextCompactionProviderRequestCount: 0, @@ -21568,7 +22201,14 @@ function isParallelReadSuite() { } function isSupervisorSwarmSuite() { - return state.suite === supervisorSwarmSuite; + return ( + state.suite === supervisorSwarmSuite || + isSupervisorSwarmTransientRetrySuite() + ); +} + +function isSupervisorSwarmTransientRetrySuite() { + return state.suite === supervisorSwarmTransientRetrySuite; } function isSteerRunnerKillSuite() { diff --git a/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs new file mode 100644 index 000000000..9c53b4265 --- /dev/null +++ b/apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs @@ -0,0 +1,580 @@ +import http from 'node:http'; +import https from 'node:https'; + +const LOOPBACK_HOST = '127.0.0.1'; +const LOOPBACK_NO_PROXY_ENTRIES = Object.freeze([ + LOOPBACK_HOST, + 'localhost', + '::1', +]); +const DEFAULT_FALLBACK_PORTS = Object.freeze( + Array.from({ length: 128 }, (_, index) => 62_000 + index), +); + +export function withLoopbackNoProxy(environment) { + const entries = [environment.NO_PROXY, environment.no_proxy] + .flatMap((value) => (typeof value === 'string' ? value.split(',') : [])) + .map((value) => value.trim()) + .filter(Boolean); + const noProxy = [...new Set([...entries, ...LOOPBACK_NO_PROXY_ENTRIES])].join( + ',', + ); + return { ...environment, NO_PROXY: noProxy, no_proxy: noProxy }; +} + +function proxyError(message) { + const error = new Error(message); + error.name = 'LlmTransientFaultProxyError'; + return error; +} + +function parseUpstreamBaseUrl(value) { + let parsed; + try { + parsed = value instanceof URL ? new URL(value.href) : new URL(value); + } catch { + throw proxyError('upstreamBaseUrl must be a valid HTTP(S) URL'); + } + + if ( + !['http:', 'https:'].includes(parsed.protocol) || + parsed.username || + parsed.password || + parsed.hash + ) { + throw proxyError( + 'upstreamBaseUrl must use HTTP(S) without userinfo or a fragment', + ); + } + + return Object.freeze({ + protocol: parsed.protocol, + hostname: parsed.hostname.replace(/^\[|\]$/gu, ''), + port: parsed.port || undefined, + hostHeader: parsed.host, + basePathname: parsed.pathname.replace(/\/+$/gu, ''), + }); +} + +function normalizeOptions(upstreamBaseUrlOrOptions, faultCount, extraOptions) { + const options = + typeof upstreamBaseUrlOrOptions === 'string' || + upstreamBaseUrlOrOptions instanceof URL + ? { + ...extraOptions, + upstreamBaseUrl: upstreamBaseUrlOrOptions, + faultCount, + } + : upstreamBaseUrlOrOptions; + + if (!options || typeof options !== 'object') { + throw proxyError('proxy options are required'); + } + + const normalizedFaultCount = options.faultCount ?? 1; + if (!Number.isSafeInteger(normalizedFaultCount) || normalizedFaultCount < 0) { + throw proxyError('faultCount must be a non-negative safe integer'); + } + if ( + options.holdAfterFault !== undefined && + typeof options.holdAfterFault !== 'boolean' + ) { + throw proxyError('holdAfterFault must be a boolean'); + } + if (options.listen !== undefined && typeof options.listen !== 'function') { + throw proxyError('listen must be a function'); + } + + const fallbackPorts = options.fallbackPorts ?? DEFAULT_FALLBACK_PORTS; + if ( + !Array.isArray(fallbackPorts) || + fallbackPorts.length === 0 || + fallbackPorts.length > 512 || + fallbackPorts.some( + (port) => !Number.isSafeInteger(port) || port < 49_152 || port > 65_535, + ) + ) { + throw proxyError('fallbackPorts must contain valid high ports'); + } + + return { + upstream: parseUpstreamBaseUrl(options.upstreamBaseUrl), + faultCount: normalizedFaultCount, + holdAfterFault: options.holdAfterFault ?? false, + fallbackPorts: [...new Set(fallbackPorts)], + listen: options.listen ?? listenOnLoopback, + }; +} + +function listenOnLoopback(server, { host, port }) { + return new Promise((resolve, reject) => { + const cleanup = () => { + server.off('error', onError); + server.off('listening', onListening); + }; + const onError = (error) => { + cleanup(); + reject(error); + }; + const onListening = () => { + cleanup(); + resolve(); + }; + + server.once('error', onError); + server.once('listening', onListening); + try { + server.listen({ host, port, exclusive: true }); + } catch (error) { + cleanup(); + reject(error); + } + }); +} + +function errorCode(error) { + return error && typeof error === 'object' && 'code' in error + ? error.code + : undefined; +} + +async function bindServer(server, listen, fallbackPorts) { + try { + await listen(server, { host: LOOPBACK_HOST, port: 0 }); + return; + } catch (error) { + if (errorCode(error) !== 'EADDRINUSE') { + throw proxyError('unable to bind transient fault proxy on loopback'); + } + } + + for (const port of fallbackPorts) { + try { + await listen(server, { host: LOOPBACK_HOST, port }); + return; + } catch (error) { + if (errorCode(error) !== 'EADDRINUSE') { + throw proxyError('unable to bind transient fault proxy on loopback'); + } + } + } + + throw proxyError('transient fault proxy fallback port pool is exhausted'); +} + +function isOriginFormPath(value) { + return ( + typeof value === 'string' && + value.startsWith('/') && + !value.startsWith('//') && + !hasAsciiControlCharacter(value) + ); +} + +function hasAsciiControlCharacter(value) { + for (const character of value) { + const codePoint = character.codePointAt(0); + if (codePoint <= 0x1f || codePoint === 0x7f) return true; + } + return false; +} + +function isWithinBasePath(value, basePathname) { + let pathname; + try { + pathname = new URL(value, 'http://proxy.invalid').pathname; + } catch { + return false; + } + return ( + basePathname === '' || + pathname === basePathname || + pathname.startsWith(`${basePathname}/`) + ); +} + +function resetSocket(socket) { + if (!socket || socket.destroyed) return; + try { + if (typeof socket.resetAndDestroy === 'function') { + socket.resetAndDestroy(); + } else { + socket.destroy(); + } + } catch { + socket.destroy(); + } +} + +function forwardHeaders(request, hostHeader) { + const headers = Object.create(null); + for (let index = 0; index < request.rawHeaders.length; index += 2) { + const name = request.rawHeaders[index]; + const value = request.rawHeaders[index + 1]; + if (!name || value === undefined) continue; + const lowerName = name.toLowerCase(); + if (lowerName === 'host' || lowerName === 'proxy-connection') continue; + const existing = headers[lowerName]; + if (existing === undefined) { + headers[lowerName] = value; + } else if (Array.isArray(existing)) { + existing.push(value); + } else { + headers[lowerName] = [existing, value]; + } + } + headers.host = hostHeader; + return headers; +} + +function failClosed(request, response, statusCode) { + const body = statusCode === 405 ? 'method not allowed' : 'request rejected'; + request.resume(); + response.shouldKeepAlive = false; + response.writeHead(statusCode, { + connection: 'close', + 'content-length': Buffer.byteLength(body), + 'content-type': 'text/plain; charset=utf-8', + }); + response.end(body); +} + +function closeServer(server) { + if (!server.listening) return Promise.resolve(); + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +/** + * Starts a loopback-only proxy that resets the first configured POST requests. + * The object form also accepts holdAfterFault, fallbackPorts, and a test listen strategy. + */ +export async function startLlmTransientFaultProxy( + upstreamBaseUrlOrOptions, + faultCount = 1, + extraOptions = {}, +) { + const options = normalizeOptions( + upstreamBaseUrlOrOptions, + faultCount, + extraOptions, + ); + const downstreamSockets = new Set(); + const upstreamRequests = new Set(); + const upstreamSockets = new Set(); + const heldForwardingWaiters = new Set(); + const counterWaiters = new Set(); + + let requestCount = 0; + let faultInjectedCount = 0; + let heldRequestCount = 0; + let forwardedRequestCount = 0; + let forwardingReleased = !options.holdAfterFault || options.faultCount === 0; + let stopping = false; + let stopped = false; + let stopPromise; + + const stats = () => + Object.freeze({ + requestCount, + faultInjectedCount, + heldRequestCount, + forwardedRequestCount, + forwardingReleased, + stopped, + }); + + const notifyCounterWaiters = (kind) => { + for (const waiter of [...counterWaiters]) { + if (waiter.kind !== kind) continue; + clearTimeout(waiter.timer); + counterWaiters.delete(waiter); + waiter.resolve(stats()); + } + }; + + const waitForCounter = (kind, timeoutMs) => { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + return Promise.reject(proxyError('timeoutMs must be a positive integer')); + } + const current = kind === 'fault' ? faultInjectedCount : heldRequestCount; + if (current > 0) return Promise.resolve(stats()); + if (stopping || stopped) { + return Promise.reject( + proxyError('proxy stopped before the wait completed'), + ); + } + + return new Promise((resolve, reject) => { + const waiter = { kind, resolve, reject, timer: undefined }; + waiter.timer = setTimeout(() => { + counterWaiters.delete(waiter); + reject(proxyError(`timed out waiting for proxy ${kind}`)); + }, timeoutMs); + waiter.timer.unref?.(); + counterWaiters.add(waiter); + }); + }; + + const releaseHeldForwarding = (shouldForward) => { + for (const waiter of [...heldForwardingWaiters]) { + heldForwardingWaiters.delete(waiter); + waiter.complete(shouldForward); + } + }; + + const waitForForwardingRelease = (request, response) => { + if (forwardingReleased) return Promise.resolve(true); + if (stopping) return Promise.resolve(false); + + return new Promise((resolve) => { + const onAborted = () => waiter.complete(false); + const onClosed = () => waiter.complete(false); + const waiter = { + complete: (shouldForward) => { + heldForwardingWaiters.delete(waiter); + request.off('aborted', onAborted); + response.off('close', onClosed); + resolve(shouldForward); + }, + }; + request.once('aborted', onAborted); + response.once('close', onClosed); + heldForwardingWaiters.add(waiter); + if (forwardingReleased) waiter.complete(true); + if (stopping) waiter.complete(false); + }); + }; + + const sendBadGateway = (request, response) => { + if (stopping || response.destroyed) return; + if (response.headersSent) { + response.destroy(); + return; + } + failClosed(request, response, 502); + }; + + const forwardRequest = (request, response) => { + const transport = options.upstream.protocol === 'https:' ? https : http; + let upstreamRequest; + try { + upstreamRequest = transport.request({ + protocol: options.upstream.protocol, + hostname: options.upstream.hostname, + port: options.upstream.port, + method: request.method, + path: request.url, + headers: forwardHeaders(request, options.upstream.hostHeader), + agent: false, + setHost: false, + }); + } catch { + sendBadGateway(request, response); + return; + } + + forwardedRequestCount += 1; + upstreamRequests.add(upstreamRequest); + upstreamRequest.once('close', () => + upstreamRequests.delete(upstreamRequest), + ); + upstreamRequest.on('socket', (socket) => { + upstreamSockets.add(socket); + socket.once('close', () => upstreamSockets.delete(socket)); + socket.on('error', () => {}); + }); + + let upstreamResponse; + const stopUpstream = () => { + upstreamResponse?.destroy(); + upstreamRequest.destroy(); + }; + request.once('aborted', stopUpstream); + request.once('error', stopUpstream); + response.once('error', stopUpstream); + response.once('close', () => { + if (!response.writableEnded) stopUpstream(); + }); + + upstreamRequest.once('response', (receivedResponse) => { + upstreamResponse = receivedResponse; + receivedResponse.once('error', () => { + if (!response.destroyed) response.destroy(); + }); + receivedResponse.once('aborted', () => { + if (!response.destroyed) response.destroy(); + }); + + if (stopping || response.destroyed) { + receivedResponse.destroy(); + return; + } + try { + if (receivedResponse.statusMessage) { + response.writeHead( + receivedResponse.statusCode ?? 502, + receivedResponse.statusMessage, + receivedResponse.rawHeaders, + ); + } else { + response.writeHead( + receivedResponse.statusCode ?? 502, + receivedResponse.rawHeaders, + ); + } + } catch { + receivedResponse.destroy(); + sendBadGateway(request, response); + return; + } + receivedResponse.pipe(response); + }); + upstreamRequest.once('error', () => { + sendBadGateway(request, response); + }); + request.pipe(upstreamRequest); + }; + + const handleRequest = async (request, response, expectsContinue) => { + requestCount += 1; + request.on('error', () => {}); + response.on('error', () => {}); + + if (request.method !== 'POST') { + failClosed(request, response, 405); + return; + } + if ( + !isOriginFormPath(request.url) || + !isWithinBasePath(request.url, options.upstream.basePathname) + ) { + failClosed(request, response, 400); + return; + } + + if ( + options.holdAfterFault && + faultInjectedCount > 0 && + !forwardingReleased + ) { + heldRequestCount += 1; + notifyCounterWaiters('held'); + const shouldForward = await waitForForwardingRelease(request, response); + if (!shouldForward) { + resetSocket(request.socket); + return; + } + } + + if (faultInjectedCount < options.faultCount) { + faultInjectedCount += 1; + notifyCounterWaiters('fault'); + resetSocket(request.socket); + return; + } + + if (stopping || request.destroyed || response.destroyed) { + resetSocket(request.socket); + return; + } + if (expectsContinue) response.writeContinue(); + forwardRequest(request, response); + }; + + const dispatchRequest = (request, response, expectsContinue = false) => { + void handleRequest(request, response, expectsContinue).catch(() => { + sendBadGateway(request, response); + }); + }; + + const server = http.createServer(); + server.on('request', (request, response) => { + dispatchRequest(request, response); + }); + server.on('checkContinue', (request, response) => { + dispatchRequest(request, response, true); + }); + server.on('checkExpectation', (request, response) => { + failClosed(request, response, 400); + }); + server.on('connect', (_request, socket) => resetSocket(socket)); + server.on('upgrade', (_request, socket) => resetSocket(socket)); + server.on('clientError', (_error, socket) => resetSocket(socket)); + server.on('connection', (socket) => { + downstreamSockets.add(socket); + socket.once('close', () => downstreamSockets.delete(socket)); + socket.on('error', () => {}); + }); + + try { + await bindServer(server, options.listen, options.fallbackPorts); + const address = server.address(); + if ( + !address || + typeof address === 'string' || + address.address !== LOOPBACK_HOST + ) { + throw proxyError('transient fault proxy did not bind to loopback'); + } + } catch (error) { + for (const socket of downstreamSockets) socket.destroy(); + await closeServer(server); + if (error?.name === 'LlmTransientFaultProxyError') throw error; + throw proxyError('unable to start transient fault proxy'); + } + + server.on('error', () => {}); + const address = server.address(); + const port = address.port; + const url = `http://${LOOPBACK_HOST}:${port}`; + const baseUrl = `${url}${options.upstream.basePathname}`; + + const releaseForwarding = () => { + if (stopping || forwardingReleased) return stats(); + forwardingReleased = true; + releaseHeldForwarding(true); + return stats(); + }; + + const stop = () => { + if (stopPromise) return stopPromise; + stopPromise = (async () => { + stopping = true; + releaseHeldForwarding(false); + for (const waiter of [...counterWaiters]) { + clearTimeout(waiter.timer); + counterWaiters.delete(waiter); + waiter.reject(proxyError('proxy stopped before the wait completed')); + } + + const closePromise = closeServer(server); + for (const request of [...upstreamRequests]) request.destroy(); + for (const socket of [...upstreamSockets]) socket.destroy(); + for (const socket of [...downstreamSockets]) socket.destroy(); + server.closeAllConnections?.(); + await closePromise; + stopped = true; + })(); + return stopPromise; + }; + + return Object.freeze({ + url, + baseUrl, + port, + get stats() { + return stats(); + }, + getStats: stats, + waitForFault(timeoutMs = 5_000) { + return waitForCounter('fault', timeoutMs); + }, + waitForHeldRequest(timeoutMs = 5_000) { + return waitForCounter('held', timeoutMs); + }, + releaseForwarding, + stop, + }); +} diff --git a/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts new file mode 100644 index 000000000..c2b05fa67 --- /dev/null +++ b/apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts @@ -0,0 +1,674 @@ +import http, { + type IncomingHttpHeaders, + type IncomingMessage, + type Server, + type ServerResponse, +} from 'node:http'; + +import { afterEach, describe, expect, it } from 'vitest'; + +import { + startLlmTransientFaultProxy, + withLoopbackNoProxy, +} from '../scripts/llm-transient-fault-proxy.mjs'; + +interface ProxyStats { + requestCount: number; + faultInjectedCount: number; + heldRequestCount: number; + forwardedRequestCount: number; + forwardingReleased: boolean; + stopped: boolean; +} + +interface ProxyHandle { + url: string; + baseUrl: string; + port: number; + stats: ProxyStats; + getStats(): ProxyStats; + waitForFault(timeoutMs?: number): Promise; + waitForHeldRequest(timeoutMs?: number): Promise; + releaseForwarding(): ProxyStats; + stop(): Promise; +} + +interface CapturedRequest { + method: string | undefined; + url: string | undefined; + headers: IncomingHttpHeaders; + body: string; +} + +interface HttpResult { + statusCode: number | undefined; + headers: IncomingHttpHeaders; + body: string; +} + +const proxies = new Set(); +const servers = new Set(); +const TEST_UPSTREAM_FALLBACK_PORTS = Array.from( + { length: 128 }, + (_, index) => 62_300 + index, +); + +afterEach(async () => { + await Promise.allSettled([...proxies].map((proxy) => proxy.stop())); + proxies.clear(); + for (const server of servers) server.closeAllConnections?.(); + await Promise.allSettled([...servers].map((server) => closeServer(server))); + servers.clear(); +}); + +function deferred() { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function listen( + server: Server, + { host, port }: { host: string; port: number }, +) { + return new Promise((resolve, reject) => { + const cleanup = () => { + server.off('error', onError); + server.off('listening', onListening); + }; + const onError = (error: Error) => { + cleanup(); + reject(error); + }; + const onListening = () => { + cleanup(); + resolve(); + }; + server.once('error', onError); + server.once('listening', onListening); + server.listen({ host, port, exclusive: true }); + }); +} + +function closeServer(server: Server) { + if (!server.listening) return Promise.resolve(); + return new Promise((resolve) => server.close(() => resolve())); +} + +function errorCode(error: unknown) { + return error && typeof error === 'object' && 'code' in error + ? error.code + : undefined; +} + +async function listenTestServer(server: Server) { + try { + await listen(server, { host: '127.0.0.1', port: 0 }); + return; + } catch (error) { + if (errorCode(error) !== 'EADDRINUSE') throw error; + } + + for (const port of TEST_UPSTREAM_FALLBACK_PORTS) { + try { + await listen(server, { host: '127.0.0.1', port }); + return; + } catch (error) { + if (errorCode(error) !== 'EADDRINUSE') throw error; + } + } + throw new Error('test upstream fallback port pool is exhausted'); +} + +async function startServer( + handler: ( + request: IncomingMessage, + response: ServerResponse, + ) => void | Promise, +) { + const server = http.createServer((request, response) => { + void Promise.resolve(handler(request, response)).catch(() => { + response.destroy(); + }); + }); + await listenTestServer(server); + servers.add(server); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('test server did not expose a TCP address'); + } + return { server, url: `http://127.0.0.1:${address.port}` }; +} + +async function startProxy(options: Record) { + const proxy = (await startLlmTransientFaultProxy(options)) as ProxyHandle; + proxies.add(proxy); + return proxy; +} + +function readBody(request: IncomingMessage) { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + request.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + request.once('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + request.once('error', reject); + }); +} + +function request( + baseUrl: string, + { + method = 'POST', + path = '/', + headers = {}, + body = '', + onChunk, + }: { + method?: string; + path?: string; + headers?: Record; + body?: string; + onChunk?: (chunk: string) => void; + } = {}, +) { + const base = new URL(baseUrl); + return new Promise((resolve, reject) => { + const outgoing = http.request( + { + hostname: base.hostname, + port: base.port, + method, + path, + headers, + agent: false, + }, + (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk) => { + const buffer = Buffer.from(chunk); + chunks.push(buffer); + onChunk?.(buffer.toString('utf8')); + }); + response.once('end', () => { + resolve({ + statusCode: response.statusCode, + headers: response.headers, + body: Buffer.concat(chunks).toString('utf8'), + }); + }); + response.once('aborted', () => reject(new Error('response aborted'))); + response.once('error', reject); + }, + ); + outgoing.once('error', reject); + outgoing.end(body); + }); +} + +describe('LLM transient fault proxy', () => { + it('pins loopback in both no-proxy variants without dropping existing entries', () => { + const source = { + HTTP_PROXY: 'http://proxy.example', + NO_PROXY: 'internal.example, 127.0.0.1', + no_proxy: 'legacy.example', + }; + + const childEnvironment = withLoopbackNoProxy(source); + + expect(childEnvironment).toEqual({ + HTTP_PROXY: 'http://proxy.example', + NO_PROXY: 'internal.example,127.0.0.1,legacy.example,localhost,::1', + no_proxy: 'internal.example,127.0.0.1,legacy.example,localhost,::1', + }); + expect(source).toEqual({ + HTTP_PROXY: 'http://proxy.example', + NO_PROXY: 'internal.example, 127.0.0.1', + no_proxy: 'legacy.example', + }); + }); + + it('resets the first POST before upstream and then streams an intact request and response', async () => { + const captured: CapturedRequest[] = []; + const releaseResponse = deferred(); + const firstResponseChunk = deferred(); + const upstream = await startServer(async (incoming, response) => { + captured.push({ + method: incoming.method, + url: incoming.url, + headers: incoming.headers, + body: await readBody(incoming), + }); + response.writeHead(201, { + 'content-type': 'text/plain', + 'x-upstream-stream': 'yes', + }); + response.write('first-'); + await releaseResponse.promise; + response.end('second'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 1, + }); + + const failedRequest = request(proxy.url, { + path: '/v1/chat/completions?mode=fault', + headers: { 'content-type': 'application/json' }, + body: '{"attempt":1}', + }); + await expect(failedRequest).rejects.toBeInstanceOf(Error); + await expect(proxy.waitForFault()).resolves.toMatchObject({ + faultInjectedCount: 1, + }); + expect(captured).toHaveLength(0); + + const forwardedRequest = request(proxy.url, { + path: '/v1/chat/completions?mode=stream', + headers: { + authorization: 'Bearer fixture-secret', + 'content-type': 'application/json', + 'x-request-marker': 'preserved', + }, + body: '{"prompt":"sensitive-body-marker"}', + onChunk: (chunk) => firstResponseChunk.resolve(chunk), + }); + await expect(firstResponseChunk.promise).resolves.toBe('first-'); + expect(captured).toEqual([ + expect.objectContaining({ + method: 'POST', + url: '/v1/chat/completions?mode=stream', + headers: expect.objectContaining({ + authorization: 'Bearer fixture-secret', + 'content-type': 'application/json', + 'x-request-marker': 'preserved', + }), + body: '{"prompt":"sensitive-body-marker"}', + }), + ]); + releaseResponse.resolve(); + + await expect(forwardedRequest).resolves.toMatchObject({ + statusCode: 201, + headers: expect.objectContaining({ 'x-upstream-stream': 'yes' }), + body: 'first-second', + }); + expect(proxy.stats).toEqual({ + requestCount: 2, + faultInjectedCount: 1, + heldRequestCount: 0, + forwardedRequestCount: 1, + forwardingReleased: true, + stopped: false, + }); + }); + + it('holds the post-fault request before upstream until forwarding is released', async () => { + const captured: CapturedRequest[] = []; + const upstream = await startServer(async (incoming, response) => { + captured.push({ + method: incoming.method, + url: incoming.url, + headers: incoming.headers, + body: await readBody(incoming), + }); + response.writeHead(200, { 'x-held-request': 'released' }); + response.end('forwarded-after-release'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 1, + holdAfterFault: true, + }); + + await expect( + request(proxy.url, { path: '/fault', body: 'first-attempt' }), + ).rejects.toBeInstanceOf(Error); + await proxy.waitForFault(); + + const heldRequest = request(proxy.url, { + path: '/v1/responses?held=1', + headers: { + authorization: 'Bearer held-secret', + 'content-type': 'application/json', + }, + body: '{"held":"request-body"}', + }); + await expect(proxy.waitForHeldRequest()).resolves.toMatchObject({ + heldRequestCount: 1, + forwardedRequestCount: 0, + }); + expect(captured).toHaveLength(0); + expect(proxy.releaseForwarding()).toMatchObject({ + forwardingReleased: true, + heldRequestCount: 1, + }); + + await expect(heldRequest).resolves.toMatchObject({ + statusCode: 200, + headers: expect.objectContaining({ 'x-held-request': 'released' }), + body: 'forwarded-after-release', + }); + expect(captured).toEqual([ + expect.objectContaining({ + method: 'POST', + url: '/v1/responses?held=1', + headers: expect.objectContaining({ + authorization: 'Bearer held-secret', + }), + body: '{"held":"request-body"}', + }), + ]); + expect(proxy.stats.forwardedRequestCount).toBe(1); + }); + + it('holds before injecting a remaining configured fault', async () => { + let upstreamRequestCount = 0; + const upstream = await startServer(async (incoming, response) => { + upstreamRequestCount += 1; + await readBody(incoming); + response.end('after-two-faults'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 2, + holdAfterFault: true, + }); + + await expect( + request(proxy.url, { body: 'fault-one' }), + ).rejects.toBeInstanceOf(Error); + const secondFault = request(proxy.url, { body: 'fault-two' }); + await proxy.waitForHeldRequest(); + expect(proxy.stats).toMatchObject({ + faultInjectedCount: 1, + heldRequestCount: 1, + forwardedRequestCount: 0, + }); + proxy.releaseForwarding(); + await expect(secondFault).rejects.toBeInstanceOf(Error); + expect(proxy.stats.faultInjectedCount).toBe(2); + expect(upstreamRequestCount).toBe(0); + + await expect( + request(proxy.url, { body: 'forward-third' }), + ).resolves.toMatchObject({ statusCode: 200, body: 'after-two-faults' }); + expect(upstreamRequestCount).toBe(1); + }); + + it('keeps sensitive upstream, header, and body values out of stats and errors', async () => { + const sensitiveQuery = 'upstream-sensitive-query-marker'; + const upstream = await startServer(async (incoming, response) => { + await readBody(incoming); + response.end('sensitive-response-marker'); + }); + const upstreamBaseUrl = `${upstream.url}/v1///?token=${sensitiveQuery}`; + const proxy = await startProxy({ + upstreamBaseUrl, + faultCount: 0, + }); + + await request(proxy.url, { + path: '/v1/chat/completions', + headers: { authorization: 'Bearer sensitive-authorization-marker' }, + body: 'sensitive-request-body-marker', + }); + let timeoutError: unknown; + try { + await proxy.waitForHeldRequest(20); + } catch (error) { + timeoutError = error; + } + + const publicSurface = JSON.stringify({ + proxy, + stats: proxy.getStats(), + error: String(timeoutError), + }); + for (const sensitiveValue of [ + upstream.url, + upstreamBaseUrl, + sensitiveQuery, + 'sensitive-authorization-marker', + 'sensitive-request-body-marker', + 'sensitive-response-marker', + ]) { + expect(publicSurface).not.toContain(sensitiveValue); + } + expect(String(timeoutError)).toBe( + 'LlmTransientFaultProxyError: timed out waiting for proxy held', + ); + }); + + it('returns a normalized local baseUrl and preserves the upstream base path', async () => { + const captured: CapturedRequest[] = []; + const upstream = await startServer(async (incoming, response) => { + captured.push({ + method: incoming.method, + url: incoming.url, + headers: incoming.headers, + body: await readBody(incoming), + }); + response.end('base-path-preserved'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: `${upstream.url}/v1///`, + faultCount: 0, + }); + + expect(proxy.baseUrl).toBe(`${proxy.url}/v1`); + await expect( + request(proxy.url, { + path: '/v10/chat/completions', + }), + ).resolves.toMatchObject({ statusCode: 400 }); + await expect( + request(proxy.url, { + path: '/v1/../chat/completions', + }), + ).resolves.toMatchObject({ statusCode: 400 }); + expect(captured).toHaveLength(0); + + const runtimeUrl = `${proxy.baseUrl}/chat/completions`; + const runtimePath = new URL(runtimeUrl).pathname; + await expect( + request(proxy.url, { + path: runtimePath, + headers: { 'content-type': 'application/json' }, + body: '{"basePath":"/v1"}', + }), + ).resolves.toMatchObject({ + statusCode: 200, + body: 'base-path-preserved', + }); + expect(captured).toEqual([ + expect.objectContaining({ + method: 'POST', + url: '/v1/chat/completions', + body: '{"basePath":"/v1"}', + }), + ]); + }); + + it('stops idempotently, unblocks held requests, and leaves no forwarded request', async () => { + let upstreamRequestCount = 0; + const upstream = await startServer(async (incoming, response) => { + upstreamRequestCount += 1; + await readBody(incoming); + response.end('unexpected'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + holdAfterFault: true, + }); + + await expect(request(proxy.url, { body: 'fault' })).rejects.toBeInstanceOf( + Error, + ); + const heldRequest = request(proxy.url, { body: 'held-until-stop' }); + await proxy.waitForHeldRequest(); + + const firstStop = proxy.stop(); + const secondStop = proxy.stop(); + expect(secondStop).toBe(firstStop); + await Promise.all([firstStop, secondStop]); + await expect(proxy.stop()).resolves.toBeUndefined(); + await expect(heldRequest).rejects.toBeInstanceOf(Error); + expect(upstreamRequestCount).toBe(0); + expect(proxy.stats).toMatchObject({ + faultInjectedCount: 1, + heldRequestCount: 1, + forwardedRequestCount: 0, + stopped: true, + }); + }); + + it('stops an active upstream request and its downstream connection', async () => { + const upstreamReceived = deferred(); + const upstreamSocketClosed = deferred(); + const upstream = await startServer(async (incoming) => { + incoming.socket.once('close', () => upstreamSocketClosed.resolve()); + await readBody(incoming); + upstreamReceived.resolve(); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 0, + }); + + const activeRequest = request(proxy.url, { body: 'active-forward' }); + await upstreamReceived.promise; + expect(proxy.stats.forwardedRequestCount).toBe(1); + + await proxy.stop(); + await expect(activeRequest).rejects.toBeInstanceOf(Error); + await expect(upstreamSocketClosed.promise).resolves.toBeUndefined(); + expect(proxy.stats.stopped).toBe(true); + }); + + it('falls back to an injected high loopback port pool after port zero exhaustion', async () => { + const upstream = await startServer(async (incoming, response) => { + await readBody(incoming); + response.end('ok'); + }); + const attempts: Array<{ host: string; port: number }> = []; + const fallbackPorts = Array.from( + { length: 64 }, + (_, index) => 62_128 + index, + ); + const injectedListen = async ( + server: Server, + options: { host: string; port: number }, + ) => { + attempts.push(options); + if (options.port === 0) { + throw Object.assign(new Error('simulated ephemeral port exhaustion'), { + code: 'EADDRINUSE', + }); + } + await listen(server, options); + }; + + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 0, + fallbackPorts, + listen: injectedListen, + }); + + expect(attempts[0]).toEqual({ host: '127.0.0.1', port: 0 }); + expect(attempts.slice(1).every(({ host }) => host === '127.0.0.1')).toBe( + true, + ); + expect(fallbackPorts).toContain(proxy.port); + await expect( + request(proxy.url, { body: 'fallback-body' }), + ).resolves.toMatchObject({ statusCode: 200, body: 'ok' }); + }); + + it('fails closed for non-POST and absolute-form requests without consuming faults', async () => { + let upstreamRequestCount = 0; + const upstream = await startServer(async (incoming, response) => { + upstreamRequestCount += 1; + await readBody(incoming); + response.end('unexpected'); + }); + const proxy = await startProxy({ upstreamBaseUrl: upstream.url }); + + await expect( + request(proxy.url, { method: 'GET', path: '/models' }), + ).resolves.toMatchObject({ statusCode: 405 }); + await expect( + request(proxy.url, { + path: 'http://second-origin.invalid/v1/chat/completions', + body: 'absolute-form', + }), + ).resolves.toMatchObject({ statusCode: 400 }); + expect(upstreamRequestCount).toBe(0); + expect(proxy.stats).toMatchObject({ + requestCount: 2, + faultInjectedCount: 0, + forwardedRequestCount: 0, + }); + + await expect( + request(proxy.url, { body: 'valid-post' }), + ).rejects.toBeInstanceOf(Error); + expect(proxy.stats.faultInjectedCount).toBe(1); + expect(upstreamRequestCount).toBe(0); + }); + + it('returns upstream redirects without following them to a second origin', async () => { + let secondOriginRequestCount = 0; + const secondOrigin = await startServer(async (incoming, response) => { + secondOriginRequestCount += 1; + await readBody(incoming); + response.end('must-not-be-requested'); + }); + const upstream = await startServer(async (incoming, response) => { + await readBody(incoming); + response.writeHead(307, { + location: `${secondOrigin.url}/redirect-target`, + }); + response.end('redirect-not-followed'); + }); + const proxy = await startProxy({ + upstreamBaseUrl: upstream.url, + faultCount: 0, + }); + + await expect( + request(proxy.url, { body: 'redirect-source' }), + ).resolves.toMatchObject({ + statusCode: 307, + headers: expect.objectContaining({ + location: `${secondOrigin.url}/redirect-target`, + }), + body: 'redirect-not-followed', + }); + expect(secondOriginRequestCount).toBe(0); + expect(proxy.stats.forwardedRequestCount).toBe(1); + }); + + it('rejects unsafe upstream URL forms without echoing them', async () => { + const unsafeUrls = [ + 'ftp://unsafe-url-marker.invalid/v1', + 'https://user:password@unsafe-url-marker.invalid/v1', + 'https://unsafe-url-marker.invalid/v1#fragment', + ]; + + for (const upstreamBaseUrl of unsafeUrls) { + let startupError: unknown; + try { + await startLlmTransientFaultProxy({ upstreamBaseUrl }); + } catch (error) { + startupError = error; + } + expect(startupError).toBeInstanceOf(Error); + expect(String(startupError)).not.toContain('unsafe-url-marker'); + } + }); +}); diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index e0d619a0c..4cb046e0d 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -16,6 +16,15 @@ --- +## 2026-07-17 AI 游戏创作 Swarm 显式重试必须使用受控真实故障门禁 + +- 背景：V1.28 `supervisor-swarm` 正式报告的 46 个 Provider request 全部 completed；确定性测试和条件式 E2E validator 虽覆盖 retry 契约，但 `failed=0 / retry=0` 仍可 PASS，不能证明真实 Provider Swarm 进入过显式重试链。 +- 决策：保留正常 `supervisor-swarm` 作为合同委派/repair/恢复协议门禁，另设 `supervisor-swarm-transient-retry`。新 suite 用 sentinel 管理的隔离 AppData 只覆盖一个专业 Agent 的 `baseUrl / maxRetries / retryBackoffMs`；本地 loopback 代理在首个 POST 转发正文前断线，后继请求先暂停。暂停期间必须证明唯一 `started -> failed`、唯一 retry audit、不同 request identity、稳定 `-transient-1` slot和相同逻辑身份，同时 action、receipt、目标 Agent 子委派、claim、assistant、pending、project revision、目标产物和 upstream forwarding 全为 0，之后才允许真实 Provider 请求继续。 +- 安全：代理不记录或返回 upstream URL、headers、Authorization、请求/响应正文或凭据，只公开计数与布尔状态；只接受 loopback origin-form POST 和原 base path，redirect 原样返回而不跟随。E2E 启动 CLI/Runner 时必须合并并同时覆盖 `NO_PROXY / no_proxy`，显式加入 `127.0.0.1 / localhost / ::1`，避免继承的系统 HTTP 代理先接触发往故障代理的凭据和正文。端口 0 耗尽时使用有界 loopback fallback，stop 必须幂等关闭全部上下游连接。隔离 AppData 创建在正式目录同级，source-dir guard 禁止本 suite 前缀进入源目录或留下残留项；源配置私有副本逐字校验，正式 Runner endpoint 身份保持不变，临时合并配置与 overlay 为 `0600` 并由 sentinel 删除。并发正式 Runner 的 heartbeat 可改变目录 mtime/ctime，不得据此把外部写入误归因给 suite。完整链后续失败时，partial report 仍必须回填已经取得的 retry checkpoint 和零副作用证据。 +- 验证：最终加强版正式 `openai_chat / gpt-5.5` 报告为 46 个 request identity、46 started/terminal、45 completed、1 failed、1 retry；受控重试前所有副作用计数为 0。代理观察到的 10 个目标 Agent 请求与该 Agent lifecycle 数量一致，其中 1 个注入失败、1 个暂停、9 个转发。放行后双专业 Agent 真重叠、2 初始 + 1 repair delivery、2 个 Observed claim、targeted contract read、pidfd Runner 强杀恢复、唯一 Supervisor assistant 和 3 条内部专业 assistant 全部成立；27/27 成功计划与 14/14 repair 全为原生工具协议，源 AppData 未被写入，重复、残留和敏感泄漏均为 0，代理、隔离 Runner/AppData/项目全部清理。 +- 范围：该门禁证明“显式重试可与既有 Swarm 完整链组合”，不证明 Supervisor 已在无 Agent ID、同轮或 repair 次数提示时自主选择编排。自主 suite、真实 `--swarm-chat`、static+isolated all-join 组合和 Tauri 宿主 E2E 保留为后续完成项。 +- 关联文档：`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`、`docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md`。 + ## 2026-07-16 AI 游戏创作 Agent Runtime V1.28 Supervisor 合同委派与单回复收束 - 背景：V1.16 已建立 Supervisor 的 durable static delivery/claim 和同一父 run 唯一回复，但旧 `agent.delegate` 只描述目标与任务，Runtime 只能确认子任务终态，不能持久证明预期产物、验证证据或返工关系；实施计划中也仍有普通用户进入单 Agent 对话的旧表述。 @@ -158,6 +167,7 @@ - 影响范围：`api-server` assets / external assets / admin 路由、后台资源查询图片放大和音视频预览、OSS 读取契约与安全测试。 - 验证方式：定向测试覆盖 curated `legacyPublicPath` 可匿名签名、任意未登记 `objectKey` 拒绝、`PublicRead` 可读、owner 私有对象仅本人可读、External 跨 owner 拒绝、Admin endpoint 仅管理员可用，以及 `read-bytes` 与 `read-url` 同授权。 - 关联文档：`docs/【后端架构】server-rs与SpacetimeDB数据契约-2026-05-15.md`。 + ## 2026-07-09 角色动作视频生成背景色统一为多色自动决策 + 阿里云抠帧 - 背景：角色动作视频抽帧过去固定 legacy `#00FF00` 绿幕 + 本地 `editor_green_screen`，与生图链路的多色自动决策不一致；实测出现背景色与前景 / 皮肤撞色（蓝撞蓝、桃 / 黄撞肤色）以及图生视频背景变白的问题。 @@ -4467,6 +4477,7 @@ - 权威查询：`asset_object` 不进入 client 长期订阅。API 通过仅 runtime service identity 可调用的 procedure，按主键或 `(bucket, object_key)` 服务端索引读取事务内 metadata；只有位置查询明确返回不存在时才允许进入 legacy curated 前缀兼容，procedure 失败、超时或重复位置一律失败关闭。 - 一致性：隐藏、删除或取消发布提交后，后续读取 procedure 的事务快照立即按新状态判断，不等待任意池连接追上订阅水位。公开派生授权、`PublicRead` 和 legacy 兼容读取签名 URL 的有效期最多 600 秒，因此该能力仍不是对既有签名的瞬时吊销机制；owner / admin 读取保持原有效期口径。 - 验证方式：公开可见作品的正式资产可匿名读取；未选候选图、参考图、跨 owner 伪造 key 仍返回不存在；隐藏、删除或取消发布后新的读取请求立即拒绝，再恢复公开可见时新的读取请求立即恢复；超长公开 `expireSeconds` 被截断为 600 秒。 + ## 2026-07-13 AI 游戏创作 Agent Runtime V1.4 Git 工作树审阅 - 决策：新增一等只读 `git.inspect`，共享 command id 为 `project.git_inspect`且默认 `auto`。工具只接受 `includeDiff / maxFiles / maxChars`，返回精确 Git top-level 的 HEAD / branch、staged / unstaged / untracked 安全路径和有界 staged / unstaged unified diff；不改项目 revision 或 verification gate。 diff --git a/docs/project-memory/shared-memory/development-workflow.md b/docs/project-memory/shared-memory/development-workflow.md index c7de2181a..fb8074fe2 100644 --- a/docs/project-memory/shared-memory/development-workflow.md +++ b/docs/project-memory/shared-memory/development-workflow.md @@ -92,6 +92,18 @@ npm run ai-game-creator-shell:typecheck 第一条覆盖固定程序 / argv 拒绝规则、输出清洗、超时和源码改写检测；第二条覆盖全局 / per-Agent 配置继承，第三条覆盖 `default / low / medium / high` 到 Provider 请求的映射；后两条覆盖共享 `confirm` 契约、配置结构和发布默认 `high`。模块级定向验证通过后，再按改动范围运行 `npm run ai-game-creator-shell:check`、`npm run check:encoding` 和 `git diff --check`。 +### AI 游戏创作 Swarm 显式重试真实复验 + +正常 `supervisor-swarm` 全部 completed 不能替代真实 retry 证据。修改 Runtime 显式重试、Provider lifecycle、隔离 AppData 或 Swarm 验收器后，先跑代理夹具和静态门禁，再运行独立真实 suite： + +```bash +npm run test -- apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts +npm run ai-game-creator-shell:typecheck +npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e -- --config-dir +``` + +真实 PASS 必须恰好包含 1 个 failed lifecycle、1 条 retry audit 和 1 个 `-transient-1` 后继 identity；forwarding gate 放行前 action、receipt、目标 Agent 子委派、claim、assistant、pending、project revision 与 upstream forwarding 全为 0。放行后仍须完成双专业 Agent 重叠、唯一 repair、Runner 强杀恢复、唯一 Supervisor assistant 和零重复/残留/泄漏。suite 只能读正式 AppData，在其同级目录写入 sentinel 管理的 `0600` 私有副本和 overlay；启动 CLI/Runner 时须把 loopback 合并进大小写两套 no-proxy 环境，防止系统 HTTP 代理绕过本地故障门禁；source-dir guard 必须证明本 suite 前缀未进入源目录，源配置和 endpoint 身份保持不变，报告不得保存 Provider URL、headers、正文、凭据或绝对配置路径。若后续 repair/恢复/终局失败，partial report 仍应保留已经取得的 retry checkpoint。 + ### AI 游戏创作 Runtime V1.10 持久进程定向复验 V1.10 的 PTY 只通过四个 Runner-owned 工具开放；不要把 V1.2 `command.exec` 改成长驻入口。最小工具输入保持结构化： diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index ebadb1bc6..e3641f5f3 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -14,6 +14,14 @@ - 关联：相关文件、文档、提交或 Issue ``` +## Provider 全成功的真实报告不能证明显式重试可用 + +- 现象：真实 Swarm 报告显示全部 Provider lifecycle completed，E2E 的 retry validator 也没有报错，于是文档把“支持瞬态重试”一并写成已真实验收。 +- 原因：validator 只在实际出现 failed lifecycle 时校验 retry audit；`failed=0 / retry=0` 会自然通过。随机等待外部网络故障既不可重复，也无法在故障和重试之间证明副作用仍为 0。 +- 处理：为重试单独建立 fail-first loopback proxy。在正式 AppData 同级创建 sentinel 管理的一次性目录，只覆盖其中一个目标 Agent 的 base URL 和重试配置；首个 POST 在正文进入 upstream 前断线，第二个请求由 forwarding gate 暂停。gate 内交叉检查 failed lifecycle、retry audit、request slot/identity、action、pending、receipt、delivery、claim、assistant、project revision 和目标产物，再显式放行真实 Provider。代理不能记录 URL、headers 或正文，不能跟随 redirect，必须可幂等清理；启动 CLI/Runner 时同时设置合并后的 `NO_PROXY / no_proxy` 并显式加入 loopback，不能假设开发机已正确配置代理绕过；source-dir guard 禁止本 suite 前缀进入源目录，配置和 endpoint 身份保持只读并逐字复核。不要用源目录 mtime/ctime 归因，正式 Runner heartbeat 会并发改变它。完整链在 checkpoint 后失败时，partial report 也要保留已取得的 identity、slot 和零副作用证据，不能退回模板默认值。 +- 验证：`npm run test -- apps/ai-game-creator-shell/tests/llmTransientFaultProxy.test.ts` 覆盖故障、暂停、base path、流式转发、fallback、隐私和清理；`npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e -- --config-dir ` 必须得到恰好 1 failed/1 retry、重试前副作用全 0，并继续通过完整 Swarm/Runner 恢复和零泄漏门禁。 +- 关联：`apps/ai-game-creator-shell/scripts/llm-transient-fault-proxy.mjs`、`apps/ai-game-creator-shell/scripts/agent-runtime-real-e2e.mjs`、`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`。 + ## Agent 真实验收的阶段等待必须同步观察 Runtime 终态 - 现象：真实 Provider 已因 transport、格式修复或其它不可恢复错误把 task/Runtime 写成 failed，专项验收仍在等待某个 pending action、observation 或 receipt，直到 30 分钟总超时才返回。 diff --git a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md index c1dacffbd..148a9ba52 100644 --- a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md +++ b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md @@ -1120,6 +1120,14 @@ Supervisor 认领回执后必须能够再次从 durable delivery 取回权威返 正式报告包含 46 个 Provider request identity，`46 started / 46 terminal / 46 completed / 0 failed`；成功计划 `24/24`、格式修复 `20/20` 均使用 `native_runtime_tools`，wrapper/text fallback 为 `0`。父计划 5 步全部 completed，2 个公开项目文件通过宿主验证；4 个 run 共 16 个 finalization stage。delivery、message、action lifecycle、executing action、receipt、Provider lifecycle 的重复计数均为 `0`，pending/batch/finalization/confirmation/user-input sidecar 均为 `0`，steer、Provider payload、私有正文、API Key、项目绝对路径、正式配置路径、报告、secret 和 lure 泄漏均为 `0`。suite 只使用 sentinel 管理的隔离 AppData，正式配置 CLI 调用为 `0`，源 Runner endpoint 未变化，隔离 Runner/AppData 已清理。 +## V1.29 Project Supervisor 受控瞬态重试真实门禁 + +2026-07-17 新增独立 `supervisor-swarm-transient-retry` 真实门禁，补足上述 `0 failed` 报告未触发显式重试的证据缺口。suite 在正式 AppData 的同级目录创建 sentinel 管理的一次性 AppData，只把 `design-director` 配置指向本地回环 fail-first 代理，并把该 Agent 设为 `maxRetries=1 / retryBackoffMs=100`；正式 AppData 目录和文件只读，其他 Agent 保留源配置。代理在首个 POST 向真实 upstream 转发正文前主动断开，第二个请求先停在 forwarding gate；验收器在放行前交叉读取 lifecycle、retry audit、action、pending、receipt、delivery、claim、conversation、project revision 和目标产物，随后才让同一重试请求进入真实 Provider。代理只保留计数，不保存或输出 upstream URL、headers、Authorization、请求/响应正文或凭据；E2E 启动 CLI/Runner 时把 `127.0.0.1 / localhost / ::1` 合并进 `NO_PROXY` 和 `no_proxy`，防止继承的系统 HTTP 代理先接触凭据或正文。源配置副本和临时 overlay 均为 `0600`，source-dir guard 必须证明源目录未出现本 suite 前缀的事件或残留项，配置 inode/内容和 Runner endpoint 身份保持不变，并由 sentinel 清理隔离目录。源目录 mtime/ctime 不能作为归因证据，因为并发正式 Runner 会合法刷新 endpoint heartbeat。 + +最终加强版正式 `openai_chat / gpt-5.5` 复验 PASS：46 个 Provider request identity 全部形成唯一终态，`46 started / 46 terminal / 45 completed / 1 failed`，恰好 1 条 retry audit；失败 attempt 与后继 `-transient-1` 使用不同 request identity，Agent/task/Session/run/source/request kind 保持一致。forwarding gate 放行前 action、receipt、专业子委派、claim、assistant、pending、project revision 和 upstream forwarding 均为 `0`。代理观察到的 10 个目标 Agent 请求与该 Agent lifecycle 数量一致，其中 1 个注入失败、1 个暂停、9 个转发。放行后仍完成 2 个初始专业 Agent 真重叠、2+1 delivery、2 个 Observed claim、1 次 targeted contract read、唯一 repair、pidfd Runner 强杀/boot 恢复、5 步父计划、唯一 Supervisor assistant 与 3 条内部专业 assistant；27/27 成功计划和 14/14 repair 均为 `native_runtime_tools`。重复、残留 sidecar、Provider payload、私有正文、API Key、项目/正式配置路径、报告、secret 与 lure 泄漏均为 `0`；source-dir suite-prefix guard 与 `sourceAppDataDirectoryUntouched` 证明正式 AppData 未被写入，物理请求/lifecycle 一一对应和失败 partial checkpoint 门禁均通过，代理、隔离 Runner/AppData/项目全部清理。 + +该受控 suite 是 V1.28 协议与恢复的故障注入门禁，不替代后续自主 Swarm 验收。现有 fixture 明确给出两个专业方向、同轮要求和一次 repair 上限；“Supervisor 在不提供 Agent ID、并行配方或 repair 次数时自主选择编排”仍需独立 `supervisor-swarm-autonomous` 真实 suite 证明。真实 `--swarm-chat`、同一 run 的 static delivery + isolated all-join 组合以及 Tauri/WebView 宿主级 Supervisor GUI 也仍是单独完成项。 + ## 验收命令 - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml structured_plan_ -- --nocapture` @@ -1148,6 +1156,7 @@ Supervisor 认领回执后必须能够再次从 durable delivery 取回权威返 - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite project-skill` - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite parallel-read` - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite supervisor-swarm` +- `npm run ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e -- --config-dir ` - `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite full` - `npm run check:encoding` - `git diff --check` diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index f3a1920b2..6e204ec40 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -585,4 +585,5 @@ game-project/ - Supervisor 认领原 delivery 后可为 needs-repair 或语义未通过创建一个 `repairOfDelegationId=<原 delegationId>` 的新委派。repair 必须完整继承原合同并交回原专业 Agent，只能留在同一父 run、深度为 1、同一原 delivery 同时最多一个非 suppressed 投递；相同重放幂等复用，不同重复/并发请求拒绝。`suppressed` repair 继续阻断，同一 action 可原地恢复；该 action 已持久失败时，新 action 只可在全部既有 repair 均 suppressed 时重做基础设施投递。所有必要 delivery/claim/repair、结构化计划、verification、确认、用户输入和其它既有 blocker 清零后，原 Supervisor run 才能写唯一用户回复。 - V1.28 对 Provider 瞬态失败采用 Runtime 显式重试：`agentLlm..maxRetries / retryBackoffMs` 表示独立物理尝试及其有界指数退避，不得恢复为 `LlmClient` 在单 lifecycle 内隐式重放。每次尝试都重建禁用自动重试的 client，并写独立单次 lifecycle；首次 request slot 不变，第 `N` 次重试稳定使用 `-transient-N` 后缀。只有 `timeout / connectivity / transport` 可重试；工具协议无效仍进入独立 `repair-N` 格式修复，其他错误与重试耗尽按原失败路径收束。重试前必须重新检查 Goal、steer、cancel、task/run 与 orphan 门禁；控制请求可阻止下一次尝试，Runner 强杀后无可信终态的 `started` 仍进入 reconciliation，不能自动补发。重试发生在解析和副作用之前，不创建 action、pending、receipt、delivery、assistant 或 revision；既有 Runner、orphan、finalization 和隐私边界均不放宽，公共审计不得保存请求/响应正文、arguments、凭据或绝对路径。 - V1.28 已于 2026-07-17 完成正式 `openai_chat / gpt-5.5` `supervisor-swarm` PASS：同一 native 批次双专业委派、真实 Provider 重叠、2 份初始 delivery、1 次 targeted contract read、1 份唯一 repair、pidfd Runner 强杀/boot 恢复、同一父 Session/run、唯一 Supervisor assistant 和 3 条内部专业 assistant 全部成立。报告包含 46/46 闭合且 completed 的 Provider lifecycle，成功计划 24/24、格式修复 20/20 全为原生工具协议；重复、残留 sidecar、Provider payload、私有正文、API Key、项目/正式配置绝对路径、报告、secret 与 lure 泄漏均为 0。正式 AppData 零 CLI 调用且源 Runner endpoint 未变化；规范复验命令为 `npm run ai-game-creator-shell:agent-runtime:real-e2e -- --config-dir --suite supervisor-swarm`。 +- 2026-07-17 追加 `supervisor-swarm-transient-retry` 受控故障门禁：一次性本地回环代理只让 `design-director` 首个请求在正文转发前断线，并暂停后继请求，直到验收器确认唯一 failed lifecycle、唯一 retry audit、新 `-transient-1` identity，以及 action/receipt/子委派/claim/assistant/pending/revision/upstream forwarding 全为 0。E2E 启动 CLI/Runner 时会把 loopback 合并进 `NO_PROXY / no_proxy`，避免继承的系统 HTTP 代理接触故障门禁请求中的凭据和正文。最终加强版正式 `gpt-5.5` 报告为 46/46 lifecycle 闭合、45 completed/1 failed/1 retry；代理观察到的 10 个目标 Agent 请求与该 Agent lifecycle 数量一致，放行后完整双 Agent、唯一 repair、Runner 强杀恢复、唯一 Supervisor assistant、零重复/残留/泄漏继续 PASS。隔离 AppData 创建在正式目录同级，source-dir guard 与 `sourceAppDataDirectoryUntouched` 证明正式 AppData 未被写入，源配置与 endpoint 身份保持只读，失败 partial report 保留已取得的 retry checkpoint，代理与隔离现场全部清理。该 suite 只证明显式重试和既有协作链可组合，不把预置双 Agent fixture 扩大解释为自主编排；无 Agent ID/并行/repair 配方的自主 suite、真实 `agc:chat`、static+isolated 组合和 Tauri 宿主 E2E 仍待单独验收。 - 开发模式可通过本地项目文件面板执行 `file.list/read/write/delete`，普通用户界面不暴露文件面板。 diff --git a/package.json b/package.json index 046d71138..22e9cd545 100644 --- a/package.json +++ b/package.json @@ -146,6 +146,7 @@ "ai-game-creator-shell:agent-run": "npm --prefix apps/ai-game-creator-shell run agent-run --", "ai-game-creator-shell:agent-run:smoke": "npm --prefix apps/ai-game-creator-shell run agent-run:smoke", "ai-game-creator-shell:agent-runtime:real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:real-e2e --", + "ai-game-creator-shell:agent-runtime:supervisor-swarm-transient-retry-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:supervisor-swarm-transient-retry-real-e2e --", "ai-game-creator-shell:agent-runtime:steer-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:steer-real-e2e --", "ai-game-creator-shell:agent-runtime:steer-runner-kill-real-e2e": "npm --prefix apps/ai-game-creator-shell run agent-runtime:steer-runner-kill-real-e2e --", "ai-game-creator-shell:typecheck": "npm --prefix apps/ai-game-creator-shell run typecheck",