From 21abd45f8600f6be91b25d6d03898857cea25030 Mon Sep 17 00:00:00 2001 From: AIGameCreator App Date: Sun, 19 Jul 2026 21:18:53 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E5=96=84Agent=20Provider=E6=8C=81?= =?UTF-8?q?=E4=B9=85=E9=87=8D=E8=AF=95=E4=B8=8E=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增按Agent和run隔离的Provider重试sidecar与原子恢复扫描 接入首次工具规划和自动上下文压缩的持久等待与到期续跑 补齐Goal暂停恢复、取消、steer、FIFO、finalization和Runner idle门禁 移除请求指纹中的刷新时间并覆盖跨秒恢复与物理请求一致性 补充sidecar、Runner、并行调度回归及V1.39技术文档 --- .../src-tauri/src/agent.rs | 1275 ++++++++++++++++- .../src-tauri/src/main.rs | 1 + .../src-tauri/src/provider_retry.rs | 809 +++++++++++ .../src-tauri/src/runner.rs | 122 +- .../src-tauri/src/tests.rs | 936 +++++++++++- .../shared-memory/decision-log.md | 8 + docs/project-memory/shared-memory/pitfalls.md | 8 + ...案】AI游戏创作Agent Runtime V1.1-2026-07-12.md | 29 + ...案】AI游戏创作智能体App实施计划-2026-06-24.md | 1 + 9 files changed, 3102 insertions(+), 87 deletions(-) create mode 100644 apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs diff --git a/apps/ai-game-creator-shell/src-tauri/src/agent.rs b/apps/ai-game-creator-shell/src-tauri/src/agent.rs index bc1e974d7..21cb3ec99 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/agent.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/agent.rs @@ -1,4 +1,7 @@ use super::*; +use crate::provider_retry::{ + self, AgentRuntimeProviderRetryIdentity, AgentRuntimeProviderRetryRecord, +}; use sha2::{Digest, Sha256}; use std::collections::BTreeSet; use std::io::{Seek, SeekFrom}; @@ -8,6 +11,9 @@ static GAME_CREATOR_AGENT_RUNTIME_UPDATE_APP_HANDLE: OnceLock static STATIC_DELEGATE_PARENT_WAKE_SINGLEFLIGHT: OnceLock< std::sync::Mutex>, > = OnceLock::new(); +static PROVIDER_RETRY_WAKE_SINGLEFLIGHT: OnceLock< + std::sync::Mutex>, +> = OnceLock::new(); fn external_agent_runner_owns_background_execution() -> bool { external_agent_runner_enabled() && !external_agent_runner_is_server_process() @@ -636,6 +642,142 @@ pub(crate) fn read_game_creator_agent_runtimes_at( .collect() } +fn mark_waiting_provider_retry_needs_reconciliation_at( + root: &Path, + agent_id: &str, + run_id: &str, + error: &str, +) -> Result { + let mut runtime = read_game_creator_agent_runtime_at(root, agent_id)?.state; + if runtime.run_id != run_id + || !matches!( + runtime.phase.as_str(), + "planning" | "waiting-for-provider-retry" + ) + { + return read_game_creator_agent_runtime_at(root, agent_id); + } + let error = redact_agent_runtime_error(root, error, 500); + runtime.status = "failed".to_string(); + runtime.phase = "needs-reconciliation".to_string(); + runtime.current_action = "Provider 重试恢复需要人工核对".to_string(); + runtime.waiting_on = "开发者核对 retry sidecar 与 Provider lifecycle".to_string(); + runtime.next_step = "修复损坏或冲突的持久记录后显式恢复或取消当前 run".to_string(); + runtime.error = Some(error.clone()); + runtime.updated_at = unix_timestamp(); + append_game_creator_agent_runtime_task(root, &runtime)?; + refresh_game_creator_agent_runtime_task_queue(root, &mut runtime)?; + write_game_creator_agent_runtime_state(root, &runtime)?; + append_game_creator_agent_runtime_event( + root, + &runtime, + "provider_request.retry_needs_reconciliation", + "failed", + "needs-reconciliation", + "Provider 重试等待态无法安全恢复，Runtime 已停止自动请求。", + Some(&error), + )?; + let _ = append_agent_db_record( + root, + serde_json::json!({ + "recordType": "agent.runtime.provider_request.retry_needs_reconciliation", + "agentId": runtime.agent_id, + "taskId": runtime.task_id, + "sessionId": runtime.session_id, + "runId": runtime.run_id, + "source": runtime.source, + "error": error, + }), + ); + emit_game_creator_agent_runtime_update(root, agent_id); + read_game_creator_agent_runtime_at(root, agent_id) +} + +fn ensure_waiting_provider_retry_projection_at( + root: &Path, + retry: &AgentRuntimeProviderRetryRecord, +) -> Result, String> { + let agent_id = retry.identity.agent_id.as_str(); + let run_id = retry.identity.run_id.as_str(); + let Some(task) = read_latest_game_creator_agent_runtime_task_by_run_id(root, agent_id, run_id)? + else { + return Err(format!( + "Provider retry sidecar 缺少所属任务：agent={agent_id} runId={run_id}" + )); + }; + if matches!(task.status.as_str(), "completed" | "failed" | "cancelled") { + provider_retry::remove_at(root, agent_id, run_id)?; + return Ok(None); + } + if retry.identity.project_id != game_creator_agent_runtime_context_project_id(root)? + || retry.identity.task_id != task.task_id + || retry.identity.session_id != task.session_id + || retry.identity.source != task.source + || retry.identity.goal_id != task.goal_id + || retry.identity.goal_revision != task.goal_revision + || retry.identity.allow_idle_context_compaction + || !matches!( + retry.identity.request_kind.as_str(), + "tool-plan" | "context-compaction" + ) + { + return mark_waiting_provider_retry_needs_reconciliation_at( + root, + agent_id, + run_id, + "Provider retry sidecar 与所属后台任务身份冲突", + ) + .map(Some); + } + let mut runtime = read_game_creator_agent_runtime_at(root, agent_id)?.state; + if runtime.run_id != run_id { + return Err(format!( + "Provider retry sidecar 不属于当前 Agent run：agent={agent_id} retryRun={run_id} currentRun={}", + runtime.run_id + )); + } + if runtime.phase == "needs-reconciliation" || task.phase == "needs-reconciliation" { + return read_game_creator_agent_runtime_at(root, agent_id).map(Some); + } + if runtime.phase == "waiting-for-provider-retry" && task.phase == "waiting-for-provider-retry" { + ensure_waiting_provider_retry_records_at(root, &runtime, retry)?; + return read_game_creator_agent_runtime_at(root, agent_id).map(Some); + } + if task.status != "running" + || !matches!( + task.phase.as_str(), + "planning" | "waiting-for-provider-retry" + ) + || !matches!( + runtime.phase.as_str(), + "planning" | "waiting-for-provider-retry" + ) + { + return mark_waiting_provider_retry_needs_reconciliation_at( + root, + agent_id, + run_id, + "Provider retry sidecar 的 task/state 投影不在可恢复窗口", + ) + .map(Some); + } + let bundle = read_game_creator_agent_runtime_context_bundle(root, &runtime)? + .ok_or_else(|| "Provider retry 等待投影缺少 Runtime context bundle".to_string())?; + let continuation = continuation_from_game_creator_agent_runtime_context_bundle(bundle); + let context_tracker = AgentRuntimeContextWindowTracker::from_continuation(&continuation); + persist_waiting_provider_retry_context_at( + root, + &mut runtime, + &task.task, + &continuation.plan, + &continuation.observations, + continuation.next_loop_index, + &context_tracker, + retry, + )?; + read_game_creator_agent_runtime_at(root, agent_id).map(Some) +} + pub(crate) fn resume_game_creator_agent_background_tasks_at( root: &Path, ) -> Result, String> { @@ -645,6 +787,21 @@ pub(crate) fn resume_game_creator_agent_background_tasks_at( return read_game_creator_agent_runtimes_at(root); } let agent_ids = collect_game_creator_agent_runtime_agent_ids(root)?; + let retry_records = provider_retry::list_at(root)?; + if retry_records + .iter() + .any(|retry| !agent_ids.contains(&retry.identity.agent_id)) + { + return Err("Provider retry sidecar 指向未知 Agent，Runner 已停止自动恢复".to_string()); + } + let mut retry_records_by_agent = + std::collections::BTreeMap::>::new(); + for retry in retry_records { + retry_records_by_agent + .entry(retry.identity.agent_id.clone()) + .or_default() + .push(retry); + } let reconciliation_records = active_process_session_records_at(root, None, None)? .into_iter() .filter(|record| record.needs_reconciliation || record.status == "needs-reconciliation") @@ -661,6 +818,40 @@ pub(crate) fn resume_game_creator_agent_background_tasks_at( else { continue; }; + let mut retry_projection_blocked = false; + if let Some(retries) = retry_records_by_agent.remove(&agent_id) { + for retry in retries { + let projection = match ensure_waiting_provider_retry_projection_at(root, &retry) { + Ok(projection) => projection, + Err(error) => { + let marked = mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &retry.identity.run_id, + &format!("恢复 Provider retry 等待投影失败：{error}"), + )?; + if marked.state.run_id != retry.identity.run_id + || marked.state.phase != "needs-reconciliation" + { + return Err(format!( + "Provider retry orphan 无法归属当前 Agent run：{error}" + )); + } + Some(marked) + } + }; + if let Some(result) = projection { + if result.state.phase == "needs-reconciliation" { + resumed.push(result); + retry_projection_blocked = true; + break; + } + } + } + } + if retry_projection_blocked { + continue; + } let runtime_lock = match resume_game_creator_agent_finalization_at(root, &agent_id, runtime_lock)? { AgentRuntimeFinalizationResume::Recovered(result, runtime_lock) => { @@ -734,6 +925,91 @@ pub(crate) fn resume_game_creator_agent_background_tasks_at( else { continue; }; + if task.phase == "waiting-for-provider-retry" { + let retry = match provider_retry::read_for_run_at(root, &agent_id, &task.run_id) { + Ok(Some(retry)) => retry, + Ok(None) => { + resumed.push(mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &task.run_id, + "Provider 重试等待态缺少 retry sidecar", + )?); + continue; + } + Err(error) => { + resumed.push(mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &task.run_id, + &format!("读取 Provider retry sidecar 失败：{error}"), + )?); + continue; + } + }; + if retry.identity.task_id != task.task_id + || retry.identity.session_id != task.session_id + || retry.identity.source != task.source + || retry.identity.goal_id != task.goal_id + || retry.identity.goal_revision != task.goal_revision + || retry.identity.allow_idle_context_compaction + || !matches!( + retry.identity.request_kind.as_str(), + "tool-plan" | "context-compaction" + ) + { + resumed.push(mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &task.run_id, + "Provider retry sidecar 与后台任务身份或请求种类冲突", + )?); + continue; + } + let snapshot = + game_creator_agent_runtime_provider_snapshot_from_retry_identity(&retry.identity); + let has_durable_control = { + let _project_lock = + acquire_game_creator_agent_runtime_project_write_lock_with_wait( + root, + "runtime.provider_retry.resume", + )?; + game_creator_agent_runtime_provider_snapshot_has_durable_control_at_locked( + root, &snapshot, + ) + }; + let has_durable_control = match has_durable_control { + Ok(value) => value, + Err(error) => { + resumed.push(mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &task.run_id, + &format!("校验 Provider retry 控制快照失败：{error}"), + )?); + continue; + } + }; + if has_durable_control { + if let Err(error) = provider_retry::remove_at(root, &agent_id, &task.run_id) { + resumed.push(mark_waiting_provider_retry_needs_reconciliation_at( + root, + &agent_id, + &task.run_id, + &format!("作废受控制请求影响的 Provider retry 失败：{error}"), + )?); + continue; + } + } else if provider_retry::remaining_ms(&retry) > 0 { + schedule_waiting_provider_retry_wake_after_lane_release( + root.to_path_buf(), + agent_id.clone(), + task.run_id.clone(), + ); + resumed.push(read_game_creator_agent_runtime_at(root, &agent_id)?); + continue; + } + } if task.phase == "waiting-for-isolated-join" { match isolated_join_completion_barrier_at(root, &agent_id, &task.run_id) { Ok(Some(detail)) if isolated_join_barrier_has_waiting_groups(&detail) => { @@ -2931,7 +3207,8 @@ fn start_game_creator_agent_background_task_with_link_in_session_lane_at( if matches!( result.state.status.as_str(), "waiting-for-confirmation" | "waiting-for-user-input" | "cancelling" - ) || game_creator_agent_runtime_has_reconciliation_barrier(root, &agent_id)? + ) || result.state.phase == "waiting-for-provider-retry" + || game_creator_agent_runtime_has_reconciliation_barrier(root, &agent_id)? { emit_game_creator_agent_runtime_update(root, &agent_id); return read_game_creator_agent_runtime_for_session_at(root, &agent_id, Some(&session_id)) @@ -3545,7 +3822,44 @@ pub(crate) fn resume_game_creator_agent_runtime_for_goal_at( } else { None }; - if pending_action.as_ref().is_some_and(|pending| { + let waiting_provider_retry = + provider_retry::read_for_run_at(root, &state.agent_id, &state.run_id)?; + if let Some(retry) = waiting_provider_retry.as_ref() { + if retry.identity.project_id != game_creator_agent_runtime_context_project_id(root)? + || retry.identity.agent_id != state.agent_id + || retry.identity.task_id != state.task_id + || retry.identity.session_id != state.session_id + || retry.identity.run_id != state.run_id + || retry.identity.source != state.source + || retry.identity.goal_id != state.goal_id + || retry.identity.goal_revision != state.goal_revision + || retry.identity.goal_snapshot_fingerprint != agent_goal_snapshot_fingerprint(goal) + || retry.identity.applied_steer_cursor != state.applied_steer_cursor + || retry.identity.allow_idle_context_compaction + || !matches!( + retry.identity.request_kind.as_str(), + "tool-plan" | "context-compaction" + ) + { + return Err("恢复 Goal 时 Provider retry sidecar 身份已变化".to_string()); + } + if provider_action_batch.is_some() + || pending_action.is_some() + || state.pending_tool_action.is_some() + { + return Err("恢复 Goal 时 Provider retry 与待处理动作状态冲突".to_string()); + } + } + if let Some(retry) = waiting_provider_retry.as_ref() { + state.status = "running".to_string(); + state.phase = "waiting-for-provider-retry".to_string(); + state.current_action = format!( + "Goal 已恢复，继续等待 Provider 瞬态重试 {}/{}", + retry.next_attempt, retry.max_retries + ); + state.waiting_on = format!("Provider {} 瞬态故障退避到期", retry.error_kind); + state.next_step = "到期后恢复同一 Session/run/loop 和 retry attempt".to_string(); + } else if pending_action.as_ref().is_some_and(|pending| { pending.status == AGENT_RUNTIME_PENDING_ACTION_STATUS_WAITING_FOR_USER_INPUT }) { state.status = "waiting-for-user-input".to_string(); @@ -3784,6 +4098,7 @@ pub(crate) fn append_game_creator_agent_runtime_queued_cancellation( remove_game_creator_agent_runtime_pending_tool_action(root, agent_id, &task.run_id)?; remove_game_creator_agent_runtime_provider_action_batch(root, agent_id, &task.run_id)?; remove_game_creator_agent_runtime_confirmations(root, agent_id, &task.run_id)?; + provider_retry::remove_at(root, agent_id, &task.run_id)?; publish_game_creator_agent_delegate_result(root, &cancelled_task, Some(summary)); emit_game_creator_agent_runtime_update(root, agent_id); Ok(()) @@ -6101,11 +6416,68 @@ pub(crate) async fn run_game_creator_agent_background_task_with_context( ); return AgentBackgroundTaskOutcome::WaitingForDelegateReceipts; } + AgentBackgroundTaskOutcome::WaitingForProviderRetry => { + schedule_waiting_provider_retry_wake_after_lane_release( + root, + agent_id, + current_run_id, + ); + return AgentBackgroundTaskOutcome::WaitingForProviderRetry; + } outcome => return outcome, } } } +fn schedule_waiting_provider_retry_wake_after_lane_release( + root: PathBuf, + agent_id: String, + run_id: String, +) { + let key = format!("{}\0{agent_id}\0{run_id}", root.to_string_lossy()); + let singleflight = PROVIDER_RETRY_WAKE_SINGLEFLIGHT + .get_or_init(|| std::sync::Mutex::new(std::collections::BTreeSet::new())); + let Ok(mut active) = singleflight.lock() else { + return; + }; + if !active.insert(key.clone()) { + return; + } + drop(active); + tauri::async_runtime::spawn(async move { + loop { + let task = match read_latest_game_creator_agent_runtime_task_by_run_id( + &root, &agent_id, &run_id, + ) { + Ok(Some(task)) => task, + _ => break, + }; + if task.status != "running" || task.phase != "waiting-for-provider-retry" { + break; + } + let retry = match provider_retry::read_for_run_at(&root, &agent_id, &run_id) { + Ok(Some(retry)) => retry, + Ok(None) | Err(_) => { + let _ = resume_game_creator_agent_background_tasks_at(&root); + break; + } + }; + let remaining_ms = provider_retry::remaining_ms(&retry); + if remaining_ms > 0 { + tokio::time::sleep(Duration::from_millis(remaining_ms)).await; + continue; + } + let _ = resume_game_creator_agent_background_tasks_at(&root); + tokio::time::sleep(Duration::from_millis(25)).await; + } + if let Some(singleflight) = PROVIDER_RETRY_WAKE_SINGLEFLIGHT.get() { + if let Ok(mut active) = singleflight.lock() { + active.remove(&key); + } + } + }); +} + fn schedule_waiting_isolated_join_parent_wake_after_lane_release( root: PathBuf, agent_id: String, @@ -6484,6 +6856,140 @@ fn persist_waiting_isolated_parent_context_at( Ok(()) } +fn persist_waiting_provider_retry_context_at( + root: &Path, + runtime: &mut AgentRuntimeState, + task: &str, + plan: &AgentRuntimeToolPlan, + observations: &[AgentRuntimeToolObservation], + next_loop_index: usize, + context_tracker: &AgentRuntimeContextWindowTracker, + retry: &AgentRuntimeProviderRetryRecord, +) -> Result<(), String> { + if retry.identity.agent_id != runtime.agent_id + || retry.identity.task_id != runtime.task_id + || retry.identity.session_id != runtime.session_id + || retry.identity.run_id != runtime.run_id + || retry.identity.source != runtime.source + || retry.identity.goal_id != runtime.goal_id + || retry.identity.goal_revision != runtime.goal_revision + || retry.identity.applied_steer_cursor != runtime.applied_steer_cursor + { + return Err("Provider 重试等待记录与当前 Runtime 身份不匹配".to_string()); + } + runtime.status = "running".to_string(); + runtime.phase = "waiting-for-provider-retry".to_string(); + runtime.current_action = format!( + "等待 Provider 瞬态重试 {}/{}", + retry.next_attempt, retry.max_retries + ); + runtime.waiting_on = format!("Provider {} 瞬态故障退避到期", retry.error_kind); + runtime.next_step = "到期后自动恢复同一 Session/run/loop，并重新检查控制状态".to_string(); + runtime.error = None; + runtime.updated_at = unix_timestamp(); + persist_game_creator_agent_runtime_context( + root, + runtime, + task, + plan, + observations, + next_loop_index, + context_tracker, + )?; + append_game_creator_agent_runtime_task(root, runtime)?; + refresh_game_creator_agent_runtime_task_queue(root, runtime)?; + write_game_creator_agent_runtime_state(root, runtime)?; + ensure_waiting_provider_retry_records_at(root, runtime, retry)?; + emit_game_creator_agent_runtime_update(root, &runtime.agent_id); + Ok(()) +} + +fn ensure_waiting_provider_retry_records_at( + root: &Path, + runtime: &AgentRuntimeState, + retry: &AgentRuntimeProviderRetryRecord, +) -> Result<(), String> { + let remaining_ms = provider_retry::remaining_ms(retry); + let detail = format!( + "requestKind={} · nextAttempt={} · maxRetries={} · backoffMs={} · remainingMs={} · errorKind={} · errorFingerprint={}", + retry.identity.request_kind, + retry.next_attempt, + retry.max_retries, + retry.backoff_ms, + remaining_ms, + retry.error_kind, + retry.error_fingerprint, + ); + if !game_creator_agent_runtime_event_exists( + root, + &runtime.agent_id, + &runtime.run_id, + "provider_request.retry_waiting", + )? { + append_game_creator_agent_runtime_event( + root, + runtime, + "provider_request.retry_waiting", + "running", + "waiting-for-provider-retry", + "Provider 瞬态失败已闭合，Agent 将在持久退避到期后继续。", + Some(&detail), + )?; + } + let (records, _) = read_agent_db_records_bounded(root, 16 * 1024 * 1024)?; + let existing = records.iter().find(|record| { + record.get("recordType").and_then(serde_json::Value::as_str) + == Some("agent.runtime.provider_request.retry_waiting") + && record.get("agentId").and_then(serde_json::Value::as_str) + == Some(runtime.agent_id.as_str()) + && record.get("runId").and_then(serde_json::Value::as_str) + == Some(runtime.run_id.as_str()) + && record + .get("nextAttempt") + .and_then(serde_json::Value::as_u64) + == Some(retry.next_attempt as u64) + }); + if let Some(existing) = existing { + if existing.get("taskId").and_then(serde_json::Value::as_str) + != Some(runtime.task_id.as_str()) + || existing + .get("sessionId") + .and_then(serde_json::Value::as_str) + != Some(runtime.session_id.as_str()) + || existing + .get("requestKind") + .and_then(serde_json::Value::as_str) + != Some(retry.identity.request_kind.as_str()) + || existing + .get("maxRetries") + .and_then(serde_json::Value::as_u64) + != Some(retry.max_retries as u64) + { + return Err("Provider retry waiting 审计身份冲突".to_string()); + } + } else { + append_agent_db_record( + root, + serde_json::json!({ + "recordType": "agent.runtime.provider_request.retry_waiting", + "agentId": runtime.agent_id, + "taskId": runtime.task_id, + "sessionId": runtime.session_id, + "runId": runtime.run_id, + "source": runtime.source, + "requestKind": retry.identity.request_kind, + "nextAttempt": retry.next_attempt, + "maxRetries": retry.max_retries, + "backoffMs": retry.backoff_ms, + "retryAtMs": retry.retry_at_ms, + "errorKind": retry.error_kind, + "errorFingerprint": retry.error_fingerprint, + }), + )?; + } + Ok(()) +} + async fn run_game_creator_agent_background_task_pass_with_context( root: PathBuf, agent_id: String, @@ -6823,7 +7329,105 @@ async fn run_game_creator_agent_background_task_pass_with_context( ) .await { - Ok(plan) => plan, + Ok(RequestedAgentRuntimeToolPlanOutcome::Ready(plan)) => plan, + Ok(RequestedAgentRuntimeToolPlanOutcome::Waiting(retry)) => { + #[cfg(test)] + { + let injection = + root.join(".agent/runtime/test-stop-after-provider-retry-sidecar"); + if injection.exists() { + let _ = fs::remove_file(injection); + return AgentBackgroundTaskOutcome::WaitingForProviderRetry; + } + } + if let Err(error) = persist_waiting_provider_retry_context_at( + &root, + &mut runtime, + &task, + &plan, + &observations, + loop_index, + &context_tracker, + &retry, + ) { + return fail_game_creator_agent_background_context_at( + &root, + &agent_id, + &session_id, + runtime, + &format!("持久化 Provider 重试等待状态失败：{error}"), + ); + } + return AgentBackgroundTaskOutcome::WaitingForProviderRetry; + } + Ok(RequestedAgentRuntimeToolPlanOutcome::Superseded) => { + let observation = AgentRuntimeToolObservation { + tool: "runtime.provider_retry".to_string(), + status: "blocked".to_string(), + summary: "Provider 重试等待期间请求或配置上下文已变化，旧 attempt 已作废。" + .to_string(), + detail: Some( + "旧 retry sidecar 已删除；当前 run 将使用新的 loop request identity 重新规划。" + .to_string(), + ), + }; + let observation_summary = observation.summary(); + runtime.observations.push(observation_summary.clone()); + context_tracker.record(&observation); + observations.push(observation.clone()); + runtime.status = "running".to_string(); + runtime.phase = "observation".to_string(); + runtime.current_action = "已作废上下文漂移前的 Provider 重试".to_string(); + runtime.waiting_on = "Agent 基于最新上下文重新规划".to_string(); + runtime.next_step = "在同一 Session/run 的下一 loop 建立新请求身份".to_string(); + runtime.error = None; + runtime.updated_at = unix_timestamp(); + let next_loop_index = loop_index.saturating_add(1); + if let Err(error) = persist_game_creator_agent_runtime_context( + &root, + &runtime, + &task, + &plan, + &observations, + next_loop_index, + &context_tracker, + ) + .and_then(|_| append_game_creator_agent_runtime_task(&root, &runtime)) + .and_then(|_| { + refresh_game_creator_agent_runtime_task_queue(&root, &mut runtime) + }) + .and_then(|_| write_game_creator_agent_runtime_state(&root, &runtime)) + .and_then(|_| { + append_game_creator_agent_runtime_event( + &root, + &runtime, + "provider_request.retry_superseded", + "running", + "observation", + &observation_summary, + observation.detail.as_deref(), + ) + }) { + return fail_game_creator_agent_background_context_at( + &root, + &agent_id, + &session_id, + runtime, + &format!("持久化 Provider 重试作废状态失败：{error}"), + ); + } + let continuation = continuation_for_game_creator_agent_runtime_steer( + &runtime, + &plan, + &observations, + next_loop_index, + &context_tracker, + ); + return AgentBackgroundTaskOutcome::ContinueSameRun { + state: runtime, + continuation, + }; + } Err(error) => { if stop_game_creator_agent_runtime_if_cancel_requested(&root, &mut runtime) { return AgentBackgroundTaskOutcome::Finished; @@ -9183,6 +9787,7 @@ pub(crate) enum AgentBackgroundTaskOutcome { Finished, WaitingForConfirmation, WaitingForUserInput, + WaitingForProviderRetry, WaitingForIsolatedJoin, WaitingForDelegateReceipts, NeedsReconciliation, @@ -9193,6 +9798,24 @@ pub(crate) enum AgentBackgroundTaskOutcome { }, } +enum AgentRuntimePersistedProviderRequestOutcome { + Response(Option), + Waiting(AgentRuntimeProviderRetryRecord), + Superseded, +} + +enum AgentRuntimeContextCompactionOutcome { + Completed(Option), + Waiting(AgentRuntimeProviderRetryRecord), + Superseded, +} + +enum RequestedAgentRuntimeToolPlanOutcome { + Ready(Option), + Waiting(AgentRuntimeProviderRetryRecord), + Superseded, +} + #[derive(Debug)] enum AgentRuntimeParallelReadBatchExecution { Executed(AgentRuntimeParallelReadBatch), @@ -10093,7 +10716,7 @@ pub(crate) fn steer_game_creator_agent_runtime_task_at( if accepted_via.is_empty() { return Err("追加指令缺少 acceptedVia".to_string()); } - let _lock = + let project_lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait(root, "runtime.steer")?; let runtime = read_game_creator_agent_runtime_for_session_at(root, &agent_id, Some(session_id))?; @@ -10196,6 +10819,30 @@ pub(crate) fn steer_game_creator_agent_runtime_task_at( latest_status = status.to_string(); } } + let was_waiting_for_provider_retry = state.phase == "waiting-for-provider-retry"; + if was_waiting_for_provider_retry { + provider_retry::remove_at(root, &agent_id, run_id)?; + let mut wake_state = state.clone(); + wake_state.status = "running".to_string(); + wake_state.phase = "planning".to_string(); + wake_state.current_action = "追加指令已作废旧 Provider retry".to_string(); + wake_state.waiting_on = "Agent Runner 应用最新 steer".to_string(); + wake_state.next_step = "在同一 Session/run 使用新 steer cursor 重新规划".to_string(); + wake_state.error = None; + wake_state.updated_at = unix_timestamp(); + append_game_creator_agent_runtime_task(root, &wake_state)?; + refresh_game_creator_agent_runtime_task_queue(root, &mut wake_state)?; + write_game_creator_agent_runtime_state(root, &wake_state)?; + append_game_creator_agent_runtime_event( + root, + &wake_state, + "provider_request.retry_superseded_by_steer", + "running", + "planning", + "运行中追加指令已作废旧 Provider retry，等待同一 run 重新规划。", + Some(&format!("steerId={steer_id}")), + )?; + } let provider_interrupted = interrupt_game_creator_agent_runtime_provider_request_at(root, &agent_id, run_id)?; ensure_game_creator_agent_runtime_steer_audit( @@ -10224,6 +10871,10 @@ pub(crate) fn steer_game_creator_agent_runtime_task_at( )), &steer_id, )?; + drop(project_lock); + if was_waiting_for_provider_retry && !external_agent_runner_owns_background_execution() { + let _ = resume_game_creator_agent_background_tasks_at(root); + } let runtime = read_game_creator_agent_runtime_for_session_at(root, &agent_id, Some(session_id))?; Ok(AgentRuntimeSteerResult { @@ -10482,6 +11133,7 @@ pub(crate) fn consume_game_creator_agent_runtime_steers( } verify_game_creator_agent_runtime_steer_conversation(root, runtime, entry)?; } + provider_retry::remove_at(root, &runtime.agent_id, &runtime.run_id)?; let previous_cursor = runtime.applied_steer_cursor; let mut refs = snapshot .entries @@ -10930,6 +11582,29 @@ fn game_creator_agent_runtime_provider_snapshot_has_durable_control_at_locked( ) } +fn game_creator_agent_runtime_provider_snapshot_is_pausing_at_locked( + root: &Path, + snapshot: &AgentRuntimeProviderRequestSnapshot, +) -> Result { + let Some(goal_id) = snapshot.goal_id.as_deref() else { + return Ok(false); + }; + let goal = read_game_creator_agent_goal_at(root, &snapshot.agent_id, &snapshot.session_id)? + .ok_or_else(|| "Provider 请求绑定的规范 Agent Goal sidecar 缺失".to_string())?; + if goal.project_id != snapshot.project_id + || goal.goal_id != goal_id + || goal.agent_id != snapshot.agent_id + || goal.session_id != snapshot.session_id + || goal.run_id != snapshot.run_id + { + return Err("Provider 请求暂停检查与规范 Agent Goal 身份冲突".to_string()); + } + Ok(matches!( + goal.status.as_str(), + AGENT_GOAL_STATUS_PAUSE_REQUESTED | AGENT_GOAL_STATUS_PAUSED + )) +} + fn append_game_creator_agent_runtime_provider_request_lifecycle( root: &Path, snapshot: &AgentRuntimeProviderRequestSnapshot, @@ -11377,6 +12052,323 @@ fn append_game_creator_agent_runtime_provider_retry_audit( Ok(backoff_ms) } +fn game_creator_agent_runtime_llm_request_fingerprint( + request: &LlmRunRequest, +) -> Result { + let reasoning_effort = request.response_reasoning_effort.map(|value| match value { + platform_llm::LlmResponseReasoningEffort::Low => "low", + platform_llm::LlmResponseReasoningEffort::Medium => "medium", + platform_llm::LlmResponseReasoningEffort::High => "high", + }); + let text_verbosity = request.response_text_verbosity.map(|value| match value { + platform_llm::LlmResponseTextVerbosity::Low => "low", + platform_llm::LlmResponseTextVerbosity::Medium => "medium", + platform_llm::LlmResponseTextVerbosity::High => "high", + }); + let stable = serde_json::json!({ + "model": request.model, + "messages": request.messages, + "maxOutputTokens": request.max_output_tokens, + "enableWebSearch": request.enable_web_search, + "apiKind": request.api_kind, + "requestTimeoutMs": request.request_timeout_ms, + "responseReasoningEffort": reasoning_effort, + "responseTextVerbosity": text_verbosity, + "functionTools": request.function_tools, + "toolChoice": request.tool_choice, + }); + let serialized = serde_json::to_vec(&stable) + .map_err(|error| format!("序列化 Provider 重试请求指纹失败：{error}"))?; + Ok(format!("{:x}", Sha256::digest(serialized))) +} + +fn game_creator_agent_runtime_provider_config_fingerprint( + llm: &GameCreatorLlmConfig, +) -> Result { + let stable = serde_json::json!({ + "apiKeySha256": format!("{:x}", Sha256::digest(llm.api_key.as_bytes())), + "baseUrl": llm.base_url, + "model": llm.model, + "apiKind": llm.api_kind, + "reasoningEffort": llm.reasoning_effort, + "stream": llm.stream, + "webSearchEnabled": llm.web_search_enabled, + "requestTimeoutMs": llm.request_timeout_ms, + "maxRetries": llm.max_retries, + "retryBackoffMs": llm.retry_backoff_ms, + }); + let serialized = serde_json::to_vec(&stable) + .map_err(|error| format!("序列化 Provider 重试配置指纹失败：{error}"))?; + Ok(format!("{:x}", Sha256::digest(serialized))) +} + +fn game_creator_agent_runtime_provider_retry_identity( + snapshot: &AgentRuntimeProviderRequestSnapshot, + llm: &GameCreatorLlmConfig, + request: &LlmRunRequest, +) -> Result { + Ok(AgentRuntimeProviderRetryIdentity { + project_id: snapshot.project_id.clone(), + agent_id: snapshot.agent_id.clone(), + task_id: snapshot.task_id.clone(), + session_id: snapshot.session_id.clone(), + run_id: snapshot.run_id.clone(), + source: snapshot.source.clone(), + goal_id: snapshot.goal_id.clone(), + goal_revision: snapshot.goal_revision, + goal_snapshot_fingerprint: snapshot.goal_snapshot_fingerprint.clone(), + applied_steer_cursor: snapshot.applied_steer_cursor, + request_kind: snapshot.request_kind.clone(), + base_request_slot: snapshot.request_slot.clone(), + request_fingerprint: game_creator_agent_runtime_llm_request_fingerprint(request)?, + provider_config_fingerprint: game_creator_agent_runtime_provider_config_fingerprint(llm)?, + web_search_enabled: snapshot.web_search_enabled, + allow_idle_context_compaction: snapshot.allow_idle_context_compaction, + }) +} + +fn game_creator_agent_runtime_provider_snapshot_from_retry_identity( + identity: &AgentRuntimeProviderRetryIdentity, +) -> AgentRuntimeProviderRequestSnapshot { + AgentRuntimeProviderRequestSnapshot { + project_id: identity.project_id.clone(), + agent_id: identity.agent_id.clone(), + task_id: identity.task_id.clone(), + session_id: identity.session_id.clone(), + run_id: identity.run_id.clone(), + source: identity.source.clone(), + goal_id: identity.goal_id.clone(), + goal_revision: identity.goal_revision, + goal_snapshot_fingerprint: identity.goal_snapshot_fingerprint.clone(), + applied_steer_cursor: identity.applied_steer_cursor, + request_kind: identity.request_kind.clone(), + request_slot: identity.base_request_slot.clone(), + web_search_enabled: identity.web_search_enabled, + allow_idle_context_compaction: identity.allow_idle_context_compaction, + } +} + +fn game_creator_agent_runtime_provider_retry_same_durable_run( + left: &AgentRuntimeProviderRetryIdentity, + right: &AgentRuntimeProviderRetryIdentity, +) -> bool { + left.project_id == right.project_id + && left.agent_id == right.agent_id + && left.task_id == right.task_id + && left.session_id == right.session_id + && left.run_id == right.run_id + && left.source == right.source +} + +async fn request_game_creator_agent_runtime_llm_with_persisted_transient_retry( + root: &Path, + provider_snapshot: &AgentRuntimeProviderRequestSnapshot, + llm: &GameCreatorLlmConfig, + config_path: &str, + operation: &str, + request: &LlmRunRequest, +) -> Result { + let identity = + game_creator_agent_runtime_provider_retry_identity(provider_snapshot, llm, request)?; + let max_retries = llm + .max_retries + .min(AGENT_RUNTIME_PROVIDER_TRANSIENT_RETRY_LIMIT); + let existing = provider_retry::read_for_run_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + let (attempt, request_slot, effective_max_retries) = if let Some(record) = existing.as_ref() { + if record.identity != identity { + if !game_creator_agent_runtime_provider_retry_same_durable_run( + &record.identity, + &identity, + ) { + return Err("Provider 重试记录与当前持久 run 身份冲突".to_string()); + } + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Ok(AgentRuntimePersistedProviderRequestOutcome::Superseded); + } + if provider_retry::remaining_ms(record) > 0 { + return Ok(AgentRuntimePersistedProviderRequestOutcome::Waiting( + record.clone(), + )); + } + ( + record.next_attempt, + record.next_request_slot.clone(), + record.max_retries, + ) + } else { + (0, provider_snapshot.request_slot.clone(), max_retries) + }; + let attempt_snapshot = provider_snapshot.with_request_slot(request_slot); + let client = match build_game_creator_agent_runtime_llm_client(llm, config_path) { + Ok(client) => client, + Err(error) => { + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Err(error); + } + }; + let provider_request = async { + client.run(request.clone()).await.map_err(|error| { + game_creator_agent_runtime_provider_error_with_transient_kind( + &error, + config_path, + operation, + ) + }) + }; + match await_game_creator_agent_runtime_provider_request_with_snapshot( + root, + attempt_snapshot.clone(), + provider_request, + ) + .await + { + Ok(response) => { + let preserve_for_pause = if response.is_none() { + let _control_lock = + acquire_game_creator_agent_runtime_project_write_lock_with_wait( + root, + "runtime.provider_retry.interrupted_control", + )?; + game_creator_agent_runtime_provider_snapshot_is_pausing_at_locked( + root, + &attempt_snapshot, + )? + } else { + false + }; + if preserve_for_pause { + return Ok(AgentRuntimePersistedProviderRequestOutcome::Response( + response, + )); + } + if let Err(error) = provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + ) { + if let Ok(_control_lock) = + acquire_game_creator_agent_runtime_project_write_lock_with_wait( + root, + "runtime.provider_retry.cleanup_reconciliation", + ) + { + let base_request_id = + game_creator_agent_runtime_provider_request_id(&attempt_snapshot); + let request_id = + resolve_game_creator_agent_runtime_provider_request_attempt_at_locked( + root, + &base_request_id, + ) + .map(|value| value.0) + .unwrap_or(base_request_id); + let _ = + mark_game_creator_agent_runtime_provider_request_needs_reconciliation_at_locked( + root, + &attempt_snapshot, + &request_id, + ); + } + return Err(format!( + "{AGENT_RUNTIME_PROVIDER_REQUEST_RECONCILIATION_PREFIX}: Provider 重试终态清理失败：{error}" + )); + } + Ok(AgentRuntimePersistedProviderRequestOutcome::Response( + response, + )) + } + Err(error) => { + let Some(encoded) = error.strip_prefix(AGENT_RUNTIME_PROVIDER_TRANSIENT_ERROR_PREFIX) + else { + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Err(error); + }; + let Some((error_kind, public_error)) = encoded.split_once('\n') else { + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Err("Provider 瞬态错误编码损坏".to_string()); + }; + if attempt >= effective_max_retries { + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Err(public_error.to_string()); + } + let control_lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait( + root, + "runtime.provider_retry.persist_wait", + )?; + let has_durable_control = + game_creator_agent_runtime_provider_snapshot_has_durable_control_at_locked( + root, + provider_snapshot, + )?; + let pausing = has_durable_control + && game_creator_agent_runtime_provider_snapshot_is_pausing_at_locked( + root, + provider_snapshot, + )?; + if has_durable_control && !pausing { + provider_retry::remove_at( + root, + &provider_snapshot.agent_id, + &provider_snapshot.run_id, + )?; + return Ok(AgentRuntimePersistedProviderRequestOutcome::Response(None)); + } + let retry_attempt = attempt.saturating_add(1); + let backoff_ms = append_game_creator_agent_runtime_provider_retry_audit( + root, + provider_snapshot, + &attempt_snapshot, + llm, + error_kind, + public_error, + retry_attempt, + effective_max_retries, + )?; + let record = provider_retry::write_next_at( + root, + &identity, + &format!( + "{}-transient-{retry_attempt}", + provider_snapshot.request_slot + ), + retry_attempt, + effective_max_retries, + backoff_ms, + error_kind, + &format!("{:x}", Sha256::digest(public_error.as_bytes())), + )?; + drop(control_lock); + if pausing { + return Ok(AgentRuntimePersistedProviderRequestOutcome::Response(None)); + } + Ok(AgentRuntimePersistedProviderRequestOutcome::Waiting(record)) + } + } +} + async fn request_game_creator_agent_runtime_llm_with_transient_retries( root: &Path, provider_snapshot: &AgentRuntimeProviderRequestSnapshot, @@ -11754,7 +12746,7 @@ pub(crate) fn sanitize_agent_runtime_context_observation( sanitized } -fn agent_runtime_json_sidecar_backup_path(path: &Path) -> PathBuf { +pub(crate) fn agent_runtime_json_sidecar_backup_path(path: &Path) -> PathBuf { path.with_file_name(format!( ".{}.previous", path.file_name() @@ -11763,7 +12755,10 @@ fn agent_runtime_json_sidecar_backup_path(path: &Path) -> PathBuf { )) } -fn remove_agent_runtime_json_sidecar_backup(path: &Path, label: &str) -> Result<(), String> { +pub(crate) fn remove_agent_runtime_json_sidecar_backup( + path: &Path, + label: &str, +) -> Result<(), String> { match fs::symlink_metadata(path) { Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => { Err(format!("{label} 恢复副本必须是普通文件")) @@ -16585,7 +17580,8 @@ fn agent_runtime_non_verification_completion_blocker_at_locked( agent_id: &str, run_id: &str, ) -> Option { - provider_action_batch_completion_blocker_at_locked(root, agent_id, run_id) + provider_retry_completion_blocker_at_locked(root, agent_id, run_id) + .or_else(|| provider_action_batch_completion_blocker_at_locked(root, agent_id, run_id)) .or_else(|| { supervisor_collaboration_policy_completion_blocker_at_locked(root, agent_id, run_id) }) @@ -16594,6 +17590,34 @@ fn agent_runtime_non_verification_completion_blocker_at_locked( .or_else(|| static_delegate_completion_blocker_at_locked(root, agent_id, run_id)) } +fn provider_retry_completion_blocker_at_locked( + root: &Path, + agent_id: &str, + run_id: &str, +) -> Option { + match provider_retry::read_for_run_at(root, agent_id, run_id) { + Ok(None) => None, + Ok(Some(retry)) => Some(AgentRuntimeToolObservation { + tool: "runtime.provider_retry".to_string(), + status: "blocked".to_string(), + summary: "Provider retry 尚未完成持久等待与请求收束，不能提交最终回复".to_string(), + detail: Some(format!( + "requestKind={} · nextAttempt={} · maxRetries={} · remainingMs={}", + retry.identity.request_kind, + retry.next_attempt, + retry.max_retries, + provider_retry::remaining_ms(&retry), + )), + }), + Err(error) => Some(AgentRuntimeToolObservation { + tool: "runtime.provider_retry".to_string(), + status: AGENT_RUNTIME_TOOL_OBSERVATION_STATUS_NEEDS_RECONCILIATION.to_string(), + summary: "Provider retry sidecar 无法通过校验，不能提交最终回复".to_string(), + detail: Some(redact_agent_runtime_project_paths(root, &error, 500)), + }), + } +} + fn provider_action_batch_completion_blocker_at_locked( root: &Path, agent_id: &str, @@ -18532,7 +19556,8 @@ async fn compact_game_creator_agent_runtime_context_at( estimated_tokens_before: u64, applied_steer_cursor: u64, allow_idle_context_compaction: bool, -) -> Result, String> { + persist_transient_retry: bool, +) -> Result { let (snapshot, source, llm, config_path, request) = { let _lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait( root, @@ -18547,11 +19572,12 @@ async fn compact_game_creator_agent_runtime_context_at( trigger, )?; if !source.has_new_source { - return source + let previous = source .previous .as_ref() .map(|sidecar| Some(context_compaction_result(sidecar, true))) .ok_or_else(|| "当前 Session 没有可压缩的旧上下文".to_string()); + return previous.map(AgentRuntimeContextCompactionOutcome::Completed); } let template_agent_id = game_creator_runtime_template_agent_id_at(root, agent_id)?; let app_config = load_game_creator_app_config()?; @@ -18595,17 +19621,40 @@ async fn compact_game_creator_agent_runtime_context_at( }; (snapshot, source, llm, config_path, request) }; - let Some(response) = request_game_creator_agent_runtime_llm_with_transient_retries( - root, - &snapshot, - &llm, - &config_path, - "上下文压缩", - &request, - ) - .await? - else { - return Ok(None); + let response = if !persist_transient_retry { + AgentRuntimePersistedProviderRequestOutcome::Response( + request_game_creator_agent_runtime_llm_with_transient_retries( + root, + &snapshot, + &llm, + &config_path, + "上下文压缩", + &request, + ) + .await?, + ) + } else { + request_game_creator_agent_runtime_llm_with_persisted_transient_retry( + root, + &snapshot, + &llm, + &config_path, + "上下文压缩", + &request, + ) + .await? + }; + let response = match response { + AgentRuntimePersistedProviderRequestOutcome::Response(Some(response)) => response, + AgentRuntimePersistedProviderRequestOutcome::Response(None) => { + return Ok(AgentRuntimeContextCompactionOutcome::Completed(None)); + } + AgentRuntimePersistedProviderRequestOutcome::Waiting(record) => { + return Ok(AgentRuntimeContextCompactionOutcome::Waiting(record)); + } + AgentRuntimePersistedProviderRequestOutcome::Superseded => { + return Ok(AgentRuntimeContextCompactionOutcome::Superseded); + } }; let control_lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait( @@ -18713,7 +19762,9 @@ async fn compact_game_creator_agent_runtime_context_at( }), ); } - Ok(Some(context_compaction_result(&sidecar, false))) + Ok(AgentRuntimeContextCompactionOutcome::Completed(Some( + context_compaction_result(&sidecar, false), + ))) } pub(crate) async fn compact_game_creator_agent_runtime_session_at( @@ -18764,7 +19815,7 @@ pub(crate) async fn compact_game_creator_agent_runtime_session_at( "manual", )?; let estimated_tokens_before = source.source_prompt_tokens.saturating_add(128); - let result = compact_game_creator_agent_runtime_context_at( + let result = match compact_game_creator_agent_runtime_context_at( root, &agent_id, &session_id, @@ -18774,9 +19825,19 @@ pub(crate) async fn compact_game_creator_agent_runtime_session_at( estimated_tokens_before, runtime.state.applied_steer_cursor, true, + false, ) .await? - .ok_or_else(|| "手动上下文压缩被新的控制指令中断".to_string())?; + { + AgentRuntimeContextCompactionOutcome::Completed(Some(result)) => result, + AgentRuntimeContextCompactionOutcome::Completed(None) => { + return Err("手动上下文压缩被新的控制指令中断".to_string()); + } + AgentRuntimeContextCompactionOutcome::Waiting(_) + | AgentRuntimeContextCompactionOutcome::Superseded => { + return Err("手动上下文压缩不应进入后台 Provider 重试等待".to_string()); + } + }; let mut state = read_game_creator_agent_runtime_for_session_at(root, &agent_id, Some(&session_id))?.state; @@ -18811,7 +19872,7 @@ async fn request_game_creator_agent_background_tool_plan_at( observations: &[AgentRuntimeToolObservation], loop_index: usize, applied_steer_cursor: u64, -) -> Result, String> { +) -> Result { let initial_request_slot = format!("loop-{loop_index}-repair-0"); let mcp_catalog = read_game_creator_mcp_catalog_at(root).await?; let mut built_request = { @@ -18833,7 +19894,7 @@ async fn request_game_creator_agent_background_tool_plan_at( let mut estimated_input_tokens = estimate_game_creator_llm_request_tokens(&built_request.2)?; let mut compaction = None; if estimated_input_tokens > built_request.0.auto_compact_token_limit { - compaction = compact_game_creator_agent_runtime_context_at( + match compact_game_creator_agent_runtime_context_at( root, agent_id, session_id, @@ -18843,10 +19904,22 @@ async fn request_game_creator_agent_background_tool_plan_at( estimated_input_tokens, applied_steer_cursor, false, + true, ) - .await?; - if compaction.is_none() { - return Ok(None); + .await? + { + AgentRuntimeContextCompactionOutcome::Completed(Some(result)) => { + compaction = Some(result); + } + AgentRuntimeContextCompactionOutcome::Completed(None) => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Ready(None)); + } + AgentRuntimeContextCompactionOutcome::Waiting(record) => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Waiting(record)); + } + AgentRuntimeContextCompactionOutcome::Superseded => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Superseded); + } } built_request = { let _lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait( @@ -18924,17 +19997,40 @@ async fn request_game_creator_agent_background_tool_plan_at( let request_snapshot = provider_snapshot .with_request_slot(&request_slot) .with_web_search_enabled(request.enable_web_search); - let Some(response) = request_game_creator_agent_runtime_llm_with_transient_retries( - root, - &request_snapshot, - &llm, - &config_path, - &operation, - &request, - ) - .await? - else { - return Ok(None); + let response = if repair_attempt == 0 { + request_game_creator_agent_runtime_llm_with_persisted_transient_retry( + root, + &request_snapshot, + &llm, + &config_path, + &operation, + &request, + ) + .await? + } else { + AgentRuntimePersistedProviderRequestOutcome::Response( + request_game_creator_agent_runtime_llm_with_transient_retries( + root, + &request_snapshot, + &llm, + &config_path, + &operation, + &request, + ) + .await?, + ) + }; + let response = match response { + AgentRuntimePersistedProviderRequestOutcome::Response(Some(response)) => response, + AgentRuntimePersistedProviderRequestOutcome::Response(None) => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Ready(None)); + } + AgentRuntimePersistedProviderRequestOutcome::Waiting(record) => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Waiting(record)); + } + AgentRuntimePersistedProviderRequestOutcome::Superseded => { + return Ok(RequestedAgentRuntimeToolPlanOutcome::Superseded); + } }; match parse_game_creator_agent_tool_plan_llm_response_with_catalog_classified( &response, @@ -18975,15 +20071,17 @@ async fn request_game_creator_agent_background_tool_plan_at( "responseId": response.response_id, }), )?; - return Ok(Some(RequestedAgentRuntimeToolPlan { - plan, - repository_context_fingerprint, - mcp_catalog_fingerprint: mcp_catalog.fingerprint.clone(), - estimated_input_tokens, - auto_compact_token_limit, - usage: response.usage, - compaction, - })); + return Ok(RequestedAgentRuntimeToolPlanOutcome::Ready(Some( + RequestedAgentRuntimeToolPlan { + plan, + repository_context_fingerprint, + mcp_catalog_fingerprint: mcp_catalog.fingerprint.clone(), + estimated_input_tokens, + auto_compact_token_limit, + usage: response.usage, + compaction, + }, + ))); } Err(error) if error.kind() == AgentRuntimeToolPlanProtocolErrorKind::CatalogBinding => { return Err(error.to_string()); @@ -19839,7 +20937,7 @@ async fn request_game_creator_agent_background_final_reply_at( let mut estimated_input_tokens = estimate_game_creator_llm_request_tokens(&built_request.2)?; let mut compaction = None; if estimated_input_tokens > built_request.0.auto_compact_token_limit { - compaction = compact_game_creator_agent_runtime_context_at( + match compact_game_creator_agent_runtime_context_at( root, agent_id, session_id, @@ -19849,10 +20947,18 @@ async fn request_game_creator_agent_background_final_reply_at( estimated_input_tokens, applied_steer_cursor, false, + false, ) - .await?; - if compaction.is_none() { - return Ok(None); + .await? + { + AgentRuntimeContextCompactionOutcome::Completed(Some(result)) => { + compaction = Some(result); + } + AgentRuntimeContextCompactionOutcome::Completed(None) => return Ok(None), + AgentRuntimeContextCompactionOutcome::Waiting(_) + | AgentRuntimeContextCompactionOutcome::Superseded => { + return Err("final-reply 上下文压缩不应进入后台 Provider 重试等待".to_string()); + } } built_request = { let _lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait( @@ -20084,8 +21190,13 @@ fn build_game_creator_agent_background_tool_plan_request( .map_err(|error| format!("序列化 Agent 工具观察失败：{error}"))? }; let tool_policy = agent_runtime_tool_policy_snapshot_at(root, agent_id)?; - let tool_policy_json = serde_json::to_string_pretty(&tool_policy) - .map_err(|error| format!("序列化 Agent 工具策略失败：{error}"))?; + let tool_policy_json = serde_json::to_string_pretty(&serde_json::json!({ + "allowedTools": tool_policy.allowed_tools, + "autoTools": tool_policy.auto_tools, + "confirmTools": tool_policy.confirm_tools, + "deniedTools": tool_policy.denied_tools, + })) + .map_err(|error| format!("序列化 Agent 工具策略失败：{error}"))?; let collaboration_policy_json = if agent_id == GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID { render_supervisor_collaboration_policy_for_prompt_at(root, agent_id, run_id)? } else { @@ -32242,6 +33353,7 @@ pub(crate) fn finish_game_creator_agent_runtime_turn_at( remove_game_creator_agent_runtime_pending_tool_action(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_provider_action_batch(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_confirmations(root, &state.agent_id, &state.run_id)?; + provider_retry::remove_at(root, &state.agent_id, &state.run_id)?; publish_game_creator_agent_delegate_result_for_state( root, &state, @@ -32488,6 +33600,7 @@ fn finish_game_creator_agent_background_runtime_turn_idempotently_at( &completed.run_id, )?; remove_game_creator_agent_runtime_confirmations(root, &completed.agent_id, &completed.run_id)?; + provider_retry::remove_at(root, &completed.agent_id, &completed.run_id)?; publish_game_creator_agent_delegate_result_for_state( root, &completed, @@ -32980,6 +34093,7 @@ pub(crate) fn fail_game_creator_agent_runtime_turn_at( remove_game_creator_agent_runtime_pending_tool_action(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_provider_action_batch(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_confirmations(root, &state.agent_id, &state.run_id)?; + provider_retry::remove_at(root, &state.agent_id, &state.run_id)?; publish_game_creator_agent_delegate_result_for_state(root, &state, state.error.as_deref()); Ok(state) } @@ -33030,6 +34144,7 @@ pub(crate) fn fail_game_creator_agent_runtime_budget_at( remove_game_creator_agent_runtime_pending_tool_action(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_provider_action_batch(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_confirmations(root, &state.agent_id, &state.run_id)?; + provider_retry::remove_at(root, &state.agent_id, &state.run_id)?; publish_game_creator_agent_delegate_result_for_state(root, &state, state.error.as_deref()); Ok(state) } @@ -33343,6 +34458,7 @@ fn agent_runtime_next_step_for_phase(phase: &str) -> &'static str { "action" => "等待工具观察结果", "waiting-for-confirmation" => "等待开发者确认工具动作", "waiting-for-user-input" => "等待用户提交全部澄清回答", + "waiting-for-provider-retry" => "等待持久退避到期后恢复同一 Provider 请求", "response" => "等待 Agent 整理最终回复", "completed" | "idle" => "等待下一轮输入", "cancelled" => "可重试该后台任务或提交新任务", @@ -33359,6 +34475,7 @@ fn agent_runtime_waiting_on_for_phase(phase: &str) -> &'static str { "action" => "工具观察结果", "waiting-for-confirmation" => "开发者确认 Agent 工具动作", "waiting-for-user-input" => "用户回答 Agent 的澄清问题", + "waiting-for-provider-retry" => "Provider 瞬态重试退避到期", "response" => "Agent 整理最终回复", "completed" | "idle" => "开发者下一轮输入", "cancelled" => "开发者下一轮输入", @@ -34387,6 +35504,7 @@ fn mark_game_creator_agent_runtime_cancelled_at_locked( remove_game_creator_agent_runtime_pending_tool_action(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_provider_action_batch(root, &state.agent_id, &state.run_id)?; remove_game_creator_agent_runtime_confirmations(root, &state.agent_id, &state.run_id)?; + provider_retry::remove_at(root, &state.agent_id, &state.run_id)?; if let Some(goal) = mark_game_creator_agent_goal_cleared_for_runtime_at_locked(root, state)? { state.goal_status = Some(goal.status); state.updated_at = unix_timestamp(); @@ -34746,6 +35864,9 @@ fn read_next_pending_game_creator_agent_runtime_task( let path = game_creator_agent_runtime_task_path(root, agent_id); let records = latest_game_creator_agent_runtime_tasks(read_all_game_creator_agent_runtime_tasks(&path)?); + if records.iter().any(|record| record.status == "running") { + return Ok(None); + } Ok(records .into_iter() .find(|record| record.status == "pending")) @@ -35279,11 +36400,41 @@ fn render_agent_runtime_prompt_context_for_session( return Ok(String::new()); } + let waiting_provider_retry = state.phase == "waiting-for-provider-retry"; + let prompt_status = if waiting_provider_retry { + "running" + } else { + state.status.as_str() + }; + let prompt_phase = if waiting_provider_retry { + "planning" + } else { + state.phase.as_str() + }; + let prompt_action = waiting_provider_retry.then(|| { + format!( + "生成 Agent 工具计划（第 {} 轮）", + state.loop_iteration.max(1) + ) + }); + let prompt_action = prompt_action + .as_deref() + .unwrap_or(state.current_action.as_str()); + let prompt_waiting_on = if waiting_provider_retry { + agent_runtime_waiting_on_for_phase("planning") + } else { + state.waiting_on.as_str() + }; + let prompt_next_step = if waiting_provider_retry { + agent_runtime_next_step_for_phase("planning") + } else { + state.next_step.as_str() + }; let mut lines = Vec::new(); lines.push(format!( "当前状态：status={}, phase={}, source={}, runId={}", - redact_agent_runtime_project_paths(root, &state.status, 80), - redact_agent_runtime_project_paths(root, &state.phase, 80), + redact_agent_runtime_project_paths(root, prompt_status, 80), + redact_agent_runtime_project_paths(root, prompt_phase, 80), redact_agent_runtime_project_paths(root, &state.source, 120), redact_agent_runtime_project_paths(root, &state.run_id, 160) )); @@ -35299,22 +36450,22 @@ fn render_agent_runtime_prompt_context_for_session( redact_agent_runtime_project_paths(root, &state.current_goal, 220) )); } - if !state.current_action.trim().is_empty() { + if !prompt_action.trim().is_empty() { lines.push(format!( "当前动作：{}", - redact_agent_runtime_project_paths(root, &state.current_action, 160) + redact_agent_runtime_project_paths(root, prompt_action, 160) )); } - if !state.waiting_on.trim().is_empty() { + if !prompt_waiting_on.trim().is_empty() { lines.push(format!( "等待：{}", - redact_agent_runtime_project_paths(root, &state.waiting_on, 160) + redact_agent_runtime_project_paths(root, prompt_waiting_on, 160) )); } - if !state.next_step.trim().is_empty() { + if !prompt_next_step.trim().is_empty() { lines.push(format!( "下一步：{}", - redact_agent_runtime_project_paths(root, &state.next_step, 160) + redact_agent_runtime_project_paths(root, prompt_next_step, 160) )); } if state.loop_iteration > 0 { diff --git a/apps/ai-game-creator-shell/src-tauri/src/main.rs b/apps/ai-game-creator-shell/src-tauri/src/main.rs index 1392549b6..6250d0070 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/main.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/main.rs @@ -66,6 +66,7 @@ mod preview; mod process_session; mod process_session_bridge; mod project; +mod provider_retry; mod repository_context; mod runner; mod swarm_cli; diff --git a/apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs b/apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs new file mode 100644 index 000000000..ae7b5ee4b --- /dev/null +++ b/apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs @@ -0,0 +1,809 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::agent::{ + agent_runtime_json_sidecar_backup_path, read_agent_runtime_json_sidecar_with_max_bytes, + remove_agent_runtime_json_sidecar_backup, write_agent_runtime_json_sidecar_with_max_bytes, +}; + +pub(crate) const PROVIDER_RETRY_SCHEMA_VERSION: &str = "game-creator-provider-retry.v1"; + +const PROVIDER_RETRY_RELATIVE_DIRECTORY: &str = ".agent/runtime/provider-retries"; +const PROVIDER_RETRY_SIDECAR_MAX_BYTES: usize = 32 * 1024; +const PROVIDER_RETRY_LABEL: &str = "Agent Runtime Provider 重试记录"; +const SAFE_PATH_COMPONENT_MAX_CHARS: usize = 160; +const HASHED_PATH_PREFIX_MAX_CHARS: usize = 80; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct AgentRuntimeProviderRetryIdentity { + pub(crate) project_id: String, + pub(crate) agent_id: String, + pub(crate) task_id: String, + pub(crate) session_id: String, + pub(crate) run_id: String, + pub(crate) source: String, + pub(crate) goal_id: Option, + pub(crate) goal_revision: u64, + pub(crate) goal_snapshot_fingerprint: String, + pub(crate) applied_steer_cursor: u64, + pub(crate) request_kind: String, + pub(crate) base_request_slot: String, + pub(crate) request_fingerprint: String, + pub(crate) provider_config_fingerprint: String, + pub(crate) web_search_enabled: bool, + pub(crate) allow_idle_context_compaction: bool, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub(crate) struct AgentRuntimeProviderRetryRecord { + pub(crate) schema_version: String, + pub(crate) identity: AgentRuntimeProviderRetryIdentity, + pub(crate) next_request_slot: String, + pub(crate) next_attempt: u32, + pub(crate) max_retries: u32, + pub(crate) backoff_ms: u64, + pub(crate) retry_at_ms: u64, + pub(crate) error_kind: String, + pub(crate) error_fingerprint: String, + pub(crate) created_at_ms: u64, + pub(crate) updated_at_ms: u64, +} + +pub(crate) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) +} + +pub(crate) fn remaining_ms(record: &AgentRuntimeProviderRetryRecord) -> u64 { + remaining_ms_at(record.retry_at_ms, now_ms()) +} + +pub(crate) fn read_for_run_at( + root: &Path, + agent_id: &str, + run_id: &str, +) -> Result, String> { + validate_path_identity(agent_id, run_id)?; + let relative_path = provider_retry_relative_path(agent_id, run_id); + let Some(record) = read_agent_runtime_json_sidecar_with_max_bytes( + root, + &relative_path, + PROVIDER_RETRY_LABEL, + PROVIDER_RETRY_SIDECAR_MAX_BYTES, + )? + else { + return Ok(None); + }; + validate_record(&record)?; + if record.identity.agent_id != agent_id || record.identity.run_id != run_id { + return Err("Provider 重试记录与路径 Agent/run 身份冲突".to_string()); + } + Ok(Some(record)) +} + +pub(crate) fn read_matching_at( + root: &Path, + identity: &AgentRuntimeProviderRetryIdentity, +) -> Result, String> { + validate_identity(identity)?; + let Some(record) = read_for_run_at(root, &identity.agent_id, &identity.run_id)? else { + return Ok(None); + }; + if record.identity != *identity { + return Err("Provider 重试记录身份冲突".to_string()); + } + Ok(Some(record)) +} + +pub(crate) fn list_at(root: &Path) -> Result, String> { + let directory = root.join(PROVIDER_RETRY_RELATIVE_DIRECTORY); + let agent_entries = match fs::read_dir(&directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => { + return Err(format!( + "读取 Provider 重试目录失败：{}: {error}", + directory.display() + )); + } + }; + let mut primary_paths = BTreeSet::new(); + for (agent_index, agent_entry) in agent_entries.enumerate() { + if agent_index >= 1024 { + return Err("Provider 重试目录超过 1024 个 Agent 上限".to_string()); + } + let agent_entry = agent_entry.map_err(|error| { + format!( + "读取 Provider 重试 Agent 目录项失败：{}: {error}", + directory.display() + ) + })?; + let agent_type = agent_entry.file_type().map_err(|error| { + format!( + "读取 Provider 重试 Agent 目录类型失败：{}: {error}", + agent_entry.path().display() + ) + })?; + if agent_type.is_symlink() || !agent_type.is_dir() { + return Err(format!( + "Provider 重试 Agent 项必须是普通目录：{}", + agent_entry.path().display() + )); + } + let run_entries = fs::read_dir(agent_entry.path()).map_err(|error| { + format!( + "读取 Provider 重试 run 目录失败：{}: {error}", + agent_entry.path().display() + ) + })?; + for (run_index, run_entry) in run_entries.enumerate() { + if run_index >= 1024 || primary_paths.len() >= 1024 { + return Err("Provider 重试记录超过 1024 条上限".to_string()); + } + let run_entry = run_entry.map_err(|error| { + format!( + "读取 Provider 重试 run 目录项失败：{}: {error}", + agent_entry.path().display() + ) + })?; + let run_type = run_entry.file_type().map_err(|error| { + format!( + "读取 Provider 重试 run 项类型失败：{}: {error}", + run_entry.path().display() + ) + })?; + if run_type.is_symlink() || !run_type.is_file() { + return Err(format!( + "Provider 重试 run 项必须是普通文件：{}", + run_entry.path().display() + )); + } + let file_name = run_entry + .file_name() + .into_string() + .map_err(|_| "Provider 重试文件名必须是 UTF-8".to_string())?; + let primary_path = if file_name.ends_with(".json") { + run_entry.path() + } else if let Some(primary_name) = file_name + .strip_prefix('.') + .and_then(|value| value.strip_suffix(".previous")) + .filter(|value| value.ends_with(".json")) + { + run_entry.path().with_file_name(primary_name) + } else { + return Err(format!("Provider 重试目录包含未知文件：{file_name}")); + }; + primary_paths.insert(primary_path); + } + } + + let mut records = Vec::with_capacity(primary_paths.len()); + for primary_path in primary_paths { + let relative_path = primary_path.strip_prefix(root).map_err(|_| { + format!( + "Provider 重试文件不在项目目录内：{}", + primary_path.display() + ) + })?; + let relative_path_text = relative_path + .to_str() + .ok_or_else(|| "Provider 重试相对路径必须是 UTF-8".to_string())?; + let record = read_agent_runtime_json_sidecar_with_max_bytes( + root, + relative_path_text, + PROVIDER_RETRY_LABEL, + PROVIDER_RETRY_SIDECAR_MAX_BYTES, + )? + .ok_or_else(|| "Provider 重试目录项在扫描期间消失".to_string())?; + validate_record(&record)?; + let expected = PathBuf::from(provider_retry_relative_path( + &record.identity.agent_id, + &record.identity.run_id, + )); + if relative_path != expected { + return Err(format!( + "Provider 重试记录身份与路径冲突：{}", + relative_path.display() + )); + } + records.push(record); + } + records.sort_by(|left, right| { + left.identity + .agent_id + .cmp(&right.identity.agent_id) + .then_with(|| left.identity.run_id.cmp(&right.identity.run_id)) + }); + Ok(records) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn write_next_at( + root: &Path, + identity: &AgentRuntimeProviderRetryIdentity, + next_request_slot: &str, + next_attempt: u32, + max_retries: u32, + backoff_ms: u64, + error_kind: &str, + error_fingerprint: &str, +) -> Result { + validate_identity(identity)?; + let existing = read_for_run_at(root, &identity.agent_id, &identity.run_id)?; + let created_at_ms; + let minimum_updated_at_ms; + if let Some(existing) = existing.as_ref() { + if existing.identity != *identity { + return Err("Provider 重试记录身份冲突".to_string()); + } + if existing.next_attempt == next_attempt { + if retry_payload_matches( + existing, + next_request_slot, + max_retries, + backoff_ms, + error_kind, + error_fingerprint, + ) { + return Ok(existing.clone()); + } + return Err("Provider 重试记录同一 attempt 的内容冲突".to_string()); + } + let expected_attempt = existing + .next_attempt + .checked_add(1) + .ok_or_else(|| "Provider 重试 attempt 溢出".to_string())?; + if next_attempt != expected_attempt || max_retries != existing.max_retries { + return Err("Provider 重试记录 attempt 或 maxRetries 冲突".to_string()); + } + created_at_ms = existing.created_at_ms; + minimum_updated_at_ms = existing.updated_at_ms; + } else { + if next_attempt != 1 { + return Err("首条 Provider 重试记录的 nextAttempt 必须为 1".to_string()); + } + created_at_ms = now_ms(); + minimum_updated_at_ms = created_at_ms; + } + + let updated_at_ms = now_ms().max(created_at_ms).max(minimum_updated_at_ms); + let retry_at_ms = updated_at_ms + .checked_add(backoff_ms) + .ok_or_else(|| "Provider 重试到期时间溢出".to_string())?; + let record = AgentRuntimeProviderRetryRecord { + schema_version: PROVIDER_RETRY_SCHEMA_VERSION.to_string(), + identity: identity.clone(), + next_request_slot: next_request_slot.to_string(), + next_attempt, + max_retries, + backoff_ms, + retry_at_ms, + error_kind: error_kind.to_string(), + error_fingerprint: error_fingerprint.to_string(), + created_at_ms, + updated_at_ms, + }; + validate_record(&record)?; + let relative_path = provider_retry_relative_path(&identity.agent_id, &identity.run_id); + write_agent_runtime_json_sidecar_with_max_bytes( + root, + &relative_path, + PROVIDER_RETRY_LABEL, + &record, + PROVIDER_RETRY_SIDECAR_MAX_BYTES, + )?; + let persisted = read_matching_at(root, identity)? + .ok_or_else(|| "Provider 重试记录写入后不存在".to_string())?; + if persisted != record { + return Err("Provider 重试记录并发写入后内容冲突".to_string()); + } + Ok(persisted) +} + +pub(crate) fn remove_at(root: &Path, agent_id: &str, run_id: &str) -> Result<(), String> { + validate_path_identity(agent_id, run_id)?; + let path = provider_retry_path(root, agent_id, run_id); + let backup_path = agent_runtime_json_sidecar_backup_path(&path); + remove_agent_runtime_json_sidecar_backup(&backup_path, PROVIDER_RETRY_LABEL)?; + remove_agent_runtime_json_sidecar_backup(&path, PROVIDER_RETRY_LABEL) +} + +#[cfg(test)] +pub(crate) fn force_provider_retry_due_for_test_at( + root: &Path, + identity: &AgentRuntimeProviderRetryIdentity, +) -> Result { + let mut record = read_matching_at(root, identity)? + .ok_or_else(|| "待强制到期的 Provider 重试记录不存在".to_string())?; + record.retry_at_ms = record.created_at_ms; + record.updated_at_ms = now_ms().max(record.updated_at_ms); + validate_record(&record)?; + let relative_path = provider_retry_relative_path(&identity.agent_id, &identity.run_id); + write_agent_runtime_json_sidecar_with_max_bytes( + root, + &relative_path, + PROVIDER_RETRY_LABEL, + &record, + PROVIDER_RETRY_SIDECAR_MAX_BYTES, + )?; + let persisted = read_matching_at(root, identity)? + .ok_or_else(|| "Provider 重试记录强制到期后不存在".to_string())?; + if persisted != record { + return Err("Provider 重试记录强制到期后内容冲突".to_string()); + } + Ok(persisted) +} + +fn validate_path_identity(agent_id: &str, run_id: &str) -> Result<(), String> { + if agent_id.trim().is_empty() || run_id.trim().is_empty() { + return Err("Provider 重试路径的 Agent/run 身份不能为空".to_string()); + } + Ok(()) +} + +fn validate_identity(identity: &AgentRuntimeProviderRetryIdentity) -> Result<(), String> { + validate_path_identity(&identity.agent_id, &identity.run_id)?; + for (label, value) in [ + ("projectId", identity.project_id.as_str()), + ("taskId", identity.task_id.as_str()), + ("sessionId", identity.session_id.as_str()), + ("source", identity.source.as_str()), + ("requestKind", identity.request_kind.as_str()), + ("baseRequestSlot", identity.base_request_slot.as_str()), + ] { + if value.trim().is_empty() { + return Err(format!("Provider 重试身份 {label} 不能为空")); + } + } + if !is_sha256(&identity.request_fingerprint) { + return Err("Provider 重试身份的 requestFingerprint 无效".to_string()); + } + if !is_sha256(&identity.provider_config_fingerprint) { + return Err("Provider 重试身份的 providerConfigFingerprint 无效".to_string()); + } + match identity.goal_id.as_deref() { + Some(goal_id) + if !goal_id.trim().is_empty() + && identity.goal_revision > 0 + && is_sha256(&identity.goal_snapshot_fingerprint) => {} + None if identity.goal_revision == 0 && identity.goal_snapshot_fingerprint.is_empty() => {} + _ => return Err("Provider 重试身份的 Goal 绑定无效".to_string()), + } + Ok(()) +} + +fn validate_record(record: &AgentRuntimeProviderRetryRecord) -> Result<(), String> { + if record.schema_version != PROVIDER_RETRY_SCHEMA_VERSION { + return Err(format!( + "不支持的 Provider 重试记录版本：{}", + record.schema_version + )); + } + validate_identity(&record.identity)?; + if record.next_attempt == 0 + || record.max_retries == 0 + || record.next_attempt > record.max_retries + { + return Err("Provider 重试记录的 attempt/maxRetries 无效".to_string()); + } + let expected_slot = format!( + "{}-transient-{}", + record.identity.base_request_slot, record.next_attempt + ); + if record.next_request_slot != expected_slot { + return Err("Provider 重试记录的 nextRequestSlot 与身份不匹配".to_string()); + } + if record.backoff_ms == 0 + || record.created_at_ms == 0 + || record.updated_at_ms < record.created_at_ms + || record.retry_at_ms < record.created_at_ms + { + return Err("Provider 重试记录的退避或时间字段无效".to_string()); + } + if record.error_kind.trim().is_empty() || !is_sha256(&record.error_fingerprint) { + return Err("Provider 重试记录的错误身份无效".to_string()); + } + Ok(()) +} + +fn retry_payload_matches( + record: &AgentRuntimeProviderRetryRecord, + next_request_slot: &str, + max_retries: u32, + backoff_ms: u64, + error_kind: &str, + error_fingerprint: &str, +) -> bool { + record.next_request_slot == next_request_slot + && record.max_retries == max_retries + && record.backoff_ms == backoff_ms + && record.error_kind == error_kind + && record.error_fingerprint == error_fingerprint +} + +fn is_sha256(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn remaining_ms_at(retry_at_ms: u64, at_ms: u64) -> u64 { + retry_at_ms.saturating_sub(at_ms) +} + +fn provider_retry_relative_path(agent_id: &str, run_id: &str) -> String { + format!( + "{PROVIDER_RETRY_RELATIVE_DIRECTORY}/{}/{}.json", + provider_retry_path_component(agent_id, "agent"), + provider_retry_path_component(run_id, "run") + ) +} + +fn provider_retry_path(root: &Path, agent_id: &str, run_id: &str) -> PathBuf { + root.join(provider_retry_relative_path(agent_id, run_id)) +} + +fn provider_retry_path_component(value: &str, fallback: &str) -> String { + if is_safe_path_component(value) { + return value.to_string(); + } + let readable = readable_path_prefix(value, fallback); + format!("{readable}--{:x}", Sha256::digest(value.as_bytes())) +} + +fn is_safe_path_component(value: &str) -> bool { + if value.is_empty() + || value.trim() != value + || value == "." + || value == ".." + || value.chars().count() > SAFE_PATH_COMPONENT_MAX_CHARS + || value.ends_with('.') + || !value.chars().all(|character| { + character.is_ascii_alphanumeric() + || character == '-' + || character == '_' + || character == '.' + }) + { + return false; + } + !is_windows_reserved_component(value) +} + +fn readable_path_prefix(value: &str, fallback: &str) -> String { + let normalized = value + .trim() + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() + || character == '-' + || character == '_' + || character == '.' + { + character + } else { + '-' + } + }) + .take(HASHED_PATH_PREFIX_MAX_CHARS) + .collect::(); + let normalized = normalized.trim_matches(|character| character == '-' || character == '.'); + if normalized.is_empty() || is_windows_reserved_component(normalized) { + fallback.to_string() + } else { + normalized.to_string() + } +} + +fn is_windows_reserved_component(value: &str) -> bool { + let stem = value + .split('.') + .next() + .unwrap_or_default() + .to_ascii_uppercase(); + matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && matches!(&stem[..3], "COM" | "LPT") + && matches!(stem.as_bytes()[3], b'1'..=b'9')) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use tempfile::tempdir; + + use super::*; + + fn identity(agent_id: &str, run_id: &str) -> AgentRuntimeProviderRetryIdentity { + AgentRuntimeProviderRetryIdentity { + project_id: "project-provider-retry".to_string(), + agent_id: agent_id.to_string(), + task_id: "task-provider-retry".to_string(), + session_id: "session-provider-retry".to_string(), + run_id: run_id.to_string(), + source: "agent-chat".to_string(), + goal_id: Some("goal-provider-retry".to_string()), + goal_revision: 3, + goal_snapshot_fingerprint: "a".repeat(64), + applied_steer_cursor: 2, + request_kind: "tool-plan".to_string(), + base_request_slot: "loop-2-repair-0".to_string(), + request_fingerprint: "d".repeat(64), + provider_config_fingerprint: "e".repeat(64), + web_search_enabled: true, + allow_idle_context_compaction: false, + } + } + + fn write_first( + root: &Path, + identity: &AgentRuntimeProviderRetryIdentity, + ) -> AgentRuntimeProviderRetryRecord { + write_next_at( + root, + identity, + "loop-2-repair-0-transient-1", + 1, + 3, + 250, + "transport", + &"b".repeat(64), + ) + .expect("write first Provider retry") + } + + #[test] + fn provider_retry_round_trips_and_same_attempt_is_idempotent() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("design-director", "run-1"); + let first = write_first(directory.path(), &identity); + let read = read_matching_at(directory.path(), &identity) + .expect("read matching Provider retry") + .expect("Provider retry exists"); + assert_eq!(read, first); + assert_eq!( + list_at(directory.path()).expect("list Provider retries"), + vec![first.clone()] + ); + let repeated = write_first(directory.path(), &identity); + assert_eq!(repeated, first); + + let second = write_next_at( + directory.path(), + &identity, + "loop-2-repair-0-transient-2", + 2, + 3, + 500, + "timeout", + &"c".repeat(64), + ) + .expect("advance Provider retry"); + assert_eq!(second.next_attempt, 2); + assert_eq!(second.created_at_ms, first.created_at_ms); + assert!(second.updated_at_ms >= first.updated_at_ms); + + let serialized = fs::read_to_string(provider_retry_path( + directory.path(), + &identity.agent_id, + &identity.run_id, + )) + .expect("read serialized Provider retry"); + assert!(serialized.contains(PROVIDER_RETRY_SCHEMA_VERSION)); + assert!(serialized.contains("\"baseRequestSlot\"")); + assert!(serialized.contains("\"allowIdleContextCompaction\"")); + + let mut unexpected = serde_json::to_value(&second).expect("serialize Provider retry"); + unexpected + .as_object_mut() + .expect("Provider retry object") + .insert("unexpected".to_string(), serde_json::json!(true)); + fs::write( + provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id), + serde_json::to_vec_pretty(&unexpected).expect("serialize unexpected Provider retry"), + ) + .expect("write unexpected Provider retry"); + let error = read_for_run_at(directory.path(), &identity.agent_id, &identity.run_id) + .expect_err("unknown Provider retry fields must fail"); + assert!(error.contains("unknown field")); + } + + #[test] + fn provider_retry_reads_atomic_previous_when_primary_is_missing() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("quality-lead", "run-previous"); + let record = write_first(directory.path(), &identity); + let path = provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id); + let backup_path = agent_runtime_json_sidecar_backup_path(&path); + fs::rename(&path, &backup_path).expect("move Provider retry to previous"); + + assert!(!path.exists()); + assert_eq!( + read_for_run_at(directory.path(), &identity.agent_id, &identity.run_id) + .expect("read previous Provider retry"), + Some(record) + ); + } + + #[test] + fn provider_retry_list_sorts_across_agents_and_runs() { + let directory = tempdir().expect("create temp directory"); + let identities = [ + identity("quality-lead", "run-2"), + identity("design-director", "run-2"), + identity("quality-lead", "run-1"), + identity("design-director", "run-1"), + ]; + for identity in &identities { + write_first(directory.path(), identity); + } + + let listed = list_at(directory.path()).expect("list sorted Provider retries"); + let listed_keys = listed + .iter() + .map(|record| { + ( + record.identity.agent_id.as_str(), + record.identity.run_id.as_str(), + ) + }) + .collect::>(); + assert_eq!( + listed_keys, + vec![ + ("design-director", "run-1"), + ("design-director", "run-2"), + ("quality-lead", "run-1"), + ("quality-lead", "run-2"), + ] + ); + } + + #[test] + fn provider_retry_list_recovers_when_only_atomic_previous_exists() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("quality-lead", "run-list-previous"); + let record = write_first(directory.path(), &identity); + let path = provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id); + let backup_path = agent_runtime_json_sidecar_backup_path(&path); + fs::rename(&path, &backup_path).expect("move Provider retry to previous"); + + assert_eq!( + list_at(directory.path()).expect("list previous Provider retry"), + vec![record] + ); + } + + #[test] + fn provider_retry_list_deduplicates_primary_and_atomic_previous() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("code-prototype", "run-list-deduplicate"); + let record = write_first(directory.path(), &identity); + let path = provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id); + let backup_path = agent_runtime_json_sidecar_backup_path(&path); + fs::copy(&path, &backup_path).expect("copy Provider retry to previous"); + + assert_eq!( + list_at(directory.path()).expect("list deduplicated Provider retry"), + vec![record] + ); + } + + #[test] + fn provider_retry_list_rejects_unknown_directory_entries() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("code-prototype", "run-list-unknown"); + write_first(directory.path(), &identity); + let agent_directory = + provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id) + .parent() + .expect("Provider retry Agent directory") + .to_path_buf(); + fs::write(agent_directory.join("unexpected.txt"), b"unexpected") + .expect("write unknown Provider retry entry"); + + let error = list_at(directory.path()).expect_err("unknown entries must fail closed"); + assert!(error.contains("Provider 重试目录包含未知文件：unexpected.txt")); + } + + #[test] + fn provider_retry_rejects_identity_and_same_attempt_conflicts() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("code-prototype", "run-conflict"); + write_first(directory.path(), &identity); + + let mut conflicting_identity = identity.clone(); + conflicting_identity.task_id = "other-task".to_string(); + let identity_error = read_matching_at(directory.path(), &conflicting_identity) + .expect_err("identity conflict must fail"); + assert!(identity_error.contains("身份冲突")); + let write_error = write_next_at( + directory.path(), + &conflicting_identity, + "loop-2-repair-0-transient-1", + 1, + 3, + 250, + "transport", + &"b".repeat(64), + ) + .expect_err("conflicting identity write must fail"); + assert!(write_error.contains("身份冲突")); + + let payload_error = write_next_at( + directory.path(), + &identity, + "loop-2-repair-0-transient-1", + 1, + 3, + 250, + "connectivity", + &"c".repeat(64), + ) + .expect_err("same attempt payload conflict must fail"); + assert!(payload_error.contains("同一 attempt")); + } + + #[test] + fn provider_retry_dangerous_path_components_do_not_collide() { + let first_agent = "design/director"; + let second_agent = "design?director"; + let run_id = "../run"; + let first = provider_retry_relative_path(first_agent, run_id); + let second = provider_retry_relative_path(second_agent, run_id); + assert_ne!(first, second); + assert!(first.contains("design-director--")); + assert!(first.contains(&format!("{:x}", Sha256::digest(first_agent.as_bytes())))); + assert!(first.contains(&format!("{:x}", Sha256::digest(run_id.as_bytes())))); + assert!(!first.contains("/../")); + assert_eq!( + provider_retry_relative_path("code-prototype", "run-safe"), + ".agent/runtime/provider-retries/code-prototype/run-safe.json" + ); + assert_ne!( + provider_retry_path_component(".", "agent"), + provider_retry_path_component("..", "agent") + ); + } + + #[test] + fn provider_retry_remaining_time_has_strict_boundaries_and_can_be_forced_due() { + assert_eq!(remaining_ms_at(1_500, 1_000), 500); + assert_eq!(remaining_ms_at(1_500, 1_500), 0); + assert_eq!(remaining_ms_at(1_500, 2_000), 0); + + let directory = tempdir().expect("create temp directory"); + let identity = identity("design-director", "run-due"); + write_first(directory.path(), &identity); + let due = force_provider_retry_due_for_test_at(directory.path(), &identity) + .expect("force Provider retry due"); + assert_eq!(remaining_ms(&due), 0); + } + + #[test] + fn provider_retry_remove_deletes_primary_and_previous_idempotently() { + let directory = tempdir().expect("create temp directory"); + let identity = identity("quality-lead", "run-remove"); + write_first(directory.path(), &identity); + let path = provider_retry_path(directory.path(), &identity.agent_id, &identity.run_id); + let backup_path = agent_runtime_json_sidecar_backup_path(&path); + fs::copy(&path, &backup_path).expect("copy Provider retry previous"); + + remove_at(directory.path(), &identity.agent_id, &identity.run_id) + .expect("remove Provider retry"); + assert!(!path.exists()); + assert!(!backup_path.exists()); + remove_at(directory.path(), &identity.agent_id, &identity.run_id) + .expect("repeat Provider retry removal"); + } +} diff --git a/apps/ai-game-creator-shell/src-tauri/src/runner.rs b/apps/ai-game-creator-shell/src-tauri/src/runner.rs index b1fea8710..76cc0bd9a 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/runner.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/runner.rs @@ -2096,7 +2096,11 @@ impl ExternalAgentRunnerTargetRunProbe { fn still_requires_wake(&self) -> bool { self.status == "pending" - || (self.status == "running" && self.phase == "waiting-for-delegate-receipts") + || (self.status == "running" + && matches!( + self.phase.as_str(), + "waiting-for-delegate-receipts" | "waiting-for-provider-retry" + )) } } @@ -2741,6 +2745,7 @@ fn external_agent_runner_root_is_idle(root: &Path) -> Result { for durable_dir in [ root.join(".agent/runtime/pending-actions"), root.join(".agent/runtime/finalizations"), + root.join(".agent/runtime/provider-retries"), ] { if external_agent_runner_directory_has_durable_files(&durable_dir)? { return Ok(false); @@ -3929,6 +3934,8 @@ mod tests { let agent_id = "project-supervisor"; let run_id = "supervisor-parent-run-1"; let waiting = probe(agent_id, run_id, "running", "waiting-for-delegate-receipts"); + let provider_retry_waiting = + probe(agent_id, run_id, "running", "waiting-for-provider-retry"); assert_eq!( classify_external_agent_runner_target_wake(agent_id, run_id, &[], Some(&waiting)), @@ -3960,6 +3967,16 @@ mod tests { Err(ExternalAgentRunnerTargetWakeRetry::StillPending), "observing the target without advancing it remains retryable" ); + assert_eq!( + classify_external_agent_runner_target_wake( + agent_id, + run_id, + std::slice::from_ref(&provider_retry_waiting), + Some(&provider_retry_waiting), + ), + Err(ExternalAgentRunnerTargetWakeRetry::StillPending), + "a durable Provider retry wait remains retryable until the target advances" + ); let advanced = probe(agent_id, run_id, "running", "planning"); assert_eq!( @@ -4450,6 +4467,109 @@ mod tests { assert!(!state.draining.load(Ordering::Acquire)); } + #[test] + fn durable_provider_retry_prevents_shutdown_and_reopens_writes() { + let directory = unique_test_directory(); + let root = directory.0.join("project"); + let identity = crate::provider_retry::AgentRuntimeProviderRetryIdentity { + project_id: "project-provider-retry-idle".to_string(), + agent_id: "code-prototype".to_string(), + task_id: "task-provider-retry-idle".to_string(), + session_id: "session-provider-retry-idle".to_string(), + run_id: "run-provider-retry-idle".to_string(), + source: "agent-chat".to_string(), + goal_id: None, + goal_revision: 0, + goal_snapshot_fingerprint: String::new(), + applied_steer_cursor: 0, + request_kind: "tool-plan".to_string(), + base_request_slot: "loop-0-repair-0".to_string(), + request_fingerprint: "a".repeat(64), + provider_config_fingerprint: "b".repeat(64), + web_search_enabled: false, + allow_idle_context_compaction: false, + }; + let retry = crate::provider_retry::write_next_at( + &root, + &identity, + "loop-0-repair-0-transient-1", + 1, + 3, + 250, + "timeout", + &"c".repeat(64), + ) + .expect("write durable Provider retry"); + + assert!(!external_agent_runner_root_is_idle(&root).expect("scan primary retry")); + let retry_path = root + .join(".agent/runtime/provider-retries/code-prototype/run-provider-retry-idle.json"); + let previous_path = crate::agent::agent_runtime_json_sidecar_backup_path(&retry_path); + fs::rename(&retry_path, &previous_path).expect("move Provider retry to previous"); + assert_eq!( + crate::provider_retry::list_at(&root).expect("scan previous Provider retry"), + vec![retry] + ); + assert!(!external_agent_runner_root_is_idle(&root).expect("scan previous retry")); + + let token = "provider-retry-shutdown-token-provider-retry-shutdown-token"; + let state = ExternalAgentRunnerServerState::new( + directory.0.join(EXTERNAL_AGENT_RUNNER_ENDPOINT_FILE_NAME), + test_endpoint(token, "provider-retry-shutdown-boot", 32324), + ); + state.remember_root(&root); + let busy_response = handle_external_agent_runner_request( + ExternalAgentRunnerRequest { + protocol_version: EXTERNAL_AGENT_RUNNER_PROTOCOL_VERSION, + request_id: "shutdown-provider-retry-busy-1".to_string(), + token: token.to_string(), + method: "runner.shutdown_if_idle".to_string(), + params: ExternalAgentRunnerRequestParams::default(), + }, + &state, + ); + + assert!(busy_response.ok); + assert_eq!( + busy_response + .result + .as_ref() + .and_then(|value| value["idle"].as_bool()), + Some(false) + ); + assert!(!state.shutdown_requested.load(Ordering::Acquire)); + assert!(!state.draining.load(Ordering::Acquire)); + + crate::provider_retry::remove_at(&root, &identity.agent_id, &identity.run_id) + .expect("remove durable Provider retry"); + assert!(crate::provider_retry::list_at(&root) + .expect("scan removed Provider retries") + .is_empty()); + assert!(external_agent_runner_root_is_idle(&root).expect("scan idle root")); + + let idle_response = handle_external_agent_runner_request( + ExternalAgentRunnerRequest { + protocol_version: EXTERNAL_AGENT_RUNNER_PROTOCOL_VERSION, + request_id: "shutdown-provider-retry-idle-1".to_string(), + token: token.to_string(), + method: "runner.shutdown_if_idle".to_string(), + params: ExternalAgentRunnerRequestParams::default(), + }, + &state, + ); + + assert!(idle_response.ok); + assert_eq!( + idle_response + .result + .as_ref() + .and_then(|value| value["idle"].as_bool()), + Some(true) + ); + assert!(state.shutdown_requested.load(Ordering::Acquire)); + assert!(state.draining.load(Ordering::Acquire)); + } + #[test] fn stale_protocol_endpoint_does_not_override_instance_lock_arbitration() { let directory = unique_test_directory(); diff --git a/apps/ai-game-creator-shell/src-tauri/src/tests.rs b/apps/ai-game-creator-shell/src-tauri/src/tests.rs index 43d6c3396..2d985bcf0 100644 --- a/apps/ai-game-creator-shell/src-tauri/src/tests.rs +++ b/apps/ai-game-creator-shell/src-tauri/src/tests.rs @@ -1308,6 +1308,22 @@ fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState runtime } +fn wait_for_agent_runtime_phase(root: &Path, agent_id: &str, phase: &str) -> AgentRuntimeState { + let mut runtime = read_game_creator_agent_runtime_at(root, agent_id) + .expect("read runtime while waiting for phase") + .state; + for _ in 0..250 { + if runtime.phase == phase { + return runtime; + } + std::thread::sleep(Duration::from_millis(20)); + runtime = read_game_creator_agent_runtime_at(root, agent_id) + .expect("read runtime while waiting for phase") + .state; + } + runtime +} + fn wait_for_response_stream_status( root: &Path, agent_id: &str, @@ -3926,41 +3942,75 @@ fn spawn_mock_llm_transport_failures_then_response( failure_count: usize, response_content: String, request_notice_sender: Option>, +) -> String { + spawn_mock_llm_transport_failures_then_responses( + failure_count, + vec![response_content], + request_notice_sender, + ) +} + +fn spawn_mock_llm_transport_failures_then_responses( + failure_count: usize, + response_contents: Vec, + request_notice_sender: Option>, +) -> String { + spawn_mock_llm_transport_failures_then_responses_with_capture( + failure_count, + response_contents, + request_notice_sender, + None, + ) +} + +fn spawn_mock_llm_transport_failures_then_responses_with_capture( + failure_count: usize, + response_contents: Vec, + request_notice_sender: Option>, + request_capture_sender: Option>, ) -> String { let listener = bind_test_tcp_listener("mock transient llm bind"); let base_url = format!("http://{}", listener.local_addr().expect("mock llm addr")); std::thread::spawn(move || { for _ in 0..failure_count { let (mut stream, _) = listener.accept().expect("mock transient llm accept"); - drop(read_mock_http_request(&mut stream)); + let request = read_mock_http_request(&mut stream); + if let Some(sender) = request_capture_sender.as_ref() { + let _ = sender.send(request); + } if let Some(sender) = request_notice_sender.as_ref() { let _ = sender.send(()); } drop(stream); } - let (mut stream, _) = listener.accept().expect("mock recovered llm accept"); - drop(read_mock_http_request(&mut stream)); - if let Some(sender) = request_notice_sender.as_ref() { - let _ = sender.send(()); + for response_content in response_contents { + let (mut stream, _) = listener.accept().expect("mock recovered llm accept"); + let request = read_mock_http_request(&mut stream); + if let Some(sender) = request_capture_sender.as_ref() { + let _ = sender.send(request); + } + if let Some(sender) = request_notice_sender.as_ref() { + let _ = sender.send(()); + } + let body = serde_json::json!({ + "id": "chatcmpl_transient_recovered", + "model": "mock-game-model", + "choices": [{ + "message": { "content": response_content }, + "finish_reason": "stop" + }], + "usage": { "prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33 } + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + stream + .write_all(response.as_bytes()) + .expect("mock recovered llm response"); } - let body = serde_json::json!({ - "id": "chatcmpl_transient_recovered", - "model": "mock-game-model", - "choices": [{ - "message": { "content": response_content }, - "finish_reason": "stop" - }], - "usage": { "prompt_tokens": 11, "completion_tokens": 22, "total_tokens": 33 } - }) - .to_string(); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ); - stream - .write_all(response.as_bytes()) - .expect("mock recovered llm response"); }); base_url } @@ -20496,7 +20546,8 @@ async fn queued_delegate_receipt_is_suppressed_when_parent_cancels_before_drain( spawn_next_game_creator_agent_background_task_drain(&root, "design-director") .expect("drain queued receipt"); let mut drained_receipt = receipt.clone(); - for _ in 0..250 { + // The full suite can saturate Tauri's shared executor with process fixtures. + for _ in 0..1_500 { let runtime = read_game_creator_agent_runtime_at(&root, "design-director") .expect("read drained receipt runtime"); drained_receipt = runtime @@ -32590,6 +32641,843 @@ async fn provider_transient_retry_transport_failure_closes_then_stable_retry_suc fs::remove_dir_all(root).ok(); } +#[tokio::test] +async fn provider_retry_waiting_tool_plan_resumes_only_after_due_and_cleans_sidecar() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 持久等待测试").expect("project init"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_response( + 1, + final_tool_plan_response("持久等待到期后已完成"), + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-persisted-wait-run"; + + let started = start_game_creator_agent_background_task_at( + &root, + "design-director", + "验证 Provider retry 等待可跨恢复并保持原 run", + run_id, + ) + .expect("start persisted retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first physical Provider request"); + + let waiting = + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + assert_eq!(waiting.status, "running"); + assert_eq!(waiting.run_id, run_id); + assert_eq!(waiting.session_id, started.state.session_id); + assert!( + game_creator_agent_runtime_task_lock_is_available(&root, "design-director") + .expect("probe released Agent lane") + ); + let retry = crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read persisted Provider retry") + .expect("persisted Provider retry exists"); + assert_eq!(retry.next_attempt, 1); + assert_eq!(retry.max_retries, 1); + assert_eq!(retry.identity.request_kind, "tool-plan"); + assert_eq!(retry.identity.base_request_slot, "loop-1-repair-0"); + assert_eq!(retry.identity.request_fingerprint.len(), 64); + assert_eq!(retry.identity.provider_config_fingerprint.len(), 64); + assert!(crate::provider_retry::remaining_ms(&retry) > 0); + + resume_game_creator_agent_background_tasks_at(&root).expect("resume before retry due"); + std::thread::sleep(Duration::from_millis(120)); + assert!(request_notice_receiver.try_recv().is_err()); + let unchanged = crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("reread persisted Provider retry") + .expect("persisted Provider retry remains"); + assert_eq!(unchanged, retry); + + crate::provider_retry::force_provider_retry_due_for_test_at(&root, &retry.identity) + .expect("force Provider retry due"); + resume_game_creator_agent_background_tasks_at(&root).expect("resume due Provider retry"); + resume_game_creator_agent_background_tasks_at(&root) + .expect("duplicate due resume remains idempotent"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("one due retry physical Provider request"); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + let completed = wait_for_agent_runtime_idle(&root, "design-director"); + assert_eq!(completed.phase, "completed"); + assert_eq!(completed.run_id, run_id); + assert_eq!( + completed.last_response.as_deref(), + Some("持久等待到期后已完成") + ); + assert!( + crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read cleaned Provider retry") + .is_none() + ); + + let records = read_agent_db_records_for_test(&root); + assert_eq!( + records + .iter() + .filter(|record| { + record["recordType"] == "agent.runtime.provider_request.retry_waiting" + && record["runId"] == run_id + }) + .count(), + 1 + ); + let lifecycle = records + .iter() + .filter(|record| { + record["recordType"] == "agent.runtime.provider_request.lifecycle" + && record["runId"] == run_id + }) + .collect::>(); + assert_eq!(lifecycle.len(), 4); + assert_eq!( + lifecycle + .iter() + .filter(|record| record["requestSlot"] == "loop-1-repair-0-transient-1") + .count(), + 2 + ); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_restart_repairs_sidecar_before_task_projection() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider torn projection 测试") + .expect("project init"); + let injection = root.join(".agent/runtime/test-stop-after-provider-retry-sidecar"); + fs::create_dir_all(injection.parent().expect("retry injection parent")) + .expect("create retry injection parent"); + fs::write(&injection, b"stop-after-sidecar").expect("write retry projection injection"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_response( + 1, + final_tool_plan_response("torn projection 恢复后已完成"), + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-retry-torn-projection-run"; + start_game_creator_agent_background_task_at( + &root, + "design-director", + "验证 sidecar 已提交而 task/state 未投影时可恢复", + run_id, + ) + .expect("start torn Provider retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first torn Provider physical request"); + + let mut retry = None; + for _ in 0..250 { + retry = crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read torn Provider retry"); + if retry.is_some() + && game_creator_agent_runtime_task_lock_is_available(&root, "design-director") + .expect("probe torn projection lane") + { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + let retry = retry.expect("Provider retry sidecar committed before projection"); + let before_resume = read_game_creator_agent_runtime_at(&root, "design-director") + .expect("read torn projection Runtime"); + assert_eq!(before_resume.state.phase, "planning"); + assert!(before_resume.recent_tasks.iter().any(|task| { + task.run_id == run_id && task.status == "running" && task.phase == "planning" + })); + + resume_game_creator_agent_background_tasks_at(&root) + .expect("repair torn Provider retry projection"); + let waiting = + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + assert_eq!(waiting.status, "running"); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + let projected = read_game_creator_agent_runtime_at(&root, "design-director") + .expect("read repaired Provider retry projection"); + assert!(projected.recent_tasks.iter().any(|task| { + task.run_id == run_id + && task.status == "running" + && task.phase == "waiting-for-provider-retry" + })); + + crate::provider_retry::force_provider_retry_due_for_test_at(&root, &retry.identity) + .expect("force repaired Provider retry due"); + resume_game_creator_agent_background_tasks_at(&root).expect("resume repaired Provider retry"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("repaired Provider retry physical request"); + let completed = wait_for_agent_runtime_idle(&root, "design-director"); + assert_eq!(completed.phase, "completed"); + assert_eq!( + completed.last_response.as_deref(), + Some("torn projection 恢复后已完成") + ); + assert!( + crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read cleaned torn Provider retry") + .is_none() + ); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_context_compaction_resumes_source_once_before_tool_plan() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider compaction retry 测试") + .expect("project init"); + let agent_id = "design-director"; + let session_id = "agent-session-design-director"; + for index in 0..60 { + append_local_conversation_message_for_session_at( + &root, + Some(agent_id), + Some(session_id), + LocalConversationMessage { + role: if index % 2 == 0 { "user" } else { "assistant" }.to_string(), + content: format!("PERSISTED_COMPACTION_MARKER_{index} {}", "x".repeat(1_200)), + agent_id: (index % 2 == 1).then(|| agent_id.to_string()), + }, + ) + .expect("append persisted compaction conversation"); + } + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_responses( + 1, + vec![ + "PERSISTED_COMPACTION_PRIVATE_SUMMARY：保留角色规范约束。".to_string(), + serde_json::json!({ + "thinkingSummary": "持久压缩后可完成回答", + "planUpdate": { + "explanation": "上下文压缩和当前回答均已完成", + "steps": [ + { "step": "恢复压缩并回答", "status": "completed" } + ] + }, + "plan": [], + "actions": [], + "response": "持久 context-compaction retry 后已完成。" + }) + .to_string(), + ], + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "compaction-retry-key", + "baseUrl": {base_url:?}, + "model": "compaction-retry-model", + "apiKind": "openai_chat", + "stream": false, + "contextWindowTokens": 128000, + "autoCompactTokenLimit": 40000, + "toolOutputTokenLimit": 8000, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-compaction-retry-run"; + start_game_creator_agent_background_task_for_session_at( + &root, + agent_id, + Some(session_id), + "验证自动 context compaction 的持久 Provider retry", + run_id, + ) + .expect("start persisted compaction retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first failed compaction Provider request"); + let waiting = wait_for_agent_runtime_phase(&root, agent_id, "waiting-for-provider-retry"); + assert_eq!(waiting.run_id, run_id); + let retry = crate::provider_retry::read_for_run_at(&root, agent_id, run_id) + .expect("read persisted compaction retry") + .expect("persisted compaction retry exists"); + assert_eq!(retry.identity.request_kind, "context-compaction"); + assert!(retry.identity.base_request_slot.starts_with("source-")); + assert!(!retry.identity.allow_idle_context_compaction); + assert!( + read_game_creator_agent_runtime_context_compaction(&root, agent_id, session_id) + .expect("read pre-retry compaction sidecar") + .is_none() + ); + + crate::provider_retry::force_provider_retry_due_for_test_at(&root, &retry.identity) + .expect("force compaction retry due"); + resume_game_creator_agent_background_tasks_at(&root) + .expect("resume persisted compaction retry"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("retried compaction Provider request"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("tool-plan request after compaction commit"); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + let completed = wait_for_agent_runtime_idle(&root, agent_id); + assert_eq!(completed.phase, "completed"); + assert_eq!( + completed.last_response.as_deref(), + Some("持久 context-compaction retry 后已完成。") + ); + let compaction = + read_game_creator_agent_runtime_context_compaction(&root, agent_id, session_id) + .expect("read committed compaction sidecar") + .expect("committed compaction sidecar exists"); + assert_eq!(compaction.revision, 1); + assert!(compaction + .summary + .contains("PERSISTED_COMPACTION_PRIVATE_SUMMARY")); + assert!( + crate::provider_retry::read_for_run_at(&root, agent_id, run_id) + .expect("read cleaned compaction retry") + .is_none() + ); + let records = read_agent_db_records_for_test(&root); + let compaction_lifecycle = records + .iter() + .filter(|record| { + record["recordType"] == "agent.runtime.provider_request.lifecycle" + && record["runId"] == run_id + && record["requestKind"] == "context-compaction" + }) + .collect::>(); + assert_eq!(compaction_lifecycle.len(), 4); + assert_eq!( + compaction_lifecycle + .iter() + .filter(|record| { + record["requestSlot"] + .as_str() + .is_some_and(|slot| slot.ends_with("-transient-1")) + }) + .count(), + 2 + ); + let public_event = fs::read_to_string(game_creator_agent_runtime_event_path(&root, agent_id)) + .expect("read compaction retry events"); + let public_agent_db = + fs::read_to_string(root.join(".agent/agent.db")).expect("read compaction retry Agent DB"); + for public_surface in [&public_event, &public_agent_db] { + assert!(!public_surface.contains("PERSISTED_COMPACTION_PRIVATE_SUMMARY")); + assert!(!public_surface.contains("PERSISTED_COMPACTION_MARKER_0")); + assert!(!public_surface.contains("compaction-retry-key")); + assert!(!public_surface.contains(root.to_string_lossy().as_ref())); + } + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_cancel_removes_sidecar_without_second_request() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 等待取消测试").expect("project init"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_response( + 1, + final_tool_plan_response("取消后不应收到此响应"), + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-wait-cancel-run"; + start_game_creator_agent_background_task_at( + &root, + "design-director", + "验证等待 Provider retry 时可立即取消", + run_id, + ) + .expect("start cancellable Provider retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first physical Provider request"); + let waiting = + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + assert_eq!(waiting.run_id, run_id); + + let cancelled = cancel_game_creator_agent_runtime_task_at(&root, "design-director", run_id) + .expect("cancel Provider retry wait"); + assert_eq!(cancelled.state.phase, "cancelled"); + assert!( + crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read Provider retry after cancel") + .is_none() + ); + std::thread::sleep(Duration::from_millis(120)); + assert!(request_notice_receiver.try_recv().is_err()); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_steer_supersedes_old_attempt_and_wakes_same_run() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 等待 steer 测试") + .expect("project init"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_response( + 1, + final_tool_plan_response("steer 已在同一 run 生效"), + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-wait-steer-run"; + let started = start_game_creator_agent_background_task_at( + &root, + "design-director", + "先进入 Provider retry 等待", + run_id, + ) + .expect("start steerable Provider retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first steerable Provider request"); + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + + let steered = steer_game_creator_agent_runtime_task_at( + &root, + "design-director", + &started.state.session_id, + run_id, + "provider-retry-steer-1", + "请把新增的触屏验收纳入当前结果", + "test", + ) + .expect("steer waiting Provider retry"); + assert!(!steered.provider_interrupted); + assert_eq!(steered.runtime.state.run_id, run_id); + assert!( + crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read superseded Provider retry") + .is_none() + ); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("steer wakes a fresh Provider request"); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + let completed = wait_for_agent_runtime_idle(&root, "design-director"); + assert_eq!(completed.phase, "completed"); + assert_eq!(completed.run_id, run_id); + assert_eq!(completed.applied_steer_cursor, 1); + assert_eq!( + completed.last_response.as_deref(), + Some("steer 已在同一 run 生效") + ); + let records = read_agent_db_records_for_test(&root); + let lifecycles = records + .iter() + .filter(|record| { + record["recordType"] == "agent.runtime.provider_request.lifecycle" + && record["runId"] == run_id + }) + .collect::>(); + assert_eq!(lifecycles.len(), 4); + assert!(lifecycles.iter().all(|record| { + !record["requestSlot"] + .as_str() + .is_some_and(|slot| slot.contains("-transient-")) + })); + let request_ids = lifecycles + .iter() + .filter_map(|record| record["requestId"].as_str()) + .collect::>(); + assert_eq!(request_ids.len(), 2); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_goal_pause_preserves_attempt_until_same_run_resume() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 等待 Goal 暂停测试") + .expect("project init"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let (request_capture_sender, request_capture_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_responses_with_capture( + 1, + vec![final_tool_plan_response("Goal 恢复后沿用持久 retry 完成")], + Some(request_notice_sender), + Some(request_capture_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "code-prototype": {{ + "apiKey": "goal-retry-key", + "baseUrl": {base_url:?}, + "model": "goal-retry-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "code-provider-wait-goal-pause-run"; + let started = start_game_creator_agent_goal_at( + &root, + "code-prototype", + None, + "验证 Provider retry 等待期间暂停并恢复同一 Goal", + vec!["保持同一 Session/run".to_string()], + vec!["最终回复唯一".to_string()], + run_id, + ) + .expect("start Goal Provider retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first Goal Provider request"); + let first_provider_request = request_capture_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("capture first Goal Provider request"); + wait_for_agent_runtime_phase(&root, "code-prototype", "waiting-for-provider-retry"); + let retry = crate::provider_retry::read_for_run_at(&root, "code-prototype", run_id) + .expect("read Goal Provider retry") + .expect("Goal Provider retry exists"); + + let paused = pause_game_creator_agent_goal_at( + &root, + "code-prototype", + &started.goal.session_id, + &started.goal.goal_id, + started.goal.revision, + ) + .expect("pause Goal during Provider retry wait"); + assert_eq!(paused.goal.status, AGENT_GOAL_STATUS_PAUSED); + assert_eq!(paused.runtime.state.status, "paused"); + assert_eq!(paused.runtime.state.run_id, run_id); + assert_eq!( + crate::provider_retry::read_for_run_at(&root, "code-prototype", run_id) + .expect("read paused Goal Provider retry"), + Some(retry.clone()) + ); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + + std::thread::sleep(Duration::from_millis(1_100)); + crate::provider_retry::force_provider_retry_due_for_test_at(&root, &retry.identity) + .expect("force paused Goal Provider retry due"); + let resumed = resume_game_creator_agent_goal_at( + &root, + "code-prototype", + &started.goal.session_id, + &started.goal.goal_id, + started.goal.revision, + ) + .expect("resume Goal with persisted Provider retry"); + assert_eq!(resumed.goal.status, AGENT_GOAL_STATUS_ACTIVE); + assert_eq!(resumed.runtime.state.run_id, run_id); + let resumed_event = fs::read_to_string(game_creator_agent_runtime_event_path( + &root, + "code-prototype", + )) + .expect("read resumed Goal Provider retry events") + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find(|event| event["runId"] == run_id && event["eventType"] == "goal.resumed") + .expect("Goal resume event exists"); + assert_eq!(resumed_event["status"], "running"); + assert_eq!(resumed_event["phase"], "waiting-for-provider-retry"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("resumed Goal Provider retry request"); + let resumed_provider_request = request_capture_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("capture resumed Goal Provider retry request"); + if first_provider_request != resumed_provider_request { + let first_body = first_provider_request + .split_once("\r\n\r\n") + .map(|(_, body)| body) + .unwrap_or(first_provider_request.as_str()); + let resumed_body = resumed_provider_request + .split_once("\r\n\r\n") + .map(|(_, body)| body) + .unwrap_or(resumed_provider_request.as_str()); + let first_difference = first_body + .bytes() + .zip(resumed_body.bytes()) + .position(|(left, right)| left != right) + .unwrap_or_else(|| first_body.len().min(resumed_body.len())); + panic!( + "Goal Provider retry request drifted at byte {first_difference}: firstSha256={:x} resumedSha256={:x}", + Sha256::digest(first_body.as_bytes()), + Sha256::digest(resumed_body.as_bytes()) + ); + } + let completed = wait_for_agent_runtime_idle(&root, "code-prototype"); + assert_eq!(completed.phase, "completed"); + assert_eq!(completed.run_id, run_id); + assert_eq!( + completed.last_response.as_deref(), + Some("Goal 恢复后沿用持久 retry 完成") + ); + assert!( + crate::provider_retry::read_for_run_at(&root, "code-prototype", run_id) + .expect("read cleaned Goal Provider retry") + .is_none() + ); + let lifecycle = read_agent_db_records_for_test(&root) + .into_iter() + .filter(|record| { + record["recordType"] == "agent.runtime.provider_request.lifecycle" + && record["runId"] == run_id + }) + .collect::>(); + assert_eq!(lifecycle.len(), 4); + assert_eq!( + lifecycle + .iter() + .filter(|record| { + record["requestSlot"] + .as_str() + .is_some_and(|slot| slot.ends_with("-transient-1")) + }) + .count(), + 2 + ); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_exhaustion_fails_and_removes_sidecar() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 等待耗尽测试").expect("project init"); + let (request_notice_sender, request_notice_receiver) = mpsc::channel(); + let base_url = spawn_mock_llm_transport_failures_then_response( + 2, + final_tool_plan_response("重试耗尽后不应收到此响应"), + Some(request_notice_sender), + ); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }} + }} +}}"# + )); + let run_id = "design-provider-wait-exhausted-run"; + start_game_creator_agent_background_task_at( + &root, + "design-director", + "验证持久 Provider retry 耗尽后干净失败", + run_id, + ) + .expect("start exhausted Provider retry task"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first physical Provider request"); + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + let retry = crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read exhausted Provider retry") + .expect("exhausted Provider retry exists before due"); + crate::provider_retry::force_provider_retry_due_for_test_at(&root, &retry.identity) + .expect("force exhausted Provider retry due"); + resume_game_creator_agent_background_tasks_at(&root).expect("resume exhausted Provider retry"); + request_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("last allowed physical Provider request"); + let failed = wait_for_agent_runtime_idle(&root, "design-director"); + assert_eq!(failed.phase, "failed"); + assert!(failed + .error + .as_deref() + .is_some_and(|error| error.contains("kind=transport"))); + assert!( + crate::provider_retry::read_for_run_at(&root, "design-director", run_id) + .expect("read Provider retry after exhaustion") + .is_none() + ); + assert!(request_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + + fs::remove_dir_all(root).ok(); +} + +#[tokio::test] +async fn provider_retry_waiting_releases_other_agent_and_preserves_same_lane_fifo() { + let root = unique_project_path(); + init_local_game_project_at(&root, "project-1", "Provider 等待并行测试").expect("project init"); + let (design_notice_sender, design_notice_receiver) = mpsc::channel(); + let design_base_url = spawn_mock_llm_transport_failures_then_response( + 1, + final_tool_plan_response("同 Agent 后继任务已完成"), + Some(design_notice_sender), + ); + let art_base_url = + spawn_mock_llm_server_responses(vec![final_tool_plan_response("另一个 Agent 已并行完成")]); + let _config_guard = write_test_local_config(format!( + r#"{{ + "agentLlm": {{ + "design-director": {{ + "apiKey": "design-key", + "baseUrl": {design_base_url:?}, + "model": "design-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 1, + "retryBackoffMs": 30000 + }}, + "art-director": {{ + "apiKey": "art-key", + "baseUrl": {art_base_url:?}, + "model": "art-runtime-model", + "apiKind": "openai_chat", + "stream": false, + "maxRetries": 0, + "retryBackoffMs": 1 + }} + }} +}}"# + )); + let waiting_run_id = "design-provider-wait-parallel-run"; + let queued_run_id = "design-provider-wait-queued-run"; + start_game_creator_agent_background_task_at( + &root, + "design-director", + "让 design Agent 进入持久重试等待", + waiting_run_id, + ) + .expect("start waiting design task"); + design_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("first design Provider request"); + wait_for_agent_runtime_phase(&root, "design-director", "waiting-for-provider-retry"); + + let queued = start_game_creator_agent_background_task_at( + &root, + "design-director", + "同 Agent 的下一条任务必须保持 FIFO", + queued_run_id, + ) + .expect("queue same-lane task"); + assert!(queued.recent_tasks.iter().any(|task| { + task.run_id == queued_run_id && task.status == "pending" && task.phase == "queued" + })); + assert!(design_notice_receiver + .recv_timeout(Duration::from_millis(100)) + .is_err()); + + start_game_creator_agent_background_task_at( + &root, + "art-director", + "另一个 Agent 应在 design 等待时独立完成", + "art-provider-wait-parallel-run", + ) + .expect("start parallel art task"); + let art_completed = wait_for_agent_runtime_idle(&root, "art-director"); + assert_eq!(art_completed.phase, "completed"); + assert_eq!( + art_completed.last_response.as_deref(), + Some("另一个 Agent 已并行完成") + ); + + cancel_game_creator_agent_runtime_task_at(&root, "design-director", waiting_run_id) + .expect("cancel waiting design task and drain queue"); + design_notice_receiver + .recv_timeout(Duration::from_secs(5)) + .expect("queued same-lane task starts after cancellation"); + let design_completed = wait_for_agent_runtime_idle(&root, "design-director"); + assert_eq!(design_completed.run_id, queued_run_id); + assert_eq!(design_completed.phase, "completed"); + let result = read_game_creator_agent_runtime_at(&root, "design-director") + .expect("read final design runtime"); + assert!(result + .recent_tasks + .iter() + .any(|task| { task.run_id == waiting_run_id && task.status == "cancelled" })); + assert!(result + .recent_tasks + .iter() + .any(|task| { task.run_id == queued_run_id && task.status == "completed" })); + + fs::remove_dir_all(root).ok(); +} + #[tokio::test] async fn provider_transient_retry_zero_max_retries_stops_after_first_failure() { let root = unique_project_path(); diff --git a/docs/project-memory/shared-memory/decision-log.md b/docs/project-memory/shared-memory/decision-log.md index 6856312d3..9b0db8e8e 100644 --- a/docs/project-memory/shared-memory/decision-log.md +++ b/docs/project-memory/shared-memory/decision-log.md @@ -4893,3 +4893,11 @@ - 漂移：snapshot 绑定后，后续 spawn、repair、新 claim、Supervisor mutation、MCP、prompt/status、completion/finalization 与恢复全部使用 snapshot。global policy 的 `matched / drifted / unreadable` 只作有界状态报告，不改变执行、revision、verification 或 reconciliation；已有 run 不重绑，新 policy 只由后续新父 run 采用。 - Claim 兼容：旧 durable claim、未观察 claim 和 legacy claimed delivery 的恢复先于 effective snapshot 解析及新 claim 门禁，继续按原 action/group 身份推进且不得取得新 delivery；global policy、snapshot 或 binding 故障不能把已提交 claim 卡死。新的 claim 必须先成功解析 effective snapshot 并核对 binding，失败发生在 journal、delivery 锁和 mutation 之前；随后仍执行 V1.35-V1.37 的全锁、预算、完整 observation 和 group 数量门禁。 - 覆盖与验收：本决策明确覆盖 V1.32 的 live drift reconciliation 旧口径，但不把 V1.32/V1.35/V1.37 历史 PASS 外推为 V1.38 证据。2026-07-19 self-test、snapshot/binding 双故障窗口与 CAS/丢失/篡改/旧 batch/危险 ID/claim 分流确定性覆盖、`supervisor_collaboration_` 52/52、`provider_action_batch_` 12/12、`project_supervisor_mixed_` 5/5 和 Tauri/Rust 全量 949 passed/4 ignored 已通过；终态 Runtime 清理后 snapshot/binding 保持原字节并继续解析为 `run-snapshot`。真实 mixed-swarm 独立功能样本已形成 2 group/3 child、policy drift、Runner 恢复、唯一最终回复和零重复/泄漏，但同轮正式 endpoint 被外部客户端重启；改用私有配置源后多轮又耗尽 transient Provider retry，最后在 `300000ms / maxRetries=3` 下于首批业务动作前形成 4 failed/3 retry 并干净终止。两类失败证据不得拼接，当前**仍不得声称 V1.38 真实 E2E 已 PASS**。 + +## 2026-07-19 AI 游戏创作 Agent Runtime V1.39 首次规划 Provider 持久重试 + +- 决策：tool-plan 首次请求及其自动 context-compaction 的可重试瞬态失败不再依赖 Runner 进程内 sleep。每个 Agent/run 使用唯一严格 v1 retry sidecar，持久绑定项目、Agent、task、Session、run、Goal、steer、请求和 Provider 配置指纹，以及下一个 `-transient-N` attempt 和绝对到期时间；一次只允许一个待重试 attempt。 +- 提交与恢复：每次物理请求仍先闭合自己的 Provider lifecycle；随后按 retry audit -> 原子 sidecar/.previous -> `running / waiting-for-provider-retry` 投影提交，再释放 lane。Runner 启动扫描 primary/.previous，未到期只重建唤醒，到期后重验 durable control 和完整指纹并恢复同一 Session/run/loop/attempt。sidecar/torn projection 冲突失败关闭，不从 Agent DB 或 UI 状态猜回请求。 +- 调度：等待态释放执行 lane，允许不同 Agent 并行；同 Agent 当前 running 等待任务继续阻挡后续 pending task，保持 FIFO。活跃 sidecar 阻断 finalization 与 `shutdown_if_idle`，cancel、steer、终态和耗尽负责清理。 +- 稳定身份：Provider 请求指纹不得包含工具策略 `updatedAt` 等非语义刷新时间。Goal pause 保留 sidecar，resume 恢复原等待态；跨秒恢复必须仍命中相同 request fingerprint 和 `-transient-N` slot。final-reply、手动压缩与 tool-plan `repair-N` 继续使用进程内重试，待具备可无歧义重建的持久请求上下文后再单独升级。 +- 证据边界：`provider_retry_` 19/19、`provider_transient_retry_` 6/6，Tauri/Rust 串行全量 968 passed/4 ignored，`cargo check`、rustfmt、编码与 diff 检查通过。本轮确定性验证与 V1.38 真实 Provider E2E 分开记账；任何旧失败轮、旧瞬态重试 PASS 或最小探针都不能拼接成 V1.39 真实 PASS。 diff --git a/docs/project-memory/shared-memory/pitfalls.md b/docs/project-memory/shared-memory/pitfalls.md index b37057a39..97de300a4 100644 --- a/docs/project-memory/shared-memory/pitfalls.md +++ b/docs/project-memory/shared-memory/pitfalls.md @@ -3357,3 +3357,11 @@ - 原因：`.admin-info-list div` 会命中列表内所有后代 `div`，把字段值内部的 `.admin-inline-identity` 和昵称容器也覆盖成双列 grid；陶泥号又允许任意位置换行，最终只剩单字符宽度。素材详情布局若始终固定为 `220px + 信息列`，移动端也没有足够空间。 - 处理：信息列表的行布局只使用直接子选择器 `.admin-info-list > div`；作者昵称与陶泥号在身份组件内分行，陶泥号保持单行并在真正不足时省略。`560px` 以下的素材详情改为单列，缩略图居中；素材查询与精选审核共用该规则。 - 验证：在桌面、560px、390px 和 320px 浏览器宽度打开素材详情，确认 `.admin-inline-identity` 的 computed `display` 为 `flex`、陶泥号横向显示、详情字段不溢出页面。 + +## Provider 瞬态重试不能只依赖进程内 sleep，也不能让刷新时间进入请求指纹 + +- 现象：Provider 首次规划请求发生 timeout/connectivity/transport 后，Runner 在 backoff 期间退出会丢失 retry，或重启后清零 attempt、提前补发；另一种偶发现象是 Goal pause/resume 看似保留 sidecar，但只要等待跨过一秒，恢复请求就被判为 context drift，旧 `-transient-N` attempt 被删除并改成新 loop 请求。 +- 原因：退避 attempt 和到期时间只存在于进程内；或虽然已有 sidecar，请求 prompt 却直接序列化 Runtime 工具策略快照，把每次刷新都会变化的 `updatedAt` 带进 request fingerprint。同一权限内容因此仅因时间变化产生不同请求身份。 +- 处理：先闭合物理请求 lifecycle，再原子持久 retry sidecar/.previous，最后投影 `waiting-for-provider-retry` 并释放 lane；Runner 启动扫描并按绝对到期时间恢复。请求指纹只绑定实际 Provider 请求的稳定语义，UI/审计时间戳、剩余等待毫秒和等待态文案不得进入 prompt。Goal pause 保留 sidecar，resume 必须校验同一 Goal/steer/request/config 身份后恢复原 attempt。 +- 验证：测试必须故意让 pause/resume 跨秒，捕获失败请求与恢复请求的 HTTP body 并比较 SHA-256，同时断言 lifecycle 使用原 `-transient-N` slot而不是新 loop；另覆盖 `.previous` 扫描、Runner idle blocker、同 Agent FIFO、跨 Agent 并行、cancel/steer/耗尽清理和重启未到期零请求。 +- 关联：`apps/ai-game-creator-shell/src-tauri/src/provider_retry.rs`、`agent.rs`、`runner.rs`、`tests.rs`、`docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md`。 diff --git a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md index 3ba549ea1..b4fa33fce 100644 --- a/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md +++ b/docs/technical/【技术方案】AI游戏创作Agent Runtime V1.1-2026-07-12.md @@ -1346,6 +1346,33 @@ V1.38 把 collaboration policy 的执行语义从“每次动作或恢复都读 - 真实 `supervisor-swarm-static-isolated-autonomous-chat` 曾在单次独立运行中完整形成 2 个 isolated group / 3 个 child、1 个 observed join claim 覆盖两组、唯一 repair、宿主验证、Runner pidfd 强杀恢复和唯一 Supervisor assistant；snapshot/binding 均为唯一、字节及字段稳定，global policy drift 被观察，重复、临时 sidecar、正文、API Key、项目路径和配置路径泄漏均为 0。但该轮运行期间正式客户端在测试外部重启了正式 Runner，source endpoint 所有权门禁按设计失败，因此该功能样本不能记为 PASS。 - 随后使用权限为 `0700/0600`、不含 endpoint/锁/会话的私有配置源副本隔离正式客户端干扰，source endpoint、源目录和清理门禁均稳定；五次最小 OpenAI-chat 探针全部 HTTP 200。然而多次独立完整运行仍在长链路耗尽 transient Provider retry。最后一轮隔离 overlay 已提高到 `requestTimeoutMs=300000 / maxRetries=3 / retryBackoffMs=500`，仍在首个业务批次前形成 4 个 failed lifecycle / 3 个 retry 后终止，child、delivery、claim 和项目 mutation 均为 0，现场清理与泄漏门禁通过。失败轮不得与前述功能完整轮拼接；截至当前，**V1.38 独立真实 Provider E2E 仍未 PASS**，需在外部 Provider 稳定后以最终代码重新独立运行。 +## V1.39 首次规划 Provider 瞬态重试持久等待态 + +V1.39 把 V1.28 的显式瞬态重试从单纯进程内退避扩展为可跨 Runner 重启恢复的持久等待态。本切片只覆盖后台 tool-plan 的首次请求（`repair-0`）以及该请求前自动触发的 context-compaction；final-reply、手动压缩和 tool-plan `repair-N` 仍使用进程内重试，因为现有持久上下文不能仅凭请求指纹无歧义重建这些请求。 + +### Sidecar 与提交顺序 + +- 每个 Agent/run 最多存在一条 `.agent/runtime/provider-retries//.json`。安全 ID 原样使用；危险 ID 使用有界安全前缀加完整原始 ID 的 SHA-256，避免 lossy 替换碰撞。记录使用严格 v1 schema，拒绝未知字段，并绑定 `projectId / agentId / taskId / sessionId / runId / source / goalId / goalRevision / goalSnapshotFingerprint / appliedSteerCursor / requestKind / baseRequestSlot / requestFingerprint / providerConfigFingerprint / webSearchEnabled / allowIdleContextCompaction`，以及下一 `requestSlot / attempt / maxRetries / backoff / retryAt / errorKind / errorFingerprint`。 +- 每个物理请求继续使用 `max_retries=0` 的 Provider client。瞬态失败先闭合当前 `started -> failed` lifecycle，再写 retry audit，随后原子提交 sidecar 和 `.previous`，最后把 task/state 投影为 `running / waiting-for-provider-retry` 并释放 Agent lane。sidecar 未可信落盘前不得宣称进入等待态；task/state 已投影但 sidecar 缺失，或两者身份冲突时进入 `needs-reconciliation`，不得猜测补发。 +- 首次请求保留原 slot；第 `N` 次持久重试固定使用 `-transient-N`。同 attempt 的重复写必须字节语义幂等，attempt 只能逐一递增，不能跳号、回退或在同一 sidecar 中排队多个请求。 +- sidecar 只保存错误类别/哈希、请求与配置指纹和恢复身份，不保存 prompt、messages、tool arguments、Provider 正文、API Key、配置文件绝对路径或项目绝对路径。 + +### 恢复、并行与控制 + +- Runner 启动扫描 `provider-retries`，primary 缺失时允许从原子 `.previous` 恢复，并对 primary/previous 去重。未到期记录只重建定时唤醒；到期后先重新校验 cancel、steer、Goal、task/run、Provider 配置和重建请求指纹，再沿原 Session/run/loop/attempt 发出一个物理请求。重启不得清零 attempt，也不得因进程启动提前发送。 +- `waiting-for-provider-retry` 释放当前执行 lane，因此其它 Agent 可继续并行；同 Agent 后续 pending task 仍被该 running task 阻挡，保持 FIFO。每个 project/Agent/run 的定时唤醒必须 singleflight，重复 resume 或到期通知不能制造第二个物理请求。 +- cancel、终态、重试耗尽和有效 steer 会删除旧 sidecar；steer 使用同一 run 的新 cursor 重新规划，不复用旧 attempt。Goal pause 保留 sidecar且零请求，resume 校验完整身份后恢复原等待态；即使暂停跨越秒级时间边界，也必须保留相同 request fingerprint 和 `-transient-N` slot。 +- 请求指纹只包含实际 Provider 请求的稳定语义。工具策略的 `updatedAt` 等刷新时间只用于 Runtime/UI 投影，不进入 planning prompt；否则一次普通 pause/resume 或 Runner 等待跨秒就会把同一请求误判为上下文漂移并作废 sidecar。 +- 活跃 sidecar 同时阻断 finalization 与 `runner.shutdown_if_idle`。Runner 只有在 retry sidecar、pending action、confirmation、finalization、进程会话和其它 durable 工作全部清空后才能进入 idle shutdown。 + +### 验收边界 + +确定性回归必须覆盖：未到期零请求、到期唯一请求、耗尽清理、cancel、steer、Goal pause/resume 跨秒保持正文与 attempt、自动 context-compaction 先恢复再进入 tool-plan、sidecar 先于 task/state 的 torn projection、Runner 重启扫描、同 Agent FIFO、跨 Agent 并行、finalization/idle blocker、`.previous` 恢复/去重/排序和危险路径身份。原有 final-reply/repair 进程内重试回归必须继续通过。 + +V1.39 当前只完成确定性实现与回归，不把 V1.38 的失败轮、旧瞬态重试真实样本或最小 HTTP 探针拼接成新的真实 Provider PASS。需要真实验收时必须在最终代码上以独立 disposable 项目和隔离 AppData 完整运行，并证明 Runner 在退避期强杀后仍只发送同一 `-transient-N` attempt、其它 Agent 真并行、同 Agent FIFO、唯一最终回复、零重复/残留/正文/密钥/路径泄漏。 + +2026-07-19 最终代码的确定性门禁已通过：`provider_retry_` 19/19、`provider_transient_retry_` 6/6；Tauri/Rust 串行全量为 968 passed、4 个环境依赖用例按设计 ignored，`cargo check`、rustfmt、编码与 diff 检查通过。以上结果不替代尚未执行的 V1.39 独立真实 Provider E2E。 + ## 验收命令 - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml structured_plan_ -- --nocapture` @@ -1354,6 +1381,8 @@ V1.38 把 collaboration policy 的执行语义从“每次动作或恢复都读 - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml response_stream_ -- --nocapture` - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml mcp_ -- --nocapture` - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml provider_action_batch_ -- --nocapture` +- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml provider_retry_ -- --nocapture --test-threads=1` +- `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml provider_transient_retry_ -- --nocapture --test-threads=1` - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml parallel_read_batch_ -- --nocapture` - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml runtime_v134_ -- --nocapture` - `cargo test --manifest-path apps/ai-game-creator-shell/src-tauri/Cargo.toml isolated -- --nocapture --test-threads=1` diff --git a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md index 9b5681ec3..8249b21dd 100644 --- a/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md +++ b/docs/technical/【技术方案】AI游戏创作智能体App实施计划-2026-06-24.md @@ -605,3 +605,4 @@ game-project/ - snapshot v1 固定且完整包含 `schemaVersion / projectId / parentAgentId / parentRunId / boundFrom / policy / policyFingerprint / snapshotFingerprint / boundAt`；snapshot fingerprint 绑定除 `snapshotFingerprint / boundAt` 外的全部稳定字段。binding v1 固定包含 `schemaVersion / projectId / parentAgentId / parentRunId / boundFrom / policyFingerprint / snapshotFingerprint / boundAt`，必须与 snapshot 逐字段一致。安全 ID 可原样作 key；不安全 Agent/run ID 必须使用有界安全前缀加原始 ID 稳定 SHA-256，锁 key 对完整父 Agent/run 身份计算稳定指纹，禁止 lossy 规范化碰撞。 - 恢复优先级为 existing valid snapshot > 完整验真的 v2 batch contract > 符合严格状态门禁的 legacy 当前有效 policy。snapshot 存在但 binding 缺失时可从 snapshot 补写；binding 存在但 snapshot 丢失时只允许可信 v2 contract 按首次身份恢复，没有可信 v2 contract 时禁止按 live policy 重绑。contractless/v1 collaboration batch 必须先失败关闭，不能伪装 fresh run。`legacy-current-project-policy` 仅允许无 snapshot/binding、无可信 v2 contract，且不存在上述旧 batch，并由 durable 身份和状态明确证明属于 `pending / running / waiting-for-confirmation / waiting-for-user-input` 的旧父 run；terminal、`needs-reconciliation` 或身份/状态未知 run 的状态读取不得新建 snapshot。 - 已有 durable/未观察 claim 与 legacy claimed delivery 继续按原 action/group 身份恢复，不要求先创建新绑定；新 claim 必须先成功解析 effective snapshot 并核对 binding，再进入 V1.35-V1.37 的全锁、预算、完整 observation 和 group 数量门禁。snapshot 绑定后 global policy 的 `matched / drifted / unreadable` 只进入有界 status/诊断，不能改变后续执行；新 policy 只由后续新父 run 采用。2026-07-19 self-test、52/52 collaboration 定向回归和 949 passed/4 ignored Rust 全量已完成，终态快照保留也有独立回归；真实 mixed-swarm 功能样本已闭合但受正式 endpoint 外部重启污染，私有配置源的后续独立运行又连续耗尽 transient Provider retry，不能拼接证据，当前仍**不得声称 V1.38 真实 E2E 已 PASS**。详细报告以 Runtime 技术方案 V1.38 节为准。 +- 2026-07-19 起，同一 Runtime 文档的“V1.39 首次规划 Provider 瞬态重试持久等待态”作为后台首次规划重试的恢复事实源。tool-plan `repair-0` 及其自动 context-compaction 在瞬态失败后先写 per-Agent/run retry sidecar，再投影 `waiting-for-provider-retry` 并释放 lane；Runner 重启按到期时间恢复同一 Session/run/loop/attempt，同 Agent 后续任务保持 FIFO，其它 Agent 可并行。final-reply、手动压缩和 tool-plan `repair-N` 暂不扩展为持久重试；工具策略刷新时间不得进入请求指纹。当前只记录确定性回归，不改变 V1.38 真实 E2E 尚未 PASS 的结论。