修复 Agent Runtime 测试的终态等待竞态

- tests/mod.rs 的 wait_for_agent_runtime_idle 原先只轮询 status 且最多等 5 秒,超时后静默返回半成品状态,高负载 CI 下会读到 running 而被调用方断言判红
- 按既有 fence 口径改为等待「状态离开 running 且 Agent lane 锁可获取」,并在释放后二次读取确认,避免状态与 lane 之间的窗口
- 将终态等待统一为 AGENT_RUNTIME_TERMINAL_WAIT_BUDGET(60 秒),同步替换两个 async 终态 helper 的 10 秒预算
- 超时改为带诊断 panic,输出最后 status/phase/run,不再把中间态当终态返回
- 非终态用途的 wait_for_captured_mock_request 与 mock listener accept 预算保持 10 秒不变
This commit is contained in:
2026-09-21 01:42:59 +08:00
parent e35917ab69
commit 288d8567c9
@@ -457,27 +457,59 @@ fn agent_goal_sidecar_path_for_test(root: &Path, agent_id: &str, session_id: &st
))
}
fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState {
/// Agent Runtime 后台任务是异步收尾的:状态先落到终态,随后才释放 Agent lane 锁。
/// 只轮询 status 会在高负载 CI 上读到「状态已终态、lane 还没放」或「任务还没跑完」的中间态,
/// 因此统一等「状态离开 running 且 lane 可获取」,并留出远大于单机耗时的硬预算。
const AGENT_RUNTIME_TERMINAL_WAIT_BUDGET: Duration = Duration::from_secs(60);
fn wait_for_agent_runtime_lane_release(
root: &Path,
agent_id: &str,
description: &str,
) -> AgentRuntimeState {
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
let mut runtime = read_game_creator_agent_runtime_at(root, agent_id)
.expect("read runtime while waiting")
.state;
for _ in 0..250 {
if runtime.status == "idle" {
return runtime;
loop {
if runtime.status != "running" {
let lane_available = game_creator_agent_runtime_task_lock_is_available(root, agent_id)
.expect("probe runtime lane while waiting");
if lane_available {
let fence = read_game_creator_agent_runtime_at(root, agent_id)
.expect("reread runtime after lane release")
.state;
if fence.status != "running" {
return fence;
}
runtime = fence;
}
}
if Instant::now() >= deadline {
panic!(
"{description}: runtime did not reach a terminal state with a released Agent lane within {}s; last status={} phase={} run={}",
AGENT_RUNTIME_TERMINAL_WAIT_BUDGET.as_secs(),
runtime.status,
runtime.phase,
runtime.run_id
);
}
std::thread::sleep(Duration::from_millis(20));
runtime = read_game_creator_agent_runtime_at(root, agent_id)
.expect("read runtime while waiting")
.state;
}
runtime
}
fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState {
wait_for_agent_runtime_lane_release(root, agent_id, "wait_for_agent_runtime_idle")
}
async fn wait_for_captured_mock_request(
receiver: &mpsc::Receiver<String>,
description: &str,
) -> String {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
let deadline = Instant::now() + Duration::from_secs(10);
loop {
match receiver.try_recv() {
Ok(request) => return request,
@@ -501,7 +533,8 @@ fn wait_for_agent_runtime_terminal_and_lane_release(
) -> AgentRuntimeResult {
let mut result = read_game_creator_agent_runtime_at(root, agent_id)
.expect("read runtime while waiting for terminal lane release");
for _ in 0..250 {
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
loop {
let matches_terminal = result.state.run_id == run_id
&& result.state.status == status
&& result.state.phase == phase;
@@ -518,14 +551,17 @@ fn wait_for_agent_runtime_terminal_and_lane_release(
return terminal;
}
}
if Instant::now() >= deadline {
panic!(
"runtime did not reach {status}/{phase} for run {run_id} before the Agent lane released within {}s; last run={} status={} phase={}",
AGENT_RUNTIME_TERMINAL_WAIT_BUDGET.as_secs(),
result.state.run_id, result.state.status, result.state.phase
);
}
std::thread::sleep(Duration::from_millis(20));
result = read_game_creator_agent_runtime_at(root, agent_id)
.expect("read runtime while waiting for terminal lane release");
}
panic!(
"runtime did not reach {status}/{phase} for run {run_id} before the Agent lane released; last run={} status={} phase={}",
result.state.run_id, result.state.status, result.state.phase
);
}
pub(crate) async fn wait_for_agent_runtime_terminal_and_lane_release_async(
@@ -535,7 +571,7 @@ pub(crate) async fn wait_for_agent_runtime_terminal_and_lane_release_async(
status: &str,
phase: &str,
) -> AgentRuntimeResult {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
let mut last_lane_probe_error = None;
let mut result = read_game_creator_agent_runtime_at(root, agent_id)
.expect("read runtime while asynchronously waiting for terminal lane release");
@@ -587,7 +623,7 @@ pub(crate) async fn wait_for_agent_runtime_manifest_projection_async(
phase: &str,
manifest_status: GameCreationAppTaskStatus,
) -> AgentRuntimeResult {
let deadline = Instant::now() + Duration::from_secs(10);
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
let mut terminal = wait_for_agent_runtime_terminal_and_lane_release_async(
root,
agent_id,
@@ -639,7 +675,7 @@ async fn wait_for_agent_runtime_lane_release_async(
root: &Path,
agent_id: &str,
) -> AgentRuntimeResult {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
let mut last_lane_probe_error = None;
loop {
match game_creator_agent_runtime_task_lock_is_available(root, agent_id) {
@@ -2794,7 +2830,7 @@ fn spawn_response_stream_mock_llm_server(
listener
.set_nonblocking(true)
.expect("response stream mock listener nonblocking");
let accept_deadline = std::time::Instant::now() + Duration::from_secs(10);
let accept_deadline = Instant::now() + Duration::from_secs(10);
let (mut final_stream, _) = loop {
match listener.accept() {
Ok(connection) => break connection,