修复 Agent Runtime 测试的终态等待竞态
- tests/mod.rs 的 wait_for_agent_runtime_idle 原先只轮询 status 且最多等 5 秒,超时后静默返回半成品状态,高负载 CI 下会读到 running 而被调用方断言判红 - 按既有 fence 口径改为等待「状态离开 running 且 Agent lane 锁可获取」,并在释放后二次读取确认,避免状态与 lane 之间的窗口 - 将终态等待统一为 AGENT_RUNTIME_TERMINAL_WAIT_BUDGET(60 秒),同步替换两个 async 终态 helper 的 10 秒预算 - 超时改为带诊断 panic,输出最后 status/phase/run,不再把中间态当终态返回 - 非终态用途的 wait_for_captured_mock_request 与 mock listener accept 预算保持 10 秒不变
This commit is contained in:
@@ -457,27 +457,59 @@ fn agent_goal_sidecar_path_for_test(root: &Path, agent_id: &str, session_id: &st
|
||||
))
|
||||
}
|
||||
|
||||
fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState {
|
||||
/// Agent Runtime 后台任务是异步收尾的:状态先落到终态,随后才释放 Agent lane 锁。
|
||||
/// 只轮询 status 会在高负载 CI 上读到「状态已终态、lane 还没放」或「任务还没跑完」的中间态,
|
||||
/// 因此统一等「状态离开 running 且 lane 可获取」,并留出远大于单机耗时的硬预算。
|
||||
const AGENT_RUNTIME_TERMINAL_WAIT_BUDGET: Duration = Duration::from_secs(60);
|
||||
|
||||
fn wait_for_agent_runtime_lane_release(
|
||||
root: &Path,
|
||||
agent_id: &str,
|
||||
description: &str,
|
||||
) -> AgentRuntimeState {
|
||||
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
|
||||
let mut runtime = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting")
|
||||
.state;
|
||||
for _ in 0..250 {
|
||||
if runtime.status == "idle" {
|
||||
return runtime;
|
||||
loop {
|
||||
if runtime.status != "running" {
|
||||
let lane_available = game_creator_agent_runtime_task_lock_is_available(root, agent_id)
|
||||
.expect("probe runtime lane while waiting");
|
||||
if lane_available {
|
||||
let fence = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("reread runtime after lane release")
|
||||
.state;
|
||||
if fence.status != "running" {
|
||||
return fence;
|
||||
}
|
||||
runtime = fence;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!(
|
||||
"{description}: runtime did not reach a terminal state with a released Agent lane within {}s; last status={} phase={} run={}",
|
||||
AGENT_RUNTIME_TERMINAL_WAIT_BUDGET.as_secs(),
|
||||
runtime.status,
|
||||
runtime.phase,
|
||||
runtime.run_id
|
||||
);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
runtime = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting")
|
||||
.state;
|
||||
}
|
||||
runtime
|
||||
}
|
||||
|
||||
fn wait_for_agent_runtime_idle(root: &Path, agent_id: &str) -> AgentRuntimeState {
|
||||
wait_for_agent_runtime_lane_release(root, agent_id, "wait_for_agent_runtime_idle")
|
||||
}
|
||||
|
||||
async fn wait_for_captured_mock_request(
|
||||
receiver: &mpsc::Receiver<String>,
|
||||
description: &str,
|
||||
) -> String {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
match receiver.try_recv() {
|
||||
Ok(request) => return request,
|
||||
@@ -501,7 +533,8 @@ fn wait_for_agent_runtime_terminal_and_lane_release(
|
||||
) -> AgentRuntimeResult {
|
||||
let mut result = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting for terminal lane release");
|
||||
for _ in 0..250 {
|
||||
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
|
||||
loop {
|
||||
let matches_terminal = result.state.run_id == run_id
|
||||
&& result.state.status == status
|
||||
&& result.state.phase == phase;
|
||||
@@ -518,14 +551,17 @@ fn wait_for_agent_runtime_terminal_and_lane_release(
|
||||
return terminal;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
panic!(
|
||||
"runtime did not reach {status}/{phase} for run {run_id} before the Agent lane released within {}s; last run={} status={} phase={}",
|
||||
AGENT_RUNTIME_TERMINAL_WAIT_BUDGET.as_secs(),
|
||||
result.state.run_id, result.state.status, result.state.phase
|
||||
);
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
result = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while waiting for terminal lane release");
|
||||
}
|
||||
panic!(
|
||||
"runtime did not reach {status}/{phase} for run {run_id} before the Agent lane released; last run={} status={} phase={}",
|
||||
result.state.run_id, result.state.status, result.state.phase
|
||||
);
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_agent_runtime_terminal_and_lane_release_async(
|
||||
@@ -535,7 +571,7 @@ pub(crate) async fn wait_for_agent_runtime_terminal_and_lane_release_async(
|
||||
status: &str,
|
||||
phase: &str,
|
||||
) -> AgentRuntimeResult {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
|
||||
let mut last_lane_probe_error = None;
|
||||
let mut result = read_game_creator_agent_runtime_at(root, agent_id)
|
||||
.expect("read runtime while asynchronously waiting for terminal lane release");
|
||||
@@ -587,7 +623,7 @@ pub(crate) async fn wait_for_agent_runtime_manifest_projection_async(
|
||||
phase: &str,
|
||||
manifest_status: GameCreationAppTaskStatus,
|
||||
) -> AgentRuntimeResult {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
|
||||
let mut terminal = wait_for_agent_runtime_terminal_and_lane_release_async(
|
||||
root,
|
||||
agent_id,
|
||||
@@ -639,7 +675,7 @@ async fn wait_for_agent_runtime_lane_release_async(
|
||||
root: &Path,
|
||||
agent_id: &str,
|
||||
) -> AgentRuntimeResult {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let deadline = Instant::now() + AGENT_RUNTIME_TERMINAL_WAIT_BUDGET;
|
||||
let mut last_lane_probe_error = None;
|
||||
loop {
|
||||
match game_creator_agent_runtime_task_lock_is_available(root, agent_id) {
|
||||
@@ -2794,7 +2830,7 @@ fn spawn_response_stream_mock_llm_server(
|
||||
listener
|
||||
.set_nonblocking(true)
|
||||
.expect("response stream mock listener nonblocking");
|
||||
let accept_deadline = std::time::Instant::now() + Duration::from_secs(10);
|
||||
let accept_deadline = Instant::now() + Duration::from_secs(10);
|
||||
let (mut final_stream, _) = loop {
|
||||
match listener.accept() {
|
||||
Ok(connection) => break connection,
|
||||
|
||||
Reference in New Issue
Block a user