审查发现1:澄清恢复的锁序重排改为让出本轮,不再掐掉整批 resume
M1C-2b 为澄清 pending 新增的「drop 执行锁 -> 取项目锁 -> 重取执行锁」重排块, 两次取锁都用裸 `?`。执行锁的重取只等 25 x 10ms,而用户刚提交澄清回答时那把锁 会被 move 进后台续跑任务、持有整个 Provider 回合,稳超等待上限;错误经 recovery_scan 的裸 `?` 上抛,掐掉 for agent_id 循环里整批 Agent 的恢复,并把一次 纯瞬时的锁竞争直接返回给前端。 锁被占恰恰说明别处正在推进,是最不该判失败的时候。同一个 commit 里的姊妹代码 (planning session 恢复窗口)已经写对:项目锁按 transient 判据 continue,执行锁 用非阻塞 try_acquire 拿不到就 continue。 - AgentRuntimePendingActionResume 新增不带锁的 Deferred,表示两把锁都已释放、 本轮让出;批量扫描的三处 match 一律 continue - 项目锁改为 transient 判据让路,其余错误仍带上下文上抛 - 执行锁改用 try_..._with_wait:等待宽限不变,超时是「本轮没轮到」而非「恢复失败」 - Runner 定向续跑不是批量扫描,Deferred 仍如实报错,文案沿用执行锁自己的措辞 - 回归 planning_clarification_recovery_defers_when_execution_lane_is_still_held: 按住项目锁把恢复卡在重排窗口,抢走执行锁后放开项目锁,断言必须 Deferred。 变异验证:回退成阻塞版 `?` 后该用例报出 "Agent Runtime 正在执行该 Agent 的其他任务:project-supervisor" Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,11 @@ use super::*;
|
||||
pub(crate) enum AgentRuntimePendingActionResume {
|
||||
NotFound(AgentRuntimeTaskLock),
|
||||
Handled(AgentRuntimeResult),
|
||||
/// 本轮无法在不破坏锁序的前提下推进:为了按 project -> execution 顺序取锁,
|
||||
/// 执行锁已经被放掉,重取时又被别处占住。返回时不带锁——两把锁都已释放。
|
||||
/// 这不是失败:锁被占恰恰说明别处正在推进,调用方应跳过该 Agent 等下一轮,
|
||||
/// 而不是把整轮恢复判失败。
|
||||
Deferred,
|
||||
}
|
||||
|
||||
pub(in crate::agent) enum AgentRuntimeFinalizationResume {
|
||||
|
||||
@@ -1078,11 +1078,32 @@ pub(crate) fn resume_game_creator_agent_pending_tool_action_at(
|
||||
let expected_action_id = pending.action_id.clone();
|
||||
let expected_action_fingerprint = pending.action_fingerprint.clone();
|
||||
drop(runtime_lock);
|
||||
let project_lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait(
|
||||
// 为守住 project -> execution 的锁序,执行锁已经在上一行放掉了。接下来这
|
||||
// 两把锁都可能正被别处占住——用户刚提交澄清回答时,执行锁会被 move 进
|
||||
// 后台续跑任务,一持有就是整个 Provider 回合,远超这里的等待上限。锁被
|
||||
// 占说明系统在前进,是最不该把整轮恢复判失败的时候:本轮让出,两把锁都
|
||||
// 释放,下一轮 resume 重来。与 `recovery_scan` 里同形状的 planning
|
||||
// session 恢复窗口保持同一套语义。
|
||||
let project_lock = match acquire_game_creator_agent_runtime_project_write_lock_with_wait(
|
||||
root,
|
||||
"planning.clarification.answer-recovery",
|
||||
)?;
|
||||
let runtime_lock = acquire_game_creator_agent_runtime_task_lock_with_wait(root, agent_id)?;
|
||||
) {
|
||||
Ok(project_lock) => project_lock,
|
||||
Err(error) if static_delegate_parent_wake_error_is_transient(&error) => {
|
||||
return Ok(AgentRuntimePendingActionResume::Deferred);
|
||||
}
|
||||
Err(error) => {
|
||||
return Err(format!("恢复 planning 澄清回答前取得项目锁失败:{error}"));
|
||||
}
|
||||
};
|
||||
// 用 `try_..._with_wait`(返回 `Option`)而不是 `acquire_..._with_wait`
|
||||
// (拿不到就 `Err`):等待宽限一样是 25 x 10ms,但超时是「本轮没轮到」而
|
||||
// 不是「恢复失败」。
|
||||
let Some(runtime_lock) =
|
||||
try_acquire_game_creator_agent_runtime_task_lock_with_wait(root, agent_id)?
|
||||
else {
|
||||
return Ok(AgentRuntimePendingActionResume::Deferred);
|
||||
};
|
||||
runtime = read_game_creator_agent_runtime_at(root, agent_id)?.state;
|
||||
if runtime.run_id != expected_run_id
|
||||
|| runtime.session_id != expected_session_id
|
||||
@@ -1875,6 +1896,7 @@ pub(crate) fn resume_game_creator_agent_provider_action_batch_for_test_at(
|
||||
match resume_game_creator_agent_provider_action_batch_at(root, agent_id, runtime_lock)? {
|
||||
AgentRuntimePendingActionResume::Handled(_) => Ok("handled"),
|
||||
AgentRuntimePendingActionResume::NotFound(_) => Ok("not-found"),
|
||||
AgentRuntimePendingActionResume::Deferred => Ok("deferred"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1204,6 +1204,7 @@ pub(in crate::agent) fn resume_game_creator_agent_background_tasks_unredacted_at
|
||||
continue;
|
||||
}
|
||||
AgentRuntimePendingActionResume::NotFound(runtime_lock) => runtime_lock,
|
||||
AgentRuntimePendingActionResume::Deferred => continue,
|
||||
};
|
||||
let runtime_lock = match resume_game_creator_agent_pending_tool_action_at(
|
||||
root,
|
||||
@@ -1215,6 +1216,8 @@ pub(in crate::agent) fn resume_game_creator_agent_background_tasks_unredacted_at
|
||||
continue;
|
||||
}
|
||||
AgentRuntimePendingActionResume::NotFound(runtime_lock) => runtime_lock,
|
||||
// 锁序重排窗口里没抢到锁。跳过该 Agent,本轮其余 Agent 照常恢复。
|
||||
AgentRuntimePendingActionResume::Deferred => continue,
|
||||
};
|
||||
match resume_game_creator_agent_provider_action_batch_at(root, &agent_id, runtime_lock)?
|
||||
{
|
||||
@@ -1223,6 +1226,7 @@ pub(in crate::agent) fn resume_game_creator_agent_background_tasks_unredacted_at
|
||||
continue;
|
||||
}
|
||||
AgentRuntimePendingActionResume::NotFound(runtime_lock) => runtime_lock,
|
||||
AgentRuntimePendingActionResume::Deferred => continue,
|
||||
}
|
||||
};
|
||||
let Some(task) =
|
||||
@@ -1607,6 +1611,11 @@ pub(crate) fn resume_game_creator_agent_pending_action_for_agent_at(
|
||||
AgentRuntimePendingActionResume::NotFound(_runtime_lock) => {
|
||||
Err("Agent Runner 未找到可继续的精确待处理动作".to_string())
|
||||
}
|
||||
// 这条入口是「继续这一个动作」的定向请求,不是批量扫描:没抢到锁只能如实
|
||||
// 报错。文案沿用执行锁自己的措辞,让上游的 transient 判据仍能认出它。
|
||||
AgentRuntimePendingActionResume::Deferred => Err(format!(
|
||||
"Agent Runtime 正在执行该 Agent 的其他任务:{agent_id}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4696,6 +4696,95 @@ fn planning_clarification_answer_prepared_recovery_releases_execution_before_pro
|
||||
cleanup_planning_clarification_fixture(fixture);
|
||||
}
|
||||
|
||||
/// 锁序重排窗口的另一半:放掉执行锁、拿到项目锁之后,重取执行锁可能撞上「用户刚
|
||||
/// 提交澄清回答、后台续跑仍握着执行锁」——那把锁会被 move 进 spawn 出去的续跑任务,
|
||||
/// 一持有就是整个 Provider 回合,远超重取的 25 x 10ms 等待上限。
|
||||
///
|
||||
/// 锁被占恰恰说明别处正在推进,是最不该判失败的时候。所以这里只能让出本轮
|
||||
/// (`Deferred`),不能让错误经 `?` 一路上抛——上抛会掐掉
|
||||
/// `resume_game_creator_agent_background_tasks_at` 对**整批** Agent 的恢复,并把一次
|
||||
/// 纯瞬时的锁竞争直接甩给前端。
|
||||
#[test]
|
||||
fn planning_clarification_recovery_defers_when_execution_lane_is_still_held() {
|
||||
let mut fixture = planning_clarification_fixture("deferred-execution-contention");
|
||||
let (pending, request, question_id) =
|
||||
prepare_first_planning_clarification_wait(&mut fixture, "deferred-execution-contention");
|
||||
fs::write(
|
||||
fixture
|
||||
.root
|
||||
.join(AGENT_RUNTIME_USER_INPUT_STOP_AFTER_PREPARED_FOR_TEST),
|
||||
b"armed",
|
||||
)
|
||||
.expect("arm answer-prepared crash window");
|
||||
let response_id = "planning-deferred-contention-response";
|
||||
let injected = answer_game_creator_agent_user_input_request_for_pending_at(
|
||||
&fixture.root,
|
||||
&pending,
|
||||
&request.request_id,
|
||||
response_id,
|
||||
BTreeMap::from([(question_id, PLAN_TEST_OPTION_A.to_string())]),
|
||||
)
|
||||
.expect_err("answer must stop after durable answer-prepared");
|
||||
assert!(injected.contains("answer-prepared"), "{injected}");
|
||||
|
||||
// 先按住项目锁,把恢复线程卡在「已经放掉执行锁、正在等项目锁」的窗口里。
|
||||
let project_lock =
|
||||
acquire_project_write_lock(&fixture.root, "test.deferred-contention.project-holder")
|
||||
.expect("hold project lock across recovery reorder");
|
||||
let thread_root = fixture.root.clone();
|
||||
let (started_sender, started_receiver) = std::sync::mpsc::channel();
|
||||
let worker = std::thread::spawn(move || {
|
||||
let runtime_lock = try_acquire_game_creator_agent_runtime_task_lock(
|
||||
&thread_root,
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
)
|
||||
.expect("acquire initial recovery execution lane")
|
||||
.expect("initial recovery execution lane available");
|
||||
started_sender
|
||||
.send(())
|
||||
.expect("signal deferred recovery start");
|
||||
resume_game_creator_agent_pending_tool_action_at(
|
||||
&thread_root,
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
runtime_lock,
|
||||
)
|
||||
});
|
||||
started_receiver
|
||||
.recv_timeout(std::time::Duration::from_secs(1))
|
||||
.expect("deferred recovery started with execution lane");
|
||||
|
||||
// 恢复线程一放掉执行锁就抢走它并按住不放,模拟用户回答的后台续跑。
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
let stolen_execution = loop {
|
||||
if let Some(runtime_lock) = try_acquire_game_creator_agent_runtime_task_lock(
|
||||
&fixture.root,
|
||||
GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID,
|
||||
)
|
||||
.expect("probe released recovery execution lane")
|
||||
{
|
||||
break runtime_lock;
|
||||
}
|
||||
assert!(
|
||||
std::time::Instant::now() < deadline,
|
||||
"恢复必须先释放 execution lane 再等待 project lock,否则本用例失去判据"
|
||||
);
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
};
|
||||
// 放开项目锁:恢复线程随即拿到它,再去重取执行锁——而执行锁在我们手上。
|
||||
drop(project_lock);
|
||||
|
||||
let recovery = worker
|
||||
.join()
|
||||
.expect("join deferred recovery")
|
||||
.expect("执行锁被别处占用是瞬时争用,恢复只能让出本轮,不得让整轮 resume 失败");
|
||||
assert!(
|
||||
matches!(recovery, AgentRuntimePendingActionResume::Deferred),
|
||||
"重取执行锁没抢到时必须让出本轮,交给下一轮 resume 重试"
|
||||
);
|
||||
drop(stolen_execution);
|
||||
cleanup_planning_clarification_fixture(fixture);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn planning_clarification_parent_wake_defers_while_execution_lane_is_busy_and_replays_once() {
|
||||
let mut fixture = planning_clarification_fixture("parent-wake-lock-order");
|
||||
@@ -5002,6 +5091,9 @@ fn planning_clarification_recovery_treats_concurrent_answer_as_obsolete_candidat
|
||||
AgentRuntimePendingActionResume::Handled(_) => {
|
||||
panic!("concurrently advanced answer must make the old recovery candidate obsolete")
|
||||
}
|
||||
AgentRuntimePendingActionResume::Deferred => {
|
||||
panic!("两把锁都已放开,本轮不该让出")
|
||||
}
|
||||
}
|
||||
let current =
|
||||
read_game_creator_agent_runtime_at(&fixture.root, GAME_CREATOR_PROJECT_SUPERVISOR_AGENT_ID)
|
||||
|
||||
Reference in New Issue
Block a user