瞬态重试恢复不再被策划用量折叠边界判成自杀
策划子 Run 第一次 tool-plan 请求撞上 HTTP 524,通用重试机制留下 retry sidecar 并 唤醒同一个 run;该 run 重新走到新请求边界时,fold_plan_provider_usage_before_new_request 把「本 run 自己那条未收口的 exchange」判成 PLAN_PROVIDER_USAGE_DEFERRED 硬失败。 于是子 Run 直接 failed,回执退化为 needs-repair,逼总控走完整的查状态→认领→返工 委派流程,一轮实测白烧四轮 Provider 请求。等于策划子 Run 撞上任何一次 502/524 都必定自杀。 延期判据本身没错——它的第一条就是「该 run 存在 retry sidecar」,而这恰恰是重试 恢复的必经状态。折叠是记账动作:exchange 没收口时本来就不该计入 session,跳过一 次是正确的,收口后的下一个边界会补上;真正裁决重放还是重发的是下游 retry / handoff 身份比对,它本来就预期 sidecar 还在。 边界函数改为接收本次请求所属的 run,只豁免「延期完全由该 run 自己造成」这一种。 三个 runtime_actions 边界传当前身份;planning_coordinator 三处在派生后继 session, 子 Run 的在途 exchange 对它们仍是硬阻塞,传 None,行为不变。内部折叠额外返回延期 来源,公开枚举与既有调用方签名不动。 用例覆盖四个方向:本 run 放行、别的 run 仍硬失败、None 调用方行为不变,以及收口 后用量仍如数折叠进 session(豁免是延后记账,不是丢账)。已实测关掉豁免后该用例 复现线上原话。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+1
-1
@@ -28,7 +28,7 @@ pub(in crate::agent) async fn compact_game_creator_agent_runtime_context_at(
|
||||
if agent_id == GAME_CREATOR_PROJECT_PLANNING_AGENT_ID {
|
||||
// Advance the immutable Provider-usage projection before any
|
||||
// request/source bytes are rebuilt from the plan session.
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, Some((agent_id, run_id)))?;
|
||||
}
|
||||
let source = build_game_creator_agent_runtime_context_compaction_source(
|
||||
root,
|
||||
|
||||
+1
-1
@@ -119,7 +119,7 @@ pub(in crate::agent) async fn request_game_creator_agent_background_final_reply_
|
||||
if agent_id == GAME_CREATOR_PROJECT_PLANNING_AGENT_ID {
|
||||
// Keep every rebuilt request field on the same budget successor
|
||||
// that will be exposed by the structured injection and binding.
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, Some((agent_id, run_id)))?;
|
||||
// Freeze the concrete final-reply request and its Provider-facing
|
||||
// planning injection under the same project lock as the durable
|
||||
// session binding. This mirrors tool-plan and prevents an older
|
||||
|
||||
+1
-1
@@ -337,7 +337,7 @@ pub(in crate::agent) async fn request_game_creator_agent_background_tool_plan_at
|
||||
if agent_id == GAME_CREATOR_PROJECT_PLANNING_AGENT_ID {
|
||||
// Fold first so the request rebuild, structured injection and
|
||||
// frozen session binding all observe one budget successor.
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, Some((agent_id, run_id)))?;
|
||||
}
|
||||
// Exact planning requests must freeze the session and the concrete
|
||||
// request object under one project lock. Rebuild once while holding
|
||||
|
||||
+3
-3
@@ -430,7 +430,7 @@ pub(crate) fn ensure_plan_session_for_planning_child_task_at_locked(
|
||||
// it advances the same session while this caller retains `project_lock`.
|
||||
// On an initial child this is a no-op (`NoSession`), and root usage is
|
||||
// folded immediately after revision 1 has established the lineage below.
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, None)?;
|
||||
let current =
|
||||
read_plan_session_with_recovery_locked(root).map_err(|error| error.to_string())?;
|
||||
let Some(previous) = current else {
|
||||
@@ -491,7 +491,7 @@ pub(crate) fn ensure_plan_session_for_planning_child_task_at_locked(
|
||||
// Root plan Provider facts predate the first planning-child session.
|
||||
// Establish revision 1 first, then fold those immutable facts under
|
||||
// the same project lock so the child never starts from budget zero.
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, None)?;
|
||||
return Ok(true);
|
||||
};
|
||||
if previous.project_id != project_id {
|
||||
@@ -607,7 +607,7 @@ pub(crate) fn project_plan_session_awaiting_user_input_at_locked(
|
||||
));
|
||||
}
|
||||
validate_project_supervisor_plan_root_binding_at(root, &runtime.agent_id, &runtime.run_id)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root)?;
|
||||
fold_plan_provider_usage_before_new_request_at_locked(root, None)?;
|
||||
let current_delivery = read_static_delegate_delivery_at(root, &delivery.delegation_id)?
|
||||
.ok_or_else(|| {
|
||||
plan_coordinator_error("PLAN_NEEDS_RECONCILIATION", "planning delivery 缺失")
|
||||
|
||||
+111
-13
@@ -260,25 +260,29 @@ fn read_plan_provider_usage_facts_at(root: &Path) -> Result<Vec<PlanProviderUsag
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn planning_child_usage_projection_is_deferred_at(
|
||||
/// 列出当前让 usage 折叠必须延期的策划 run。返回列表而不是布尔,是因为调用方
|
||||
/// 需要区分「别的 run 有在途 exchange」和「延期就是本 run 自己造成的」——后者是
|
||||
/// 瞬态重试恢复的必经状态,不是冲突。
|
||||
fn planning_child_usage_projection_deferring_runs_at(
|
||||
root: &Path,
|
||||
facts: &[PlanProviderUsageFactV1],
|
||||
) -> Result<bool, String> {
|
||||
) -> Result<Vec<(String, String)>, String> {
|
||||
let mut runs = BTreeMap::<(String, String), ()>::new();
|
||||
for fact in facts {
|
||||
if fact.planning_session_binding.is_some() {
|
||||
runs.insert((fact.agent_id.clone(), fact.run_id.clone()), ());
|
||||
}
|
||||
}
|
||||
let mut deferring = Vec::new();
|
||||
for ((agent_id, run_id), ()) in runs {
|
||||
if crate::provider_retry::read_for_run_at(root, &agent_id, &run_id)?.is_some()
|
||||
|| crate::provider_handoff::read_for_run_at(root, &agent_id, &run_id)?.is_some()
|
||||
|| game_creator_agent_runtime_provider_action_batch_exists(root, &agent_id, &run_id)
|
||||
{
|
||||
return Ok(true);
|
||||
deferring.push((agent_id, run_id));
|
||||
}
|
||||
}
|
||||
Ok(false)
|
||||
Ok(deferring)
|
||||
}
|
||||
|
||||
fn plan_provider_usage_fact_matches_session(
|
||||
@@ -336,10 +340,16 @@ fn plan_provider_usage_fact_matches_session(
|
||||
pub(crate) fn fold_plan_provider_usage_into_session_at_locked(
|
||||
root: &Path,
|
||||
) -> Result<PlanProviderUsageFoldOutcome, String> {
|
||||
Ok(fold_plan_provider_usage_into_session_at_locked_with_deferring_runs(root)?.0)
|
||||
}
|
||||
|
||||
fn fold_plan_provider_usage_into_session_at_locked_with_deferring_runs(
|
||||
root: &Path,
|
||||
) -> Result<(PlanProviderUsageFoldOutcome, Vec<(String, String)>), String> {
|
||||
let Some(previous) =
|
||||
read_plan_session_with_recovery_locked(root).map_err(|error| error.to_string())?
|
||||
else {
|
||||
return Ok(PlanProviderUsageFoldOutcome::NoSession);
|
||||
return Ok((PlanProviderUsageFoldOutcome::NoSession, Vec::new()));
|
||||
};
|
||||
let all_facts = read_plan_provider_usage_facts_at(root)?;
|
||||
let mut unique = BTreeMap::<String, PlanProviderUsageFactV1>::new();
|
||||
@@ -363,8 +373,9 @@ pub(crate) fn fold_plan_provider_usage_into_session_at_locked(
|
||||
matching.push(fact);
|
||||
}
|
||||
}
|
||||
if planning_child_usage_projection_is_deferred_at(root, &matching)? {
|
||||
return Ok(PlanProviderUsageFoldOutcome::Deferred);
|
||||
let deferring_runs = planning_child_usage_projection_deferring_runs_at(root, &matching)?;
|
||||
if !deferring_runs.is_empty() {
|
||||
return Ok((PlanProviderUsageFoldOutcome::Deferred, deferring_runs));
|
||||
}
|
||||
|
||||
let mut total = 0_u64;
|
||||
@@ -426,7 +437,7 @@ pub(crate) fn fold_plan_provider_usage_into_session_at_locked(
|
||||
);
|
||||
}
|
||||
if previous.accumulated_agent_millis == total {
|
||||
return Ok(PlanProviderUsageFoldOutcome::Unchanged);
|
||||
return Ok((PlanProviderUsageFoldOutcome::Unchanged, Vec::new()));
|
||||
}
|
||||
let mut next = previous.clone();
|
||||
next.session_revision = previous
|
||||
@@ -441,20 +452,48 @@ pub(crate) fn fold_plan_provider_usage_into_session_at_locked(
|
||||
plan_session_fingerprint(&next).map_err(|error| error.to_string())?;
|
||||
validate_plan_session_successor(&previous, &next).map_err(|error| error.to_string())?;
|
||||
write_plan_session_atomic_locked(root, &next).map_err(|error| error.to_string())?;
|
||||
Ok(PlanProviderUsageFoldOutcome::Advanced)
|
||||
Ok((PlanProviderUsageFoldOutcome::Advanced, Vec::new()))
|
||||
}
|
||||
|
||||
/// A fresh Provider request may only freeze bytes after all prior planning
|
||||
/// exchange state has cleared. Retry replay keeps using its already-frozen
|
||||
/// bytes elsewhere; reaching this boundary while folding is deferred must
|
||||
/// block instead of quietly issuing a request from the older session.
|
||||
///
|
||||
/// `resuming_run` 是本次请求所属的 run。瞬态上游失败后,通用重试机制会留下 retry
|
||||
/// sidecar 再唤醒同一个 run;这个 run 随后必然重新走到本边界,而它自己那条尚未
|
||||
/// 收口的 exchange 正是延期判据的第一条。把这种情况也判成硬失败,等于让策划子 Run
|
||||
/// 撞上任何一次 502/524 都必定自杀——现场就是子 Run 在第一次 tool-plan 请求超时后
|
||||
/// 直接 failed,回执退化成 needs-repair,逼总控多烧一整轮返工委派。
|
||||
///
|
||||
/// 折叠本身是记账动作:exchange 还没收口时本来就不该把它计入 session,跳过一次是
|
||||
/// 正确的,等收口后的下一个边界会补上。真正裁决重放还是重发的是下游的 retry /
|
||||
/// handoff 身份比对。因此只豁免「延期完全由 `resuming_run` 自己造成」这一种;别的
|
||||
/// 策划 run 有在途 exchange 时仍然硬失败,传 `None` 的调用方行为不变。
|
||||
pub(crate) fn fold_plan_provider_usage_before_new_request_at_locked(
|
||||
root: &Path,
|
||||
resuming_run: Option<(&str, &str)>,
|
||||
) -> Result<(), String> {
|
||||
match fold_plan_provider_usage_into_session_at_locked(root)? {
|
||||
PlanProviderUsageFoldOutcome::Deferred => Err(
|
||||
"PLAN_PROVIDER_USAGE_DEFERRED: 上一条 planning Provider exchange 尚未收口".to_string(),
|
||||
),
|
||||
let (outcome, deferring_runs) =
|
||||
fold_plan_provider_usage_into_session_at_locked_with_deferring_runs(root)?;
|
||||
match outcome {
|
||||
PlanProviderUsageFoldOutcome::Deferred => {
|
||||
if let Some((agent_id, run_id)) = resuming_run {
|
||||
if !deferring_runs.is_empty()
|
||||
&& deferring_runs
|
||||
.iter()
|
||||
.all(|(deferring_agent, deferring_run)| {
|
||||
deferring_agent == agent_id && deferring_run == run_id
|
||||
})
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
Err(
|
||||
"PLAN_PROVIDER_USAGE_DEFERRED: 上一条 planning Provider exchange 尚未收口"
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
PlanProviderUsageFoldOutcome::NoSession
|
||||
| PlanProviderUsageFoldOutcome::Unchanged
|
||||
| PlanProviderUsageFoldOutcome::Advanced => Ok(()),
|
||||
@@ -775,6 +814,65 @@ mod tests {
|
||||
assert_eq!(advanced.accumulated_agent_millis, 41);
|
||||
}
|
||||
|
||||
/// 瞬态上游失败会留下 sidecar 再唤醒同一个 run;该 run 重新走到新请求边界时,
|
||||
/// 唯一的延期来源就是它自己那条未收口的 exchange。把这判成硬失败等于让策划子
|
||||
/// Run 撞上任何一次 502/524 都必定自杀。别的 run 造成的延期必须照旧硬失败。
|
||||
#[test]
|
||||
fn new_request_boundary_lets_the_resuming_run_past_its_own_deferral() {
|
||||
let fixture = usage_fixture();
|
||||
let snapshot = planning_snapshot(&fixture, "tool-plan", "usage-retry-resume");
|
||||
persist_usage(&fixture, &snapshot, "failed", 17, true);
|
||||
|
||||
let _lock = acquire_game_creator_agent_runtime_project_write_lock_with_wait(
|
||||
&fixture.root,
|
||||
"test.planning_provider_usage.retry_resume_boundary",
|
||||
)
|
||||
.expect("retry-resume boundary lock");
|
||||
let batch_path = game_creator_agent_runtime_provider_action_batch_path(
|
||||
&fixture.root,
|
||||
&snapshot.agent_id,
|
||||
&snapshot.run_id,
|
||||
);
|
||||
std::fs::create_dir_all(batch_path.parent().expect("batch parent"))
|
||||
.expect("create batch parent");
|
||||
std::fs::write(&batch_path, b"in-flight-exchange").expect("create in-flight sentinel");
|
||||
|
||||
fold_plan_provider_usage_before_new_request_at_locked(
|
||||
&fixture.root,
|
||||
Some((&snapshot.agent_id, &snapshot.run_id)),
|
||||
)
|
||||
.expect("本 run 自己的在途 exchange 不能拦住它自己的重试恢复");
|
||||
|
||||
let other_run = fold_plan_provider_usage_before_new_request_at_locked(
|
||||
&fixture.root,
|
||||
Some((&snapshot.agent_id, "delegated-some-other-run")),
|
||||
)
|
||||
.expect_err("别的 run 的在途 exchange 仍须硬失败");
|
||||
assert!(
|
||||
other_run.starts_with("PLAN_PROVIDER_USAGE_DEFERRED"),
|
||||
"{other_run}"
|
||||
);
|
||||
|
||||
let no_owner = fold_plan_provider_usage_before_new_request_at_locked(&fixture.root, None)
|
||||
.expect_err("未声明归属的调用方行为不变");
|
||||
assert!(
|
||||
no_owner.starts_with("PLAN_PROVIDER_USAGE_DEFERRED"),
|
||||
"{no_owner}"
|
||||
);
|
||||
|
||||
// 豁免只是本轮跳过记账,不是把这笔用量丢掉:exchange 收口后仍须折叠进来。
|
||||
std::fs::remove_file(&batch_path).expect("remove settled sentinel");
|
||||
fold_plan_provider_usage_before_new_request_at_locked(
|
||||
&fixture.root,
|
||||
Some((&snapshot.agent_id, &snapshot.run_id)),
|
||||
)
|
||||
.expect("exchange 收口后折叠");
|
||||
let session = read_plan_session_with_recovery_locked(&fixture.root)
|
||||
.expect("read folded session")
|
||||
.expect("folded session exists");
|
||||
assert_eq!(session.accumulated_agent_millis, 17);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fold_waits_for_submit_anchor_cleanup_then_advances() {
|
||||
let fixture = usage_fixture();
|
||||
|
||||
Reference in New Issue
Block a user