宿主:连接死亡的失败事实先于看门狗落地

- `CodexAppServerInner` 新增私有去重标志 `connection_end_claimed`,与 `closed` 分开:认领只保证死亡收口只跑一次,"看门狗可以开始收束"必须等失败事实写进执行适配器
- `fail_game_creator_codex_app_server_connection` 改用新标志去重,不再顺带置 `closed`;`closed` 交给 `shutdown_game_creator_codex_app_server_inner` 在 `record_execution_turn_failure` 之后置位,看门狗在事实落地前没有可观测信号
- 补一条把看门狗真正跑起来的回归用例 `connection_death_records_the_failure_fact_before_the_watchdog_seals_the_turn`:卡住 stderr 摘要锁把窗口拉成确定性,断言终态仍带 `transport-failed` 载荷(顺序反了就红)
- 失败事实是在模型终态那一刻被快照进终态上下文的,晚补记无用,所以只修"事实先于可见性"这一条落点
This commit is contained in:
2026-09-24 16:51:36 +08:00
parent 2f5e0b0ede
commit ab970b9fdb
@@ -1343,7 +1343,13 @@ struct CodexAppServerInner {
execution: std::sync::Mutex<Option<Arc<ExecutionAdapter>>>,
next_request_id: AtomicU64,
last_used: AtomicU64,
/// 连接已死。**它在语义上是"这一段已经收束 / 失败事实已经记下"**,看门狗就盯着它(见
/// `ExecutionAdapter::start_watchdog`);所以置位必须发生在失败事实落地之后——别拿它当去重标志用,
/// 那是 [`Self::connection_end_claimed`] 的事。
closed: AtomicBool,
/// 谁的连接死亡收口第一个到(去重)。它与 `closed` 是两件事:认领只保证"这一段只跑一次",
/// 而"看门狗可以开始收束了"必须等到失败事实写进执行适配器之后,否则终态判定拿不到原因。
connection_end_claimed: AtomicBool,
_working_dir: tempfile::TempDir,
workspace_path: std::path::PathBuf,
workspace_mode: CodexAppServerWorkspaceMode,
@@ -2869,6 +2875,7 @@ impl CodexAppServerConnection {
next_request_id: AtomicU64::new(1),
last_used: AtomicU64::new(next_game_creator_codex_app_server_usage_tick()),
closed: AtomicBool::new(false),
connection_end_claimed: AtomicBool::new(false),
_working_dir: working_dir,
workspace_path,
workspace_mode,
@@ -5128,7 +5135,9 @@ async fn fail_game_creator_codex_app_server_connection(
let Some(inner) = inner.upgrade() else {
return;
};
if inner.closed.swap(true, Ordering::AcqRel) {
// 去重只看这个私有标志:**不能**用 `inner.closed` 顺手去重——它是看门狗的信号,先置上就等于
// "失败事实还没记,收束已经可以开始"(见下面注释与 `CodexAppServerInner::closed` 的说明)。
if inner.connection_end_claimed.swap(true, Ordering::AcqRel) {
return;
}
let exit_status = inner
@@ -5142,9 +5151,12 @@ async fn fail_game_creator_codex_app_server_connection(
let stderr = inner.stderr_summary.lock().await.diagnostic();
let diagnostic = format!("{error}exitStatus={exit_status}{stderr}");
app_log!("agent.runner.failed: Codex app-server 连接终止:{diagnostic}");
// 连接是在回合进行中断掉的:先把"本轮以传输失败收口"和这份诊断记到执行适配器上,再去收束
// 连接。顺序不能反——执行适配器的看门狗盯着同一个 `closed` 标志,它可能先一步把本轮收束成
// "被中断";终态一旦算出来,失败原因就只剩日志,界面只会看到"本轮已结束、没有原因"。
// 连接是在回合进行中断掉的:先把"本轮以传输失败收口"和这份诊断记到执行适配器上,再让"连接
// 已死"对看门狗可见(`shutdown_game_creator_codex_app_server_inner` 才置 `closed`)。顺序不能
// 反——执行适配器的看门狗盯着 `closed`,它一旦先醒就会把本轮收束成"被中断";而失败事实是在
// 模型终态那一刻被**快照**进终态上下文的(见 `run_turn` 里的 `DirectTurnTerminalContext`),
// 晚一步补记没有意义,界面只会看到"本轮已结束、没有原因"。
// 这一段中间有两次加锁和一个日志写,都可能让出线程;认领标志保证只有第一个观察者走到这里。
record_execution_turn_failure(
&inner,
DirectTurnError::TransportClosed {
@@ -7946,6 +7958,170 @@ while IFS= read -r line; do :; done
);
}
/// 连接在回合进行中死掉时,失败事实必须**先于**看门狗可见。
///
/// 连接死亡的收口路径在 `inner.closed` 置位之前要做两次加锁和一个日志写;适配器的看门狗盯着
/// 同一个标志,它一旦先醒就会把这一轮收束成 `Interrupted`typed `TransportClosed` 记不进去,
/// 终态退化成"本轮已结束、没有原因"。这条用例把那个窗口拉开成确定性的(卡住 stderr 摘要的锁,
/// 于是收口路径停在记录之前,看门狗至少跑完一个 200ms 周期),断言终态仍带 `transport-failed`
/// 载荷——顺序反了这条就红。
#[cfg(unix)]
#[tokio::test]
async fn connection_death_records_the_failure_fact_before_the_watchdog_seals_the_turn() {
use std::os::unix::fs::PermissionsExt;
let temp = tempfile::tempdir().expect("temp dir");
let project = temp.path().join("direct-connection-end-project");
crate::init_local_game_project_at(&project, "direct-connection-end", "连接收尾")
.expect("init project");
let turn_started_marker = temp.path().join("turn-started");
let exit_marker = temp.path().join("exit-now");
let executable = temp.path().join("fake-codex-app-server-connection-end");
std::fs::write(
&executable,
format!(
r#"#!/bin/sh
case " $* " in *" debug models "*) printf '%s\n' '{{"models":[{{"slug":"fixture-model","apply_patch_tool_type":"freeform","supports_parallel_tool_calls":true,"model_messages":{{"instructions_template":"fixture"}}}}]}}'; exit 0 ;; esac
while IFS= read -r line; do
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9][0-9]*\).*/\1/p')
case "$line" in
*'"method":"initialize"'*) printf '{{"id":%s,"result":{{"codexHome":"/tmp","platformFamily":"unix","platformOs":"linux","userAgent":"fixture"}}}}\n' "$id" ;;
*'"method":"skills/extraRoots/set"'*) printf '{{"id":%s,"result":{{}}}}\n' "$id" ;;
*'"method":"skills/list"'*) printf '{{"id":%s,"result":{{"data":[{{"skills":[{{"name":"agc-browser-playtest"}},{{"name":"agc-client-projection"}},{{"name":"agc-game-production-workflow"}},{{"name":"agc-godot-editor"}},{{"name":"agc-project-structure"}},{{"name":"agc-unity-editor"}},{{"name":"agc-web-game-development"}},{{"name":"taonier-art-assets"}}],"errors":[]}}]}}}}\n' "$id" ;;
*'"method":"thread/start"'*) printf '{{"id":%s,"result":{{"thread":{{"id":"thread-1"}}}}}}\n' "$id" ;;
*'"method":"thread/inject_items"'*) printf '{{"id":%s,"result":{{}}}}\n' "$id" ;;
*'"method":"turn/start"'*)
printf '{{"id":%s,"result":{{"turn":{{"id":"turn-1","items":[],"status":"inProgress"}}}}}}\n' "$id"
: > "{turn_started}"
while [ ! -f "{exit_marker}" ]; do sleep 0.05; done
exit 0
;;
esac
done
"#,
turn_started = turn_started_marker.display(),
exit_marker = exit_marker.display(),
),
)
.expect("write fake app-server");
let mut permissions = std::fs::metadata(&executable)
.expect("fake metadata")
.permissions();
permissions.set_mode(0o700);
std::fs::set_permissions(&executable, permissions).expect("chmod fake app-server");
let llm = test_llm();
let credential = CodexAppServerCredential::AppDataKey {
fingerprint: "fixture-credential".to_string(),
};
let connection =
CodexAppServerConnection::spawn_with_executable_and_credential_at_workspace(
&llm,
&credential,
executable.as_os_str(),
Some(&project),
CodexAppServerWorkspaceMode::DirectProject,
)
.await
.expect("spawn direct-project app-server");
let user_item = serde_json::json!({
"type": "message",
"role": "user",
"id": "direct-codex:turn-0001:user",
"content": [{ "type": "input_text", "text": "请创建菜单" }]
});
let thread_id = direct_thread_id_for_project(&project);
let bootstrap = crate::agent::subscribe_direct_thread(&thread_id);
let _active_invocation =
crate::agent::DirectTaonierActiveInvocationGuard::enter(&project, "turn-0001")
.expect("enter direct invocation");
let _reservation = crate::agent::direct_turn_accept::DirectTurnReservation::accept(
&thread_id,
"turn-0001",
Some("direct-codex:turn-0001:user"),
)
.expect("accept logical turn");
crate::agent::append_direct_project_user_message_at(&project, &user_item)
.expect("persist opener user item");
let execution = super::super::direct_execution::open_at(
&temp.path().join("host"),
&project,
"turn-0001",
&format!("{:x}", Sha256::digest("请创建菜单".as_bytes())),
false,
&super::super::direct_validation::DirectValidationConfig::default(),
)
.expect("open host execution");
let mut snapshot = test_snapshot();
snapshot.project_id = direct_codex_canonical_project_identity(&project)
.expect("canonical Provider snapshot identity")
.1;
let _execution_guard = super::super::direct_execution::register_for_test(execution)
.expect("register host execution");
let turn_connection = connection.clone();
let turn = tokio::spawn(async move {
let mut observer = |_observation| {};
turn_connection
.run_turn_with_direct_observer_and_history(
&snapshot,
&llm,
LlmRunRequest::single_turn("系统", "请创建菜单"),
Some(&project),
Some("turn-0001"),
Some(&user_item),
DirectCodexTurnKind::User,
None,
Some(&mut observer),
)
.await
});
tokio::time::timeout(Duration::from_secs(10), async {
while !turn_started_marker.exists() {
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("fake app-server must answer turn/start");
// 卡住"取 stderr 摘要"这一步:连接死亡的收口路径会停在这里,看门狗至少跑完一个周期。
let stderr_guard = connection.inner.stderr_summary.lock().await;
std::fs::write(&exit_marker, "1").expect("let the fake app-server exit");
tokio::time::sleep(Duration::from_millis(500)).await;
drop(stderr_guard);
let _ = tokio::time::timeout(Duration::from_secs(20), turn)
.await
.expect("the turn must finish after the connection is reclaimed");
let consumed = crate::agent::consume_direct_thread(&bootstrap.subscription_id)
.expect("consume events");
let terminal = consumed
.events
.iter()
.find_map(|event| match event {
DirectThreadEvent::TurnCompleted {
status, failure, ..
} => Some((status.clone(), failure.clone())),
_ => None,
})
.expect("连接死亡之后逻辑回合必须有终态");
assert_eq!(
terminal.0, "failed",
"失败事实必须先落地:{:?}",
consumed.events
);
let failure = terminal
.1
.expect("连接死亡必须带失败载荷,否则界面只会看到「本轮已结束」");
assert_eq!(
failure.kind,
crate::agent::DirectTurnFailureKind::TransportFailed
);
assert!(failure.message.contains("已退出"), "{}", failure.message);
}
#[cfg(unix)]
#[tokio::test]
async fn direct_project_turn_does_not_forward_codex_user_echo_as_chat_items() {