宿主:连接死亡的失败事实先于看门狗落地
- `CodexAppServerInner` 新增私有去重标志 `connection_end_claimed`,与 `closed` 分开:认领只保证死亡收口只跑一次,"看门狗可以开始收束"必须等失败事实写进执行适配器 - `fail_game_creator_codex_app_server_connection` 改用新标志去重,不再顺带置 `closed`;`closed` 交给 `shutdown_game_creator_codex_app_server_inner` 在 `record_execution_turn_failure` 之后置位,看门狗在事实落地前没有可观测信号 - 补一条把看门狗真正跑起来的回归用例 `connection_death_records_the_failure_fact_before_the_watchdog_seals_the_turn`:卡住 stderr 摘要锁把窗口拉成确定性,断言终态仍带 `transport-failed` 载荷(顺序反了就红) - 失败事实是在模型终态那一刻被快照进终态上下文的,晚补记无用,所以只修"事实先于可见性"这一条落点
This commit is contained in:
@@ -1343,7 +1343,13 @@ struct CodexAppServerInner {
|
||||
execution: std::sync::Mutex<Option<Arc<ExecutionAdapter>>>,
|
||||
next_request_id: AtomicU64,
|
||||
last_used: AtomicU64,
|
||||
/// 连接已死。**它在语义上是"这一段已经收束 / 失败事实已经记下"**,看门狗就盯着它(见
|
||||
/// `ExecutionAdapter::start_watchdog`);所以置位必须发生在失败事实落地之后——别拿它当去重标志用,
|
||||
/// 那是 [`Self::connection_end_claimed`] 的事。
|
||||
closed: AtomicBool,
|
||||
/// 谁的连接死亡收口第一个到(去重)。它与 `closed` 是两件事:认领只保证"这一段只跑一次",
|
||||
/// 而"看门狗可以开始收束了"必须等到失败事实写进执行适配器之后,否则终态判定拿不到原因。
|
||||
connection_end_claimed: AtomicBool,
|
||||
_working_dir: tempfile::TempDir,
|
||||
workspace_path: std::path::PathBuf,
|
||||
workspace_mode: CodexAppServerWorkspaceMode,
|
||||
@@ -2869,6 +2875,7 @@ impl CodexAppServerConnection {
|
||||
next_request_id: AtomicU64::new(1),
|
||||
last_used: AtomicU64::new(next_game_creator_codex_app_server_usage_tick()),
|
||||
closed: AtomicBool::new(false),
|
||||
connection_end_claimed: AtomicBool::new(false),
|
||||
_working_dir: working_dir,
|
||||
workspace_path,
|
||||
workspace_mode,
|
||||
@@ -5128,7 +5135,9 @@ async fn fail_game_creator_codex_app_server_connection(
|
||||
let Some(inner) = inner.upgrade() else {
|
||||
return;
|
||||
};
|
||||
if inner.closed.swap(true, Ordering::AcqRel) {
|
||||
// 去重只看这个私有标志:**不能**用 `inner.closed` 顺手去重——它是看门狗的信号,先置上就等于
|
||||
// "失败事实还没记,收束已经可以开始"(见下面注释与 `CodexAppServerInner::closed` 的说明)。
|
||||
if inner.connection_end_claimed.swap(true, Ordering::AcqRel) {
|
||||
return;
|
||||
}
|
||||
let exit_status = inner
|
||||
@@ -5142,9 +5151,12 @@ async fn fail_game_creator_codex_app_server_connection(
|
||||
let stderr = inner.stderr_summary.lock().await.diagnostic();
|
||||
let diagnostic = format!("{error};exitStatus={exit_status};{stderr}");
|
||||
app_log!("agent.runner.failed: Codex app-server 连接终止:{diagnostic}");
|
||||
// 连接是在回合进行中断掉的:先把"本轮以传输失败收口"和这份诊断记到执行适配器上,再去收束
|
||||
// 连接。顺序不能反——执行适配器的看门狗盯着同一个 `closed` 标志,它可能先一步把本轮收束成
|
||||
// "被中断";终态一旦算出来,失败原因就只剩日志,界面只会看到"本轮已结束、没有原因"。
|
||||
// 连接是在回合进行中断掉的:先把"本轮以传输失败收口"和这份诊断记到执行适配器上,再让"连接
|
||||
// 已死"对看门狗可见(`shutdown_game_creator_codex_app_server_inner` 才置 `closed`)。顺序不能
|
||||
// 反——执行适配器的看门狗盯着 `closed`,它一旦先醒就会把本轮收束成"被中断";而失败事实是在
|
||||
// 模型终态那一刻被**快照**进终态上下文的(见 `run_turn` 里的 `DirectTurnTerminalContext`),
|
||||
// 晚一步补记没有意义,界面只会看到"本轮已结束、没有原因"。
|
||||
// 这一段中间有两次加锁和一个日志写,都可能让出线程;认领标志保证只有第一个观察者走到这里。
|
||||
record_execution_turn_failure(
|
||||
&inner,
|
||||
DirectTurnError::TransportClosed {
|
||||
@@ -7946,6 +7958,170 @@ while IFS= read -r line; do :; done
|
||||
);
|
||||
}
|
||||
|
||||
/// 连接在回合进行中死掉时,失败事实必须**先于**看门狗可见。
|
||||
///
|
||||
/// 连接死亡的收口路径在 `inner.closed` 置位之前要做两次加锁和一个日志写;适配器的看门狗盯着
|
||||
/// 同一个标志,它一旦先醒就会把这一轮收束成 `Interrupted`,typed `TransportClosed` 记不进去,
|
||||
/// 终态退化成"本轮已结束、没有原因"。这条用例把那个窗口拉开成确定性的(卡住 stderr 摘要的锁,
|
||||
/// 于是收口路径停在记录之前,看门狗至少跑完一个 200ms 周期),断言终态仍带 `transport-failed`
|
||||
/// 载荷——顺序反了这条就红。
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn connection_death_records_the_failure_fact_before_the_watchdog_seals_the_turn() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let temp = tempfile::tempdir().expect("temp dir");
|
||||
let project = temp.path().join("direct-connection-end-project");
|
||||
crate::init_local_game_project_at(&project, "direct-connection-end", "连接收尾")
|
||||
.expect("init project");
|
||||
let turn_started_marker = temp.path().join("turn-started");
|
||||
let exit_marker = temp.path().join("exit-now");
|
||||
let executable = temp.path().join("fake-codex-app-server-connection-end");
|
||||
std::fs::write(
|
||||
&executable,
|
||||
format!(
|
||||
r#"#!/bin/sh
|
||||
case " $* " in *" debug models "*) printf '%s\n' '{{"models":[{{"slug":"fixture-model","apply_patch_tool_type":"freeform","supports_parallel_tool_calls":true,"model_messages":{{"instructions_template":"fixture"}}}}]}}'; exit 0 ;; esac
|
||||
while IFS= read -r line; do
|
||||
id=$(printf '%s' "$line" | sed -n 's/.*"id":\([0-9][0-9]*\).*/\1/p')
|
||||
case "$line" in
|
||||
*'"method":"initialize"'*) printf '{{"id":%s,"result":{{"codexHome":"/tmp","platformFamily":"unix","platformOs":"linux","userAgent":"fixture"}}}}\n' "$id" ;;
|
||||
*'"method":"skills/extraRoots/set"'*) printf '{{"id":%s,"result":{{}}}}\n' "$id" ;;
|
||||
*'"method":"skills/list"'*) printf '{{"id":%s,"result":{{"data":[{{"skills":[{{"name":"agc-browser-playtest"}},{{"name":"agc-client-projection"}},{{"name":"agc-game-production-workflow"}},{{"name":"agc-godot-editor"}},{{"name":"agc-project-structure"}},{{"name":"agc-unity-editor"}},{{"name":"agc-web-game-development"}},{{"name":"taonier-art-assets"}}],"errors":[]}}]}}}}\n' "$id" ;;
|
||||
*'"method":"thread/start"'*) printf '{{"id":%s,"result":{{"thread":{{"id":"thread-1"}}}}}}\n' "$id" ;;
|
||||
*'"method":"thread/inject_items"'*) printf '{{"id":%s,"result":{{}}}}\n' "$id" ;;
|
||||
*'"method":"turn/start"'*)
|
||||
printf '{{"id":%s,"result":{{"turn":{{"id":"turn-1","items":[],"status":"inProgress"}}}}}}\n' "$id"
|
||||
: > "{turn_started}"
|
||||
while [ ! -f "{exit_marker}" ]; do sleep 0.05; done
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
done
|
||||
"#,
|
||||
turn_started = turn_started_marker.display(),
|
||||
exit_marker = exit_marker.display(),
|
||||
),
|
||||
)
|
||||
.expect("write fake app-server");
|
||||
let mut permissions = std::fs::metadata(&executable)
|
||||
.expect("fake metadata")
|
||||
.permissions();
|
||||
permissions.set_mode(0o700);
|
||||
std::fs::set_permissions(&executable, permissions).expect("chmod fake app-server");
|
||||
|
||||
let llm = test_llm();
|
||||
let credential = CodexAppServerCredential::AppDataKey {
|
||||
fingerprint: "fixture-credential".to_string(),
|
||||
};
|
||||
let connection =
|
||||
CodexAppServerConnection::spawn_with_executable_and_credential_at_workspace(
|
||||
&llm,
|
||||
&credential,
|
||||
executable.as_os_str(),
|
||||
Some(&project),
|
||||
CodexAppServerWorkspaceMode::DirectProject,
|
||||
)
|
||||
.await
|
||||
.expect("spawn direct-project app-server");
|
||||
|
||||
let user_item = serde_json::json!({
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"id": "direct-codex:turn-0001:user",
|
||||
"content": [{ "type": "input_text", "text": "请创建菜单" }]
|
||||
});
|
||||
let thread_id = direct_thread_id_for_project(&project);
|
||||
let bootstrap = crate::agent::subscribe_direct_thread(&thread_id);
|
||||
let _active_invocation =
|
||||
crate::agent::DirectTaonierActiveInvocationGuard::enter(&project, "turn-0001")
|
||||
.expect("enter direct invocation");
|
||||
let _reservation = crate::agent::direct_turn_accept::DirectTurnReservation::accept(
|
||||
&thread_id,
|
||||
"turn-0001",
|
||||
Some("direct-codex:turn-0001:user"),
|
||||
)
|
||||
.expect("accept logical turn");
|
||||
crate::agent::append_direct_project_user_message_at(&project, &user_item)
|
||||
.expect("persist opener user item");
|
||||
let execution = super::super::direct_execution::open_at(
|
||||
&temp.path().join("host"),
|
||||
&project,
|
||||
"turn-0001",
|
||||
&format!("{:x}", Sha256::digest("请创建菜单".as_bytes())),
|
||||
false,
|
||||
&super::super::direct_validation::DirectValidationConfig::default(),
|
||||
)
|
||||
.expect("open host execution");
|
||||
let mut snapshot = test_snapshot();
|
||||
snapshot.project_id = direct_codex_canonical_project_identity(&project)
|
||||
.expect("canonical Provider snapshot identity")
|
||||
.1;
|
||||
let _execution_guard = super::super::direct_execution::register_for_test(execution)
|
||||
.expect("register host execution");
|
||||
|
||||
let turn_connection = connection.clone();
|
||||
let turn = tokio::spawn(async move {
|
||||
let mut observer = |_observation| {};
|
||||
turn_connection
|
||||
.run_turn_with_direct_observer_and_history(
|
||||
&snapshot,
|
||||
&llm,
|
||||
LlmRunRequest::single_turn("系统", "请创建菜单"),
|
||||
Some(&project),
|
||||
Some("turn-0001"),
|
||||
Some(&user_item),
|
||||
DirectCodexTurnKind::User,
|
||||
None,
|
||||
Some(&mut observer),
|
||||
)
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !turn_started_marker.exists() {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("fake app-server must answer turn/start");
|
||||
|
||||
// 卡住"取 stderr 摘要"这一步:连接死亡的收口路径会停在这里,看门狗至少跑完一个周期。
|
||||
let stderr_guard = connection.inner.stderr_summary.lock().await;
|
||||
std::fs::write(&exit_marker, "1").expect("let the fake app-server exit");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
drop(stderr_guard);
|
||||
|
||||
let _ = tokio::time::timeout(Duration::from_secs(20), turn)
|
||||
.await
|
||||
.expect("the turn must finish after the connection is reclaimed");
|
||||
|
||||
let consumed = crate::agent::consume_direct_thread(&bootstrap.subscription_id)
|
||||
.expect("consume events");
|
||||
let terminal = consumed
|
||||
.events
|
||||
.iter()
|
||||
.find_map(|event| match event {
|
||||
DirectThreadEvent::TurnCompleted {
|
||||
status, failure, ..
|
||||
} => Some((status.clone(), failure.clone())),
|
||||
_ => None,
|
||||
})
|
||||
.expect("连接死亡之后逻辑回合必须有终态");
|
||||
assert_eq!(
|
||||
terminal.0, "failed",
|
||||
"失败事实必须先落地:{:?}",
|
||||
consumed.events
|
||||
);
|
||||
let failure = terminal
|
||||
.1
|
||||
.expect("连接死亡必须带失败载荷,否则界面只会看到「本轮已结束」");
|
||||
assert_eq!(
|
||||
failure.kind,
|
||||
crate::agent::DirectTurnFailureKind::TransportFailed
|
||||
);
|
||||
assert!(failure.message.contains("已退出"), "{}", failure.message);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn direct_project_turn_does_not_forward_codex_user_echo_as_chat_items() {
|
||||
|
||||
Reference in New Issue
Block a user