拆分 Agent Host 运行职责
将 durable 取消、失败收口、审批决议和外部结果对账迁入 RuntimeService 把 Host 工具、MCP、Skill 与外部 backend 桥接拆为私有模块并保持公开 API 补充 Runtime 控制测试、依赖反向门禁和架构验收文档
This commit is contained in:
@@ -0,0 +1,350 @@
|
||||
//! Runtime-only control operations formerly implemented by `agent-host`.
|
||||
//!
|
||||
//! These methods deliberately depend only on Core values and the existing
|
||||
//! `RuntimeService` facade. They do not construct an Engine or call an
|
||||
//! external adapter; Host remains responsible for deciding when to invoke
|
||||
//! them.
|
||||
|
||||
use super::{
|
||||
ApprovalRecord, CheckpointRecord, RunRecord, RuntimeService, RuntimeServiceError, StorageError,
|
||||
};
|
||||
use agent_runtime_core::{
|
||||
ApprovalDecision, ContentPart, Message, MessageRole, RunStatus, RuntimeEvent, RuntimeEventKind,
|
||||
SystemClock,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
impl RuntimeService {
|
||||
/// Resolve a pending approval without starting a worker.
|
||||
pub fn resolve_approval_decision(
|
||||
&self,
|
||||
approval_id: &str,
|
||||
decision: ApprovalDecision,
|
||||
) -> super::Result<ApprovalRecord> {
|
||||
let (status, payload) = match decision {
|
||||
ApprovalDecision::Allow => ("allowed", json!({"decision": "allow"})),
|
||||
ApprovalDecision::Deny { reason } => {
|
||||
if reason.trim().is_empty() {
|
||||
return Err(RuntimeServiceError::InvalidInput(
|
||||
"审批拒绝原因不能为空".to_owned(),
|
||||
));
|
||||
}
|
||||
("denied", json!({"decision": "deny", "reason": reason}))
|
||||
}
|
||||
ApprovalDecision::Ask => {
|
||||
return Err(RuntimeServiceError::InvalidInput(
|
||||
"不能把 Ask 作为已决 approval 写回".to_owned(),
|
||||
));
|
||||
}
|
||||
};
|
||||
self.resolve_approval(approval_id, "pending", status, payload)
|
||||
}
|
||||
|
||||
/// Apply the Host cancellation policy at the Runtime boundary.
|
||||
///
|
||||
/// A queued run is cancelled atomically before requesting cancellation;
|
||||
/// an active worker receives a cooperative request, while a stale worker
|
||||
/// is moved to reconciliation rather than being guessed safe.
|
||||
pub fn cancel_run(&self, run_id: &str) -> super::Result<RunRecord> {
|
||||
let before = self
|
||||
.get_run(run_id)?
|
||||
.ok_or_else(|| invalid(format!("找不到指定 run: {run_id}")))?;
|
||||
|
||||
if before.status == "queued"
|
||||
&& let Some(cancelled) = self.finish_queued_cancelled_if_unclaimed(run_id)?
|
||||
{
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
return Ok(cancelled);
|
||||
}
|
||||
|
||||
let record = self.request_cancel(run_id)?;
|
||||
if matches!(record.status.as_str(), "completed" | "failed" | "cancelled") {
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
return Ok(record);
|
||||
}
|
||||
|
||||
let checkpoint = self.read_checkpoint(run_id)?;
|
||||
let lease = self.get_run_lease(run_id)?;
|
||||
let now = SystemClock.now_millis().min(i64::MAX as u64) as i64;
|
||||
let lease_active = lease
|
||||
.as_ref()
|
||||
.is_some_and(|value| value.lease_expires_at > now);
|
||||
let safe_checkpoint = checkpoint
|
||||
.as_ref()
|
||||
.is_some_and(|value| matches!(value.phase.as_str(), "safe" | "awaiting_approval"));
|
||||
|
||||
if !lease_active && safe_checkpoint {
|
||||
if lease.is_some() && self.reconcile_expired_run_if_stale(run_id)?.is_none() {
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
return Ok(self.get_run(run_id)?.unwrap_or(record));
|
||||
}
|
||||
if self.get_run_lease(run_id)?.is_none() {
|
||||
let cancelled = self.finish_unclaimed_cancelled_if_safe(run_id)?;
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
return Ok(cancelled);
|
||||
}
|
||||
}
|
||||
|
||||
if !lease_active && let Some(recovered) = self.reconcile_expired_run_if_stale(run_id)? {
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
return Ok(recovered);
|
||||
}
|
||||
|
||||
self.cancel_pending_approvals(run_id)?;
|
||||
Ok(record)
|
||||
}
|
||||
|
||||
/// Atomically fail a queued/reconciling run before an Engine starts.
|
||||
pub fn fail_unclaimed_run(
|
||||
&self,
|
||||
run_id: &str,
|
||||
error: impl Into<String>,
|
||||
) -> super::Result<RunRecord> {
|
||||
let error = error.into();
|
||||
if error.trim().is_empty() {
|
||||
return Err(invalid("failed 原因不能为空"));
|
||||
}
|
||||
let record = self
|
||||
.get_run(run_id)?
|
||||
.ok_or_else(|| invalid(format!("找不到指定 run: {run_id}")))?;
|
||||
if record.status == "failed" {
|
||||
return Ok(record);
|
||||
}
|
||||
if matches!(
|
||||
record.status.as_str(),
|
||||
"completed" | "cancelled" | "canceled"
|
||||
) {
|
||||
return Err(RuntimeServiceError::Storage(StorageError::TerminalRun {
|
||||
id: run_id.to_owned(),
|
||||
status: record.status,
|
||||
}));
|
||||
}
|
||||
if !matches!(record.status.as_str(), "queued" | "reconciling") {
|
||||
return Err(invalid(format!(
|
||||
"只有无 lease 的 queued/reconciling run 可以失败收口，当前为 {}: {run_id}",
|
||||
record.status
|
||||
)));
|
||||
}
|
||||
if record.cancel_requested || self.get_run_lease(run_id)?.is_some() {
|
||||
return Err(invalid(format!(
|
||||
"run 已请求取消或仍由 worker 持有 lease，不能无 lease 失败收口: {run_id}"
|
||||
)));
|
||||
}
|
||||
|
||||
let runtime_id = self
|
||||
.runtime_id_for_run(run_id)?
|
||||
.ok_or_else(|| invalid(format!("run 缺少 runtime 身份: {run_id}")))?;
|
||||
let runtime_snapshot = self
|
||||
.load_runtime_snapshot(&runtime_id)?
|
||||
.ok_or_else(|| invalid(format!("找不到 runtime: {runtime_id}")))?;
|
||||
let run_snapshot = runtime_snapshot
|
||||
.run(run_id)
|
||||
.cloned()
|
||||
.ok_or_else(|| invalid(format!("runtime 中找不到 run: {run_id}")))?;
|
||||
if run_snapshot.status().is_terminal() {
|
||||
return Err(invalid(format!(
|
||||
"runtime run 已处于终态 {:?}，不能失败收口: {run_id}",
|
||||
run_snapshot.status()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut next_runtime = runtime_snapshot.clone();
|
||||
let mut events = Vec::new();
|
||||
if run_snapshot.status() == RunStatus::Pending {
|
||||
let started = RuntimeEvent::status_changed(
|
||||
&runtime_id,
|
||||
next_runtime.revision() + 1,
|
||||
SystemClock.now_millis(),
|
||||
run_id,
|
||||
RuntimeEventKind::RunStarted,
|
||||
)
|
||||
.map_err(core_error)?;
|
||||
next_runtime =
|
||||
agent_runtime_core::reduce(&next_runtime, &started).map_err(core_error)?;
|
||||
events.push(started);
|
||||
}
|
||||
if let Some(run) = next_runtime.run(run_id)
|
||||
&& !run.status().is_terminal()
|
||||
{
|
||||
let failed = RuntimeEvent::failed(
|
||||
&runtime_id,
|
||||
next_runtime.revision() + 1,
|
||||
SystemClock.now_millis(),
|
||||
run_id,
|
||||
error.clone(),
|
||||
)
|
||||
.map_err(core_error)?;
|
||||
next_runtime =
|
||||
agent_runtime_core::reduce(&next_runtime, &failed).map_err(core_error)?;
|
||||
events.push(failed);
|
||||
}
|
||||
if events.is_empty() {
|
||||
return Err(invalid(format!("runtime 没有可失败收口的事件: {run_id}")));
|
||||
}
|
||||
|
||||
self.fail_run_with_runtime(
|
||||
run_id,
|
||||
Some(json!({"error": error})),
|
||||
&runtime_id,
|
||||
Some(runtime_snapshot.revision()),
|
||||
&next_runtime,
|
||||
&events,
|
||||
)
|
||||
}
|
||||
|
||||
/// Record a verified Provider result and make its checkpoint safe.
|
||||
pub fn reconcile_provider_result(
|
||||
&self,
|
||||
run_id: &str,
|
||||
provider_request_id: &str,
|
||||
messages: Vec<Message>,
|
||||
) -> super::Result<CheckpointRecord> {
|
||||
self.reconcile_external_result(run_id, "provider_in_flight", provider_request_id, messages)
|
||||
}
|
||||
|
||||
/// Record a verified Tool result and make its checkpoint safe.
|
||||
pub fn reconcile_tool_result(
|
||||
&self,
|
||||
run_id: &str,
|
||||
tool_call_id: &str,
|
||||
messages: Vec<Message>,
|
||||
) -> super::Result<CheckpointRecord> {
|
||||
self.reconcile_external_result(run_id, "tool_in_flight", tool_call_id, messages)
|
||||
}
|
||||
|
||||
fn reconcile_external_result(
|
||||
&self,
|
||||
run_id: &str,
|
||||
phase: &str,
|
||||
external_id: &str,
|
||||
messages: Vec<Message>,
|
||||
) -> super::Result<CheckpointRecord> {
|
||||
let checkpoint = self
|
||||
.read_checkpoint(run_id)?
|
||||
.ok_or_else(|| invalid(format!("run 没有可对账 checkpoint: {run_id}")))?;
|
||||
validate_reconciliation_messages(&checkpoint, phase, external_id, &messages)?;
|
||||
let encoded = serde_json::to_value(&messages)
|
||||
.map_err(|error| invalid(format!("对账消息无法编码: {error}")))?;
|
||||
self.record_reconciliation_result(
|
||||
run_id,
|
||||
phase,
|
||||
external_id,
|
||||
checkpoint.step,
|
||||
checkpoint.attempt,
|
||||
encoded,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(message: impl Into<String>) -> RuntimeServiceError {
|
||||
RuntimeServiceError::InvalidInput(message.into())
|
||||
}
|
||||
|
||||
fn core_error(error: impl std::fmt::Display) -> RuntimeServiceError {
|
||||
RuntimeServiceError::Core(error.to_string())
|
||||
}
|
||||
|
||||
/// Validate the complete message history supplied by an external reconciler.
|
||||
/// Storage receives only a validated JSON wire value and performs its own CAS.
|
||||
fn validate_reconciliation_messages(
|
||||
checkpoint: &CheckpointRecord,
|
||||
phase: &str,
|
||||
external_id: &str,
|
||||
messages: &[Message],
|
||||
) -> super::Result<()> {
|
||||
if !matches!(phase, "provider_in_flight" | "tool_in_flight") {
|
||||
return Err(invalid(format!("不支持的对账 checkpoint phase: {phase}")));
|
||||
}
|
||||
if messages.is_empty() {
|
||||
return Err(invalid("对账消息不能为空"));
|
||||
}
|
||||
if checkpoint.phase != phase {
|
||||
return Err(invalid(format!(
|
||||
"checkpoint phase 不匹配：expected={phase} actual={}",
|
||||
checkpoint.phase
|
||||
)));
|
||||
}
|
||||
if checkpoint.step != checkpoint.next_step {
|
||||
return Err(invalid("in-flight checkpoint 的 step/next_step 游标无效"));
|
||||
}
|
||||
match phase {
|
||||
"provider_in_flight"
|
||||
if checkpoint.provider_request_id.as_deref() != Some(external_id)
|
||||
|| checkpoint.tool_call_id.is_some() =>
|
||||
{
|
||||
return Err(invalid("Provider request identity 与 checkpoint 不匹配"));
|
||||
}
|
||||
"tool_in_flight" if checkpoint.tool_call_id.as_deref() != Some(external_id) => {
|
||||
return Err(invalid("tool call identity 与 checkpoint 不匹配"));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let checkpoint_messages = serde_json::from_value::<Vec<Message>>(checkpoint.messages.clone())
|
||||
.map_err(|error| invalid(format!("checkpoint 消息无效: {error}")))?;
|
||||
if messages.len() <= checkpoint_messages.len() {
|
||||
return Err(invalid("对账消息必须包含完整 checkpoint 前缀和新增结果"));
|
||||
}
|
||||
if !checkpoint_messages
|
||||
.iter()
|
||||
.zip(messages)
|
||||
.all(|(expected, actual)| expected == actual)
|
||||
{
|
||||
return Err(invalid("对账消息没有保留 checkpoint 的完整前缀"));
|
||||
}
|
||||
|
||||
let mut calls = std::collections::BTreeSet::new();
|
||||
let mut results = std::collections::BTreeSet::new();
|
||||
let mut suffix_has_assistant = false;
|
||||
let mut suffix_has_matching_tool_result = false;
|
||||
for (message_index, message) in messages.iter().enumerate() {
|
||||
let in_suffix = message_index >= checkpoint_messages.len();
|
||||
if in_suffix && message.role() == MessageRole::Assistant {
|
||||
suffix_has_assistant = true;
|
||||
}
|
||||
for part in message.content() {
|
||||
match part {
|
||||
ContentPart::ToolCall { id, .. } => {
|
||||
if message.role() != MessageRole::Assistant {
|
||||
return Err(invalid(format!(
|
||||
"tool call 必须位于 assistant 消息: index={message_index}"
|
||||
)));
|
||||
}
|
||||
if !calls.insert(id.clone()) {
|
||||
return Err(invalid(format!("对账消息重复 tool call: {id}")));
|
||||
}
|
||||
}
|
||||
ContentPart::ToolResult { tool_call_id, .. } => {
|
||||
if message.role() != MessageRole::Tool {
|
||||
return Err(invalid(format!(
|
||||
"tool result 必须位于 tool 消息: index={message_index}"
|
||||
)));
|
||||
}
|
||||
if !calls.contains(tool_call_id) {
|
||||
return Err(invalid(format!(
|
||||
"tool result 引用了尚未出现的 call: {tool_call_id}"
|
||||
)));
|
||||
}
|
||||
if !results.insert(tool_call_id.clone()) {
|
||||
return Err(invalid(format!("对账消息重复 tool result: {tool_call_id}")));
|
||||
}
|
||||
if in_suffix && phase == "tool_in_flight" && tool_call_id == external_id {
|
||||
suffix_has_matching_tool_result = true;
|
||||
}
|
||||
}
|
||||
ContentPart::Text { .. } | ContentPart::Image { .. } => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if calls.iter().any(|call_id| !results.contains(call_id)) {
|
||||
return Err(invalid("对账消息仍包含未完成的 tool call，不能标记 safe"));
|
||||
}
|
||||
if phase == "provider_in_flight" && !suffix_has_assistant {
|
||||
return Err(invalid("Provider 对账后缀必须包含 assistant 响应"));
|
||||
}
|
||||
if phase == "tool_in_flight" && !suffix_has_matching_tool_result {
|
||||
return Err(invalid("工具对账后缀必须包含对应 tool result"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -45,6 +45,7 @@ pub use agent_runtime::{
|
||||
DurableToolCallCheckpointRuntimeCommit, DurableToolCallInput, DurableToolCallRuntimeCommit,
|
||||
DurableToolCallView,
|
||||
};
|
||||
mod control;
|
||||
mod durable_sqlite;
|
||||
pub use durable_sqlite::{SqliteDurableStore, SqliteDurableStoreError};
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use agent_runtime_core::{ApprovalDecision, Message, RunStatus};
|
||||
use agent_runtime_sqlite::{RuntimeService, RuntimeServiceError, WorkerLease};
|
||||
|
||||
fn user(text: &str) -> Message {
|
||||
Message::user(text).expect("valid user message")
|
||||
}
|
||||
|
||||
fn wait_until_epoch_ms(target: i64) {
|
||||
let deadline = Instant::now() + Duration::from_secs(2);
|
||||
loop {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.expect("system clock")
|
||||
.as_millis() as i64;
|
||||
if now >= target {
|
||||
return;
|
||||
}
|
||||
assert!(Instant::now() < deadline, "lease did not expire");
|
||||
std::thread::sleep(Duration::from_millis(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queued_cancel_is_atomic_and_runtime_visible() {
|
||||
let service = RuntimeService::in_memory().expect("runtime");
|
||||
let handle = service.prepare_run("queued cancel").expect("prepare");
|
||||
|
||||
let cancelled = service.cancel_run(&handle.run_id).expect("cancel");
|
||||
assert_eq!(cancelled.status, "cancelled");
|
||||
assert_eq!(
|
||||
service
|
||||
.get_run(&handle.run_id)
|
||||
.expect("run")
|
||||
.unwrap()
|
||||
.status,
|
||||
"cancelled"
|
||||
);
|
||||
let snapshot = service
|
||||
.load_runtime_snapshot(&handle.runtime_id)
|
||||
.expect("snapshot")
|
||||
.expect("runtime exists");
|
||||
assert_eq!(
|
||||
snapshot.run(&handle.run_id).expect("run snapshot").status(),
|
||||
RunStatus::Cancelled
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_and_expired_cancel_stay_on_the_reconciliation_gate() {
|
||||
let service = RuntimeService::in_memory().expect("runtime");
|
||||
let active = service.prepare_run("active cancel").expect("prepare");
|
||||
let active_lease = WorkerLease::new(&active.run_id);
|
||||
service
|
||||
.claim_run_with_lease(&active.run_id, &active_lease, Duration::from_secs(30))
|
||||
.expect("claim active");
|
||||
let requested = service.cancel_run(&active.run_id).expect("request cancel");
|
||||
assert_eq!(requested.status, "cancel_requested");
|
||||
assert_eq!(
|
||||
service
|
||||
.get_run(&active.run_id)
|
||||
.expect("run")
|
||||
.unwrap()
|
||||
.status,
|
||||
"cancel_requested"
|
||||
);
|
||||
|
||||
// A short lease represents a worker that disappeared before its first
|
||||
// heartbeat; cancellation must reconcile it instead of guessing safe.
|
||||
let stale = service.prepare_run("stale cancel").expect("prepare");
|
||||
let stale_lease = WorkerLease::new(&stale.run_id);
|
||||
let (_, stale_record) = service
|
||||
.claim_run_with_lease(&stale.run_id, &stale_lease, Duration::from_millis(1))
|
||||
.expect("claim stale");
|
||||
wait_until_epoch_ms(stale_record.lease_expires_at);
|
||||
let reconciled = service.cancel_run(&stale.run_id).expect("reconcile cancel");
|
||||
assert_eq!(reconciled.status, "reconciling");
|
||||
let snapshot = service
|
||||
.load_runtime_snapshot(&stale.runtime_id)
|
||||
.expect("snapshot")
|
||||
.expect("runtime exists");
|
||||
assert_eq!(
|
||||
snapshot.run(&stale.run_id).expect("run snapshot").status(),
|
||||
RunStatus::Reconciling
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fail_unclaimed_run_closes_run_runtime_and_session() {
|
||||
let service = RuntimeService::in_memory().expect("runtime");
|
||||
let handle = service.prepare_run("setup failure").expect("prepare");
|
||||
|
||||
let failed = service
|
||||
.fail_unclaimed_run(&handle.run_id, "provider 配置失败")
|
||||
.expect("fail run");
|
||||
assert_eq!(failed.status, "failed");
|
||||
assert_eq!(
|
||||
service
|
||||
.get_session(&handle.session_id)
|
||||
.expect("session")
|
||||
.unwrap()
|
||||
.status,
|
||||
"failed"
|
||||
);
|
||||
let snapshot = service
|
||||
.load_runtime_snapshot(&handle.runtime_id)
|
||||
.expect("snapshot")
|
||||
.expect("runtime exists");
|
||||
assert_eq!(
|
||||
snapshot.run(&handle.run_id).expect("run snapshot").status(),
|
||||
RunStatus::Failed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_approval_and_reconciliation_leave_runtime_unchanged() {
|
||||
let service = RuntimeService::in_memory().expect("runtime");
|
||||
let handle = service
|
||||
.prepare_run_with_messages("invalid control", vec![user("invalid control")])
|
||||
.expect("prepare");
|
||||
let before = service
|
||||
.load_runtime_snapshot(&handle.runtime_id)
|
||||
.expect("snapshot")
|
||||
.expect("runtime exists");
|
||||
|
||||
let approval_error = service
|
||||
.resolve_approval_decision("missing-approval", ApprovalDecision::Allow)
|
||||
.expect_err("missing approval must fail");
|
||||
assert!(!matches!(approval_error, RuntimeServiceError::Core(_)));
|
||||
let reconcile_error = service
|
||||
.reconcile_provider_result(&handle.run_id, "missing-provider-request", vec![user("x")])
|
||||
.expect_err("missing checkpoint must fail");
|
||||
assert!(matches!(
|
||||
reconcile_error,
|
||||
RuntimeServiceError::InvalidInput(_)
|
||||
));
|
||||
|
||||
let after = service
|
||||
.load_runtime_snapshot(&handle.runtime_id)
|
||||
.expect("snapshot")
|
||||
.expect("runtime exists");
|
||||
assert_eq!(after.revision(), before.revision());
|
||||
assert_eq!(
|
||||
after.run(&handle.run_id).expect("run snapshot").status(),
|
||||
before.run(&handle.run_id).expect("run snapshot").status()
|
||||
);
|
||||
assert!(
|
||||
service
|
||||
.get_approval("missing-approval")
|
||||
.expect("approval lookup")
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user