修复 Router 凭据并发一致性问题
Project CI / Repository checks (pull_request) Failing after 8m28s
Project CI / Backend tests (pull_request) Has been cancelled
Project CI / Native shell tests (pull_request) Has been cancelled
Project CI / Frontend tests (pull_request) Has been cancelled

移除 Router Key 进程内 TTL 缓存,改为每次读取权威账号状态并即时解密当前密文

Router 账号写入改为同主键原地更新,避免 delete+insert 读取缺口

新增 owner/route 映射冲突校验,防止并发写入插入重复账号行

同步更新后端数据契约文档中的凭据读取语义
This commit is contained in:
2026-09-04 14:08:16 +08:00
parent aa47fa833d
commit 020f7d7a7a
3 changed files with 50 additions and 65 deletions
@@ -642,7 +642,7 @@ Responses 的终态载荷既是工具调用的恢复源,也是正文的恢复
### `llm_router_account`
- 当前 AGC Router 需求由 `llm_router_account` 表单独承载:API Key 核心字段、加密凭据、Router 账号元数据、生命周期与 provisioning 状态均在该表;不依赖 `external_api_key`。api-server 首次需要上游调用时解密凭据,并按 owner + route 使用 10 分钟进程内缓存;轮换或撤销时清理缓存
- 当前 AGC Router 需求由 `llm_router_account` 表单独承载:API Key 核心字段、加密凭据、Router 账号元数据、生命周期与 provisioning 状态均在该表;不依赖 `external_api_key`。api-server 每次上游调用都读取权威 `llm_router_account` active/revoked 状态并即时解密当前密文,不做 TTL 凭据缓存,避免任意实例轮换或撤销后继续使用旧 Key
- Rust 结构体:`LlmRouterAccount`
- 源码:`server-rs/crates/spacetime-module/src/llm_router_account.rs`
@@ -23,7 +23,6 @@ use spacetime_client::{
};
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use crate::{
@@ -55,21 +54,10 @@ const LLM_ROUTER_SUBSCRIPTION_PLAN_ID: i64 = 1;
const LLM_ROUTER_SUBSCRIPTION_RENEWAL_THRESHOLD_SECONDS: i64 = 24 * 60 * 60;
const LLM_ROUTER_RECONCILIATION_ERROR_PREFIX: &str = "llm-router-reconciliation-required:";
const LLM_ROUTER_PROVISIONING_CREDENTIAL_VERSION: u32 = 1;
const LLM_ROUTER_CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600);
type HmacSha256 = Hmac<Sha256>;
static LLM_ROUTER_PROVISION_LOCKS: OnceLock<Mutex<HashMap<String, Arc<Mutex<()>>>>> =
OnceLock::new();
static LLM_ROUTER_CREDENTIAL_CACHE: OnceLock<Mutex<HashMap<String, CachedRouterCredential>>> =
OnceLock::new();
struct CachedRouterCredential {
expires_at: Instant,
api_key: String,
key_id: String,
}
struct ProvisionedRouterCredential {
provider_account_id: String,
provider_account_json: Option<String>,
@@ -107,10 +95,6 @@ fn llm_router_provision_locks() -> &'static Mutex<HashMap<String, Arc<Mutex<()>>
LLM_ROUTER_PROVISION_LOCKS.get_or_init(|| Mutex::new(HashMap::new()))
}
fn llm_router_credential_cache() -> &'static Mutex<HashMap<String, CachedRouterCredential>> {
LLM_ROUTER_CREDENTIAL_CACHE.get_or_init(|| Mutex::new(HashMap::new()))
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyCreateRequest {
@@ -607,7 +591,7 @@ async fn upsert_llm_router_account_state(
)
})
});
state
let account = state
.spacetime_client()
.upsert_llm_router_account(LlmRouterAccountUpsertRecordInput {
account_key: account_key.to_string(),
@@ -637,7 +621,8 @@ async fn upsert_llm_router_account_state(
.map(|_| current_utc_micros()),
})
.await
.map_err(|error| format!("保存 LLM Router 账号状态失败:{error}"))
.map_err(|error| format!("保存 LLM Router 账号状态失败:{error}"))?;
Ok(account)
}
/// Ensures the Router user has the fixed AGC subscription before the server
@@ -992,29 +977,6 @@ pub(crate) async fn read_active_llm_router_credentials(
return Err("LLM Router 账号缺少 owner_user_id".to_string());
}
ensure_llm_router_target_allowed(state)?;
let cache_key = format!(
"{}|{}",
owner_user_id,
state.config.llm_router_base_url.trim_end_matches('/')
);
let now = Instant::now();
let mut cache = llm_router_credential_cache().lock().await;
if let Some(cached) = cache
.get(cache_key.as_str())
.filter(|entry| entry.expires_at > now)
{
return Ok(Some((
state
.config
.llm_router_base_url
.trim_end_matches('/')
.to_string(),
cached.api_key.clone(),
cached.key_id.clone(),
)));
}
cache.remove(cache_key.as_str());
drop(cache);
let encryption_secret = state
.config
.effective_llm_router_api_key_encryption_secret()
@@ -1038,15 +1000,12 @@ pub(crate) async fn read_active_llm_router_credentials(
let ciphertext = account.credential_ciphertext.as_deref().ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 密文缺失")
})?;
// Router account state is read from SpacetimeDB immediately before every
// upstream request and the current ciphertext is decrypted on demand. This
// deliberately avoids a TTL cache: another api-server instance can rotate
// or revoke the durable row at any time, and no process-local entry can
// outlive that authoritative state.
let api_key = decrypt_router_api_key(ciphertext, encryption_secret.as_str())?;
llm_router_credential_cache().lock().await.insert(
cache_key,
CachedRouterCredential {
expires_at: Instant::now() + LLM_ROUTER_CREDENTIAL_CACHE_TTL,
api_key: api_key.clone(),
key_id: account.key_id.clone(),
},
);
Ok(Some((
state
.config
@@ -1064,10 +1023,6 @@ pub(crate) async fn revoke_llm_router_account(
key_id: &str,
) -> Result<(), String> {
let route_origin = state.config.llm_router_base_url.trim_end_matches('/');
llm_router_credential_cache()
.lock()
.await
.remove(format!("{}|{}", owner_user_id, route_origin).as_str());
if let Some(account) = state
.spacetime_client()
.get_llm_router_account(owner_user_id.to_string(), route_origin.to_string())
@@ -174,19 +174,34 @@ fn upsert_llm_router_account(
.revoked_at_micros
.map(Timestamp::from_micros_since_unix_epoch),
};
if ctx
.db
.llm_router_account()
.account_key()
.find(&account_key)
.is_some()
{
ctx.db
// SpacetimeDB procedures serialize writes in the same transaction. Update
// the durable primary-key row in place so readers can never observe the
// delete+insert gap and a concurrent owner/route write cannot resurrect an
// old row through a separate primary key.
if let Some(existing) = ctx.db.llm_router_account().account_key().find(&account_key) {
ensure_llm_router_account_identity(
Some((existing.owner_user_id, existing.route_origin)),
row.owner_user_id.as_str(),
row.route_origin.as_str(),
)?;
let mut updated = row;
updated.created_at = existing.created_at;
ctx.db.llm_router_account().account_key().update(updated);
} else {
let conflicting_owner_route = ctx
.db
.llm_router_account()
.account_key()
.delete(&account_key);
.by_llm_router_account_owner_route()
.filter((row.owner_user_id.as_str(), row.route_origin.as_str()))
.next()
.map(|row| (row.owner_user_id, row.route_origin));
ensure_llm_router_account_identity(
conflicting_owner_route,
row.owner_user_id.as_str(),
row.route_origin.as_str(),
)?;
ctx.db.llm_router_account().try_insert(row)?;
}
ctx.db.llm_router_account().insert(row);
ctx.db
.llm_router_account()
.account_key()
@@ -195,6 +210,21 @@ fn upsert_llm_router_account(
.ok_or_else(|| "Router 账号状态保存失败".to_string())
}
fn ensure_llm_router_account_identity(
existing: Option<(String, String)>,
owner_user_id: &str,
route_origin: &str,
) -> Result<(), String> {
if let Some((existing_owner, existing_route)) = existing {
if existing_owner != owner_user_id || existing_route != route_origin {
return Err(format!(
"Router 账号 owner/route 映射冲突:{existing_owner}|{existing_route} != {owner_user_id}|{route_origin}"
));
}
}
Ok(())
}
fn llm_router_account_snapshot_from_row(row: LlmRouterAccount) -> LlmRouterAccountSnapshot {
LlmRouterAccountSnapshot {
account_key: row.account_key,