修复 Router 凭据并发一致性问题
移除 Router Key 进程内 TTL 缓存,改为每次读取权威账号状态并即时解密当前密文 Router 账号写入改为同主键原地更新,避免 delete+insert 读取缺口 新增 owner/route 映射冲突校验,防止并发写入插入重复账号行 同步更新后端数据契约文档中的凭据读取语义
This commit is contained in:
@@ -642,7 +642,7 @@ Responses 的终态载荷既是工具调用的恢复源,也是正文的恢复
|
||||
|
||||
### `llm_router_account`
|
||||
|
||||
- 当前 AGC Router 需求由 `llm_router_account` 表单独承载:API Key 核心字段、加密凭据、Router 账号元数据、生命周期与 provisioning 状态均在该表;不依赖 `external_api_key`。api-server 首次需要上游调用时解密凭据,并按 owner + route 使用 10 分钟进程内缓存;轮换或撤销时清理缓存。
|
||||
- 当前 AGC Router 需求由 `llm_router_account` 表单独承载:API Key 核心字段、加密凭据、Router 账号元数据、生命周期与 provisioning 状态均在该表;不依赖 `external_api_key`。api-server 每次上游调用都读取权威 `llm_router_account` active/revoked 状态并即时解密当前密文,不做 TTL 凭据缓存,避免任意实例轮换或撤销后继续使用旧 Key。
|
||||
|
||||
- Rust 结构体:`LlmRouterAccount`
|
||||
- 源码:`server-rs/crates/spacetime-module/src/llm_router_account.rs`
|
||||
|
||||
@@ -23,7 +23,6 @@ use spacetime_client::{
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use crate::{
|
||||
@@ -55,21 +54,10 @@ const LLM_ROUTER_SUBSCRIPTION_PLAN_ID: i64 = 1;
|
||||
const LLM_ROUTER_SUBSCRIPTION_RENEWAL_THRESHOLD_SECONDS: i64 = 24 * 60 * 60;
|
||||
const LLM_ROUTER_RECONCILIATION_ERROR_PREFIX: &str = "llm-router-reconciliation-required:";
|
||||
const LLM_ROUTER_PROVISIONING_CREDENTIAL_VERSION: u32 = 1;
|
||||
const LLM_ROUTER_CREDENTIAL_CACHE_TTL: Duration = Duration::from_secs(600);
|
||||
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
static LLM_ROUTER_PROVISION_LOCKS: OnceLock<Mutex<HashMap<String, Arc<Mutex<()>>>>> =
|
||||
OnceLock::new();
|
||||
static LLM_ROUTER_CREDENTIAL_CACHE: OnceLock<Mutex<HashMap<String, CachedRouterCredential>>> =
|
||||
OnceLock::new();
|
||||
|
||||
struct CachedRouterCredential {
|
||||
expires_at: Instant,
|
||||
api_key: String,
|
||||
key_id: String,
|
||||
}
|
||||
|
||||
struct ProvisionedRouterCredential {
|
||||
provider_account_id: String,
|
||||
provider_account_json: Option<String>,
|
||||
@@ -107,10 +95,6 @@ fn llm_router_provision_locks() -> &'static Mutex<HashMap<String, Arc<Mutex<()>>
|
||||
LLM_ROUTER_PROVISION_LOCKS.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
fn llm_router_credential_cache() -> &'static Mutex<HashMap<String, CachedRouterCredential>> {
|
||||
LLM_ROUTER_CREDENTIAL_CACHE.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ExternalApiKeyCreateRequest {
|
||||
@@ -607,7 +591,7 @@ async fn upsert_llm_router_account_state(
|
||||
)
|
||||
})
|
||||
});
|
||||
state
|
||||
let account = state
|
||||
.spacetime_client()
|
||||
.upsert_llm_router_account(LlmRouterAccountUpsertRecordInput {
|
||||
account_key: account_key.to_string(),
|
||||
@@ -637,7 +621,8 @@ async fn upsert_llm_router_account_state(
|
||||
.map(|_| current_utc_micros()),
|
||||
})
|
||||
.await
|
||||
.map_err(|error| format!("保存 LLM Router 账号状态失败:{error}"))
|
||||
.map_err(|error| format!("保存 LLM Router 账号状态失败:{error}"))?;
|
||||
Ok(account)
|
||||
}
|
||||
|
||||
/// Ensures the Router user has the fixed AGC subscription before the server
|
||||
@@ -992,29 +977,6 @@ pub(crate) async fn read_active_llm_router_credentials(
|
||||
return Err("LLM Router 账号缺少 owner_user_id".to_string());
|
||||
}
|
||||
ensure_llm_router_target_allowed(state)?;
|
||||
let cache_key = format!(
|
||||
"{}|{}",
|
||||
owner_user_id,
|
||||
state.config.llm_router_base_url.trim_end_matches('/')
|
||||
);
|
||||
let now = Instant::now();
|
||||
let mut cache = llm_router_credential_cache().lock().await;
|
||||
if let Some(cached) = cache
|
||||
.get(cache_key.as_str())
|
||||
.filter(|entry| entry.expires_at > now)
|
||||
{
|
||||
return Ok(Some((
|
||||
state
|
||||
.config
|
||||
.llm_router_base_url
|
||||
.trim_end_matches('/')
|
||||
.to_string(),
|
||||
cached.api_key.clone(),
|
||||
cached.key_id.clone(),
|
||||
)));
|
||||
}
|
||||
cache.remove(cache_key.as_str());
|
||||
drop(cache);
|
||||
let encryption_secret = state
|
||||
.config
|
||||
.effective_llm_router_api_key_encryption_secret()
|
||||
@@ -1038,15 +1000,12 @@ pub(crate) async fn read_active_llm_router_credentials(
|
||||
let ciphertext = account.credential_ciphertext.as_deref().ok_or_else(|| {
|
||||
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 密文缺失")
|
||||
})?;
|
||||
// Router account state is read from SpacetimeDB immediately before every
|
||||
// upstream request and the current ciphertext is decrypted on demand. This
|
||||
// deliberately avoids a TTL cache: another api-server instance can rotate
|
||||
// or revoke the durable row at any time, and no process-local entry can
|
||||
// outlive that authoritative state.
|
||||
let api_key = decrypt_router_api_key(ciphertext, encryption_secret.as_str())?;
|
||||
llm_router_credential_cache().lock().await.insert(
|
||||
cache_key,
|
||||
CachedRouterCredential {
|
||||
expires_at: Instant::now() + LLM_ROUTER_CREDENTIAL_CACHE_TTL,
|
||||
api_key: api_key.clone(),
|
||||
key_id: account.key_id.clone(),
|
||||
},
|
||||
);
|
||||
Ok(Some((
|
||||
state
|
||||
.config
|
||||
@@ -1064,10 +1023,6 @@ pub(crate) async fn revoke_llm_router_account(
|
||||
key_id: &str,
|
||||
) -> Result<(), String> {
|
||||
let route_origin = state.config.llm_router_base_url.trim_end_matches('/');
|
||||
llm_router_credential_cache()
|
||||
.lock()
|
||||
.await
|
||||
.remove(format!("{}|{}", owner_user_id, route_origin).as_str());
|
||||
if let Some(account) = state
|
||||
.spacetime_client()
|
||||
.get_llm_router_account(owner_user_id.to_string(), route_origin.to_string())
|
||||
|
||||
@@ -174,19 +174,34 @@ fn upsert_llm_router_account(
|
||||
.revoked_at_micros
|
||||
.map(Timestamp::from_micros_since_unix_epoch),
|
||||
};
|
||||
if ctx
|
||||
.db
|
||||
.llm_router_account()
|
||||
.account_key()
|
||||
.find(&account_key)
|
||||
.is_some()
|
||||
{
|
||||
ctx.db
|
||||
// SpacetimeDB procedures serialize writes in the same transaction. Update
|
||||
// the durable primary-key row in place so readers can never observe the
|
||||
// delete+insert gap and a concurrent owner/route write cannot resurrect an
|
||||
// old row through a separate primary key.
|
||||
if let Some(existing) = ctx.db.llm_router_account().account_key().find(&account_key) {
|
||||
ensure_llm_router_account_identity(
|
||||
Some((existing.owner_user_id, existing.route_origin)),
|
||||
row.owner_user_id.as_str(),
|
||||
row.route_origin.as_str(),
|
||||
)?;
|
||||
let mut updated = row;
|
||||
updated.created_at = existing.created_at;
|
||||
ctx.db.llm_router_account().account_key().update(updated);
|
||||
} else {
|
||||
let conflicting_owner_route = ctx
|
||||
.db
|
||||
.llm_router_account()
|
||||
.account_key()
|
||||
.delete(&account_key);
|
||||
.by_llm_router_account_owner_route()
|
||||
.filter((row.owner_user_id.as_str(), row.route_origin.as_str()))
|
||||
.next()
|
||||
.map(|row| (row.owner_user_id, row.route_origin));
|
||||
ensure_llm_router_account_identity(
|
||||
conflicting_owner_route,
|
||||
row.owner_user_id.as_str(),
|
||||
row.route_origin.as_str(),
|
||||
)?;
|
||||
ctx.db.llm_router_account().try_insert(row)?;
|
||||
}
|
||||
ctx.db.llm_router_account().insert(row);
|
||||
ctx.db
|
||||
.llm_router_account()
|
||||
.account_key()
|
||||
@@ -195,6 +210,21 @@ fn upsert_llm_router_account(
|
||||
.ok_or_else(|| "Router 账号状态保存失败".to_string())
|
||||
}
|
||||
|
||||
fn ensure_llm_router_account_identity(
|
||||
existing: Option<(String, String)>,
|
||||
owner_user_id: &str,
|
||||
route_origin: &str,
|
||||
) -> Result<(), String> {
|
||||
if let Some((existing_owner, existing_route)) = existing {
|
||||
if existing_owner != owner_user_id || existing_route != route_origin {
|
||||
return Err(format!(
|
||||
"Router 账号 owner/route 映射冲突:{existing_owner}|{existing_route} != {owner_user_id}|{route_origin}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn llm_router_account_snapshot_from_row(row: LlmRouterAccount) -> LlmRouterAccountSnapshot {
|
||||
LlmRouterAccountSnapshot {
|
||||
account_key: row.account_key,
|
||||
|
||||
Reference in New Issue
Block a user