Files
Genarrative/server-rs/crates/api-server/src/external_api_keys.rs
T
suzmii 736a1b6ac6
Project CI / Repository checks (pull_request) Successful in 2m36s
Project CI / Frontend tests (pull_request) Successful in 3m15s
Project CI / Backend tests (pull_request) Successful in 7m35s
Project CI / Native shell tests (pull_request) Failing after 7m43s
接入 LLM Router 累计额度结算
按 Router used_quota 累计值与首次基线结算泥点
新增原子 checkpoint 事务及 llm_router_consume 钱包流水
同步额度查询校验、前端展示、生成绑定和技术文档
2026-09-06 00:36:31 +08:00

3082 lines
113 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use axum::{
Json,
extract::{Extension, Path, State},
http::StatusCode,
};
use base64::{
Engine as _,
engine::general_purpose::{STANDARD_NO_PAD, URL_SAFE_NO_PAD},
};
use hmac::{Hmac, Mac};
use platform_auth::hash_refresh_session_token;
use ring::{
aead,
rand::{SecureRandom, SystemRandom},
};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use shared_kernel::{build_prefixed_uuid_id, new_uuid_simple_string};
use spacetime_client::{
ExternalApiKeyCreateRecordInput, ExternalApiKeyRecord, ExternalApiKeyRevokeRecordInput,
LlmRouterAccountRecord, LlmRouterAccountUpsertRecordInput, SpacetimeClientError,
};
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use tokio::sync::Mutex;
use crate::{
api_response::json_success_body,
auth::AuthenticatedAccessToken,
config::{OFFICIAL_LLM_ROUTER_BASE_URL, OFFICIAL_LLM_ROUTER_MODEL},
editor_project::current_utc_micros,
http_error::AppError,
request_context::RequestContext,
state::AppState,
};
const EXTERNAL_API_KEY_ID_PREFIX: &str = "external-api-key-";
const EXTERNAL_API_KEY_SECRET_PREFIX: &str = "tnr_sk_";
const EXTERNAL_API_KEY_PREFIX_VISIBLE_CHARS: usize = 18;
const EXTERNAL_API_KEY_SCOPES: [&str; 4] = [
"editor:project",
"editor:canvas",
"editor:image-generate",
"editor:asset",
];
/// New API token 标识。它只用于在每个 Router 用户账号内定位同一个 Token,
/// 不承载产品展示语义;Router 用户本身通过完整 owner id 的稳定短哈希区分。
const LLM_ROUTER_TOKEN_IDENTIFIER: &str = "agc_auto_generate";
const LLM_ROUTER_USER_GROUP: &str = "taonier";
const LLM_ROUTER_TOKEN_GROUP: &str = "default";
const LLM_ROUTER_API_KEY_SCOPES: [&str; 1] = ["llm:responses"];
const LLM_ROUTER_SUBSCRIPTION_PLAN_ID: i64 = 1;
const LLM_ROUTER_SUBSCRIPTION_RENEWAL_THRESHOLD_SECONDS: i64 = 24 * 60 * 60;
const LLM_ROUTER_RECONCILIATION_ERROR_PREFIX: &str = "llm-router-reconciliation-required:";
const LLM_ROUTER_PROVISIONING_CREDENTIAL_VERSION: u32 = 1;
type HmacSha256 = Hmac<Sha256>;
static LLM_ROUTER_PROVISION_LOCKS: OnceLock<Mutex<HashMap<String, Arc<Mutex<()>>>>> =
OnceLock::new();
struct ProvisionedRouterCredential {
provider_account_id: String,
provider_account_json: Option<String>,
raw_key: String,
router_username: Option<String>,
router_password: Option<String>,
router_access_token: Option<String>,
}
#[derive(Clone, Debug)]
struct RouterAccountLogin {
username: String,
password: String,
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct RouterTokenMatch {
id: String,
group: Option<String>,
}
/// The Router API key and the credentials used to obtain it are one private
/// server-side secret. The dedicated `llm_router_account.credential_ciphertext`
/// column stores this bundle; plaintext never leaves api-server.
#[derive(Debug, Deserialize, Serialize)]
struct RouterCredentialSecret {
version: u32,
api_key: String,
username: Option<String>,
password: Option<String>,
access_token: Option<String>,
}
fn llm_router_provision_locks() -> &'static Mutex<HashMap<String, Arc<Mutex<()>>>> {
LLM_ROUTER_PROVISION_LOCKS.get_or_init(|| Mutex::new(HashMap::new()))
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyCreateRequest {
name: Option<String>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyPayload {
key_id: String,
name: String,
key_prefix: String,
scopes: Vec<String>,
created_at: String,
last_used_at: Option<String>,
revoked_at: Option<String>,
updated_at: String,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyCreateResponse {
api_key: String,
key: ExternalApiKeyPayload,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyListResponse {
keys: Vec<ExternalApiKeyPayload>,
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExternalApiKeyResponse {
key: ExternalApiKeyPayload,
}
pub async fn list_external_api_keys(
State(state): State<AppState>,
Extension(request_context): Extension<RequestContext>,
Extension(authenticated): Extension<AuthenticatedAccessToken>,
) -> Result<Json<Value>, AppError> {
let keys = state
.spacetime_client()
.list_external_api_keys(authenticated.claims().user_id().to_string())
.await
.map_err(map_external_api_key_error)?
.into_iter()
.map(external_api_key_payload_from_record)
.collect();
Ok(json_success_body(
Some(&request_context),
ExternalApiKeyListResponse { keys },
))
}
pub async fn create_external_api_key(
State(state): State<AppState>,
Extension(request_context): Extension<RequestContext>,
Extension(authenticated): Extension<AuthenticatedAccessToken>,
Json(payload): Json<ExternalApiKeyCreateRequest>,
) -> Result<Json<Value>, AppError> {
let raw_key = generate_external_api_key_secret();
let key_prefix = external_api_key_prefix(raw_key.as_str());
let key = state
.spacetime_client()
.create_external_api_key(ExternalApiKeyCreateRecordInput {
key_id: build_prefixed_uuid_id(EXTERNAL_API_KEY_ID_PREFIX),
owner_user_id: authenticated.claims().user_id().to_string(),
name: payload
.name
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.unwrap_or_else(|| "外部 API Key".to_string()),
key_prefix,
key_hash: hash_external_api_key(raw_key.as_str()),
scopes: default_external_api_key_scopes(),
now_micros: current_utc_micros(),
})
.await
.map_err(map_external_api_key_error)?;
Ok(json_success_body(
Some(&request_context),
ExternalApiKeyCreateResponse {
api_key: raw_key,
key: external_api_key_payload_from_record(key),
},
))
}
/// Creates the fixed-purpose LLM Router account key. The request intentionally has
/// no body so ordinary clients cannot choose a provider, URL, model or scope.
pub async fn ensure_llm_router_api_key(
State(state): State<AppState>,
Extension(request_context): Extension<RequestContext>,
Extension(authenticated): Extension<AuthenticatedAccessToken>,
) -> Result<Json<Value>, AppError> {
let account = ensure_llm_router_account(&state, authenticated.claims().user_id())
.await
.map_err(|message| {
AppError::from_status(StatusCode::SERVICE_UNAVAILABLE).with_message(message)
})?;
Ok(json_success_body(
Some(&request_context),
json!({ "key": external_api_key_payload_from_router_account(account) }),
))
}
/// Ensures that the authenticated account has one server-owned Router key.
/// The plaintext key never leaves this function and is persisted only as
/// authenticated encryption ciphertext in the dedicated llm_router_account row.
pub(crate) async fn ensure_llm_router_account(
state: &AppState,
owner_user_id: &str,
) -> Result<LlmRouterAccountRecord, String> {
let owner_user_id = owner_user_id.trim();
if owner_user_id.is_empty() {
return Err("LLM Router 账号缺少 owner_user_id".to_string());
}
ensure_llm_router_target_allowed(state)?;
// Registration and the first LLM request can race in the same process.
// Serialize provisioning per owner so a transient empty read cannot create
// multiple Router accounts/keys for one user.
let provision_lock = {
let mut locks = llm_router_provision_locks().lock().await;
Arc::clone(
locks
.entry(owner_user_id.to_string())
.or_insert_with(|| Arc::new(Mutex::new(()))),
)
};
let _guard = provision_lock.lock().await;
let configured_route = state.config.llm_router_base_url.trim_end_matches('/');
let encryption_secret = state
.config
.effective_llm_router_api_key_encryption_secret()
.ok_or_else(|| "LLM Router 账号密钥加密配置缺失".to_string())?;
let account_key = derive_llm_router_account_key(owner_user_id, configured_route);
let mut persisted = state
.spacetime_client()
.get_llm_router_account(owner_user_id.to_string(), configured_route.to_string())
.await
.map_err(|error| format!("读取 LLM Router 账号状态失败:{error}"))?;
if let Some(existing) = persisted.as_ref().filter(|account| {
account.status == "active"
&& account.revoked_at.is_none()
&& account
.credential_ciphertext
.as_deref()
.is_some_and(|value| !value.trim().is_empty())
}) {
tracing::debug!(
owner_user_id,
key_id = %existing.key_id,
"复用已存在的 LLM Router 账号 Key"
);
ensure_existing_llm_router_account_contract(state, existing).await?;
return Ok(existing.clone());
}
// A previous process may have revoked the Router account before it crashed
// while updating the saga row. Repair that local state before deciding whether
// the account is blocked or reusable.
if let Some(account) = persisted.as_ref()
&& account.revoked_at.is_some()
{
let repaired = upsert_llm_router_account_state(
state,
account.account_key.as_str(),
owner_user_id,
configured_route,
account.key_id.as_str(),
account.router_account_id.clone(),
account.credential_ciphertext.clone(),
account.account_json.clone(),
"retryable",
account.attempt_count,
None,
Some(current_utc_micros()),
Some("Router Key 已撤销,准备重新签发".to_string()),
)
.await?;
persisted = Some(repaired);
}
if let Some(account) = persisted.as_ref() {
let deterministic_credential_error = account.router_account_id.is_none()
&& account
.last_error
.as_deref()
.is_some_and(is_deterministic_router_credential_error);
if (account.status == "reconciliation_required" || account.status == "unknown")
&& !deterministic_credential_error
{
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 账号状态需要对账:{}",
account.last_error.as_deref().unwrap_or("未知错误")
));
}
}
let mut bundle = persisted
.as_ref()
.and_then(|account| account.credential_ciphertext.as_deref())
.map(|ciphertext| {
decrypt_router_credential_secret_allow_pending(ciphertext, encryption_secret.as_str())
})
.transpose()
.map_err(|error| format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{error}"))?;
// A previous build could have persisted credentials rejected by New API's
// username/password length validator. When no remote account id exists,
// those credentials never identified a committed remote user and may be
// safely replaced with the current deterministic values. Unknown remote
// outcomes with a known account id remain reconciliation-blocked above.
let expected_username = router_username_for_owner(owner_user_id);
let provisioning_secret = llm_router_provisioning_secret(state)?;
let expected_password = generate_router_account_password_with_secret(
owner_user_id,
provisioning_secret.as_bytes(),
)?;
let stale_pending_credentials = persisted.as_ref().is_some_and(|account| {
account.router_account_id.is_none()
&& bundle.as_ref().is_some_and(|value| {
value.api_key.trim().is_empty()
&& (value.username.as_deref() != Some(expected_username.as_str())
|| value.password.as_deref() != Some(expected_password.as_str()))
})
});
if stale_pending_credentials {
bundle = None;
}
let key_id = persisted
.as_ref()
.map(|account| account.key_id.clone())
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| derive_llm_router_key_id(owner_user_id, configured_route));
let mut reusable_login = None;
let mut generated_new_pending_credentials = false;
if bundle.is_none() {
// A historical `llm-router` row may predate the saga table. Reuse its
// encrypted username/password when the old key was revoked; otherwise
// a fresh Router user would be created for the same Genarrative owner.
// The dedicated row already owns the encrypted login bundle; no
// historical API-key row is consulted for Router recovery.
}
if bundle.is_none() {
generated_new_pending_credentials = true;
let username = expected_username.clone();
let password = expected_password.clone();
let pending = RouterCredentialSecret {
version: 1,
api_key: String::new(),
username: Some(username),
password: Some(password),
access_token: None,
};
let ciphertext = encrypt_router_credential_secret(&pending, encryption_secret.as_str())?;
upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
None,
Some(ciphertext),
persisted
.as_ref()
.and_then(|value| value.account_json.clone()),
"pending",
persisted.as_ref().map_or(0, |value| value.attempt_count),
None,
None,
None,
)
.await?;
bundle = Some(pending);
}
if let Some(existing_bundle) = bundle.as_ref() {
if !existing_bundle.api_key.trim().is_empty()
&& persisted
.as_ref()
.is_some_and(|value| matches!(value.status.as_str(), "key_issued" | "active"))
{
let active_account = upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
persisted
.as_ref()
.and_then(|value| value.router_account_id.clone()),
Some(encrypt_router_credential_secret(
existing_bundle,
encryption_secret.as_str(),
)?),
persisted
.as_ref()
.and_then(|value| value.account_json.clone()),
"active",
persisted.as_ref().map_or(0, |value| value.attempt_count),
None,
None,
None,
)
.await?;
ensure_existing_llm_router_account_contract(state, &active_account).await?;
return Ok(active_account);
}
}
// A persisted Router account id proves that the remote user was created;
// its encrypted username/password may therefore be reused after a key
// expiry or an interrupted login/token step. A freshly generated pending
// bundle is also checked against the public Router first, so a missing local
// database row can recover an already-existing deterministic account.
if !generated_new_pending_credentials
&& reusable_login.is_none()
&& persisted
.as_ref()
.is_some_and(|value| value.router_account_id.is_some())
{
reusable_login = bundle.as_ref().and_then(|value| {
Some(RouterAccountLogin {
username: value.username.clone()?,
password: value.password.clone()?,
})
});
}
let attempt_count = persisted
.as_ref()
.map_or(0, |value| value.attempt_count)
.saturating_add(1);
let pending_ciphertext = bundle
.as_ref()
.map(|value| encrypt_router_credential_secret(value, encryption_secret.as_str()))
.transpose()?;
let pending_login = bundle.as_ref().and_then(|value| {
Some(RouterAccountLogin {
username: value.username.clone()?,
password: value.password.clone()?,
})
});
upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
persisted
.as_ref()
.and_then(|value| value.router_account_id.clone()),
pending_ciphertext.clone(),
persisted
.as_ref()
.and_then(|value| value.account_json.clone()),
"registering",
attempt_count,
None,
None,
None,
)
.await?;
let provisioned = match provision_router_account(
state,
owner_user_id,
reusable_login.as_ref(),
pending_login.as_ref(),
)
.await
{
Ok(value) => value,
Err(error) => {
let status = if error.starts_with(LLM_ROUTER_RECONCILIATION_ERROR_PREFIX) {
"unknown"
} else {
"retryable"
};
let _ = upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
persisted
.as_ref()
.and_then(|value| value.router_account_id.clone()),
pending_ciphertext,
persisted
.as_ref()
.and_then(|value| value.account_json.clone()),
status,
attempt_count,
None,
Some(current_utc_micros().saturating_add(60_000_000)),
Some(error.clone()),
)
.await;
return Err(error);
}
};
let credential_secret = RouterCredentialSecret {
version: 1,
api_key: provisioned.raw_key.clone(),
username: provisioned
.router_username
.clone()
.or_else(|| reusable_login.as_ref().map(|v| v.username.clone())),
password: provisioned
.router_password
.clone()
.or_else(|| reusable_login.as_ref().map(|v| v.password.clone())),
access_token: provisioned.router_access_token.clone(),
};
let secret_ciphertext =
encrypt_router_credential_secret(&credential_secret, encryption_secret.as_str())
.map_err(|error| format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{error}"))?;
upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
Some(provisioned.provider_account_id.clone()),
Some(secret_ciphertext.clone()),
provisioned.provider_account_json.clone(),
"key_issued",
attempt_count,
None,
None,
None,
)
.await?;
let account = upsert_llm_router_account_state(
state,
&account_key,
owner_user_id,
configured_route,
&key_id,
Some(provisioned.provider_account_id.clone()),
Some(secret_ciphertext),
provisioned.provider_account_json.clone(),
"active",
attempt_count,
None,
None,
None,
)
.await?;
tracing::info!(
owner_user_id,
key_id = %account.key_id,
router_account_id = ?provisioned.provider_account_id,
"LLM Router 账号与 API Key provisioning 完成"
);
Ok(account)
}
async fn upsert_llm_router_account_state(
state: &AppState,
account_key: &str,
owner_user_id: &str,
route_origin: &str,
key_id: &str,
router_account_id: Option<String>,
credential_ciphertext: Option<String>,
account_json: Option<String>,
status: &str,
attempt_count: u32,
lease_until_micros: Option<i64>,
next_retry_at_micros: Option<i64>,
last_error: Option<String>,
) -> Result<LlmRouterAccountRecord, String> {
let key_material = credential_ciphertext.as_deref().and_then(|ciphertext| {
state
.config
.effective_llm_router_api_key_encryption_secret()
.and_then(|secret| decrypt_router_credential_secret(ciphertext, secret.as_str()).ok())
.filter(|bundle| !bundle.api_key.trim().is_empty())
.map(|bundle| {
let raw_key = bundle.api_key.trim().to_string();
(
external_api_key_prefix(raw_key.as_str()),
hash_llm_router_key(owner_user_id, key_id, raw_key.as_str()),
)
})
});
let account = state
.spacetime_client()
.upsert_llm_router_account(LlmRouterAccountUpsertRecordInput {
account_key: account_key.to_string(),
owner_user_id: owner_user_id.to_string(),
route_origin: route_origin.to_string(),
idempotency_key: key_id.to_string(),
router_account_id,
credential_ciphertext,
account_json,
status: status.to_string(),
attempt_count,
lease_until_micros,
next_retry_at_micros,
last_error: last_error.clone(),
credential_version: 1,
now_micros: current_utc_micros(),
key_id: key_id.to_string(),
key_name: LLM_ROUTER_TOKEN_IDENTIFIER.to_string(),
key_prefix: key_material.as_ref().map(|(prefix, _)| prefix.clone()),
key_hash: key_material.map(|(_, hash)| hash),
last_used_at_micros: None,
revoked_at_micros: last_error
.as_deref()
.filter(|value| {
value.contains("Router 返回 401/403") || value.contains("Router Key 已撤销")
})
.map(|_| current_utc_micros()),
})
.await
.map_err(|error| format!("保存 LLM Router 账号状态失败:{error}"))?;
Ok(account)
}
/// Ensures the Router user has the fixed AGC subscription before the server
/// exposes/reuses its API key. Subscription management is deliberately tied to
/// authentication/key preparation rather than the streaming response path.
async fn ensure_existing_llm_router_account_contract(
state: &AppState,
record: &LlmRouterAccountRecord,
) -> Result<(), String> {
let Some(admin_token) = state
.config
.llm_router_admin_token
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
else {
return Err(
"LLM Router 管理员 Token 未配置,无法确保用户分组和 plan_id=1 订阅".to_string(),
);
};
let encryption_secret = state
.config
.effective_llm_router_api_key_encryption_secret()
.ok_or_else(|| "LLM Router 账号密钥加密配置缺失".to_string())?;
let bundle = record
.credential_ciphertext
.as_deref()
.filter(|value| !value.trim().is_empty())
.map(|ciphertext| decrypt_router_credential_secret(ciphertext, encryption_secret.as_str()))
.transpose()
.map_err(|error| format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{error}"))?
.ok_or_else(|| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 缺少 Router 账号凭据"
)
})?;
let username = bundle.username.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 缺少 Router 用户名")
})?;
let password = bundle.password.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 缺少 Router 密码")
})?;
let origin = router_control_origin(state.config.llm_router_base_url.as_str())?;
let client = router_admin_client()?;
let router_user_id = search_router_user_id(&client, origin.as_str(), admin_token, &username)
.await?
.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}无法通过用户名查询 Router 用户 ID")
})?;
if let Some(recorded_router_user_id) = router_user_id_from_account(record)
&& recorded_router_user_id != router_user_id
{
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}本地 Router 用户 ID 与远端查询结果不一致"
));
}
// The paid group belongs to the Router user, not to the API token. This
// repair is intentionally run at account preparation/login time so an old
// active row created before the group split cannot keep using `taonier` as
// its token group.
update_router_user(
&client,
origin.as_str(),
admin_token,
router_user_id,
username.as_str(),
record.owner_user_id.as_str(),
)
.await?;
let login_token = login_router_account(
&client,
origin.as_str(),
username.as_str(),
password.as_str(),
)
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router 用户无法使用保存的凭据登录:{error}"
)
})?;
let token = find_router_token_id(&client, origin.as_str(), login_token.as_str())
.await?
.ok_or_else(|| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 对应的固定 Token 不存在"
)
})?;
ensure_router_token_contract(
&client,
origin.as_str(),
login_token.as_str(),
token.id.as_str(),
)
.await?;
ensure_router_subscription(&client, origin.as_str(), admin_token, router_user_id).await
}
fn router_admin_client() -> Result<reqwest::Client, String> {
reqwest::Client::builder()
.connect_timeout(std::time::Duration::from_secs(10))
.timeout(std::time::Duration::from_secs(30))
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| format!("创建 LLM Router 管理客户端失败:{error}"))
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct RouterSubscriptionSummary {
plan_id: i64,
status: String,
end_time: i64,
}
fn subscription_requires_renewal(
subscriptions: &[RouterSubscriptionSummary],
now_unix_seconds: i64,
) -> bool {
let latest_active_end = subscriptions
.iter()
.filter(|subscription| {
subscription.plan_id == LLM_ROUTER_SUBSCRIPTION_PLAN_ID
&& subscription.status.eq_ignore_ascii_case("active")
})
.map(|subscription| subscription.end_time)
.max();
latest_active_end.is_none_or(|end_time| {
end_time
<= now_unix_seconds.saturating_add(LLM_ROUTER_SUBSCRIPTION_RENEWAL_THRESHOLD_SECONDS)
})
}
fn extract_router_subscriptions(payload: &Value) -> Vec<RouterSubscriptionSummary> {
fn search(value: &Value, matches: &mut Vec<RouterSubscriptionSummary>) {
match value {
Value::Object(object) => {
let subscription = object.get("subscription").unwrap_or(value);
if let Value::Object(subscription) = subscription {
let plan_id = subscription
.get("plan_id")
.or_else(|| subscription.get("planId"))
.and_then(provider_value_to_i64);
let status = subscription
.get("status")
.and_then(Value::as_str)
.map(str::trim)
.filter(|value| !value.is_empty());
let end_time = subscription
.get("end_time")
.or_else(|| subscription.get("endTime"))
.and_then(provider_value_to_i64);
if let (Some(plan_id), Some(status), Some(end_time)) =
(plan_id, status, end_time)
{
matches.push(RouterSubscriptionSummary {
plan_id,
status: status.to_string(),
end_time,
});
}
}
for child in object.values() {
search(child, matches);
}
}
Value::Array(values) => {
for child in values {
search(child, matches);
}
}
_ => {}
}
}
let mut matches = Vec::new();
search(payload, &mut matches);
matches.sort_by_key(|subscription| {
(
subscription.plan_id,
subscription.status.to_ascii_lowercase(),
subscription.end_time,
)
});
matches.dedup();
matches
}
async fn ensure_router_subscription(
client: &reqwest::Client,
origin: &str,
admin_token: &str,
router_user_id: i64,
) -> Result<(), String> {
if router_user_id <= 0 {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户 ID 无效,无法检查订阅"
));
}
let response = client
.get(format!(
"{origin}/api/subscription/admin/users/{router_user_id}/subscriptions"
))
.bearer_auth(admin_token)
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 订阅查询未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() || provider_payload_failed(&payload) {
let message = format!(
"LLM Router 订阅查询失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
);
return Err(
if !status.is_success() && router_http_failure_requires_reconciliation(status) {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{message}")
} else {
message
},
);
}
let subscriptions = extract_router_subscriptions(&payload);
let now_unix_seconds = current_utc_micros().div_euclid(1_000_000);
if !subscription_requires_renewal(&subscriptions, now_unix_seconds) {
return Ok(());
}
let response = client
.post(format!(
"{origin}/api/subscription/admin/users/{router_user_id}/subscriptions"
))
.bearer_auth(admin_token)
.json(&json!({"plan_id": LLM_ROUTER_SUBSCRIPTION_PLAN_ID}))
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 订阅创建未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() || provider_payload_failed(&payload) {
let message = format!(
"LLM Router 订阅创建失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
);
return Err(
if !status.is_success() && router_http_failure_requires_reconciliation(status) {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{message}")
} else {
message
},
);
}
tracing::info!(
router_user_id,
plan_id = LLM_ROUTER_SUBSCRIPTION_PLAN_ID,
"LLM Router 订阅已开通或续期"
);
Ok(())
}
pub(crate) fn router_user_id_from_account(record: &LlmRouterAccountRecord) -> Option<i64> {
record
.router_account_id
.as_deref()
.and_then(|value| value.trim().parse::<i64>().ok())
.or_else(|| {
record
.account_json
.as_deref()
.and_then(|value| serde_json::from_str::<Value>(value).ok())
.and_then(|value| {
value
.get("routerUserId")
.or_else(|| value.get("router_user_id"))
.or_else(|| value.get("userId"))
.or_else(|| value.get("user_id"))
.and_then(provider_value_to_i64)
})
})
}
/// Starts best-effort Router-account preparation after successful
/// authentication. Router control-plane availability must not gate the main
/// site session; the LLM hot path validates the locally persisted credential
/// when a request is actually made.
pub(crate) async fn ensure_llm_router_account_after_auth_success(
state: &AppState,
_request_context: &RequestContext,
owner_user_id: &str,
) -> Result<(), String> {
#[cfg(test)]
if std::env::var_os("GENARRATIVE_LLM_ROUTER_AUTH_PROVISION_TEST").is_none() {
// Authentication tests should not contact the production Router. The
// provisioning path remains covered by its dedicated tests and can
// be enabled explicitly with this test-only fixture switch.
return Ok(());
}
#[cfg(test)]
{
if let Err(error) = ensure_llm_router_account(state, owner_user_id).await {
tracing::warn!(
user_id = %owner_user_id,
error = %error,
"测试环境登录后 LLM Router 账号准备失败;已允许主站登录"
);
}
return Ok(());
}
#[cfg(not(test))]
{
let state = state.clone();
let owner_user_id = owner_user_id.to_string();
tokio::spawn(async move {
if let Err(error) = ensure_llm_router_account(&state, owner_user_id.as_str()).await {
tracing::warn!(
user_id = %owner_user_id,
error = %error,
"登录后 LLM Router 账号准备失败;已允许主站登录,后续 LLM 请求将按需失败关闭"
);
}
});
Ok(())
}
}
/// Reads an already-provisioned Router credential without contacting the
/// Router control plane. Account registration, user-group repair and
/// subscription renewal belong to the authentication/key-preparation anchor;
/// the hot LLM forwarding path only needs this validated local row.
pub(crate) async fn read_active_llm_router_credentials(
state: &AppState,
owner_user_id: &str,
) -> Result<Option<(String, String, String)>, String> {
let owner_user_id = owner_user_id.trim();
if owner_user_id.is_empty() {
return Err("LLM Router 账号缺少 owner_user_id".to_string());
}
ensure_llm_router_target_allowed(state)?;
let encryption_secret = state
.config
.effective_llm_router_api_key_encryption_secret()
.ok_or_else(|| "LLM Router 账号密钥加密配置缺失".to_string())?;
let Some(account) = state
.spacetime_client()
.get_llm_router_account(
owner_user_id.to_string(),
state
.config
.llm_router_base_url
.trim_end_matches('/')
.to_string(),
)
.await
.map_err(|error| format!("读取 LLM Router 账号失败:{error}"))?
.filter(|account| account.status == "active" && account.revoked_at.is_none())
else {
return Ok(None);
};
let ciphertext = account.credential_ciphertext.as_deref().ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}active Router Key 密文缺失")
})?;
// Router account state is read from SpacetimeDB immediately before every
// upstream request and the current ciphertext is decrypted on demand. This
// deliberately avoids a TTL cache: another api-server instance can rotate
// or revoke the durable row at any time, and no process-local entry can
// outlive that authoritative state.
let api_key = decrypt_router_api_key(ciphertext, encryption_secret.as_str())?;
Ok(Some((
state
.config
.llm_router_base_url
.trim_end_matches('/')
.to_string(),
api_key,
account.key_id,
)))
}
pub(crate) async fn revoke_llm_router_account(
state: &AppState,
owner_user_id: &str,
key_id: &str,
) -> Result<(), String> {
let route_origin = state.config.llm_router_base_url.trim_end_matches('/');
if let Some(account) = state
.spacetime_client()
.get_llm_router_account(owner_user_id.to_string(), route_origin.to_string())
.await
.map_err(|error| format!("读取 LLM Router 账号失效状态失败:{error}"))?
.filter(|account| account.key_id == key_id)
{
upsert_llm_router_account_state(
state,
account.account_key.as_str(),
owner_user_id,
route_origin,
key_id,
account.router_account_id.clone(),
account.credential_ciphertext.clone(),
account.account_json.clone(),
"retryable",
account.attempt_count,
None,
Some(current_utc_micros()),
Some("Router 返回 401/403,等待重新签发账号 Key".to_string()),
)
.await?;
}
Ok(())
}
async fn provision_router_account(
state: &AppState,
owner_user_id: &str,
reusable_login: Option<&RouterAccountLogin>,
pending_login: Option<&RouterAccountLogin>,
) -> Result<ProvisionedRouterCredential, String> {
// 走 Router New API 的正式账号链路:先按稳定用户名/密码恢复已有用户,确认不存在
// 后才注册,再登录、复用或创建无限额度 token、签发 API Key。网络/5xx/格式不确定
// 等情况保持 reconciliation,避免重复注册。
provision_router_account_via_new_api(state, owner_user_id, reusable_login, pending_login).await
}
async fn provision_router_account_via_new_api(
state: &AppState,
owner_user_id: &str,
reusable_login: Option<&RouterAccountLogin>,
pending_login: Option<&RouterAccountLogin>,
) -> Result<ProvisionedRouterCredential, String> {
let origin = router_control_origin(state.config.llm_router_base_url.as_str())?;
let client = reqwest::Client::builder()
.connect_timeout(std::time::Duration::from_secs(10))
.timeout(std::time::Duration::from_secs(30))
.redirect(reqwest::redirect::Policy::none())
.build()
.map_err(|error| format!("创建 LLM Router 账号客户端失败:{error}"))?;
let (username, password) = reusable_login
.map(|login| (login.username.clone(), login.password.clone()))
.or_else(|| pending_login.map(|login| (login.username.clone(), login.password.clone())))
.unwrap_or((
router_username_for_owner(owner_user_id),
generate_router_account_password_with_secret(
owner_user_id,
llm_router_provisioning_secret(state)?.as_bytes(),
)?,
));
let admin_token = state
.config
.llm_router_admin_token
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty());
let admin_token = admin_token.ok_or_else(|| {
"LLM Router 管理员 Token 未配置,无法完成用户分组、订阅和 API Key 签发".to_string()
})?;
let (login_token, router_user_id) = if reusable_login.is_some() {
let login_token = login_router_account(
&client,
origin.as_str(),
username.as_str(),
password.as_str(),
)
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}已存在的 Router 用户无法使用保存的凭据登录:{error}"
)
})?;
let router_user_id = search_router_user_id(
&client,
origin.as_str(),
admin_token,
username.as_str(),
)
.await?
.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}已登录的 Router 用户无法查询用户 ID")
})?;
(login_token, router_user_id)
} else {
let existing_router_user_id =
search_router_user_id(&client, origin.as_str(), admin_token, username.as_str()).await?;
if let Some(remote_router_user_id) = existing_router_user_id {
// Existing deterministic accounts may have been created before the
// group-update step completed (for example after a process crash).
// Reapply the complete user update so the account is always in the
// paid `taonier` group.
update_router_user(
&client,
origin.as_str(),
admin_token,
remote_router_user_id,
username.as_str(),
owner_user_id,
)
.await?;
let login_token = login_router_account(&client, origin.as_str(), username.as_str(), password.as_str())
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户已存在但稳定密码无法登录:{error}"
)
})?;
(login_token, remote_router_user_id)
} else {
let register_error = register_router_user(
&client,
origin.as_str(),
admin_token,
username.as_str(),
password.as_str(),
owner_user_id,
)
.await
.err();
let remote_router_user_id =
search_router_user_id(&client, origin.as_str(), admin_token, username.as_str())
.await?
.ok_or_else(|| {
register_error.unwrap_or_else(|| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 注册成功后无法查询到用户 ID"
)
})
})?;
update_router_user(
&client,
origin.as_str(),
admin_token,
remote_router_user_id,
username.as_str(),
owner_user_id,
)
.await?;
let login_token = login_router_account(
&client,
origin.as_str(),
username.as_str(),
password.as_str(),
)
.await?;
(login_token, remote_router_user_id)
}
};
ensure_router_subscription(&client, origin.as_str(), admin_token, router_user_id).await?;
let (token_id, token_payload) = if let Some(token) =
find_router_token_id(&client, origin.as_str(), login_token.as_str()).await?
{
// Keep the fixed token contract authoritative even when the Router
// returns an incomplete token summary (for example without `group`).
// The user remains in `taonier`; only this API token must be in the
// `default` group with unlimited quota and no expiry.
// PUT the complete fixed token contract, not just the group. This
// repairs old tokens that were created with `taonier`, and also
// restores unlimited quota/permanent expiry if an operator changed
// either field. The request is cheap because this path only runs when
// provisioning or recovering an account, not for every LLM call.
ensure_router_token_contract(
&client,
origin.as_str(),
login_token.as_str(),
token.id.as_str(),
)
.await?;
(token.id, json!({}))
} else {
let token_response = client
.post(format!("{origin}/api/token/"))
.bearer_auth(login_token.as_str())
.json(&router_token_request())
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 创建未取得确定响应:{error}"
)
})?;
let token_status = token_response.status();
let token_payload = response_json_or_text(token_response).await?;
if !token_status.is_success() {
if router_http_failure_requires_reconciliation(token_status) {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}LLM Router token 创建返回 HTTP {},结果可能未知{}",
token_status.as_u16(),
provider_payload_error_suffix(&token_payload)
));
}
return Err(format!(
"LLM Router token 创建失败:HTTP {}{}",
token_status.as_u16(),
provider_payload_error_suffix(&token_payload)
));
}
if provider_payload_failed(&token_payload) {
return Err(format!(
"LLM Router token 创建失败:HTTP {}{}",
token_status.as_u16(),
provider_payload_error_suffix(&token_payload)
));
}
let token_id = if let Some(token_id) =
extract_provider_string(&token_payload, &["id", "tokenId", "token_id"])
{
token_id
} else {
find_router_token_id(&client, origin.as_str(), login_token.as_str())
.await?
.map(|token| token.id)
.ok_or_else(|| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 查询响应缺少 token id"
)
})?
};
// New API may accept the create request while applying the user's
// default group. Normalize the freshly-created token before issuing a
// key; otherwise a successful POST can still produce a token routed
// through `taonier` and later fail with `model_not_found`.
ensure_router_token_contract(
&client,
origin.as_str(),
login_token.as_str(),
token_id.as_str(),
)
.await?;
(token_id, token_payload)
};
let key_response = client
.post(format!("{origin}/api/token/{token_id}/key"))
.bearer_auth(login_token.as_str())
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router API Key 签发未取得确定响应:{error}"
)
})?;
let key_status = key_response.status();
let key_payload = response_json_or_text(key_response).await?;
if !key_status.is_success() {
if router_http_failure_requires_reconciliation(key_status) {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}LLM Router API Key 签发返回 HTTP {},结果可能未知{}",
key_status.as_u16(),
provider_payload_error_suffix(&key_payload)
));
}
return Err(format!(
"LLM Router API Key 签发失败:HTTP {}{}",
key_status.as_u16(),
provider_payload_error_suffix(&key_payload)
));
}
if provider_payload_failed(&key_payload) {
return Err(format!(
"LLM Router API Key 签发失败:HTTP {}{}",
key_status.as_u16(),
provider_payload_error_suffix(&key_payload)
));
}
let raw_key = extract_provider_string(&key_payload, &["apiKey", "api_key", "key", "token"])
.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router API Key 响应缺少 key")
})?;
let account_id = extract_provider_string(
&token_payload,
&["accountId", "account_id", "userId", "user_id"],
)
.unwrap_or_else(|| username.clone());
Ok(ProvisionedRouterCredential {
provider_account_id: account_id.clone(),
provider_account_json: Some(
json!({
"version": 1,
"accountId": account_id,
"routerUserId": router_user_id,
"username": username,
"routeOrigin": state.config.llm_router_base_url,
"model": state.config.llm_router_model,
"credentialSource": "router-new-api",
})
.to_string(),
),
raw_key: normalize_new_api_relay_api_key(raw_key.as_str())?,
router_username: Some(username),
router_password: Some(password),
router_access_token: None,
})
}
async fn search_router_user_id(
client: &reqwest::Client,
origin: &str,
admin_token: &str,
username: &str,
) -> Result<Option<i64>, String> {
let mut search_url = reqwest::Url::parse(format!("{origin}/api/user/search").as_str())
.map_err(|error| format!("Router 用户查询地址无效:{error}"))?;
search_url
.query_pairs_mut()
.append_pair("keyword", username);
let response = client
.get(search_url)
.bearer_auth(admin_token)
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户查询未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_empty(response).await?;
if !status.is_success() {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户查询返回 HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
));
}
if provider_payload_failed(&payload) {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户查询失败{}",
provider_payload_error_suffix(&payload)
));
}
Ok(extract_router_user_id(&payload, username))
}
async fn register_router_user(
client: &reqwest::Client,
origin: &str,
admin_token: &str,
username: &str,
password: &str,
owner_user_id: &str,
) -> Result<(), String> {
let response = client
.post(format!("{origin}/api/user/"))
.bearer_auth(admin_token)
.json(&json!({
"username": username,
"password": password,
"display_name": router_user_display_name(username),
"remark": owner_user_id,
"role": 1,
}))
.send()
.await
.map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 注册未取得确定响应:{error}")
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() || provider_payload_failed(&payload) {
let message = format!(
"LLM Router 注册失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
);
// A successful HTTP response with `success=false` is a deterministic
// business/validation failure. Only transport, redirect, and server
// side failures require reconciliation because the remote write may
// have committed before the response was lost.
return Err(
if !status.is_success() && router_http_failure_requires_reconciliation(status) {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{message}")
} else {
message
},
);
}
Ok(())
}
async fn update_router_user(
client: &reqwest::Client,
origin: &str,
admin_token: &str,
router_user_id: i64,
username: &str,
owner_user_id: &str,
) -> Result<(), String> {
let response = client
.put(format!("{origin}/api/user/"))
.bearer_auth(admin_token)
.json(&router_user_update_request(
router_user_id,
username,
owner_user_id,
))
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 用户分组设置未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() || provider_payload_failed(&payload) {
let message = format!(
"LLM Router 用户分组设置失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
);
return Err(if !status.is_client_error() {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{message}")
} else {
message
});
}
Ok(())
}
async fn login_router_account(
client: &reqwest::Client,
origin: &str,
username: &str,
password: &str,
) -> Result<String, String> {
let response = client
.post(format!("{origin}/api/user/login"))
.json(&json!({"username": username, "password": password}))
.send()
.await
.map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 登录未取得确定响应:{error}")
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() {
if router_http_failure_requires_reconciliation(status) {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}LLM Router 登录返回 HTTP {},结果可能未知{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
));
}
return Err(format!(
"LLM Router 登录失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
));
}
if provider_payload_failed(&payload) {
return Err(format!(
"LLM Router 登录失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
));
}
extract_provider_string(
&payload,
&["token", "accessToken", "access_token", "userToken"],
)
.ok_or_else(|| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 登录响应缺少 access token")
})
}
fn router_token_request() -> Value {
json!({
"name": LLM_ROUTER_TOKEN_IDENTIFIER,
"expired_time": -1,
"unlimited_quota": true,
"group": LLM_ROUTER_TOKEN_GROUP,
})
}
fn router_token_update_request(token_id: i64) -> Value {
json!({
"id": token_id,
"name": LLM_ROUTER_TOKEN_IDENTIFIER,
"expired_time": -1,
"unlimited_quota": true,
"group": LLM_ROUTER_TOKEN_GROUP,
})
}
fn router_user_update_request(user_id: i64, username: &str, owner_user_id: &str) -> Value {
json!({
"id": user_id,
"username": username,
"display_name": router_user_display_name(username),
"remark": owner_user_id,
"group": LLM_ROUTER_USER_GROUP,
"role": 1,
})
}
fn router_user_display_name(username: &str) -> String {
username.to_string()
}
async fn find_router_token_id(
client: &reqwest::Client,
origin: &str,
login_token: &str,
) -> Result<Option<RouterTokenMatch>, String> {
let mut search_url = reqwest::Url::parse(format!("{origin}/api/token/search").as_str())
.map_err(|error| format!("Router token 查询地址无效:{error}"))?;
search_url
.query_pairs_mut()
.append_pair("keyword", LLM_ROUTER_TOKEN_IDENTIFIER);
let response = client
.get(search_url)
.bearer_auth(login_token)
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 查询未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_empty(response).await?;
if !status.is_success() {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 查询返回 HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
));
}
if provider_payload_failed(&payload) {
return Err(format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 查询失败{}",
provider_payload_error_suffix(&payload)
));
}
let token =
extract_router_token_info(&payload, LLM_ROUTER_TOKEN_IDENTIFIER).map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 标识不唯一:{error}")
})?;
Ok(token)
}
async fn ensure_router_token_contract(
client: &reqwest::Client,
origin: &str,
login_token: &str,
token_id: &str,
) -> Result<(), String> {
let numeric_token_id = token_id.parse::<i64>().map_err(|_| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token id 格式无效,无法修正分组")
})?;
let response = client
.put(format!("{origin}/api/token/"))
.bearer_auth(login_token)
.json(&router_token_update_request(numeric_token_id))
.send()
.await
.map_err(|error| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router token 分组修正未取得确定响应:{error}"
)
})?;
let status = response.status();
let payload = response_json_or_text(response).await?;
if !status.is_success() || provider_payload_failed(&payload) {
let message = format!(
"LLM Router token 分组修正失败:HTTP {}{}",
status.as_u16(),
provider_payload_error_suffix(&payload)
);
return Err(
if !status.is_success() && router_http_failure_requires_reconciliation(status) {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}{message}")
} else {
message
},
);
}
Ok(())
}
fn router_control_origin(base_url: &str) -> Result<String, String> {
let mut url = reqwest::Url::parse(base_url.trim_end_matches('/'))
.map_err(|error| format!("LLM Router 地址无效:{error}"))?;
let is_loopback = url.host_str().is_some_and(|host| {
host.eq_ignore_ascii_case("localhost")
|| host
.parse::<std::net::IpAddr>()
.is_ok_and(|address| address.is_loopback())
});
let allowed_scheme = url.scheme() == "https" || (url.scheme() == "http" && is_loopback);
if !allowed_scheme || url.host_str().is_none() {
return Err("LLM Router 账号链路只允许 HTTPS 或 loopback HTTP 地址".to_string());
}
url.set_path("");
url.set_query(None);
url.set_fragment(None);
Ok(url.to_string().trim_end_matches('/').to_string())
}
fn ensure_llm_router_target_allowed(state: &AppState) -> Result<(), String> {
let base_url = state.config.llm_router_base_url.trim_end_matches('/');
let url =
reqwest::Url::parse(base_url).map_err(|error| format!("LLM Router 地址无效:{error}"))?;
let host = url
.host_str()
.ok_or_else(|| "LLM Router 地址缺少主机名".to_string())?;
let is_loopback = host.eq_ignore_ascii_case("localhost")
|| host
.parse::<std::net::IpAddr>()
.is_ok_and(|address| address.is_loopback());
#[cfg(test)]
if !is_loopback {
return Err("单元测试禁止连接非 loopback 的 LLM Router".to_string());
}
if state.config.is_production() {
if base_url != OFFICIAL_LLM_ROUTER_BASE_URL {
return Err("生产环境 LLM Router 必须使用官方固定路由".to_string());
}
if state.config.llm_router_model.trim() != OFFICIAL_LLM_ROUTER_MODEL {
return Err("生产环境 LLM Router 必须使用官方固定模型".to_string());
}
if url.scheme() != "https" {
return Err("生产环境 LLM Router 只允许 HTTPS 地址".to_string());
}
return Ok(());
}
if base_url == OFFICIAL_LLM_ROUTER_BASE_URL {
if state.config.llm_router_model.trim() != OFFICIAL_LLM_ROUTER_MODEL {
return Err("LLM Router 必须使用官方固定模型".to_string());
}
if url.scheme() != "https" {
return Err("官方 LLM Router 只允许 HTTPS 地址".to_string());
}
// Shared Router is intentionally usable by development deployments too:
// each deployment may have an independent database, while the stable
// owner-derived agc_user_ account and agc_auto_generate Token are shared.
return Ok(());
}
if !is_loopback {
return Err(format!(
"当前环境 {} 只允许官方固定 LLM Router 或 loopback Router;请检查路由配置",
state.config.environment
));
}
if !matches!(url.scheme(), "http" | "https") {
return Err("非生产环境 LLM Router 只允许 HTTP/HTTPS loopback 地址".to_string());
}
Ok(())
}
fn router_username_for_owner(owner_user_id: &str) -> String {
// New API 的 User.Username 校验上限是 20 个字符。保留可读前缀后只
// 能放 11 个字符;使用完整 owner id 做 SHA-256,再编码成 8 字节的
// URL-safe 短码(11 个 ASCII 字符),得到稳定且看起来随机的用户名。
let digest = Sha256::digest(
format!("genarrative-agc-router-user:v1\n{}", owner_user_id.trim()).as_bytes(),
);
let short_code = URL_SAFE_NO_PAD.encode(&digest[..8]);
debug_assert_eq!(short_code.len(), 11);
format!("agc_user_{short_code}")
}
fn llm_router_provisioning_secret(state: &AppState) -> Result<String, String> {
if let Some(secret) = state
.config
.llm_router_provisioning_secret
.as_deref()
.map(str::trim)
.filter(|value| !value.is_empty())
{
return Ok(secret.to_string());
}
#[cfg(test)]
{
return Ok("test-only-router-provisioning-secret".to_string());
}
#[cfg(not(test))]
{
Err("LLM Router 账号密码派生密钥未配置".to_string())
}
}
#[cfg(test)]
fn generate_router_account_password(owner_user_id: &str) -> Result<String, String> {
generate_router_account_password_with_secret(
owner_user_id,
b"test-only-router-provisioning-secret",
)
}
fn generate_router_account_password_with_secret(
owner_user_id: &str,
provisioning_secret: &[u8],
) -> Result<String, String> {
let owner_user_id = owner_user_id.trim();
if owner_user_id.is_empty() {
return Err("LLM Router 账号密码派生参数缺失".to_string());
}
if provisioning_secret.is_empty() {
return Err("LLM Router 账号密码派生密钥未配置".to_string());
}
let mut signer = HmacSha256::new_from_slice(provisioning_secret)
.map_err(|_| "LLM Router 账号密码派生密钥无效".to_string())?;
signer.update(
format!(
"genarrative-llm-router-password:v{}\n{}",
LLM_ROUTER_PROVISIONING_CREDENTIAL_VERSION, owner_user_id
)
.as_bytes(),
);
let digest = signer.finalize().into_bytes();
// New API 的 User.Password 校验范围是 8..=20。使用完整 owner id
// 派生 10 字节 HMAC 并编码为 20 位 hex;不再附加人为的 Router 前缀。
Ok(hex::encode(&digest[..10]))
}
async fn response_json_or_empty(response: reqwest::Response) -> Result<Value, String> {
let bytes = response.bytes().await.map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 响应读取失败:{error}")
})?;
if bytes.is_empty() {
return Ok(json!({}));
}
serde_json::from_slice::<Value>(&bytes).map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 响应格式无效:{error}")
})
}
async fn response_json_or_text(response: reqwest::Response) -> Result<Value, String> {
let bytes = response.bytes().await.map_err(|error| {
format!("{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 响应读取失败:{error}")
})?;
if bytes.is_empty() {
return Ok(json!({}));
}
match serde_json::from_slice::<Value>(&bytes) {
Ok(value) => Ok(value),
Err(_) => String::from_utf8(bytes.to_vec())
.map(|value| Value::String(value.trim().to_string()))
.ok()
.filter(|value| value.as_str().is_some_and(|text| !text.is_empty()))
.ok_or_else(|| {
format!(
"{LLM_ROUTER_RECONCILIATION_ERROR_PREFIX}Router 响应既不是合法 JSON 也不是非空文本"
)
}),
}
}
fn extract_router_user_id(payload: &Value, username: &str) -> Option<i64> {
fn search(value: &Value, username: &str) -> Option<i64> {
match value {
Value::Object(object) => {
let object_username = object
.get("username")
.or_else(|| object.get("userName"))
.and_then(Value::as_str);
if object_username.is_some_and(|value| value == username) {
return object
.get("id")
.or_else(|| object.get("userId"))
.or_else(|| object.get("user_id"))
.and_then(provider_value_to_i64);
}
object.values().find_map(|value| search(value, username))
}
Value::Array(values) => values.iter().find_map(|value| search(value, username)),
_ => None,
}
}
search(payload, username)
}
fn provider_value_to_i64(value: &Value) -> Option<i64> {
value
.as_i64()
.or_else(|| value.as_u64().and_then(|value| i64::try_from(value).ok()))
.or_else(|| value.as_str().and_then(|value| value.trim().parse().ok()))
}
fn extract_router_token_info(
payload: &Value,
token_name: &str,
) -> Result<Option<RouterTokenMatch>, String> {
fn search(value: &Value, token_name: &str, matches: &mut Vec<RouterTokenMatch>) {
match value {
Value::Object(object) => {
let object_name = object.get("name").and_then(Value::as_str).map(str::trim);
if object_name == Some(token_name) {
if let Some(token_id) = object
.get("id")
.or_else(|| object.get("tokenId"))
.or_else(|| object.get("token_id"))
.and_then(provider_value_to_string)
{
matches.push(RouterTokenMatch {
id: token_id,
group: object
.get("group")
.or_else(|| object.get("groupName"))
.and_then(Value::as_str)
.map(str::trim)
.filter(|group| !group.is_empty())
.map(str::to_string),
});
}
return;
}
for value in object.values() {
search(value, token_name, matches);
}
}
Value::Array(values) => {
for value in values {
search(value, token_name, matches);
}
}
_ => {}
}
}
let mut matches = Vec::new();
search(payload, token_name, &mut matches);
matches.sort_by(|left, right| left.id.cmp(&right.id));
matches.dedup();
let mut ids = matches
.iter()
.map(|token| token.id.as_str())
.collect::<Vec<_>>();
ids.sort_unstable();
ids.dedup();
match ids.as_slice() {
[] => Ok(None),
[token_id] => {
let groups = matches
.iter()
.filter(|token| token.id == *token_id)
.filter_map(|token| token.group.as_deref())
.collect::<Vec<_>>();
if groups.windows(2).any(|window| window[0] != window[1]) {
return Err(format!(
"固定 Token 标识 {token_name} 对应同一 id 但分组不一致"
));
}
Ok(matches.iter().find(|token| token.id == *token_id).cloned())
}
_ => Err(format!(
"固定 Token 标识 {token_name} 对应多个不同 token id"
)),
}
}
fn extract_router_token_id(payload: &Value, token_name: &str) -> Result<Option<String>, String> {
extract_router_token_info(payload, token_name).map(|token| token.map(|token| token.id))
}
fn provider_payload_data(value: &Value) -> &Value {
value.get("data").unwrap_or(value)
}
/// A 4xx response is a deterministic rejection and may be retried with the
/// same durable account intent. A 3xx/5xx response is not safe to interpret as
/// a failed provisioning operation because the remote side may have committed
/// the account before the response was lost or redirected; keep it in
/// reconciliation instead of registering another account.
fn router_http_failure_requires_reconciliation(status: reqwest::StatusCode) -> bool {
!status.is_client_error()
}
fn is_deterministic_router_credential_error(message: &str) -> bool {
let normalized = message.to_ascii_lowercase();
normalized.contains("field validation")
&& (normalized.contains("username")
|| normalized.contains("password")
|| normalized.contains("display_name"))
&& (normalized.contains("max") || normalized.contains("min"))
}
fn provider_payload_failed(value: &Value) -> bool {
value
.get("success")
.and_then(Value::as_bool)
.is_some_and(|success| !success)
|| value
.get("ok")
.and_then(Value::as_bool)
.is_some_and(|ok| !ok)
|| value
.get("code")
.and_then(provider_value_to_i64)
.is_some_and(|code| code != 0 && code != 200)
}
fn provider_payload_error_suffix(value: &Value) -> String {
extract_provider_string(value, &["message", "error", "msg"])
.map(|message| format!("{message}"))
.unwrap_or_default()
}
fn extract_provider_string(value: &Value, keys: &[&str]) -> Option<String> {
let data = provider_payload_data(value);
if let Some(result) = keys
.iter()
.find_map(|key| data.get(*key).and_then(provider_value_to_string))
{
return Some(result);
}
// New API uses both object envelopes (`data: { key: ... }`) and direct
// string envelopes (`data: "opaque-key"`) depending on the endpoint and
// version. A direct string is only accepted for callers that explicitly
// requested a scalar credential/id field, never for arbitrary messages.
if data.is_string()
&& keys.iter().any(|key| {
matches!(
*key,
"token"
| "accessToken"
| "access_token"
| "userToken"
| "apiKey"
| "api_key"
| "key"
| "id"
| "tokenId"
| "token_id"
)
})
{
return provider_value_to_string(data);
}
None
}
fn provider_value_to_string(value: &Value) -> Option<String> {
match value {
Value::String(value) => Some(value.trim().to_string()),
Value::Number(value) => Some(value.to_string()),
_ => None,
}
.filter(|value| !value.is_empty())
}
fn derive_llm_router_key_id(owner_user_id: &str, route_origin: &str) -> String {
let fingerprint = Sha256::digest(
format!(
"{}\n{}",
owner_user_id.trim(),
route_origin.trim_end_matches('/')
)
.as_bytes(),
);
format!(
"{EXTERNAL_API_KEY_ID_PREFIX}llm-router-{}",
&hex::encode(fingerprint)[..32]
)
}
fn derive_llm_router_account_key(owner_user_id: &str, route_origin: &str) -> String {
let fingerprint = Sha256::digest(
format!(
"llm-router-account\n{}\n{}",
owner_user_id.trim(),
route_origin.trim_end_matches('/')
)
.as_bytes(),
);
format!("llm-router-account-{}", &hex::encode(fingerprint)[..40])
}
fn hash_llm_router_key(owner_user_id: &str, key_id: &str, raw_key: &str) -> String {
// LLM Router rows are server-owned credentials and are never authenticated through
// the public external-key endpoint. Bind the uniqueness hash to owner and
// row id so each deterministic provisioning generation has one local row.
hash_external_api_key(&format!("llm-router:{owner_user_id}:{key_id}:{raw_key}"))
}
fn normalize_router_api_key(value: &str) -> Result<String, String> {
let value = value.trim();
if value.is_empty()
|| value.len() > 1024
|| value
.chars()
.any(|character| character.is_whitespace() || character.is_control())
{
return Err("LLM Router 返回的 API Key 格式无效".to_string());
}
Ok(value.to_string())
}
/// Preserve the opaque key returned by Router. Key prefixes are owned by the
/// Router deployment; api-server must not invent or rewrite one.
fn normalize_new_api_relay_api_key(value: &str) -> Result<String, String> {
normalize_router_api_key(value)
}
fn encrypt_router_credential_secret(
secret_payload: &RouterCredentialSecret,
secret: &str,
) -> Result<String, String> {
let payload = serde_json::to_vec(secret_payload)
.map_err(|error| format!("LLM Router 账号凭据序列化失败:{error}"))?;
encrypt_router_secret_payload(payload.as_slice(), secret)
}
fn encrypt_router_secret_payload(payload: &[u8], secret: &str) -> Result<String, String> {
let key_material = Sha256::digest(secret.trim().as_bytes());
let unbound = aead::UnboundKey::new(&aead::AES_256_GCM, key_material.as_ref())
.map_err(|_| "LLM Router API Key 加密密钥无效".to_string())?;
let key = aead::LessSafeKey::new(unbound);
let mut nonce_bytes = [0_u8; aead::NONCE_LEN];
SystemRandom::new()
.fill(&mut nonce_bytes)
.map_err(|_| "LLM Router API Key 随机数生成失败".to_string())?;
let nonce = aead::Nonce::assume_unique_for_key(nonce_bytes);
let mut encrypted_payload = payload.to_vec();
key.seal_in_place_append_tag(nonce, aead::Aad::empty(), &mut encrypted_payload)
.map_err(|_| "LLM Router API Key 加密失败".to_string())?;
let mut encoded = nonce_bytes.to_vec();
encoded.extend_from_slice(&encrypted_payload);
Ok(STANDARD_NO_PAD.encode(encoded))
}
pub(crate) fn decrypt_router_api_key(ciphertext: &str, secret: &str) -> Result<String, String> {
let plaintext = decrypt_router_secret_payload(ciphertext, secret)?;
if let Ok(bundle) = serde_json::from_slice::<RouterCredentialSecret>(&plaintext) {
return normalize_router_api_key(bundle.api_key.as_str());
}
normalize_router_api_key(
std::str::from_utf8(&plaintext).map_err(|_| "LLM Router API Key 编码无效".to_string())?,
)
}
fn decrypt_router_credential_secret(
ciphertext: &str,
secret: &str,
) -> Result<RouterCredentialSecret, String> {
let plaintext = decrypt_router_secret_payload(ciphertext, secret)?;
let bundle = serde_json::from_slice::<RouterCredentialSecret>(&plaintext)
.map_err(|_| "LLM Router 账号凭据格式无效".to_string())?;
normalize_router_api_key(bundle.api_key.as_str())?;
Ok(bundle)
}
fn decrypt_router_credential_secret_allow_pending(
ciphertext: &str,
secret: &str,
) -> Result<RouterCredentialSecret, String> {
let plaintext = decrypt_router_secret_payload(ciphertext, secret)?;
let bundle = serde_json::from_slice::<RouterCredentialSecret>(&plaintext)
.map_err(|_| "LLM Router 账号凭据格式无效".to_string())?;
if bundle.api_key.trim().is_empty()
&& (bundle.username.as_deref().unwrap_or("").trim().is_empty()
|| bundle.password.as_deref().unwrap_or("").trim().is_empty())
{
return Err("LLM Router 待完成账号凭据缺少用户名或密码".to_string());
}
if !bundle.api_key.trim().is_empty() {
normalize_router_api_key(bundle.api_key.as_str())?;
}
Ok(bundle)
}
fn decrypt_router_secret_payload(ciphertext: &str, secret: &str) -> Result<Vec<u8>, String> {
let mut encoded = STANDARD_NO_PAD
.decode(ciphertext.trim())
.map_err(|_| "LLM Router API Key 密文格式无效".to_string())?;
if encoded.len() < aead::NONCE_LEN + 16 {
return Err("LLM Router API Key 密文长度无效".to_string());
}
let nonce_bytes: [u8; aead::NONCE_LEN] = encoded[..aead::NONCE_LEN]
.try_into()
.map_err(|_| "LLM Router API Key nonce 无效".to_string())?;
let nonce = aead::Nonce::assume_unique_for_key(nonce_bytes);
let payload = &mut encoded[aead::NONCE_LEN..];
let key_material = Sha256::digest(secret.trim().as_bytes());
let unbound = aead::UnboundKey::new(&aead::AES_256_GCM, key_material.as_ref())
.map_err(|_| "LLM Router API Key 解密密钥无效".to_string())?;
let key = aead::LessSafeKey::new(unbound);
let plaintext = key
.open_in_place(nonce, aead::Aad::empty(), payload)
.map_err(|_| "LLM Router API Key 解密失败".to_string())?;
Ok(plaintext.to_vec())
}
pub async fn revoke_external_api_key(
State(state): State<AppState>,
Path(key_id): Path<String>,
Extension(request_context): Extension<RequestContext>,
Extension(authenticated): Extension<AuthenticatedAccessToken>,
) -> Result<Json<Value>, AppError> {
let key = state
.spacetime_client()
.revoke_external_api_key(ExternalApiKeyRevokeRecordInput {
key_id,
owner_user_id: authenticated.claims().user_id().to_string(),
revoked_at_micros: current_utc_micros(),
})
.await
.map_err(map_external_api_key_error)?;
Ok(json_success_body(
Some(&request_context),
ExternalApiKeyResponse {
key: external_api_key_payload_from_record(key),
},
))
}
pub(crate) fn hash_external_api_key(raw_key: &str) -> String {
hash_refresh_session_token(raw_key)
}
pub(crate) fn default_external_api_key_scopes() -> Vec<String> {
EXTERNAL_API_KEY_SCOPES
.iter()
.map(|scope| (*scope).to_string())
.collect()
}
fn generate_external_api_key_secret() -> String {
format!(
"{EXTERNAL_API_KEY_SECRET_PREFIX}{}.{}",
new_uuid_simple_string(),
new_uuid_simple_string()
)
}
fn external_api_key_prefix(raw_key: &str) -> String {
raw_key
.chars()
.take(EXTERNAL_API_KEY_PREFIX_VISIBLE_CHARS)
.collect()
}
fn external_api_key_payload_from_record(record: ExternalApiKeyRecord) -> ExternalApiKeyPayload {
ExternalApiKeyPayload {
key_id: record.key_id,
name: record.name,
key_prefix: record.key_prefix,
scopes: record.scopes,
created_at: record.created_at,
last_used_at: record.last_used_at,
revoked_at: record.revoked_at,
updated_at: record.updated_at,
}
}
fn external_api_key_payload_from_router_account(
account: LlmRouterAccountRecord,
) -> ExternalApiKeyPayload {
ExternalApiKeyPayload {
key_id: account.key_id,
name: account.key_name,
key_prefix: account.key_prefix.unwrap_or_default(),
scopes: LLM_ROUTER_API_KEY_SCOPES
.iter()
.map(|scope| (*scope).to_string())
.collect(),
created_at: account.created_at,
last_used_at: account.last_used_at,
revoked_at: account.revoked_at,
updated_at: account.updated_at,
}
}
pub(crate) fn map_external_api_key_error(error: SpacetimeClientError) -> AppError {
match error {
SpacetimeClientError::Procedure(message)
if message.contains("不存在") || message.contains("已失效") =>
{
AppError::from_status(StatusCode::UNAUTHORIZED).with_details(json!({
"provider": "external-api-key",
"message": message,
}))
}
SpacetimeClientError::Procedure(message) if message.contains("无权") => {
AppError::from_status(StatusCode::FORBIDDEN).with_details(json!({
"provider": "external-api-key",
"message": message,
}))
}
SpacetimeClientError::Runtime(message) | SpacetimeClientError::Procedure(message) => {
AppError::from_status(StatusCode::BAD_REQUEST).with_details(json!({
"provider": "external-api-key",
"message": message,
}))
}
other => AppError::from_status(StatusCode::BAD_GATEWAY).with_details(json!({
"provider": "spacetimedb",
"message": other.to_string(),
})),
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::{
io::{Read, Write},
net::{TcpListener, TcpStream},
sync::{Arc, Mutex},
thread,
time::Duration,
};
use crate::{config::AppConfig, state::AppState};
#[test]
fn generated_external_api_key_uses_public_prefix_only_for_display() {
let raw_key = generate_external_api_key_secret();
let key_prefix = external_api_key_prefix(raw_key.as_str());
assert!(raw_key.starts_with(EXTERNAL_API_KEY_SECRET_PREFIX));
assert_eq!(
key_prefix.chars().count(),
EXTERNAL_API_KEY_PREFIX_VISIBLE_CHARS
);
assert!(raw_key.starts_with(key_prefix.as_str()));
assert_ne!(hash_external_api_key(raw_key.as_str()), raw_key);
}
#[test]
fn router_api_key_accepts_provider_owned_opaque_formats() {
assert_eq!(
normalize_router_api_key(" sk-eNuq-test-key ").expect("sk key should be accepted"),
"sk-eNuq-test-key"
);
assert_eq!(
normalize_router_api_key("tnr_sk_fixture_123").expect("tnr key should remain accepted"),
"tnr_sk_fixture_123"
);
assert!(normalize_router_api_key(" ").is_err());
assert!(normalize_router_api_key("router-key\nwith-newline").is_err());
}
#[test]
fn new_api_relay_key_preserves_router_owned_opaque_format() {
assert_eq!(
normalize_new_api_relay_api_key("raw-router-token").unwrap(),
"raw-router-token"
);
assert_eq!(
normalize_new_api_relay_api_key(" sk-already-prefixed ").unwrap(),
"sk-already-prefixed"
);
}
#[test]
fn new_api_token_request_is_unlimited_and_bound_to_default_group() {
let payload = router_token_request();
assert_eq!(payload["name"], LLM_ROUTER_TOKEN_IDENTIFIER);
assert_eq!(payload["expired_time"], -1);
assert_eq!(payload["unlimited_quota"], true);
assert_eq!(payload["group"], "default");
assert!(payload.get("idempotencyKey").is_none());
}
#[test]
fn router_subscription_renews_when_missing_expired_or_within_one_day() {
let now = 1_700_000_000;
assert!(subscription_requires_renewal(&[], now));
assert!(subscription_requires_renewal(
&[RouterSubscriptionSummary {
plan_id: LLM_ROUTER_SUBSCRIPTION_PLAN_ID,
status: "expired".to_string(),
end_time: now + 10 * 24 * 60 * 60,
}],
now,
));
assert!(subscription_requires_renewal(
&[RouterSubscriptionSummary {
plan_id: LLM_ROUTER_SUBSCRIPTION_PLAN_ID,
status: "active".to_string(),
end_time: now + 24 * 60 * 60,
}],
now,
));
assert!(!subscription_requires_renewal(
&[RouterSubscriptionSummary {
plan_id: LLM_ROUTER_SUBSCRIPTION_PLAN_ID,
status: "active".to_string(),
end_time: now + 24 * 60 * 60 + 1,
}],
now,
));
assert!(subscription_requires_renewal(
&[RouterSubscriptionSummary {
plan_id: 99,
status: "active".to_string(),
end_time: now + 365 * 24 * 60 * 60,
}],
now,
));
}
#[test]
fn router_subscription_payload_reads_new_api_summary_envelope() {
let payload = json!({
"success": true,
"data": [
{"subscription": {
"plan_id": 1,
"status": "active",
"start_time": 1700000000,
"end_time": 1800000000
}}
]
});
assert_eq!(
extract_router_subscriptions(&payload),
vec![RouterSubscriptionSummary {
plan_id: 1,
status: "active".to_string(),
end_time: 1800000000,
}]
);
}
#[test]
fn existing_router_token_group_is_normalized_to_default() {
let payload = router_token_update_request(77);
assert_eq!(payload["id"], 77);
assert_eq!(payload["name"], LLM_ROUTER_TOKEN_IDENTIFIER);
assert_eq!(payload["expired_time"], -1);
assert_eq!(payload["unlimited_quota"], true);
assert_eq!(payload["group"], "default");
}
#[test]
fn duplicate_fixed_router_tokens_require_reconciliation() {
let payload = json!({
"data": [
{"id": 77, "name": LLM_ROUTER_TOKEN_IDENTIFIER},
{"id": 88, "name": LLM_ROUTER_TOKEN_IDENTIFIER}
]
});
let error = extract_router_token_id(&payload, LLM_ROUTER_TOKEN_IDENTIFIER)
.expect_err("multiple fixed-name tokens must be rejected");
assert!(error.contains("多个不同 token id"));
let duplicate_same_id = json!({
"data": [
{"id": 77, "name": LLM_ROUTER_TOKEN_IDENTIFIER},
{"id": 77, "name": LLM_ROUTER_TOKEN_IDENTIFIER}
]
});
assert_eq!(
extract_router_token_id(&duplicate_same_id, LLM_ROUTER_TOKEN_IDENTIFIER)
.expect("same token repeated in an envelope is not ambiguous"),
Some("77".to_string())
);
}
#[test]
fn new_api_user_update_uses_numeric_id_and_fixed_group() {
let payload = router_user_update_request(42, "router_abcd", "user_full-id");
assert_eq!(payload["id"], 42);
assert_eq!(payload["username"], "router_abcd");
assert_eq!(payload["remark"], "user_full-id");
assert_eq!(payload["group"], "taonier");
assert_eq!(payload["role"], 1);
assert!(
payload["display_name"]
.as_str()
.is_some_and(|value| value.contains("abcd"))
);
}
#[tokio::test]
async fn new_api_provisioning_uses_admin_user_flow_and_preserves_opaque_key() {
let username = router_username_for_owner("owner-1");
let password =
generate_router_account_password("owner-1").expect("Router password should derive");
let (base_url, captured) = spawn_new_api_provisioning_mock(username.clone());
let state = AppState::new(AppConfig {
llm_router_base_url: base_url,
llm_router_admin_token: Some("admin-secret".to_string()),
..AppConfig::default()
})
.expect("state should build");
let pending_login = RouterAccountLogin {
username: username.clone(),
password: password.clone(),
};
let provisioned =
provision_router_account_via_new_api(&state, "owner-1", None, Some(&pending_login))
.await
.expect("New API provisioning should succeed");
assert_eq!(provisioned.raw_key, "opaque-router-key");
assert_eq!(provisioned.provider_account_id, username);
assert_eq!(
provisioned.router_username.as_deref(),
Some(username.as_str())
);
assert!(provisioned.router_password.is_some());
let requests = captured.lock().expect("captured requests lock").clone();
assert_eq!(requests.len(), 11);
assert!(requests[0].starts_with("GET /api/user/search?keyword="));
assert!(requests[1].starts_with("POST /api/user/ HTTP/1.1"));
assert!(requests[2].starts_with("GET /api/user/search?keyword="));
assert!(requests[3].starts_with("PUT /api/user/ HTTP/1.1"));
assert!(requests[4].starts_with("POST /api/user/login HTTP/1.1"));
assert!(
requests[5].starts_with("GET /api/subscription/admin/users/42/subscriptions HTTP/1.1")
);
assert!(
requests[6].starts_with("POST /api/subscription/admin/users/42/subscriptions HTTP/1.1")
);
assert!(requests[7].starts_with("GET /api/token/search?keyword="));
assert!(requests[8].starts_with("POST /api/token/ HTTP/1.1"));
assert!(requests[9].starts_with("PUT /api/token/ HTTP/1.1"));
assert!(requests[10].starts_with("POST /api/token/77/key HTTP/1.1"));
assert!(
requests[0]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer admin-secret"))
);
assert!(
requests[1]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer admin-secret"))
);
assert!(
requests[2]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer admin-secret"))
);
let create_body = request_body(&requests[1]);
let create_payload: Value = serde_json::from_str(create_body).expect("create json");
assert_eq!(create_payload["username"], username);
assert_eq!(create_payload["password"], password);
assert_eq!(create_payload["role"], 1);
assert!(
create_payload["remark"]
.as_str()
.is_some_and(|value| value == "owner-1")
);
assert!(
create_payload["password"]
.as_str()
.is_some_and(|value| value.len() <= 20)
);
let update_payload: Value =
serde_json::from_str(request_body(&requests[3])).expect("update json");
assert_eq!(update_payload["id"], 42);
assert_eq!(update_payload["remark"], "owner-1");
assert_eq!(update_payload["group"], "taonier");
assert_eq!(update_payload["role"], 1);
let token_payload: Value =
serde_json::from_str(request_body(&requests[8])).expect("token json");
assert_eq!(token_payload["unlimited_quota"], true);
assert_eq!(token_payload["expired_time"], -1);
assert_eq!(token_payload["group"], "default");
let token_update_payload: Value =
serde_json::from_str(request_body(&requests[9])).expect("token update json");
assert_eq!(token_update_payload["id"], 77);
assert_eq!(token_update_payload["unlimited_quota"], true);
assert_eq!(token_update_payload["expired_time"], -1);
assert_eq!(token_update_payload["group"], "default");
assert!(
requests[7]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer router-login-token"))
);
assert!(
requests[8]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer router-login-token"))
);
assert!(
requests[10]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer router-login-token"))
);
assert!(
requests[5]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer admin-secret"))
);
assert!(
requests[6]
.lines()
.any(|line| line.eq_ignore_ascii_case("authorization: Bearer admin-secret"))
);
let subscription_payload: Value =
serde_json::from_str(request_body(&requests[6])).expect("subscription json");
assert_eq!(
subscription_payload["plan_id"],
LLM_ROUTER_SUBSCRIPTION_PLAN_ID
);
}
#[tokio::test]
async fn missing_local_account_recovers_existing_router_user_and_reuses_token() {
let username = router_username_for_owner("owner-recover");
let (base_url, captured) = spawn_existing_router_account_mock(username.clone());
let state = AppState::new(AppConfig {
llm_router_base_url: base_url,
llm_router_admin_token: Some("admin-secret".to_string()),
..AppConfig::default()
})
.expect("state should build");
let pending_login = RouterAccountLogin {
username: username.clone(),
password: generate_router_account_password("owner-recover")
.expect("password should derive"),
};
let provisioned = provision_router_account_via_new_api(
&state,
"owner-recover",
None,
Some(&pending_login),
)
.await
.expect("existing Router account should be recovered");
assert_eq!(provisioned.raw_key, "recovered-router-key");
assert_eq!(provisioned.provider_account_id, username);
let requests = captured.lock().expect("captured requests lock").clone();
assert_eq!(requests.len(), 7);
assert!(requests[0].starts_with("GET /api/user/search?keyword="));
assert!(requests[1].starts_with("PUT /api/user/ HTTP/1.1"));
assert!(requests[2].starts_with("POST /api/user/login HTTP/1.1"));
assert!(
requests[3].starts_with("GET /api/subscription/admin/users/42/subscriptions HTTP/1.1")
);
assert!(requests[4].starts_with("GET /api/token/search?keyword="));
assert!(requests[5].starts_with("PUT /api/token/ HTTP/1.1"));
assert!(requests[6].starts_with("POST /api/token/77/key HTTP/1.1"));
let update_payload: Value =
serde_json::from_str(request_body(&requests[1])).expect("update json");
assert_eq!(update_payload["id"], 42);
assert_eq!(update_payload["username"], username);
assert_eq!(update_payload["group"], "taonier");
assert_eq!(update_payload["role"], 1);
let token_update_payload: Value =
serde_json::from_str(request_body(&requests[5])).expect("token update json");
assert_eq!(token_update_payload["id"], 77);
assert_eq!(token_update_payload["group"], "default");
assert!(
!requests
.iter()
.any(|request| request.starts_with("POST /api/user/ HTTP/1.1"))
);
assert!(
!requests
.iter()
.any(|request| request.starts_with("POST /api/token/ HTTP/1.1"))
);
}
#[tokio::test]
async fn active_router_key_repair_keeps_user_taonier_and_token_default() {
let owner_user_id = "owner-active-repair";
let username = router_username_for_owner(owner_user_id);
let password =
generate_router_account_password(owner_user_id).expect("password should derive");
let (base_url, captured) = spawn_active_router_contract_mock(username.clone());
let state = AppState::new(AppConfig {
llm_router_base_url: base_url.clone(),
llm_router_admin_token: Some("admin-secret".to_string()),
llm_router_api_key_encryption_secret: Some("fixture-encryption-secret".to_string()),
..AppConfig::default()
})
.expect("state should build");
let ciphertext = encrypt_router_credential_secret(
&RouterCredentialSecret {
version: 1,
api_key: "existing-router-key".to_string(),
username: Some(username.clone()),
password: Some(password),
access_token: None,
},
"fixture-encryption-secret",
)
.expect("credential should encrypt");
let account = LlmRouterAccountRecord {
account_key: "fixture-account".to_string(),
owner_user_id: owner_user_id.to_string(),
route_origin: base_url.trim_end_matches('/').to_string(),
idempotency_key: "key-active".to_string(),
router_account_id: Some("42".to_string()),
credential_ciphertext: Some(ciphertext),
account_json: Some(json!({"routerUserId": 42, "username": username}).to_string()),
status: "active".to_string(),
attempt_count: 0,
lease_until_micros: None,
next_retry_at_micros: None,
last_error: None,
credential_version: 1,
created_at: "2026-08-29T00:00:00Z".to_string(),
updated_at: "2026-08-29T00:00:00Z".to_string(),
key_id: "key-active".to_string(),
key_name: LLM_ROUTER_TOKEN_IDENTIFIER.to_string(),
key_prefix: Some("tnr_sk_fixture".to_string()),
key_hash: None,
last_used_at: None,
revoked_at: None,
};
ensure_existing_llm_router_account_contract(&state, &account)
.await
.expect("active Router contract should repair");
let requests = captured.lock().expect("captured requests lock").clone();
assert_eq!(requests.len(), 7);
assert!(requests[0].starts_with("GET /api/user/search?keyword="));
assert!(requests[1].starts_with("PUT /api/user/ HTTP/1.1"));
assert!(requests[2].starts_with("POST /api/user/login HTTP/1.1"));
assert!(requests[3].starts_with("GET /api/token/search?keyword="));
assert!(requests[4].starts_with("PUT /api/token/ HTTP/1.1"));
assert!(
requests[5].starts_with("GET /api/subscription/admin/users/42/subscriptions HTTP/1.1")
);
assert!(
requests[6].starts_with("POST /api/subscription/admin/users/42/subscriptions HTTP/1.1")
);
let user_payload: Value =
serde_json::from_str(request_body(&requests[1])).expect("user update json");
assert_eq!(user_payload["group"], "taonier");
assert_eq!(user_payload["remark"], owner_user_id);
let token_payload: Value =
serde_json::from_str(request_body(&requests[4])).expect("token update json");
assert_eq!(token_payload["group"], "default");
assert_eq!(token_payload["unlimited_quota"], true);
assert_eq!(token_payload["expired_time"], -1);
let subscription_payload: Value =
serde_json::from_str(request_body(&requests[6])).expect("subscription json");
assert_eq!(
subscription_payload["plan_id"],
LLM_ROUTER_SUBSCRIPTION_PLAN_ID
);
}
#[tokio::test]
async fn router_provisioning_requires_admin_token_for_subscription_guarantee() {
let base_url = "http://127.0.0.1:3100/v1".to_string();
let state = AppState::new(AppConfig {
llm_router_base_url: base_url,
llm_router_admin_token: None,
..AppConfig::default()
})
.expect("state should build");
let pending_login = RouterAccountLogin {
username: router_username_for_owner("owner-no-admin"),
password: "stable-password".to_string(),
};
let result = provision_router_account_via_new_api(
&state,
"owner-no-admin",
None,
Some(&pending_login),
)
.await;
let error = match result {
Ok(_) => panic!("provisioning must not bypass subscription checks"),
Err(error) => error,
};
assert!(error.contains("管理员 Token 未配置"));
}
#[test]
fn router_account_credentials_are_stable_per_owner_across_routes() {
let first = generate_router_account_password("owner-1").expect("password should derive");
let same = generate_router_account_password(" owner-1 ").expect("password should derive");
let different_owner =
generate_router_account_password("owner-2").expect("password should derive");
assert_eq!(first, same);
assert_ne!(first, different_owner);
// New API 的 User.Username/User.Password 字段校验是 20 字符上限,
// 因此派生值必须保持短且确定:`agc_user_` + 11 字符 = 20
// 密码为 20 位 hex,且二者都基于完整 owner id。
let username = router_username_for_owner("owner-1");
assert!(username.starts_with("agc_user_"));
assert_eq!(username.len(), 20);
assert_ne!(router_username_for_owner("owner-2"), username);
assert_ne!(router_username_for_owner("other-owner"), username);
assert_eq!(first.len(), 20);
}
#[test]
fn default_external_api_key_scopes_cover_editor_openapi_v1() {
let scopes = default_external_api_key_scopes();
assert_eq!(
scopes,
vec![
"editor:project".to_string(),
"editor:canvas".to_string(),
"editor:image-generate".to_string(),
"editor:asset".to_string(),
]
);
}
#[test]
fn router_credential_bundle_round_trip_keeps_password_server_side() {
let bundle = RouterCredentialSecret {
version: 1,
api_key: "sk-router-test".to_string(),
username: Some("router_user".to_string()),
password: Some("Router-random-password".to_string()),
access_token: None,
};
let ciphertext = encrypt_router_credential_secret(&bundle, "test-secret")
.expect("credential bundle should encrypt");
assert_eq!(
decrypt_router_api_key(&ciphertext, "test-secret").unwrap(),
"sk-router-test"
);
let plaintext = decrypt_router_secret_payload(&ciphertext, "test-secret")
.expect("credential bundle should decrypt");
let restored: RouterCredentialSecret =
serde_json::from_slice(&plaintext).expect("credential bundle should be json");
assert_eq!(restored.password.as_deref(), Some("Router-random-password"));
assert!(!ciphertext.contains("Router-random-password"));
}
#[test]
fn router_control_origin_removes_responses_path() {
assert_eq!(
router_control_origin("https://router.example/v1").unwrap(),
"https://router.example"
);
assert!(router_control_origin("ftp://router.example/v1").is_err());
}
#[test]
fn router_payload_success_and_key_extraction_accept_new_api_shapes() {
let payload = json!({"success": true, "data": {"id": 42, "key": "sk-router"}});
assert!(!provider_payload_failed(&payload));
assert_eq!(
extract_provider_string(&payload, &["id"]),
Some("42".to_string())
);
assert_eq!(
extract_provider_string(&payload, &["key"]),
Some("sk-router".to_string())
);
assert!(provider_payload_failed(
&json!({"success": false, "message": "denied"})
));
}
#[test]
fn new_api_scalar_data_envelopes_are_supported_for_login_and_key_issuance() {
assert_eq!(
extract_provider_string(
&json!({"success": true, "data": "router-login-token"}),
&["token", "accessToken"]
),
Some("router-login-token".to_string())
);
assert_eq!(
extract_provider_string(
&json!({"success": true, "data": "sk-router-key"}),
&["apiKey", "key"]
),
Some("sk-router-key".to_string())
);
assert_eq!(
extract_provider_string(&json!("sk-router-plain-text"), &["apiKey", "key"]),
Some("sk-router-plain-text".to_string())
);
}
#[test]
fn router_http_failures_mark_non_client_responses_as_uncertain() {
assert!(!router_http_failure_requires_reconciliation(
reqwest::StatusCode::BAD_REQUEST
));
assert!(!router_http_failure_requires_reconciliation(
reqwest::StatusCode::FORBIDDEN
));
assert!(router_http_failure_requires_reconciliation(
reqwest::StatusCode::MOVED_PERMANENTLY
));
assert!(router_http_failure_requires_reconciliation(
reqwest::StatusCode::BAD_GATEWAY
));
assert!(router_http_failure_requires_reconciliation(
reqwest::StatusCode::SERVICE_UNAVAILABLE
));
}
#[test]
fn deterministic_router_credential_validation_errors_can_be_retried() {
assert!(is_deterministic_router_credential_error(
"Invalid input Key: 'User.Username' Error:Field validation for 'Username' failed on the 'max' tag"
));
assert!(is_deterministic_router_credential_error(
"Field validation for 'Password' failed on the 'min' tag"
));
assert!(!is_deterministic_router_credential_error(
"connection reset by peer"
));
assert!(!is_deterministic_router_credential_error(
"Router registration may have committed"
));
}
#[test]
fn non_production_router_target_is_loopback_only() {
let state = AppState::new(AppConfig {
environment: "test".to_string(),
llm_router_base_url: "https://router.genarrative.world/v1".to_string(),
..AppConfig::default()
})
.expect("test state should build");
let error = ensure_llm_router_target_allowed(&state)
.expect_err("test environment must reject the online Router target");
assert!(error.contains("loopback"));
let state = AppState::new(AppConfig {
environment: "test".to_string(),
llm_router_base_url: "http://127.0.0.1:3100/v1".to_string(),
..AppConfig::default()
})
.expect("test state should build");
ensure_llm_router_target_allowed(&state)
.expect("test environment should allow a loopback Router fixture");
}
#[test]
fn production_router_target_requires_https_but_not_admin_token_for_reuse() {
let state = AppState::new(AppConfig {
environment: "production".to_string(),
llm_router_base_url: "http://router.example/v1".to_string(),
..AppConfig::default()
})
.expect("test state should build");
ensure_llm_router_target_allowed(&state)
.expect_err("production Router control plane must use HTTPS");
}
fn spawn_new_api_provisioning_mock(username: String) -> (String, Arc<Mutex<Vec<String>>>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("mock listener should bind");
let address = listener
.local_addr()
.expect("mock address should be available");
let captured = Arc::new(Mutex::new(Vec::new()));
let captured_for_thread = Arc::clone(&captured);
thread::spawn(move || {
let responses = vec![
r#"{"success":true,"data":[]}"#.to_string(),
"success".to_string(),
format!(
r#"{{"success":true,"data":[{{"id":42,"username":"{}"}}]}}"#,
username
),
"success".to_string(),
r#"{"success":true,"data":{"token":"router-login-token"}}"#.to_string(),
r#"{"success":true,"data":[]}"#.to_string(),
"success".to_string(),
r#"{"success":true,"data":[]}"#.to_string(),
r#"{"success":true,"data":{"id":77}}"#.to_string(),
"success".to_string(),
r#"{"success":true,"data":{"key":"opaque-router-key"}}"#.to_string(),
];
for response_body in responses {
let (mut stream, _) = listener.accept().expect("mock request should connect");
let request = read_http_request(&mut stream);
captured_for_thread
.lock()
.expect("captured requests lock")
.push(request);
write_http_response(&mut stream, response_body.as_str());
}
});
(format!("http://{address}"), captured)
}
fn spawn_existing_router_account_mock(username: String) -> (String, Arc<Mutex<Vec<String>>>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("mock listener should bind");
let address = listener
.local_addr()
.expect("mock address should be available");
let captured = Arc::new(Mutex::new(Vec::new()));
let captured_for_thread = Arc::clone(&captured);
thread::spawn(move || {
let responses = vec![
format!(
r#"{{"success":true,"data":[{{"id":42,"username":"{}"}}]}}"#,
username
),
"success".to_string(),
r#"{"success":true,"data":{"token":"router-login-token"}}"#.to_string(),
r#"{"success":true,"data":[{"subscription":{"plan_id":1,"status":"active","end_time":4102444800}}]}"#.to_string(),
format!(
r#"{{"success":true,"data":[{{"id":77,"name":"{}","group":"taonier"}}]}}"#,
LLM_ROUTER_TOKEN_IDENTIFIER
),
"success".to_string(),
r#"{"success":true,"data":{"key":"recovered-router-key"}}"#.to_string(),
];
for response_body in responses {
let (mut stream, _) = listener.accept().expect("mock request should connect");
let request = read_http_request(&mut stream);
captured_for_thread
.lock()
.expect("captured requests lock")
.push(request);
write_http_response(&mut stream, response_body.as_str());
}
});
(format!("http://{address}"), captured)
}
fn spawn_active_router_contract_mock(username: String) -> (String, Arc<Mutex<Vec<String>>>) {
let listener = TcpListener::bind("127.0.0.1:0").expect("mock listener should bind");
let address = listener
.local_addr()
.expect("mock address should be available");
let captured = Arc::new(Mutex::new(Vec::new()));
let captured_for_thread = Arc::clone(&captured);
thread::spawn(move || {
let responses = vec![
format!(
r#"{{"success":true,"data":[{{"id":42,"username":"{}"}}]}}"#,
username
),
"success".to_string(),
r#"{"success":true,"data":{"token":"router-login-token"}}"#.to_string(),
format!(
r#"{{"success":true,"data":[{{"id":77,"name":"{}","group":"taonier"}}]}}"#,
LLM_ROUTER_TOKEN_IDENTIFIER
),
"success".to_string(),
r#"{"success":true,"data":[]}"#.to_string(),
"success".to_string(),
];
for response_body in responses {
let (mut stream, _) = listener.accept().expect("mock request should connect");
let request = read_http_request(&mut stream);
captured_for_thread
.lock()
.expect("captured requests lock")
.push(request);
write_http_response(&mut stream, response_body.as_str());
}
});
(format!("http://{address}"), captured)
}
fn request_body(request: &str) -> &str {
request
.split_once("\r\n\r\n")
.map(|(_, body)| body)
.unwrap_or_default()
}
fn read_http_request(stream: &mut TcpStream) -> String {
stream
.set_read_timeout(Some(Duration::from_secs(2)))
.expect("mock read timeout should set");
let mut buffer = Vec::new();
let mut chunk = [0_u8; 4096];
let mut expected_total = None;
loop {
match stream.read(&mut chunk) {
Ok(0) => break,
Ok(read) => {
buffer.extend_from_slice(&chunk[..read]);
if expected_total.is_none()
&& let Some(header_end) =
buffer.windows(4).position(|value| value == b"\r\n\r\n")
{
let header_end = header_end + 4;
let headers = String::from_utf8_lossy(&buffer[..header_end]);
let content_length = headers
.lines()
.find_map(|line| {
line.strip_prefix("Content-Length:")
.or_else(|| line.strip_prefix("content-length:"))
})
.and_then(|value| value.trim().parse::<usize>().ok())
.unwrap_or(0);
expected_total = Some(header_end + content_length);
}
if expected_total.is_some_and(|total| buffer.len() >= total) {
break;
}
}
Err(error)
if matches!(
error.kind(),
std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut
) =>
{
break;
}
Err(error) => panic!("mock request read failed: {error}"),
}
}
String::from_utf8_lossy(&buffer).into_owned()
}
fn write_http_response(stream: &mut TcpStream, body: &str) {
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
stream
.write_all(response.as_bytes())
.expect("mock response should write");
}
}