GenarrativeAI/Genarrative
10
0
Fork 0
Code Issues 21 Pull Requests 6 Actions Packages Projects 1 Releases Wiki Activity

修复鉴权投影并发与退款 outbox 冲突

Browse Source 
绑定认证工作集投影版本并在 SpacetimeDB 内执行 CAS

为认证 procedure 增加 runtime service identity 校验

防止退款 outbox 跨进程覆盖并校验临时文件事实

更新后端契约与项目决策记录
This commit is contained in:
段舒康
2026-08-27 14:47:22 +08:00
parent 72d7214646
commit 36a4d37d24
10 changed files with 441 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server-rs/crates/api-server/src/state.rs
+146 -24
View File
@@ -8,7 +8,7 @@ use std::{
fmt,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering},
},
};
@@ -35,7 +35,7 @@ use spacetime_client::{
SpacetimeClient, SpacetimeClientConfig, SpacetimeClientError, SpacetimeClientHealthSnapshot,
};
use time::OffsetDateTime;
use tokio::sync::{Semaphore, broadcast};
use tokio::sync::{Mutex as AsyncMutex, Semaphore, broadcast};
use tracing::{info, warn};
use crate::config::AppConfig;
@@ -273,6 +273,14 @@ pub struct AppStateInner {
oss_client: Option<OssClient>,
#[cfg_attr(test, allow(dead_code))]
auth_store: InMemoryAuthStore,
/// 当前进程工作集所基于的正式认证投影版本；跨节点写入使用它做 CAS。
#[cfg_attr(test, allow(dead_code))]
auth_projection_version: AtomicI64,
/// 最近一次确认写入正式投影时对应的工作集 revision；不一致表示有待重试的本地变更。
#[cfg_attr(test, allow(dead_code))]
auth_projection_synced_revision: AtomicU64,
#[cfg_attr(test, allow(dead_code))]
auth_projection_sync_lock: AsyncMutex<()>,
password_entry_service: PasswordEntryService,
refresh_session_service: RefreshSessionService,
auth_user_service: AuthUserService,
@@ -505,12 +513,13 @@ impl AppState {
pub fn new_with_empty_auth_store(config: AppConfig) -> Result<Self, AppStateInitError> {
// 中文注释：api-server 不再把本地 auth-store.json 当作用户认证真相源，启动恢复只允许来自 SpacetimeDB。
Self::new_with_auth_store(config, InMemoryAuthStore::default())
Self::new_with_auth_store(config, InMemoryAuthStore::default(), 0)
}
fn new_with_auth_store(
config: AppConfig,
auth_store: InMemoryAuthStore,
auth_projection_version: i64,
) -> Result<Self, AppStateInitError> {
let auth_jwt_config = JwtConfig::new(
config.jwt_issuer.clone(),
@@ -629,6 +638,9 @@ impl AppState {
test_external_background_removal_enqueue: Arc::new(Mutex::new(None)),
oss_client,
auth_store,
auth_projection_version: AtomicI64::new(auth_projection_version),
auth_projection_synced_revision: AtomicU64::new(0),
auth_projection_sync_lock: AsyncMutex::new(()),
password_entry_service,
refresh_session_service,
auth_user_service,
@@ -1288,30 +1300,102 @@ impl AppState {
return Ok(());
#[cfg(not(test))]
let updated_at_micros = i64::try_from(
OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000,
)
.map_err(|_| SpacetimeClientError::Runtime("认证状态更新时间超出 i64 范围".to_string()))?;
let _sync_guard = self.auth_projection_sync_lock.lock().await;
#[cfg(not(test))]
let projection = self
.auth_store
.export_projection_view(updated_at_micros)
.map_err(SpacetimeClientError::Runtime)?;
// 当前仍由 module-auth 的进程内工作集执行业务规则；这里只用 typed projection 同步正式认证表。
let sync_start_revision = self.auth_store.revision();
#[cfg(not(test))]
if let Err(error) = self
.spacetime_client
.sync_auth_store_projection(projection)
.await
{
warn!(
error = %error,
"认证投影同步 SpacetimeDB 正式表失败，当前认证流程中止"
);
return Err(error);
let mut local_changed_during_sync =
self.auth_projection_synced_revision.load(Ordering::Acquire) != sync_start_revision;
#[cfg(not(test))]
for attempt in 0..3 {
let base_updated_at_micros = self.auth_projection_version.load(Ordering::Acquire);
let now_updated_at_micros =
i64::try_from(OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000).map_err(
|_| SpacetimeClientError::Runtime("认证状态更新时间超出 i64 范围".to_string()),
)?;
let updated_at_micros = if now_updated_at_micros > base_updated_at_micros {
now_updated_at_micros
} else {
base_updated_at_micros.checked_add(1).ok_or_else(|| {
SpacetimeClientError::Runtime("认证状态版本超出 i64 范围".to_string())
})?
};
let (mut projection, attempted_revision) = self
.auth_store
.export_projection_view_with_revision(updated_at_micros)
.map_err(SpacetimeClientError::Runtime)?;
if attempted_revision != sync_start_revision {
local_changed_during_sync = true;
}
projection.base_updated_at_micros = base_updated_at_micros;
// 当前仍由 module-auth 的进程内工作集执行业务规则；这里只用 typed projection 同步正式认证表。
match self
.spacetime_client
.sync_auth_store_projection(projection)
.await
{
Ok(_) => {
self.auth_projection_version
.store(updated_at_micros, Ordering::Release);
if self.auth_store.revision() == attempted_revision {
self.auth_projection_synced_revision
.store(attempted_revision, Ordering::Release);
return Ok(());
}
local_changed_during_sync = true;
warn!(
attempt,
"认证投影同步期间工作集发生变化，将继续同步最新工作集"
);
continue;
}
Err(error) => {
warn!(
error = %error,
"认证投影同步 SpacetimeDB 正式表失败，当前认证流程中止"
);
if local_changed_during_sync || self.auth_store.revision() != attempted_revision
{
warn!(
"认证投影同步失败期间工作集发生并发变化，跳过自动恢复以避免覆盖未提交变更"
);
} else if let Ok(current_projection) = self
.spacetime_client
.export_auth_store_projection_from_tables()
.await
{
match self.auth_store.refresh_from_projection_view_if_revision(
current_projection.clone(),
attempted_revision,
) {
Ok(true) => {
self.auth_projection_version
.store(current_projection.updated_at_micros, Ordering::Release);
self.auth_projection_synced_revision
.store(self.auth_store.revision(), Ordering::Release);
}
Ok(false) => {
warn!(
"认证投影同步冲突期间工作集发生并发变化，跳过自动恢复以避免覆盖未提交变更"
);
}
Err(refresh_error) => {
warn!(
error = %refresh_error,
"认证投影同步冲突后恢复进程内工作集失败"
);
}
}
}
return Err(error);
}
}
}
#[cfg(not(test))]
Ok(())
Err(SpacetimeClientError::Runtime(
"认证工作集在同步期间持续发生变化，未能完成投影同步".to_string(),
))
}
pub async fn try_restore_auth_store_from_spacetime(
@@ -1319,6 +1403,11 @@ impl AppState {
) -> Result<Self, AppStateInitError> {
let spacetime_client =
SpacetimeClient::new(spacetime_client_config_for_startup_restore(&config));
initialize_editor_generation_runtime_service_identity_for_startup(
&config,
&spacetime_client,
)
.await?;
let mut spacetime_restore_available = false;
let mut restore_errors = Vec::new();
@@ -1332,7 +1421,11 @@ impl AppState {
projection,
AuthStoreRestoreSource::SpacetimeTables,
)? {
let state = Self::new_with_auth_store(config, candidate.auth_store)?;
let state = Self::new_with_auth_store(
config,
candidate.auth_store,
candidate.updated_at_micros.unwrap_or_default(),
)?;
info!(
source = candidate.source.as_str(),
updated_at_micros = candidate.updated_at_micros,
@@ -1845,6 +1938,7 @@ fn auth_store_candidate_from_projection_view(
if projection.users.is_empty()
&& projection.identities.is_empty()
&& projection.refresh_sessions.is_empty()
&& projection.updated_at_micros == 0
{
return Ok(None);
}
@@ -1886,6 +1980,34 @@ fn spacetime_client_config_for_startup_restore(config: &AppConfig) -> SpacetimeC
}
}
async fn initialize_editor_generation_runtime_service_identity_for_startup(
config: &AppConfig,
spacetime_client: &SpacetimeClient,
) -> Result<(), AppStateInitError> {
let pricing_store =
EditorGenerationPricingStore::load(config.editor_generation_pricing_override_path.clone())
.map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?;
let fallback = pricing_store
.snapshot()
.map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?;
let models = editor_generation_pricing_to_records(&fallback)
.map_err(|error| AppStateInitError::DependencyUnavailable(error.to_string()))?;
spacetime_client
.initialize_editor_generation_pricing_config_if_missing(
editor_generation_pricing_upsert_input(
config,
"system:editor-generation-pricing".to_string(),
models,
crate::editor_project::current_utc_micros(),
),
)
.await
.map_err(|error| {
AppStateInitError::DependencyUnavailable(format!("初始化模型定价服务身份失败：{error}"))
})?;
Ok(())
}
impl fmt::Display for AppStateInitError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
server-rs/crates/api-server/src/wallet_refund_outbox.rs
+154 -13
View File
@@ -34,7 +34,7 @@ pub struct WalletRefundOutbox {
flush_notify: Arc<Notify>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub(crate) struct WalletRefundOutboxRecord {
pub owner_user_id: String,
pub amount: u64,
@@ -85,7 +85,7 @@ impl WalletRefundOutbox {
let pending_path = self.pending_path_for_ledger(&record.ledger_id);
if self
.reuse_existing_pending_file(&pending_path, &record.ledger_id)
.reuse_existing_pending_file(&pending_path, &record)
.await?
{
self.flush_notify.notify_one();
@@ -112,23 +112,48 @@ impl WalletRefundOutbox {
file.sync_data().await?;
drop(file);
if self
.reuse_existing_pending_file(&pending_path, &record.ledger_id)
.reuse_existing_pending_file(&pending_path, &record)
.await?
{
let _ = fs::remove_file(&temp_path).await;
self.flush_notify.notify_one();
return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued);
}
fs::rename(&temp_path, &pending_path).await?;
sync_directory_metadata(&self.dir).await?;
self.flush_notify.notify_one();
Ok(WalletRefundOutboxEnqueueOutcome::Enqueued)
for _ in 0..2 {
match fs::hard_link(&temp_path, &pending_path).await {
Ok(()) => {
sync_directory_metadata(&self.dir).await?;
remove_file_and_sync(&temp_path, &self.dir).await?;
self.flush_notify.notify_one();
return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued);
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
if self
.reuse_existing_pending_file(&pending_path, &record)
.await?
{
remove_file_and_sync(&temp_path, &self.dir).await?;
self.flush_notify.notify_one();
return Ok(WalletRefundOutboxEnqueueOutcome::Enqueued);
}
}
Err(error) => return Err(error.into()),
}
}
Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
format!(
"refund pending path could not be installed: {}",
pending_path.display()
),
)
.into())
}
async fn reuse_existing_pending_file(
&self,
pending_path: &Path,
ledger_id: &str,
expected: &WalletRefundOutboxRecord,
) -> Result<bool, WalletRefundOutboxError> {
let metadata = match fs::metadata(pending_path).await {
Ok(metadata) => metadata,
@@ -147,7 +172,16 @@ impl WalletRefundOutbox {
}
match read_refund_record(pending_path).await {
Ok(existing) if existing.ledger_id == ledger_id => Ok(true),
Ok(existing) if existing == *expected => Ok(true),
Ok(existing) if existing.ledger_id == expected.ledger_id => Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
format!(
"refund ledger {} 已存在但退款事实不一致: {}",
expected.ledger_id,
pending_path.display()
),
)
.into()),
Ok(_) => Err(std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
format!("refund ledger hash collision at {}", pending_path.display()),
@@ -272,7 +306,7 @@ impl WalletRefundOutbox {
match fs::metadata(&pending_path).await {
Ok(metadata) if metadata.is_file() => {
match read_refund_record(&pending_path).await {
Ok(existing) if existing.ledger_id == record.ledger_id => {
Ok(existing) if existing == record => {
remove_file_and_sync(&path, &self.dir).await?;
debug!(
ledger_id = %record.ledger_id,
@@ -282,6 +316,16 @@ impl WalletRefundOutbox {
);
continue;
}
Ok(existing) if existing.ledger_id == record.ledger_id => {
self.quarantine_file(&path).await?;
warn!(
ledger_id = %record.ledger_id,
source = %path.display(),
target = %pending_path.display(),
"wallet refund outbox 临时文件与已有幂等文件事实不一致，已隔离临时文件"
);
continue;
}
Err(error) if error.is_data_corruption() => {
self.quarantine_file(&pending_path).await?;
}
@@ -324,9 +368,51 @@ impl WalletRefundOutbox {
);
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
// Another writer won the same ledger id. Keep the first durable file and
// remove only this duplicate temporary link.
remove_file_and_sync(&path, &self.dir).await?;
match self
.reuse_existing_pending_file(&pending_path, &record)
.await
{
Ok(true) => {
// Another writer won the same ledger id with identical facts. Keep
// the first durable file and remove only this duplicate temporary
// link.
remove_file_and_sync(&path, &self.dir).await?;
}
Ok(false) => match fs::hard_link(&path, &pending_path).await {
Ok(()) => {
sync_directory_metadata(&self.dir).await?;
remove_file_and_sync(&path, &self.dir).await?;
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
// The file may have been completed by another process after the
// scan.
continue;
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
self.quarantine_file(&path).await?;
warn!(
ledger_id = %record.ledger_id,
source = %path.display(),
target = %pending_path.display(),
"wallet refund outbox 临时文件无法与现有幂等文件合并，已隔离临时文件"
);
}
Err(error) => return Err(error.into()),
},
Err(WalletRefundOutboxError::Io(error))
if error.kind() == std::io::ErrorKind::AlreadyExists =>
{
self.quarantine_file(&path).await?;
warn!(
ledger_id = %record.ledger_id,
source = %path.display(),
target = %pending_path.display(),
error = %error,
"wallet refund outbox 临时文件与现有幂等文件事实冲突，已隔离临时文件"
);
}
Err(error) => return Err(error),
}
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
// The file may have been completed by another process after the scan.
@@ -588,6 +674,25 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn enqueue_rejects_conflicting_existing_ledger_file() {
let dir = test_dir("conflicting-ledger");
let outbox = test_outbox(dir.clone(), 1024 * 1024);
outbox.enqueue(sample_record("ledger-1")).await.unwrap();
let mut conflicting = sample_record("ledger-1");
conflicting.amount += 1;
let error = outbox
.enqueue(conflicting)
.await
.expect_err("conflicting refund must fail");
assert!(
matches!(error, WalletRefundOutboxError::Io(error) if error.kind() == std::io::ErrorKind::AlreadyExists)
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn enqueue_drops_when_outbox_exceeds_max_bytes() {
let dir = test_dir("max-bytes");
@@ -702,6 +807,42 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn flush_quarantines_conflicting_crash_left_temp_file() {
let dir = test_dir("recover-conflicting-temp");
std::fs::create_dir_all(&dir).unwrap();
let outbox = test_outbox(dir.clone(), 1024 * 1024);
let record = sample_record("ledger-temp-conflict");
let mut conflicting = record.clone();
conflicting.amount += 1;
let pending_path = outbox.pending_path_for_ledger(&record.ledger_id);
let temp_path = outbox.temp_path();
std::fs::write(&pending_path, serde_json::to_vec(&conflicting).unwrap()).unwrap();
std::fs::write(&temp_path, serde_json::to_vec(&record).unwrap()).unwrap();
let result = outbox.flush_pending_files_once().await;
assert!(matches!(result, Err(WalletRefundOutboxError::Spacetime(_))));
assert!(!temp_path.exists());
assert_eq!(
read_refund_record(&pending_path).await.unwrap(),
conflicting
);
let corrupt_count = std::fs::read_dir(&dir)
.unwrap()
.filter_map(Result::ok)
.filter(|entry| {
entry
.file_name()
.to_str()
.is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX))
})
.count();
assert_eq!(corrupt_count, 1);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn flush_quarantines_corrupt_crash_left_temp_file() {
let dir = test_dir("recover-corrupt-temp");