perf(api-server): batch route tracking through local outbox
This commit is contained in:
@@ -46,7 +46,7 @@ shared-kernel = { workspace = true }
|
||||
shared-logging = { workspace = true }
|
||||
socket2 = { workspace = true }
|
||||
spacetime-client = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "time", "sync"] }
|
||||
tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "time", "sync", "fs", "io-util"] }
|
||||
tokio-stream = { workspace = true }
|
||||
futures-util = { workspace = true }
|
||||
time = { workspace = true, features = ["formatting"] }
|
||||
|
||||
@@ -26,6 +26,11 @@ pub struct AppConfig {
|
||||
pub gallery_max_concurrent_requests: Option<usize>,
|
||||
pub detail_max_concurrent_requests: Option<usize>,
|
||||
pub admin_max_concurrent_requests: Option<usize>,
|
||||
pub tracking_outbox_enabled: bool,
|
||||
pub tracking_outbox_dir: PathBuf,
|
||||
pub tracking_outbox_batch_size: usize,
|
||||
pub tracking_outbox_flush_interval: Duration,
|
||||
pub tracking_outbox_max_bytes: u64,
|
||||
pub log_filter: String,
|
||||
pub otel_enabled: bool,
|
||||
pub admin_username: Option<String>,
|
||||
@@ -160,6 +165,11 @@ impl Default for AppConfig {
|
||||
gallery_max_concurrent_requests: None,
|
||||
detail_max_concurrent_requests: None,
|
||||
admin_max_concurrent_requests: None,
|
||||
tracking_outbox_enabled: true,
|
||||
tracking_outbox_dir: PathBuf::from("server-rs/.data/tracking-outbox"),
|
||||
tracking_outbox_batch_size: 500,
|
||||
tracking_outbox_flush_interval: Duration::from_millis(1_000),
|
||||
tracking_outbox_max_bytes: 256 * 1024 * 1024,
|
||||
log_filter: "info,tower_http=info".to_string(),
|
||||
otel_enabled: false,
|
||||
admin_username: None,
|
||||
@@ -343,6 +353,26 @@ impl AppConfig {
|
||||
{
|
||||
config.admin_max_concurrent_requests = Some(max_concurrent_requests);
|
||||
}
|
||||
if let Some(enabled) = read_first_bool_env(&["GENARRATIVE_TRACKING_OUTBOX_ENABLED"]) {
|
||||
config.tracking_outbox_enabled = enabled;
|
||||
}
|
||||
if let Some(dir) = read_first_non_empty_env(&["GENARRATIVE_TRACKING_OUTBOX_DIR"]) {
|
||||
config.tracking_outbox_dir = PathBuf::from(dir);
|
||||
}
|
||||
if let Some(batch_size) = read_first_usize_env(&["GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE"])
|
||||
{
|
||||
config.tracking_outbox_batch_size = batch_size;
|
||||
}
|
||||
if let Some(flush_interval_ms) =
|
||||
read_first_positive_u64_env(&["GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS"])
|
||||
{
|
||||
config.tracking_outbox_flush_interval = Duration::from_millis(flush_interval_ms);
|
||||
}
|
||||
if let Some(max_bytes) =
|
||||
read_first_positive_u64_env(&["GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES"])
|
||||
{
|
||||
config.tracking_outbox_max_bytes = max_bytes;
|
||||
}
|
||||
if let Some(otel_enabled) = read_first_bool_env(&["GENARRATIVE_OTEL_ENABLED"]) {
|
||||
config.otel_enabled = otel_enabled;
|
||||
}
|
||||
@@ -1230,6 +1260,11 @@ mod tests {
|
||||
std::env::remove_var("GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_ENABLED");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_DIR");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES");
|
||||
std::env::remove_var("GENARRATIVE_OTEL_ENABLED");
|
||||
std::env::set_var("GENARRATIVE_API_LISTEN_BACKLOG", "2048");
|
||||
std::env::set_var("GENARRATIVE_API_WORKER_THREADS", "6");
|
||||
@@ -1237,6 +1272,14 @@ mod tests {
|
||||
std::env::set_var("GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS", "64");
|
||||
std::env::set_var("GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS", "32");
|
||||
std::env::set_var("GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS", "16");
|
||||
std::env::set_var("GENARRATIVE_TRACKING_OUTBOX_ENABLED", "false");
|
||||
std::env::set_var(
|
||||
"GENARRATIVE_TRACKING_OUTBOX_DIR",
|
||||
"/tmp/genarrative-tracking-outbox",
|
||||
);
|
||||
std::env::set_var("GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE", "250");
|
||||
std::env::set_var("GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS", "2000");
|
||||
std::env::set_var("GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES", "1048576");
|
||||
std::env::set_var("GENARRATIVE_OTEL_ENABLED", "true");
|
||||
}
|
||||
|
||||
@@ -1247,6 +1290,17 @@ mod tests {
|
||||
assert_eq!(config.gallery_max_concurrent_requests, Some(64));
|
||||
assert_eq!(config.detail_max_concurrent_requests, Some(32));
|
||||
assert_eq!(config.admin_max_concurrent_requests, Some(16));
|
||||
assert!(!config.tracking_outbox_enabled);
|
||||
assert_eq!(
|
||||
config.tracking_outbox_dir,
|
||||
std::path::PathBuf::from("/tmp/genarrative-tracking-outbox")
|
||||
);
|
||||
assert_eq!(config.tracking_outbox_batch_size, 250);
|
||||
assert_eq!(
|
||||
config.tracking_outbox_flush_interval,
|
||||
std::time::Duration::from_millis(2_000)
|
||||
);
|
||||
assert_eq!(config.tracking_outbox_max_bytes, 1_048_576);
|
||||
assert!(config.otel_enabled);
|
||||
|
||||
unsafe {
|
||||
@@ -1256,6 +1310,11 @@ mod tests {
|
||||
std::env::remove_var("GENARRATIVE_API_GALLERY_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_API_DETAIL_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_API_ADMIN_MAX_CONCURRENT_REQUESTS");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_ENABLED");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_DIR");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_BATCH_SIZE");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_FLUSH_INTERVAL_MS");
|
||||
std::env::remove_var("GENARRATIVE_TRACKING_OUTBOX_MAX_BYTES");
|
||||
std::env::remove_var("GENARRATIVE_OTEL_ENABLED");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,8 +55,8 @@ mod password_entry;
|
||||
mod password_management;
|
||||
mod phone_auth;
|
||||
mod platform_errors;
|
||||
mod profile_identity;
|
||||
mod process_metrics;
|
||||
mod profile_identity;
|
||||
mod prompt;
|
||||
mod puzzle;
|
||||
mod puzzle_agent_turn;
|
||||
@@ -80,6 +80,7 @@ mod story_battles;
|
||||
mod story_sessions;
|
||||
mod telemetry;
|
||||
mod tracking;
|
||||
mod tracking_outbox;
|
||||
mod vector_engine_audio_generation;
|
||||
mod visual_novel;
|
||||
mod volcengine_speech;
|
||||
@@ -154,6 +155,9 @@ async fn run_server(config: AppConfig) -> Result<(), io::Error> {
|
||||
.await
|
||||
.map_err(|error| std::io::Error::other(format!("初始化应用状态失败:{error}")))?;
|
||||
state.puzzle_gallery_cache().spawn_cleanup_task();
|
||||
if let Some(outbox) = state.tracking_outbox() {
|
||||
outbox.spawn_worker();
|
||||
}
|
||||
let router = build_router(state);
|
||||
|
||||
info!(
|
||||
|
||||
@@ -33,6 +33,7 @@ use tracing::{info, warn};
|
||||
|
||||
use crate::config::AppConfig;
|
||||
use crate::puzzle_gallery_cache::PuzzleGalleryCache;
|
||||
use crate::tracking_outbox::TrackingOutbox;
|
||||
use crate::wechat_pay::{WechatPayClient, map_wechat_pay_init_error};
|
||||
use crate::wechat_provider::build_wechat_provider;
|
||||
|
||||
@@ -167,6 +168,7 @@ pub struct AppStateInner {
|
||||
ai_task_service: AiTaskService,
|
||||
spacetime_client: SpacetimeClient,
|
||||
puzzle_gallery_cache: PuzzleGalleryCache,
|
||||
tracking_outbox: Option<Arc<TrackingOutbox>>,
|
||||
llm_client: Option<LlmClient>,
|
||||
creative_agent_gpt5_client: Option<LlmClient>,
|
||||
creative_agent_executor: Arc<MockLangChainRustAgentExecutor>,
|
||||
@@ -297,6 +299,7 @@ impl AppState {
|
||||
pool_size: config.spacetime_pool_size,
|
||||
procedure_timeout: config.spacetime_procedure_timeout,
|
||||
});
|
||||
let tracking_outbox = TrackingOutbox::from_config(&config, spacetime_client.clone());
|
||||
let llm_client = build_llm_client(&config)?;
|
||||
let creative_agent_gpt5_client = build_creative_agent_gpt5_client(&config)?;
|
||||
let http_request_permit_pools = HttpRequestPermitPools::from_config(&config);
|
||||
@@ -324,6 +327,7 @@ impl AppState {
|
||||
ai_task_service,
|
||||
spacetime_client,
|
||||
puzzle_gallery_cache: PuzzleGalleryCache::new(),
|
||||
tracking_outbox,
|
||||
llm_client,
|
||||
creative_agent_gpt5_client,
|
||||
creative_agent_executor: Arc::new(MockLangChainRustAgentExecutor),
|
||||
@@ -582,6 +586,10 @@ impl AppState {
|
||||
&self.puzzle_gallery_cache
|
||||
}
|
||||
|
||||
pub fn tracking_outbox(&self) -> Option<Arc<TrackingOutbox>> {
|
||||
self.tracking_outbox.clone()
|
||||
}
|
||||
|
||||
pub fn llm_client(&self) -> Option<&LlmClient> {
|
||||
self.llm_client.as_ref()
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ use crate::{
|
||||
};
|
||||
|
||||
static HTTP_RESPONSE_BODY_IN_FLIGHT: AtomicI64 = AtomicI64::new(0);
|
||||
static TRACKING_OUTBOX_PENDING_BYTES: AtomicI64 = AtomicI64::new(0);
|
||||
static TRACKING_OUTBOX_PENDING_FILES: AtomicI64 = AtomicI64::new(0);
|
||||
static HTTP_REQUEST_PERMITS_AVAILABLE: OnceLock<HttpRequestPermitsAvailableGauges> =
|
||||
OnceLock::new();
|
||||
|
||||
@@ -123,6 +125,53 @@ pub(crate) fn record_puzzle_gallery_cache_rebuild(
|
||||
.record(data_bytes.min(u64::MAX as usize) as u64, &[]);
|
||||
}
|
||||
|
||||
pub(crate) fn record_tracking_outbox_enqueued() {
|
||||
tracking_outbox_metrics().enqueued.add(1, &[]);
|
||||
}
|
||||
|
||||
pub(crate) fn record_tracking_outbox_dropped(reason: &'static str) {
|
||||
tracking_outbox_metrics()
|
||||
.dropped
|
||||
.add(1, &[KeyValue::new("reason", reason)]);
|
||||
}
|
||||
|
||||
pub(crate) fn record_tracking_outbox_sealed(reason: &'static str) {
|
||||
tracking_outbox_metrics()
|
||||
.sealed_files
|
||||
.add(1, &[KeyValue::new("reason", reason)]);
|
||||
}
|
||||
|
||||
pub(crate) fn record_tracking_outbox_corrupt_file() {
|
||||
tracking_outbox_metrics().corrupt_files.add(1, &[]);
|
||||
}
|
||||
|
||||
pub(crate) fn record_tracking_outbox_flush(
|
||||
duration: std::time::Duration,
|
||||
accepted_count: u32,
|
||||
file_bytes: u64,
|
||||
failed: bool,
|
||||
) {
|
||||
let status_class = if failed { "error" } else { "ok" };
|
||||
let labels = [KeyValue::new("status_class", status_class)];
|
||||
let metrics = tracking_outbox_metrics();
|
||||
metrics.flushes.add(1, &labels);
|
||||
metrics
|
||||
.flush_duration
|
||||
.record(duration.as_secs_f64(), &labels);
|
||||
metrics
|
||||
.flushed_events
|
||||
.add(u64::from(accepted_count), &labels);
|
||||
metrics.flushed_bytes.add(file_bytes, &labels);
|
||||
}
|
||||
|
||||
pub(crate) fn update_tracking_outbox_pending_bytes(bytes: u64) {
|
||||
TRACKING_OUTBOX_PENDING_BYTES.store(bytes.min(i64::MAX as u64) as i64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn update_tracking_outbox_pending_files(files: usize) {
|
||||
TRACKING_OUTBOX_PENDING_FILES.store(files.min(i64::MAX as usize) as i64, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn track_response_body_in_flight(response: Response<Body>) -> Response<Body> {
|
||||
response.map(|body| {
|
||||
HTTP_RESPONSE_BODY_IN_FLIGHT.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -151,6 +200,17 @@ struct PuzzleGalleryCacheMetrics {
|
||||
data_json_bytes: opentelemetry::metrics::Histogram<u64>,
|
||||
}
|
||||
|
||||
struct TrackingOutboxMetrics {
|
||||
enqueued: Counter<u64>,
|
||||
dropped: Counter<u64>,
|
||||
sealed_files: Counter<u64>,
|
||||
corrupt_files: Counter<u64>,
|
||||
flushes: Counter<u64>,
|
||||
flush_duration: opentelemetry::metrics::Histogram<f64>,
|
||||
flushed_events: Counter<u64>,
|
||||
flushed_bytes: Counter<u64>,
|
||||
}
|
||||
|
||||
struct HttpRequestPermitsAvailableGauges {
|
||||
default: Arc<AtomicI64>,
|
||||
gallery: Arc<AtomicI64>,
|
||||
@@ -254,6 +314,51 @@ fn puzzle_gallery_cache_metrics() -> &'static PuzzleGalleryCacheMetrics {
|
||||
})
|
||||
}
|
||||
|
||||
fn tracking_outbox_metrics() -> &'static TrackingOutboxMetrics {
|
||||
static METRICS: std::sync::OnceLock<TrackingOutboxMetrics> = std::sync::OnceLock::new();
|
||||
METRICS.get_or_init(|| {
|
||||
let meter = global::meter("genarrative-api");
|
||||
TrackingOutboxMetrics {
|
||||
enqueued: meter
|
||||
.u64_counter("genarrative.tracking_outbox.events.enqueued")
|
||||
.with_description("Tracking events appended to the local outbox")
|
||||
.build(),
|
||||
dropped: meter
|
||||
.u64_counter("genarrative.tracking_outbox.events.dropped")
|
||||
.with_description("Tracking events dropped by local outbox protection")
|
||||
.build(),
|
||||
sealed_files: meter
|
||||
.u64_counter("genarrative.tracking_outbox.files.sealed")
|
||||
.with_description("Tracking outbox active files sealed for flushing")
|
||||
.build(),
|
||||
corrupt_files: meter
|
||||
.u64_counter("genarrative.tracking_outbox.files.corrupt")
|
||||
.with_description(
|
||||
"Tracking outbox sealed files quarantined because they could not be parsed",
|
||||
)
|
||||
.build(),
|
||||
flushes: meter
|
||||
.u64_counter("genarrative.tracking_outbox.flushes")
|
||||
.with_description("Tracking outbox sealed file flush attempts")
|
||||
.build(),
|
||||
flush_duration: meter
|
||||
.f64_histogram("genarrative.tracking_outbox.flush.duration")
|
||||
.with_unit("s")
|
||||
.with_description("Tracking outbox sealed file flush duration")
|
||||
.build(),
|
||||
flushed_events: meter
|
||||
.u64_counter("genarrative.tracking_outbox.events.flushed")
|
||||
.with_description("Tracking events accepted by SpacetimeDB batch procedure")
|
||||
.build(),
|
||||
flushed_bytes: meter
|
||||
.u64_counter("genarrative.tracking_outbox.bytes.flushed")
|
||||
.with_unit("By")
|
||||
.with_description("Tracking outbox bytes removed after successful flush")
|
||||
.build(),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn register_http_request_permits_available_metric() -> HttpRequestPermitsAvailableGauges {
|
||||
let gauges = HttpRequestPermitsAvailableGauges::new();
|
||||
let meter = global::meter("genarrative-api");
|
||||
@@ -311,6 +416,22 @@ pub(crate) fn register_http_runtime_metrics() {
|
||||
observer.observe(HTTP_RESPONSE_BODY_IN_FLIGHT.load(Ordering::Relaxed), &[]);
|
||||
})
|
||||
.build();
|
||||
meter
|
||||
.i64_observable_up_down_counter("genarrative.tracking_outbox.pending.bytes")
|
||||
.with_unit("By")
|
||||
.with_description("Tracking outbox bytes waiting on local disk")
|
||||
.with_callback(|observer| {
|
||||
observer.observe(TRACKING_OUTBOX_PENDING_BYTES.load(Ordering::Relaxed), &[]);
|
||||
})
|
||||
.build();
|
||||
meter
|
||||
.i64_observable_up_down_counter("genarrative.tracking_outbox.pending.files")
|
||||
.with_unit("{file}")
|
||||
.with_description("Tracking outbox sealed files waiting for flush")
|
||||
.with_callback(|observer| {
|
||||
observer.observe(TRACKING_OUTBOX_PENDING_FILES.load(Ordering::Relaxed), &[]);
|
||||
})
|
||||
.build();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -85,7 +85,7 @@ pub async fn record_route_tracking_event_after_success(
|
||||
draft.owner_user_id = draft.user_id.clone();
|
||||
}
|
||||
|
||||
record_tracking_event_after_success(state, request_context, draft).await;
|
||||
record_route_tracking_event_via_outbox_after_success(state, request_context, draft).await;
|
||||
}
|
||||
|
||||
fn resolve_route_tracking_spec(method: &Method, path: &str) -> Option<RouteTrackingSpec> {
|
||||
@@ -524,26 +524,101 @@ pub async fn record_tracking_event_after_success(
|
||||
request_context: &RequestContext,
|
||||
draft: TrackingEventDraft,
|
||||
) {
|
||||
let occurred_at_micros = OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000;
|
||||
let event_id = build_tracking_event_id(&draft, occurred_at_micros);
|
||||
let event_key = draft.event_key.to_string();
|
||||
let scope_kind = draft.scope_kind;
|
||||
let scope_id = draft.scope_id;
|
||||
let metadata_json = draft.metadata.to_string();
|
||||
record_tracking_event_input_after_success(
|
||||
state,
|
||||
request_context,
|
||||
build_tracking_event_input(draft),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn record_route_tracking_event_via_outbox_after_success(
|
||||
state: &AppState,
|
||||
request_context: &RequestContext,
|
||||
draft: TrackingEventDraft,
|
||||
) {
|
||||
let event = build_tracking_event_input(draft);
|
||||
let event_key = event.event_key.clone();
|
||||
let scope_kind = event.scope_kind;
|
||||
let scope_id = event.scope_id.clone();
|
||||
|
||||
if let Some(outbox) = state.tracking_outbox() {
|
||||
match outbox.enqueue(event.clone()).await {
|
||||
Ok(crate::tracking_outbox::TrackingOutboxEnqueueOutcome::Enqueued) => {
|
||||
tracing::debug!(
|
||||
request_id = request_context.request_id(),
|
||||
operation = request_context.operation(),
|
||||
event_key = %event_key,
|
||||
scope_kind = %scope_kind.as_str(),
|
||||
scope_id = %scope_id,
|
||||
"后端 route 埋点已写入本机 outbox"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Ok(crate::tracking_outbox::TrackingOutboxEnqueueOutcome::Dropped { reason }) => {
|
||||
tracing::warn!(
|
||||
request_id = request_context.request_id(),
|
||||
operation = request_context.operation(),
|
||||
event_key = %event_key,
|
||||
scope_kind = %scope_kind.as_str(),
|
||||
scope_id = %scope_id,
|
||||
reason,
|
||||
"后端 route 埋点因 outbox 保护阈值被丢弃,主业务流程继续"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!(
|
||||
request_id = request_context.request_id(),
|
||||
operation = request_context.operation(),
|
||||
event_key = %event_key,
|
||||
scope_kind = %scope_kind.as_str(),
|
||||
scope_id = %scope_id,
|
||||
error = %error,
|
||||
"后端 route 埋点写入 outbox 失败,回退同步直写 SpacetimeDB"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
record_tracking_event_input_after_success(state, request_context, event).await;
|
||||
}
|
||||
|
||||
async fn record_tracking_event_input_after_success(
|
||||
state: &AppState,
|
||||
request_context: &RequestContext,
|
||||
event: module_runtime::RuntimeTrackingEventInput,
|
||||
) {
|
||||
let event_key = event.event_key.clone();
|
||||
let log_scope_kind = event.scope_kind;
|
||||
let scope_id = event.scope_id.clone();
|
||||
|
||||
let module_runtime::RuntimeTrackingEventInput {
|
||||
event_id,
|
||||
event_key: procedure_event_key,
|
||||
scope_kind: procedure_scope_kind,
|
||||
scope_id: procedure_scope_id,
|
||||
user_id,
|
||||
owner_user_id,
|
||||
profile_id,
|
||||
module_key,
|
||||
metadata_json,
|
||||
occurred_at_micros,
|
||||
} = event;
|
||||
|
||||
match state
|
||||
.spacetime_client()
|
||||
.record_tracking_event(
|
||||
event_id,
|
||||
event_key.clone(),
|
||||
scope_kind,
|
||||
scope_id.clone(),
|
||||
draft.user_id,
|
||||
draft.owner_user_id,
|
||||
draft.profile_id,
|
||||
draft.module_key.map(str::to_string),
|
||||
procedure_event_key,
|
||||
procedure_scope_kind,
|
||||
procedure_scope_id,
|
||||
user_id,
|
||||
owner_user_id,
|
||||
profile_id,
|
||||
module_key,
|
||||
metadata_json,
|
||||
occurred_at_micros as i64,
|
||||
occurred_at_micros,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -551,7 +626,7 @@ pub async fn record_tracking_event_after_success(
|
||||
request_id = request_context.request_id(),
|
||||
operation = request_context.operation(),
|
||||
event_key = %event_key,
|
||||
scope_kind = %scope_kind.as_str(),
|
||||
scope_kind = %log_scope_kind.as_str(),
|
||||
scope_id = %scope_id,
|
||||
"后端埋点已记录"
|
||||
),
|
||||
@@ -559,7 +634,7 @@ pub async fn record_tracking_event_after_success(
|
||||
request_id = request_context.request_id(),
|
||||
operation = request_context.operation(),
|
||||
event_key = %event_key,
|
||||
scope_kind = %scope_kind.as_str(),
|
||||
scope_kind = %log_scope_kind.as_str(),
|
||||
scope_id = %scope_id,
|
||||
error = %error,
|
||||
"后端埋点记录失败,主业务流程继续"
|
||||
@@ -567,6 +642,26 @@ pub async fn record_tracking_event_after_success(
|
||||
}
|
||||
}
|
||||
|
||||
fn build_tracking_event_input(
|
||||
draft: TrackingEventDraft,
|
||||
) -> module_runtime::RuntimeTrackingEventInput {
|
||||
let occurred_at_micros = OffsetDateTime::now_utc().unix_timestamp_nanos() / 1_000;
|
||||
let event_id = build_tracking_event_id(&draft, occurred_at_micros);
|
||||
|
||||
module_runtime::RuntimeTrackingEventInput {
|
||||
event_id,
|
||||
event_key: draft.event_key.to_string(),
|
||||
scope_kind: draft.scope_kind,
|
||||
scope_id: draft.scope_id,
|
||||
user_id: draft.user_id,
|
||||
owner_user_id: draft.owner_user_id,
|
||||
profile_id: draft.profile_id,
|
||||
module_key: draft.module_key.map(str::to_string),
|
||||
metadata_json: draft.metadata.to_string(),
|
||||
occurred_at_micros: occurred_at_micros as i64,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_tracking_event_id(draft: &TrackingEventDraft, occurred_at_micros: i128) -> String {
|
||||
if draft.event_key == "daily_login"
|
||||
&& draft.scope_kind == RuntimeTrackingScopeKind::User
|
||||
|
||||
594
server-rs/crates/api-server/src/tracking_outbox.rs
Normal file
594
server-rs/crates/api-server/src/tracking_outbox.rs
Normal file
@@ -0,0 +1,594 @@
|
||||
use std::{
|
||||
fmt,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
|
||||
};
|
||||
|
||||
use module_runtime::RuntimeTrackingEventInput;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use spacetime_client::{SpacetimeClient, SpacetimeClientError};
|
||||
use tokio::{
|
||||
fs::{self, File, OpenOptions},
|
||||
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
|
||||
sync::Mutex,
|
||||
time::sleep,
|
||||
};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::config::AppConfig;
|
||||
|
||||
const ACTIVE_FILE_NAME: &str = "active.ndjson";
|
||||
const SEALED_FILE_PREFIX: &str = "sealed-";
|
||||
const CORRUPT_FILE_PREFIX: &str = "corrupt-";
|
||||
const SEALED_FILE_EXTENSION: &str = ".ndjson";
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct TrackingOutbox {
|
||||
dir: PathBuf,
|
||||
batch_size: usize,
|
||||
flush_interval: Duration,
|
||||
max_bytes: u64,
|
||||
spacetime_client: SpacetimeClient,
|
||||
inner: Arc<Mutex<TrackingOutboxInner>>,
|
||||
}
|
||||
|
||||
struct TrackingOutboxInner {
|
||||
initialized: bool,
|
||||
active_file: Option<File>,
|
||||
active_count: usize,
|
||||
active_bytes: u64,
|
||||
total_bytes: u64,
|
||||
last_sealed_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum TrackingOutboxEnqueueOutcome {
|
||||
Enqueued,
|
||||
Dropped { reason: &'static str },
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum TrackingOutboxError {
|
||||
Io(std::io::Error),
|
||||
Json(serde_json::Error),
|
||||
Spacetime(SpacetimeClientError),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||
struct TrackingOutboxRecord {
|
||||
event: RuntimeTrackingEventInput,
|
||||
}
|
||||
|
||||
impl TrackingOutbox {
|
||||
pub fn from_config(config: &AppConfig, spacetime_client: SpacetimeClient) -> Option<Arc<Self>> {
|
||||
if !config.tracking_outbox_enabled {
|
||||
return None;
|
||||
}
|
||||
|
||||
let total_bytes = directory_size_if_exists(&config.tracking_outbox_dir).unwrap_or(0);
|
||||
let outbox = Self {
|
||||
dir: config.tracking_outbox_dir.clone(),
|
||||
batch_size: config.tracking_outbox_batch_size.max(1),
|
||||
flush_interval: config.tracking_outbox_flush_interval,
|
||||
max_bytes: config.tracking_outbox_max_bytes,
|
||||
spacetime_client,
|
||||
inner: Arc::new(Mutex::new(TrackingOutboxInner {
|
||||
initialized: false,
|
||||
active_file: None,
|
||||
active_count: 0,
|
||||
active_bytes: 0,
|
||||
total_bytes,
|
||||
last_sealed_at: Instant::now(),
|
||||
})),
|
||||
};
|
||||
crate::telemetry::update_tracking_outbox_pending_bytes(total_bytes);
|
||||
Some(Arc::new(outbox))
|
||||
}
|
||||
|
||||
pub async fn enqueue(
|
||||
&self,
|
||||
event: RuntimeTrackingEventInput,
|
||||
) -> Result<TrackingOutboxEnqueueOutcome, TrackingOutboxError> {
|
||||
let record = TrackingOutboxRecord { event };
|
||||
let mut line = serde_json::to_vec(&record)?;
|
||||
line.push(b'\n');
|
||||
let line_bytes = line.len().min(u64::MAX as usize) as u64;
|
||||
|
||||
let mut inner = self.inner.lock().await;
|
||||
self.ensure_initialized_locked(&mut inner).await?;
|
||||
|
||||
if inner.total_bytes.saturating_add(line_bytes) > self.max_bytes {
|
||||
crate::telemetry::record_tracking_outbox_dropped("max_bytes");
|
||||
return Ok(TrackingOutboxEnqueueOutcome::Dropped {
|
||||
reason: "max_bytes",
|
||||
});
|
||||
}
|
||||
|
||||
let active_path = self.active_path();
|
||||
if inner.active_file.is_none() {
|
||||
inner.active_file = Some(
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&active_path)
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
let file = inner
|
||||
.active_file
|
||||
.as_mut()
|
||||
.expect("active file should be open before append");
|
||||
file.write_all(&line).await?;
|
||||
inner.active_count = inner.active_count.saturating_add(1);
|
||||
inner.active_bytes = inner.active_bytes.saturating_add(line_bytes);
|
||||
inner.total_bytes = inner.total_bytes.saturating_add(line_bytes);
|
||||
crate::telemetry::record_tracking_outbox_enqueued();
|
||||
crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
|
||||
|
||||
if inner.active_count >= self.batch_size {
|
||||
self.seal_active_locked(&mut inner, "batch_size").await?;
|
||||
}
|
||||
|
||||
Ok(TrackingOutboxEnqueueOutcome::Enqueued)
|
||||
}
|
||||
|
||||
pub fn spawn_worker(self: Arc<Self>) {
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
sleep(self.flush_interval).await;
|
||||
if let Err(error) = self.seal_active_if_due().await {
|
||||
warn!(error = %error, "tracking outbox 定时封存 active 文件失败");
|
||||
}
|
||||
if let Err(error) = self.flush_sealed_files_once().await {
|
||||
warn!(error = %error, "tracking outbox 批量写入 SpacetimeDB 失败,将保留 sealed 文件等待重试");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn seal_active_if_due(&self) -> Result<(), TrackingOutboxError> {
|
||||
let mut inner = self.inner.lock().await;
|
||||
self.ensure_initialized_locked(&mut inner).await?;
|
||||
if inner.active_count == 0 || inner.last_sealed_at.elapsed() < self.flush_interval {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.seal_active_locked(&mut inner, "flush_interval").await
|
||||
}
|
||||
|
||||
async fn flush_sealed_files_once(&self) -> Result<(), TrackingOutboxError> {
|
||||
self.ensure_initialized().await?;
|
||||
|
||||
let sealed_files = self.list_sealed_files().await?;
|
||||
crate::telemetry::update_tracking_outbox_pending_files(sealed_files.len());
|
||||
for path in sealed_files {
|
||||
let started_at = Instant::now();
|
||||
let metadata = fs::metadata(&path).await?;
|
||||
let file_bytes = metadata.len();
|
||||
let events = match read_outbox_events(&path).await {
|
||||
Ok(events) => events,
|
||||
Err(error) if error.is_data_corruption() => {
|
||||
let corrupt_path = self.corrupt_path_for(&path);
|
||||
fs::rename(&path, &corrupt_path).await?;
|
||||
self.subtract_total_bytes(file_bytes).await;
|
||||
crate::telemetry::record_tracking_outbox_corrupt_file();
|
||||
warn!(
|
||||
error = %error,
|
||||
source = %path.display(),
|
||||
target = %corrupt_path.display(),
|
||||
"tracking outbox sealed 文件含无法解析的记录,已隔离并继续处理后续文件"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if events.is_empty() {
|
||||
fs::remove_file(&path).await?;
|
||||
self.subtract_total_bytes(file_bytes).await;
|
||||
continue;
|
||||
}
|
||||
|
||||
match self.spacetime_client.record_tracking_events(events).await {
|
||||
Ok(accepted_count) => {
|
||||
fs::remove_file(&path).await?;
|
||||
self.subtract_total_bytes(file_bytes).await;
|
||||
crate::telemetry::record_tracking_outbox_flush(
|
||||
started_at.elapsed(),
|
||||
accepted_count,
|
||||
file_bytes,
|
||||
false,
|
||||
);
|
||||
debug!(
|
||||
accepted_count,
|
||||
file_bytes,
|
||||
path = %path.display(),
|
||||
"tracking outbox sealed 文件已批量入库并删除"
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
crate::telemetry::record_tracking_outbox_flush(
|
||||
started_at.elapsed(),
|
||||
0,
|
||||
file_bytes,
|
||||
true,
|
||||
);
|
||||
return Err(TrackingOutboxError::Spacetime(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn ensure_initialized(&self) -> Result<(), TrackingOutboxError> {
|
||||
let mut inner = self.inner.lock().await;
|
||||
self.ensure_initialized_locked(&mut inner).await
|
||||
}
|
||||
|
||||
async fn ensure_initialized_locked(
|
||||
&self,
|
||||
inner: &mut TrackingOutboxInner,
|
||||
) -> Result<(), TrackingOutboxError> {
|
||||
if inner.initialized {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
fs::create_dir_all(&self.dir).await?;
|
||||
self.seal_existing_active_file().await?;
|
||||
inner.total_bytes = directory_size(&self.dir).await?;
|
||||
inner.initialized = true;
|
||||
inner.last_sealed_at = Instant::now();
|
||||
crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn seal_active_locked(
|
||||
&self,
|
||||
inner: &mut TrackingOutboxInner,
|
||||
reason: &'static str,
|
||||
) -> Result<(), TrackingOutboxError> {
|
||||
if inner.active_count == 0 && inner.active_bytes == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(mut file) = inner.active_file.take() {
|
||||
file.flush().await?;
|
||||
file.sync_data().await?;
|
||||
drop(file);
|
||||
}
|
||||
|
||||
let active_path = self.active_path();
|
||||
match fs::metadata(&active_path).await {
|
||||
Ok(metadata) if metadata.len() > 0 => {
|
||||
let sealed_path = self.next_sealed_path();
|
||||
fs::rename(&active_path, &sealed_path).await?;
|
||||
crate::telemetry::record_tracking_outbox_sealed(reason);
|
||||
debug!(
|
||||
reason,
|
||||
event_count = inner.active_count,
|
||||
file_bytes = metadata.len(),
|
||||
path = %sealed_path.display(),
|
||||
"tracking outbox active 文件已封存"
|
||||
);
|
||||
}
|
||||
Ok(_) => {
|
||||
let _ = fs::remove_file(&active_path).await;
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
|
||||
inner.active_count = 0;
|
||||
inner.active_bytes = 0;
|
||||
inner.last_sealed_at = Instant::now();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn seal_existing_active_file(&self) -> Result<(), TrackingOutboxError> {
|
||||
let active_path = self.active_path();
|
||||
match fs::metadata(&active_path).await {
|
||||
Ok(metadata) if metadata.len() > 0 => {
|
||||
fs::rename(&active_path, self.next_sealed_path()).await?;
|
||||
crate::telemetry::record_tracking_outbox_sealed("startup");
|
||||
}
|
||||
Ok(_) => {
|
||||
let _ = fs::remove_file(&active_path).await;
|
||||
}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_sealed_files(&self) -> Result<Vec<PathBuf>, TrackingOutboxError> {
|
||||
let mut entries = fs::read_dir(&self.dir).await?;
|
||||
let mut files = Vec::new();
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
let path = entry.path();
|
||||
let Some(name) = path.file_name().and_then(|value| value.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
if name.starts_with(SEALED_FILE_PREFIX) && name.ends_with(SEALED_FILE_EXTENSION) {
|
||||
files.push(path);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
async fn subtract_total_bytes(&self, bytes: u64) {
|
||||
let mut inner = self.inner.lock().await;
|
||||
inner.total_bytes = inner.total_bytes.saturating_sub(bytes);
|
||||
crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
|
||||
}
|
||||
|
||||
fn active_path(&self) -> PathBuf {
|
||||
self.dir.join(ACTIVE_FILE_NAME)
|
||||
}
|
||||
|
||||
fn next_sealed_path(&self) -> PathBuf {
|
||||
self.dir.join(format!(
|
||||
"{SEALED_FILE_PREFIX}{}-{uuid}{SEALED_FILE_EXTENSION}",
|
||||
current_unix_micros(),
|
||||
uuid = uuid::Uuid::new_v4()
|
||||
))
|
||||
}
|
||||
|
||||
fn corrupt_path_for(&self, path: &Path) -> PathBuf {
|
||||
let name = path
|
||||
.file_name()
|
||||
.and_then(|value| value.to_str())
|
||||
.unwrap_or("unknown.ndjson");
|
||||
self.dir.join(format!(
|
||||
"{CORRUPT_FILE_PREFIX}{}-{uuid}-{name}",
|
||||
current_unix_micros(),
|
||||
uuid = uuid::Uuid::new_v4()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for TrackingOutbox {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("TrackingOutbox")
|
||||
.field("dir", &self.dir)
|
||||
.field("batch_size", &self.batch_size)
|
||||
.field("flush_interval", &self.flush_interval)
|
||||
.field("max_bytes", &self.max_bytes)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for TrackingOutboxError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Io(error) => write!(f, "{error}"),
|
||||
Self::Json(error) => write!(f, "{error}"),
|
||||
Self::Spacetime(error) => write!(f, "{error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for TrackingOutboxError {
|
||||
fn from(value: std::io::Error) -> Self {
|
||||
Self::Io(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for TrackingOutboxError {
|
||||
fn from(value: serde_json::Error) -> Self {
|
||||
Self::Json(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl TrackingOutboxError {
|
||||
fn is_data_corruption(&self) -> bool {
|
||||
matches!(self, Self::Json(_))
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_outbox_events(
|
||||
path: &Path,
|
||||
) -> Result<Vec<RuntimeTrackingEventInput>, TrackingOutboxError> {
|
||||
let file = File::open(path).await?;
|
||||
let mut lines = BufReader::new(file).lines();
|
||||
let mut events = Vec::new();
|
||||
while let Some(line) = lines.next_line().await? {
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
}
|
||||
let record = serde_json::from_str::<TrackingOutboxRecord>(&line)?;
|
||||
events.push(record.event);
|
||||
}
|
||||
Ok(events)
|
||||
}
|
||||
|
||||
async fn directory_size(path: &Path) -> Result<u64, TrackingOutboxError> {
|
||||
let mut total = 0u64;
|
||||
let mut entries = fs::read_dir(path).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if !is_pending_outbox_file_name(&entry.file_name()) {
|
||||
continue;
|
||||
}
|
||||
let metadata = entry.metadata().await?;
|
||||
if metadata.is_file() {
|
||||
total = total.saturating_add(metadata.len());
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn directory_size_if_exists(path: &Path) -> Result<u64, std::io::Error> {
|
||||
if !path.is_dir() {
|
||||
return Ok(0);
|
||||
}
|
||||
|
||||
let mut total = 0u64;
|
||||
for entry in std::fs::read_dir(path)? {
|
||||
let entry = entry?;
|
||||
if !is_pending_outbox_file_name(&entry.file_name()) {
|
||||
continue;
|
||||
}
|
||||
let metadata = entry.metadata()?;
|
||||
if metadata.is_file() {
|
||||
total = total.saturating_add(metadata.len());
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn current_unix_micros() -> u128 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_micros()
|
||||
}
|
||||
|
||||
fn is_pending_outbox_file_name(name: &std::ffi::OsStr) -> bool {
|
||||
name.to_str().is_some_and(|value| {
|
||||
value == ACTIVE_FILE_NAME
|
||||
|| (value.starts_with(SEALED_FILE_PREFIX) && value.ends_with(SEALED_FILE_EXTENSION))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sample_event(event_id: &str) -> RuntimeTrackingEventInput {
|
||||
RuntimeTrackingEventInput {
|
||||
event_id: event_id.to_string(),
|
||||
event_key: "puzzle_route_success".to_string(),
|
||||
scope_kind: module_runtime::RuntimeTrackingScopeKind::Site,
|
||||
scope_id: "site".to_string(),
|
||||
user_id: None,
|
||||
owner_user_id: None,
|
||||
profile_id: None,
|
||||
module_key: Some("puzzle".to_string()),
|
||||
metadata_json: "{}".to_string(),
|
||||
occurred_at_micros: 1_713_680_000_000_000,
|
||||
}
|
||||
}
|
||||
|
||||
fn test_dir(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"genarrative-tracking-outbox-{name}-{}",
|
||||
current_unix_micros()
|
||||
));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
dir
|
||||
}
|
||||
|
||||
fn test_outbox(dir: PathBuf, batch_size: usize, max_bytes: u64) -> Arc<TrackingOutbox> {
|
||||
let config = AppConfig {
|
||||
tracking_outbox_dir: dir,
|
||||
tracking_outbox_batch_size: batch_size,
|
||||
tracking_outbox_max_bytes: max_bytes,
|
||||
tracking_outbox_flush_interval: Duration::from_secs(60),
|
||||
..AppConfig::default()
|
||||
};
|
||||
TrackingOutbox::from_config(
|
||||
&config,
|
||||
SpacetimeClient::new(spacetime_client::SpacetimeClientConfig {
|
||||
server_url: "http://127.0.0.1:1".to_string(),
|
||||
database: "missing".to_string(),
|
||||
token: None,
|
||||
pool_size: 1,
|
||||
procedure_timeout: Duration::from_millis(10),
|
||||
}),
|
||||
)
|
||||
.expect("outbox should be enabled")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_seals_active_file_when_batch_size_reached() {
|
||||
let dir = test_dir("batch");
|
||||
let outbox = test_outbox(dir.clone(), 2, 1024 * 1024);
|
||||
|
||||
outbox.enqueue(sample_event("event-1")).await.unwrap();
|
||||
outbox.enqueue(sample_event("event-2")).await.unwrap();
|
||||
|
||||
assert!(!dir.join(ACTIVE_FILE_NAME).exists());
|
||||
let sealed_count = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with(SEALED_FILE_PREFIX))
|
||||
})
|
||||
.count();
|
||||
assert_eq!(sealed_count, 1);
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn enqueue_drops_when_outbox_exceeds_max_bytes() {
|
||||
let dir = test_dir("max-bytes");
|
||||
let outbox = test_outbox(dir.clone(), 500, 1);
|
||||
|
||||
let outcome = outbox.enqueue(sample_event("event-1")).await.unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
TrackingOutboxEnqueueOutcome::Dropped {
|
||||
reason: "max_bytes"
|
||||
}
|
||||
));
|
||||
assert!(!dir.join(ACTIVE_FILE_NAME).exists());
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn flush_quarantines_corrupt_sealed_file() {
|
||||
let dir = test_dir("corrupt");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let sealed_path = dir.join(format!("{SEALED_FILE_PREFIX}bad{SEALED_FILE_EXTENSION}"));
|
||||
std::fs::write(&sealed_path, b"{not-json}\n").unwrap();
|
||||
let outbox = test_outbox(dir.clone(), 500, 1024 * 1024);
|
||||
|
||||
outbox.flush_sealed_files_once().await.unwrap();
|
||||
|
||||
assert!(!sealed_path.exists());
|
||||
let corrupt_count = std::fs::read_dir(&dir)
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX))
|
||||
})
|
||||
.count();
|
||||
assert_eq!(corrupt_count, 1);
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_size_excludes_quarantined_corrupt_files() {
|
||||
let dir = test_dir("directory-size");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join(ACTIVE_FILE_NAME), b"active").unwrap();
|
||||
std::fs::write(
|
||||
dir.join(format!("{SEALED_FILE_PREFIX}one{SEALED_FILE_EXTENSION}")),
|
||||
b"sealed",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
dir.join(format!("{CORRUPT_FILE_PREFIX}one{SEALED_FILE_EXTENSION}")),
|
||||
b"corrupt",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let total = directory_size_if_exists(&dir).unwrap();
|
||||
|
||||
assert_eq!(total, 12);
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user