perf(api-server): batch route tracking through local outbox

server-rs/crates/api-server/src/tracking_outbox.rs

@@ -0,0 +1,594 @@
+use std::{
+    fmt,
+    path::{Path, PathBuf},
+    sync::Arc,
+    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
+};
+
+use module_runtime::RuntimeTrackingEventInput;
+use serde::{Deserialize, Serialize};
+use spacetime_client::{SpacetimeClient, SpacetimeClientError};
+use tokio::{
+    fs::{self, File, OpenOptions},
+    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    sync::Mutex,
+    time::sleep,
+};
+use tracing::{debug, warn};
+
+use crate::config::AppConfig;
+
+const ACTIVE_FILE_NAME: &str = "active.ndjson";
+const SEALED_FILE_PREFIX: &str = "sealed-";
+const CORRUPT_FILE_PREFIX: &str = "corrupt-";
+const SEALED_FILE_EXTENSION: &str = ".ndjson";
+
+#[derive(Clone)]
+pub struct TrackingOutbox {
+    dir: PathBuf,
+    batch_size: usize,
+    flush_interval: Duration,
+    max_bytes: u64,
+    spacetime_client: SpacetimeClient,
+    inner: Arc<Mutex<TrackingOutboxInner>>,
+}
+
+struct TrackingOutboxInner {
+    initialized: bool,
+    active_file: Option<File>,
+    active_count: usize,
+    active_bytes: u64,
+    total_bytes: u64,
+    last_sealed_at: Instant,
+}
+
+#[derive(Debug)]
+pub enum TrackingOutboxEnqueueOutcome {
+    Enqueued,
+    Dropped { reason: &'static str },
+}
+
+#[derive(Debug)]
+pub enum TrackingOutboxError {
+    Io(std::io::Error),
+    Json(serde_json::Error),
+    Spacetime(SpacetimeClientError),
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+struct TrackingOutboxRecord {
+    event: RuntimeTrackingEventInput,
+}
+
+impl TrackingOutbox {
+    pub fn from_config(config: &AppConfig, spacetime_client: SpacetimeClient) -> Option<Arc<Self>> {
+        if !config.tracking_outbox_enabled {
+            return None;
+        }
+
+        let total_bytes = directory_size_if_exists(&config.tracking_outbox_dir).unwrap_or(0);
+        let outbox = Self {
+            dir: config.tracking_outbox_dir.clone(),
+            batch_size: config.tracking_outbox_batch_size.max(1),
+            flush_interval: config.tracking_outbox_flush_interval,
+            max_bytes: config.tracking_outbox_max_bytes,
+            spacetime_client,
+            inner: Arc::new(Mutex::new(TrackingOutboxInner {
+                initialized: false,
+                active_file: None,
+                active_count: 0,
+                active_bytes: 0,
+                total_bytes,
+                last_sealed_at: Instant::now(),
+            })),
+        };
+        crate::telemetry::update_tracking_outbox_pending_bytes(total_bytes);
+        Some(Arc::new(outbox))
+    }
+
+    pub async fn enqueue(
+        &self,
+        event: RuntimeTrackingEventInput,
+    ) -> Result<TrackingOutboxEnqueueOutcome, TrackingOutboxError> {
+        let record = TrackingOutboxRecord { event };
+        let mut line = serde_json::to_vec(&record)?;
+        line.push(b'\n');
+        let line_bytes = line.len().min(u64::MAX as usize) as u64;
+
+        let mut inner = self.inner.lock().await;
+        self.ensure_initialized_locked(&mut inner).await?;
+
+        if inner.total_bytes.saturating_add(line_bytes) > self.max_bytes {
+            crate::telemetry::record_tracking_outbox_dropped("max_bytes");
+            return Ok(TrackingOutboxEnqueueOutcome::Dropped {
+                reason: "max_bytes",
+            });
+        }
+
+        let active_path = self.active_path();
+        if inner.active_file.is_none() {
+            inner.active_file = Some(
+                OpenOptions::new()
+                    .create(true)
+                    .append(true)
+                    .open(&active_path)
+                    .await?,
+            );
+        }
+
+        let file = inner
+            .active_file
+            .as_mut()
+            .expect("active file should be open before append");
+        file.write_all(&line).await?;
+        inner.active_count = inner.active_count.saturating_add(1);
+        inner.active_bytes = inner.active_bytes.saturating_add(line_bytes);
+        inner.total_bytes = inner.total_bytes.saturating_add(line_bytes);
+        crate::telemetry::record_tracking_outbox_enqueued();
+        crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
+
+        if inner.active_count >= self.batch_size {
+            self.seal_active_locked(&mut inner, "batch_size").await?;
+        }
+
+        Ok(TrackingOutboxEnqueueOutcome::Enqueued)
+    }
+
+    pub fn spawn_worker(self: Arc<Self>) {
+        tokio::spawn(async move {
+            loop {
+                sleep(self.flush_interval).await;
+                if let Err(error) = self.seal_active_if_due().await {
+                    warn!(error = %error, "tracking outbox 定时封存 active 文件失败");
+                }
+                if let Err(error) = self.flush_sealed_files_once().await {
+                    warn!(error = %error, "tracking outbox 批量写入 SpacetimeDB 失败，将保留 sealed 文件等待重试");
+                }
+            }
+        });
+    }
+
+    async fn seal_active_if_due(&self) -> Result<(), TrackingOutboxError> {
+        let mut inner = self.inner.lock().await;
+        self.ensure_initialized_locked(&mut inner).await?;
+        if inner.active_count == 0 || inner.last_sealed_at.elapsed() < self.flush_interval {
+            return Ok(());
+        }
+
+        self.seal_active_locked(&mut inner, "flush_interval").await
+    }
+
+    async fn flush_sealed_files_once(&self) -> Result<(), TrackingOutboxError> {
+        self.ensure_initialized().await?;
+
+        let sealed_files = self.list_sealed_files().await?;
+        crate::telemetry::update_tracking_outbox_pending_files(sealed_files.len());
+        for path in sealed_files {
+            let started_at = Instant::now();
+            let metadata = fs::metadata(&path).await?;
+            let file_bytes = metadata.len();
+            let events = match read_outbox_events(&path).await {
+                Ok(events) => events,
+                Err(error) if error.is_data_corruption() => {
+                    let corrupt_path = self.corrupt_path_for(&path);
+                    fs::rename(&path, &corrupt_path).await?;
+                    self.subtract_total_bytes(file_bytes).await;
+                    crate::telemetry::record_tracking_outbox_corrupt_file();
+                    warn!(
+                        error = %error,
+                        source = %path.display(),
+                        target = %corrupt_path.display(),
+                        "tracking outbox sealed 文件含无法解析的记录，已隔离并继续处理后续文件"
+                    );
+                    continue;
+                }
+                Err(error) => return Err(error),
+            };
+            if events.is_empty() {
+                fs::remove_file(&path).await?;
+                self.subtract_total_bytes(file_bytes).await;
+                continue;
+            }
+
+            match self.spacetime_client.record_tracking_events(events).await {
+                Ok(accepted_count) => {
+                    fs::remove_file(&path).await?;
+                    self.subtract_total_bytes(file_bytes).await;
+                    crate::telemetry::record_tracking_outbox_flush(
+                        started_at.elapsed(),
+                        accepted_count,
+                        file_bytes,
+                        false,
+                    );
+                    debug!(
+                        accepted_count,
+                        file_bytes,
+                        path = %path.display(),
+                        "tracking outbox sealed 文件已批量入库并删除"
+                    );
+                }
+                Err(error) => {
+                    crate::telemetry::record_tracking_outbox_flush(
+                        started_at.elapsed(),
+                        0,
+                        file_bytes,
+                        true,
+                    );
+                    return Err(TrackingOutboxError::Spacetime(error));
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    async fn ensure_initialized(&self) -> Result<(), TrackingOutboxError> {
+        let mut inner = self.inner.lock().await;
+        self.ensure_initialized_locked(&mut inner).await
+    }
+
+    async fn ensure_initialized_locked(
+        &self,
+        inner: &mut TrackingOutboxInner,
+    ) -> Result<(), TrackingOutboxError> {
+        if inner.initialized {
+            return Ok(());
+        }
+
+        fs::create_dir_all(&self.dir).await?;
+        self.seal_existing_active_file().await?;
+        inner.total_bytes = directory_size(&self.dir).await?;
+        inner.initialized = true;
+        inner.last_sealed_at = Instant::now();
+        crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
+        Ok(())
+    }
+
+    async fn seal_active_locked(
+        &self,
+        inner: &mut TrackingOutboxInner,
+        reason: &'static str,
+    ) -> Result<(), TrackingOutboxError> {
+        if inner.active_count == 0 && inner.active_bytes == 0 {
+            return Ok(());
+        }
+
+        if let Some(mut file) = inner.active_file.take() {
+            file.flush().await?;
+            file.sync_data().await?;
+            drop(file);
+        }
+
+        let active_path = self.active_path();
+        match fs::metadata(&active_path).await {
+            Ok(metadata) if metadata.len() > 0 => {
+                let sealed_path = self.next_sealed_path();
+                fs::rename(&active_path, &sealed_path).await?;
+                crate::telemetry::record_tracking_outbox_sealed(reason);
+                debug!(
+                    reason,
+                    event_count = inner.active_count,
+                    file_bytes = metadata.len(),
+                    path = %sealed_path.display(),
+                    "tracking outbox active 文件已封存"
+                );
+            }
+            Ok(_) => {
+                let _ = fs::remove_file(&active_path).await;
+            }
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+            Err(error) => return Err(error.into()),
+        }
+
+        inner.active_count = 0;
+        inner.active_bytes = 0;
+        inner.last_sealed_at = Instant::now();
+        Ok(())
+    }
+
+    async fn seal_existing_active_file(&self) -> Result<(), TrackingOutboxError> {
+        let active_path = self.active_path();
+        match fs::metadata(&active_path).await {
+            Ok(metadata) if metadata.len() > 0 => {
+                fs::rename(&active_path, self.next_sealed_path()).await?;
+                crate::telemetry::record_tracking_outbox_sealed("startup");
+            }
+            Ok(_) => {
+                let _ = fs::remove_file(&active_path).await;
+            }
+            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
+            Err(error) => return Err(error.into()),
+        }
+        Ok(())
+    }
+
+    async fn list_sealed_files(&self) -> Result<Vec<PathBuf>, TrackingOutboxError> {
+        let mut entries = fs::read_dir(&self.dir).await?;
+        let mut files = Vec::new();
+        while let Some(entry) = entries.next_entry().await? {
+            let path = entry.path();
+            let Some(name) = path.file_name().and_then(|value| value.to_str()) else {
+                continue;
+            };
+            if name.starts_with(SEALED_FILE_PREFIX) && name.ends_with(SEALED_FILE_EXTENSION) {
+                files.push(path);
+            }
+        }
+        files.sort();
+        Ok(files)
+    }
+
+    async fn subtract_total_bytes(&self, bytes: u64) {
+        let mut inner = self.inner.lock().await;
+        inner.total_bytes = inner.total_bytes.saturating_sub(bytes);
+        crate::telemetry::update_tracking_outbox_pending_bytes(inner.total_bytes);
+    }
+
+    fn active_path(&self) -> PathBuf {
+        self.dir.join(ACTIVE_FILE_NAME)
+    }
+
+    fn next_sealed_path(&self) -> PathBuf {
+        self.dir.join(format!(
+            "{SEALED_FILE_PREFIX}{}-{uuid}{SEALED_FILE_EXTENSION}",
+            current_unix_micros(),
+            uuid = uuid::Uuid::new_v4()
+        ))
+    }
+
+    fn corrupt_path_for(&self, path: &Path) -> PathBuf {
+        let name = path
+            .file_name()
+            .and_then(|value| value.to_str())
+            .unwrap_or("unknown.ndjson");
+        self.dir.join(format!(
+            "{CORRUPT_FILE_PREFIX}{}-{uuid}-{name}",
+            current_unix_micros(),
+            uuid = uuid::Uuid::new_v4()
+        ))
+    }
+}
+
+impl fmt::Debug for TrackingOutbox {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TrackingOutbox")
+            .field("dir", &self.dir)
+            .field("batch_size", &self.batch_size)
+            .field("flush_interval", &self.flush_interval)
+            .field("max_bytes", &self.max_bytes)
+            .finish()
+    }
+}
+
+impl fmt::Display for TrackingOutboxError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Io(error) => write!(f, "{error}"),
+            Self::Json(error) => write!(f, "{error}"),
+            Self::Spacetime(error) => write!(f, "{error}"),
+        }
+    }
+}
+
+impl From<std::io::Error> for TrackingOutboxError {
+    fn from(value: std::io::Error) -> Self {
+        Self::Io(value)
+    }
+}
+
+impl From<serde_json::Error> for TrackingOutboxError {
+    fn from(value: serde_json::Error) -> Self {
+        Self::Json(value)
+    }
+}
+
+impl TrackingOutboxError {
+    fn is_data_corruption(&self) -> bool {
+        matches!(self, Self::Json(_))
+    }
+}
+
+async fn read_outbox_events(
+    path: &Path,
+) -> Result<Vec<RuntimeTrackingEventInput>, TrackingOutboxError> {
+    let file = File::open(path).await?;
+    let mut lines = BufReader::new(file).lines();
+    let mut events = Vec::new();
+    while let Some(line) = lines.next_line().await? {
+        if line.trim().is_empty() {
+            continue;
+        }
+        let record = serde_json::from_str::<TrackingOutboxRecord>(&line)?;
+        events.push(record.event);
+    }
+    Ok(events)
+}
+
+async fn directory_size(path: &Path) -> Result<u64, TrackingOutboxError> {
+    let mut total = 0u64;
+    let mut entries = fs::read_dir(path).await?;
+    while let Some(entry) = entries.next_entry().await? {
+        if !is_pending_outbox_file_name(&entry.file_name()) {
+            continue;
+        }
+        let metadata = entry.metadata().await?;
+        if metadata.is_file() {
+            total = total.saturating_add(metadata.len());
+        }
+    }
+    Ok(total)
+}
+
+fn directory_size_if_exists(path: &Path) -> Result<u64, std::io::Error> {
+    if !path.is_dir() {
+        return Ok(0);
+    }
+
+    let mut total = 0u64;
+    for entry in std::fs::read_dir(path)? {
+        let entry = entry?;
+        if !is_pending_outbox_file_name(&entry.file_name()) {
+            continue;
+        }
+        let metadata = entry.metadata()?;
+        if metadata.is_file() {
+            total = total.saturating_add(metadata.len());
+        }
+    }
+    Ok(total)
+}
+
+fn current_unix_micros() -> u128 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_micros()
+}
+
+fn is_pending_outbox_file_name(name: &std::ffi::OsStr) -> bool {
+    name.to_str().is_some_and(|value| {
+        value == ACTIVE_FILE_NAME
+            || (value.starts_with(SEALED_FILE_PREFIX) && value.ends_with(SEALED_FILE_EXTENSION))
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn sample_event(event_id: &str) -> RuntimeTrackingEventInput {
+        RuntimeTrackingEventInput {
+            event_id: event_id.to_string(),
+            event_key: "puzzle_route_success".to_string(),
+            scope_kind: module_runtime::RuntimeTrackingScopeKind::Site,
+            scope_id: "site".to_string(),
+            user_id: None,
+            owner_user_id: None,
+            profile_id: None,
+            module_key: Some("puzzle".to_string()),
+            metadata_json: "{}".to_string(),
+            occurred_at_micros: 1_713_680_000_000_000,
+        }
+    }
+
+    fn test_dir(name: &str) -> PathBuf {
+        let dir = std::env::temp_dir().join(format!(
+            "genarrative-tracking-outbox-{name}-{}",
+            current_unix_micros()
+        ));
+        let _ = std::fs::remove_dir_all(&dir);
+        dir
+    }
+
+    fn test_outbox(dir: PathBuf, batch_size: usize, max_bytes: u64) -> Arc<TrackingOutbox> {
+        let config = AppConfig {
+            tracking_outbox_dir: dir,
+            tracking_outbox_batch_size: batch_size,
+            tracking_outbox_max_bytes: max_bytes,
+            tracking_outbox_flush_interval: Duration::from_secs(60),
+            ..AppConfig::default()
+        };
+        TrackingOutbox::from_config(
+            &config,
+            SpacetimeClient::new(spacetime_client::SpacetimeClientConfig {
+                server_url: "http://127.0.0.1:1".to_string(),
+                database: "missing".to_string(),
+                token: None,
+                pool_size: 1,
+                procedure_timeout: Duration::from_millis(10),
+            }),
+        )
+        .expect("outbox should be enabled")
+    }
+
+    #[tokio::test]
+    async fn enqueue_seals_active_file_when_batch_size_reached() {
+        let dir = test_dir("batch");
+        let outbox = test_outbox(dir.clone(), 2, 1024 * 1024);
+
+        outbox.enqueue(sample_event("event-1")).await.unwrap();
+        outbox.enqueue(sample_event("event-2")).await.unwrap();
+
+        assert!(!dir.join(ACTIVE_FILE_NAME).exists());
+        let sealed_count = std::fs::read_dir(&dir)
+            .unwrap()
+            .filter_map(Result::ok)
+            .filter(|entry| {
+                entry
+                    .file_name()
+                    .to_str()
+                    .is_some_and(|name| name.starts_with(SEALED_FILE_PREFIX))
+            })
+            .count();
+        assert_eq!(sealed_count, 1);
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
+    #[tokio::test]
+    async fn enqueue_drops_when_outbox_exceeds_max_bytes() {
+        let dir = test_dir("max-bytes");
+        let outbox = test_outbox(dir.clone(), 500, 1);
+
+        let outcome = outbox.enqueue(sample_event("event-1")).await.unwrap();
+
+        assert!(matches!(
+            outcome,
+            TrackingOutboxEnqueueOutcome::Dropped {
+                reason: "max_bytes"
+            }
+        ));
+        assert!(!dir.join(ACTIVE_FILE_NAME).exists());
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
+    #[tokio::test]
+    async fn flush_quarantines_corrupt_sealed_file() {
+        let dir = test_dir("corrupt");
+        std::fs::create_dir_all(&dir).unwrap();
+        let sealed_path = dir.join(format!("{SEALED_FILE_PREFIX}bad{SEALED_FILE_EXTENSION}"));
+        std::fs::write(&sealed_path, b"{not-json}\n").unwrap();
+        let outbox = test_outbox(dir.clone(), 500, 1024 * 1024);
+
+        outbox.flush_sealed_files_once().await.unwrap();
+
+        assert!(!sealed_path.exists());
+        let corrupt_count = std::fs::read_dir(&dir)
+            .unwrap()
+            .filter_map(Result::ok)
+            .filter(|entry| {
+                entry
+                    .file_name()
+                    .to_str()
+                    .is_some_and(|name| name.starts_with(CORRUPT_FILE_PREFIX))
+            })
+            .count();
+        assert_eq!(corrupt_count, 1);
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+
+    #[test]
+    fn directory_size_excludes_quarantined_corrupt_files() {
+        let dir = test_dir("directory-size");
+        std::fs::create_dir_all(&dir).unwrap();
+        std::fs::write(dir.join(ACTIVE_FILE_NAME), b"active").unwrap();
+        std::fs::write(
+            dir.join(format!("{SEALED_FILE_PREFIX}one{SEALED_FILE_EXTENSION}")),
+            b"sealed",
+        )
+        .unwrap();
+        std::fs::write(
+            dir.join(format!("{CORRUPT_FILE_PREFIX}one{SEALED_FILE_EXTENSION}")),
+            b"corrupt",
+        )
+        .unwrap();
+
+        let total = directory_size_if_exists(&dir).unwrap();
+
+        assert_eq!(total, 12);
+
+        let _ = std::fs::remove_dir_all(dir);
+    }
+}