feat(api-server): add container loadtest observability

server-rs/crates/spacetime-client/src/telemetry.rs

@@ -10,6 +10,11 @@ pub(crate) struct ProcedureMetricsGuard {
     started_at: std::time::Instant,
 }
 
+pub(crate) struct ReadMetricsGuard {
+    read: &'static str,
+    started_at: std::time::Instant,
+}
+
 pub(crate) fn begin_procedure(procedure: &'static str) -> ProcedureMetricsGuard {
     ProcedureMetricsGuard {
         procedure,
@@ -17,6 +22,13 @@ pub(crate) fn begin_procedure(procedure: &'static str) -> ProcedureMetricsGuard
     }
 }
 
+pub(crate) fn begin_read(read: &'static str) -> ReadMetricsGuard {
+    ReadMetricsGuard {
+        read,
+        started_at: std::time::Instant::now(),
+    }
+}
+
 impl ProcedureMetricsGuard {
     pub(crate) fn finish<T>(&self, result: &Result<T, SpacetimeClientError>) {
         let duration = self.started_at.elapsed();
@@ -24,10 +36,20 @@ impl ProcedureMetricsGuard {
     }
 }
 
+impl ReadMetricsGuard {
+    pub(crate) fn finish<T>(&self, result: &Result<T, SpacetimeClientError>) {
+        let duration = self.started_at.elapsed();
+        record_read(self.read, duration, result.is_err());
+    }
+}
+
 struct SpacetimeMetrics {
     calls: Counter<u64>,
     errors: Counter<u64>,
     duration_ms: opentelemetry::metrics::Histogram<f64>,
+    read_calls: Counter<u64>,
+    read_errors: Counter<u64>,
+    read_duration_ms: opentelemetry::metrics::Histogram<f64>,
 }
 
 fn spacetime_metrics() -> &'static SpacetimeMetrics {
@@ -48,6 +70,19 @@ fn spacetime_metrics() -> &'static SpacetimeMetrics {
                 .with_unit("ms")
                 .with_description("SpacetimeDB procedure duration in milliseconds")
                 .build(),
+            read_calls: meter
+                .u64_counter("genarrative.spacetime.read.calls")
+                .with_description("SpacetimeDB local subscription cache read count")
+                .build(),
+            read_errors: meter
+                .u64_counter("genarrative.spacetime.read.errors")
+                .with_description("SpacetimeDB local subscription cache read error count")
+                .build(),
+            read_duration_ms: meter
+                .f64_histogram("genarrative.spacetime.read.duration_ms")
+                .with_unit("ms")
+                .with_description("SpacetimeDB local subscription cache read duration in milliseconds")
+                .build(),
         }
     })
 }
@@ -66,3 +101,18 @@ fn record_procedure(procedure: &'static str, duration: Duration, failed: bool) {
         metrics.errors.add(1, &labels);
     }
 }
+
+fn record_read(read: &'static str, duration: Duration, failed: bool) {
+    let labels = vec![
+        KeyValue::new("read", read),
+        KeyValue::new("status_class", if failed { "error" } else { "ok" }),
+    ];
+    let metrics = spacetime_metrics();
+    metrics.read_calls.add(1, &labels);
+    metrics
+        .read_duration_ms
+        .record(duration.as_secs_f64() * 1000.0, &labels);
+    if failed {
+        metrics.read_errors.add(1, &labels);
+    }
+}