GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api-server): add container loadtest observability

Browse Source
This commit is contained in:
kdletters
2026-05-17 20:52:15 +08:00
parent 73f937d78a
commit 5a4a8a4892
36 changed files with 1325 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server-rs/crates/spacetime-client/src/big_fish.rs
View File

@@ -74,7 +74,7 @@ impl SpacetimeClient {
pub async fn list_big_fish_gallery(
&self,
) -> Result<Vec<BigFishWorkSummaryRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_big_fish_gallery", move |connection| {
let recent_play_counts = public_work_recent_play_counts(connection, "big-fish");
let mut items = connection
.db()

2
server-rs/crates/spacetime-client/src/custom_world.rs
View File

@@ -199,7 +199,7 @@ impl SpacetimeClient {
async fn read_custom_world_gallery_entries_from_cache(
&self,
) -> Result<Vec<CustomWorldGalleryEntryRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_custom_world_gallery", move |connection| {
let recent_play_counts = public_work_recent_play_counts(connection, "custom-world");
let mut entries = connection
.db()

12
server-rs/crates/spacetime-client/src/lib.rs
View File

@@ -407,12 +407,21 @@ impl SpacetimeClient {
async fn read_after_connect<T>(
&self,
read_name: &'static str,
read: impl FnOnce(&DbConnection) -> Result<T, SpacetimeClientError> + Send + 'static,
) -> Result<T, SpacetimeClientError>
where
T: Send + 'static,
{
let lease = self.acquire_connection().await?;
let metrics_guard = telemetry::begin_read(read_name);
let lease = match self.acquire_connection().await {
Ok(lease) => lease,
Err(error) => {
let final_result = Err(error);
metrics_guard.finish(&final_result);
return final_result;
}
};
let final_result = if let Some(connection) = lease.connection.as_ref() {
read(&connection.connection)
} else {
@@ -422,6 +431,7 @@ impl SpacetimeClient {
};
self.release_connection(lease).await;
metrics_guard.finish(&final_result);
final_result
}

2
server-rs/crates/spacetime-client/src/match3d.rs
View File

@@ -225,7 +225,7 @@ impl SpacetimeClient {
pub async fn list_match3d_gallery(
&self,
) -> Result<Vec<Match3DWorkProfileRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_match3d_gallery", move |connection| {
let mut items = connection
.db()
.match_3_d_gallery_view()

2
server-rs/crates/spacetime-client/src/puzzle.rs
View File

@@ -403,7 +403,7 @@ impl SpacetimeClient {
pub async fn list_puzzle_gallery(
&self,
) -> Result<Vec<PuzzleGalleryCardRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_puzzle_gallery", move |connection| {
let mut items = connection
.db()
.puzzle_gallery_card_view()

2
server-rs/crates/spacetime-client/src/runtime.rs
View File

@@ -5,7 +5,7 @@ impl SpacetimeClient {
&self,
) -> Result<CreationEntryConfigRecord, SpacetimeClientError> {
match self
.read_after_connect(move |connection| {
.read_after_connect("get_creation_entry_config", move |connection| {
let config_id = module_runtime::CREATION_ENTRY_CONFIG_GLOBAL_ID.to_string();
let header = connection
.db()

2
server-rs/crates/spacetime-client/src/square_hole.rs
View File

@@ -228,7 +228,7 @@ impl SpacetimeClient {
pub async fn list_square_hole_gallery(
&self,
) -> Result<Vec<SquareHoleWorkProfileRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_square_hole_gallery", move |connection| {
let mut items = connection
.db()
.square_hole_gallery_view()

50
server-rs/crates/spacetime-client/src/telemetry.rs
View File

@@ -10,6 +10,11 @@ pub(crate) struct ProcedureMetricsGuard {
started_at: std::time::Instant,
}
pub(crate) struct ReadMetricsGuard {
read: &'static str,
started_at: std::time::Instant,
}
pub(crate) fn begin_procedure(procedure: &'static str) -> ProcedureMetricsGuard {
ProcedureMetricsGuard {
procedure,
@@ -17,6 +22,13 @@ pub(crate) fn begin_procedure(procedure: &'static str) -> ProcedureMetricsGuard
}
}
pub(crate) fn begin_read(read: &'static str) -> ReadMetricsGuard {
ReadMetricsGuard {
read,
started_at: std::time::Instant::now(),
}
}
impl ProcedureMetricsGuard {
pub(crate) fn finish<T>(&self, result: &Result<T, SpacetimeClientError>) {
let duration = self.started_at.elapsed();
@@ -24,10 +36,20 @@ impl ProcedureMetricsGuard {
}
}
impl ReadMetricsGuard {
pub(crate) fn finish<T>(&self, result: &Result<T, SpacetimeClientError>) {
let duration = self.started_at.elapsed();
record_read(self.read, duration, result.is_err());
}
}
struct SpacetimeMetrics {
calls: Counter<u64>,
errors: Counter<u64>,
duration_ms: opentelemetry::metrics::Histogram<f64>,
read_calls: Counter<u64>,
read_errors: Counter<u64>,
read_duration_ms: opentelemetry::metrics::Histogram<f64>,
}
fn spacetime_metrics() -> &'static SpacetimeMetrics {
@@ -48,6 +70,19 @@ fn spacetime_metrics() -> &'static SpacetimeMetrics {
.with_unit("ms")
.with_description("SpacetimeDB procedure duration in milliseconds")
.build(),
read_calls: meter
.u64_counter("genarrative.spacetime.read.calls")
.with_description("SpacetimeDB local subscription cache read count")
.build(),
read_errors: meter
.u64_counter("genarrative.spacetime.read.errors")
.with_description("SpacetimeDB local subscription cache read error count")
.build(),
read_duration_ms: meter
.f64_histogram("genarrative.spacetime.read.duration_ms")
.with_unit("ms")
.with_description("SpacetimeDB local subscription cache read duration in milliseconds")
.build(),
}
})
}
@@ -66,3 +101,18 @@ fn record_procedure(procedure: &'static str, duration: Duration, failed: bool) {
metrics.errors.add(1, &labels);
}
}
fn record_read(read: &'static str, duration: Duration, failed: bool) {
let labels = vec![
KeyValue::new("read", read),
KeyValue::new("status_class", if failed { "error" } else { "ok" }),
];
let metrics = spacetime_metrics();
metrics.read_calls.add(1, &labels);
metrics
.read_duration_ms
.record(duration.as_secs_f64() * 1000.0, &labels);
if failed {
metrics.read_errors.add(1, &labels);
}
}

2
server-rs/crates/spacetime-client/src/visual_novel.rs
View File

@@ -239,7 +239,7 @@ impl SpacetimeClient {
pub async fn list_visual_novel_gallery(
&self,
) -> Result<Vec<VisualNovelWorkProfileRecord>, SpacetimeClientError> {
self.read_after_connect(move |connection| {
self.read_after_connect("list_visual_novel_gallery", move |connection| {
let mut items = connection
.db()
.visual_novel_gallery_view()