GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(api-server): audit external api failures

Browse Source
This commit is contained in:
kdletters
2026-05-21 16:33:13 +08:00
parent 487efff9c4
commit cc23b6020d
19 changed files with 2266 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
server-rs/crates/api-server/src/telemetry.rs
View File

@@ -172,6 +172,23 @@ pub(crate) fn update_tracking_outbox_pending_files(files: usize) {
TRACKING_OUTBOX_PENDING_FILES.store(files.min(i64::MAX as usize) as i64, Ordering::Relaxed);
}
pub(crate) fn record_external_api_failure(
provider: &'static str,
failure_stage: &'static str,
status_class: &'static str,
retryable: bool,
) {
external_api_metrics().failures.add(
1,
&[
KeyValue::new("provider", provider),
KeyValue::new("failure_stage", failure_stage),
KeyValue::new("status_class", status_class),
KeyValue::new("retryable", retryable),
],
);
}
fn track_response_body_in_flight(response: Response<Body>) -> Response<Body> {
response.map(|body| {
HTTP_RESPONSE_BODY_IN_FLIGHT.fetch_add(1, Ordering::Relaxed);
@@ -211,6 +228,10 @@ struct TrackingOutboxMetrics {
flushed_bytes: Counter<u64>,
}
struct ExternalApiMetrics {
failures: Counter<u64>,
}
struct HttpRequestPermitsAvailableGauges {
default: Arc<AtomicI64>,
gallery: Arc<AtomicI64>,
@@ -359,6 +380,21 @@ fn tracking_outbox_metrics() -> &'static TrackingOutboxMetrics {
})
}
fn external_api_metrics() -> &'static ExternalApiMetrics {
static METRICS: std::sync::OnceLock<ExternalApiMetrics> = std::sync::OnceLock::new();
METRICS.get_or_init(|| {
let meter = global::meter("genarrative-api");
ExternalApiMetrics {
failures: meter
.u64_counter("genarrative.external_api.failures")
.with_description(
"External API call failures grouped by provider and failure stage",
)
.build(),
}
})
}
fn register_http_request_permits_available_metric() -> HttpRequestPermitsAvailableGauges {
let gauges = HttpRequestPermitsAvailableGauges::new();
let meter = global::meter("genarrative-api");