GenarrativeAI/Genarrative
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: add loadtest observability setup

Browse Source
This commit is contained in:
kdletters
2026-05-16 22:44:30 +08:00
parent 7f16e88e57
commit 0305b79440
55 changed files with 2867 additions and 1622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
server-rs/crates/api-server/src/app.rs
View File

@@ -11,7 +11,7 @@ use tower_http::{
classify::ServerErrorsFailureClass,
trace::{DefaultOnRequest, TraceLayer},
};
use tracing::{Level, Span, error, info, info_span, warn};
use tracing::{Level, Span, error, info_span};
use crate::{
auth::{AuthenticatedAccessToken, require_bearer_auth},
@@ -22,6 +22,7 @@ use crate::{
response_headers::propagate_request_id_header,
runtime_inventory::get_runtime_inventory_state,
state::AppState,
telemetry::record_http_observability,
tracking::record_route_tracking_event_after_success,
vector_engine_audio_generation::{
create_background_music_task, create_sound_effect_task,
@@ -42,8 +43,6 @@ use crate::{
// 统一由这里构造 Axum 路由树，后续再逐项挂接中间件与业务路由。
pub fn build_router(state: AppState) -> Router {
let slow_request_threshold_ms = state.config.slow_request_threshold_ms;
Router::new()
.merge(modules::admin::router(state.clone()))
.merge(modules::health::router(state.clone()))
@@ -86,47 +85,55 @@ pub fn build_router(state: AppState) -> Router {
state.clone(),
record_api_tracking_after_success,
))
// HTTP 指标与请求完成日志放在 tracing span 内侧，日志事件可以继承当前 trace/span context。
.layer(middleware::from_fn_with_state(
state.clone(),
record_http_observability,
))
// 当前阶段先统一挂接 HTTP tracing，后续 request_id、响应头与错误中间件继续在这里扩展。
.layer(
TraceLayer::new_for_http()
.make_span_with(|request: &Request<Body>| {
let request_id =
resolve_request_id(request).unwrap_or_else(|| "unknown".to_string());
let route = crate::telemetry::observability_route(request.uri().path());
let scheme = crate::telemetry::resolve_request_scheme(request.headers());
let span_name = format!("{} {}", request.method(), route);
info_span!(
"http.request",
otel.kind = "server",
otel.name = %span_name,
otel.status_code = tracing::field::Empty,
http.response.status_code = tracing::field::Empty,
method = %request.method(),
uri = %request.uri(),
http.request.method = %request.method(),
http.route = %route,
url.scheme = %scheme,
url.path = %request.uri().path(),
request_id = %request_id,
status = tracing::field::Empty,
latency_ms = tracing::field::Empty,
)
})
.on_request(DefaultOnRequest::new().level(Level::INFO))
.on_response(
move |response: &axum::response::Response,
latency: std::time::Duration,
span: &Span| {
|response: &axum::response::Response,
latency: std::time::Duration,
span: &Span| {
let latency_ms = latency.as_millis().min(u64::MAX as u128) as u64;
let status = response.status().as_u16();
let slow_request = latency_ms >= slow_request_threshold_ms;
span.record("status", status);
span.record("http.response.status_code", status);
span.record(
"otel.status_code",
if response.status().is_server_error() {
"ERROR"
} else {
"OK"
},
);
span.record("latency_ms", latency_ms);
if slow_request {
warn!(
parent: span,
status,
latency_ms,
slow_request = true,
"http request completed slowly"
);
} else {
info!(
parent: span,
status,
latency_ms,
slow_request = false,
"http request completed"
);
}
},
)
.on_failure(