GenarrativeAI/Genarrative
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: add loadtest observability setup

Browse Source
This commit is contained in:
kdletters
2026-05-16 22:44:30 +08:00
parent 7f16e88e57
commit 0305b79440
55 changed files with 2867 additions and 1622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
server-rs/crates/shared-logging/Cargo.toml
View File

@@ -5,4 +5,10 @@ version.workspace = true
license.workspace = true
[dependencies]
tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] }
opentelemetry = { workspace = true }
opentelemetry-otlp = { workspace = true }
opentelemetry-appender-tracing = { workspace = true }
opentelemetry_sdk = { workspace = true }
tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter", "fmt", "registry"] }

213
server-rs/crates/shared-logging/src/lib.rs
View File

@@ -1,6 +1,23 @@
use std::io;
use tracing_subscriber::{EnvFilter, fmt};
use opentelemetry::{KeyValue, global, trace::TracerProvider};
use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge;
use opentelemetry_otlp::WithExportConfig;
use opentelemetry_sdk::{
Resource,
logs::SdkLoggerProvider,
metrics::SdkMeterProvider,
trace::SdkTracerProvider,
};
use tracing::warn;
use tracing_subscriber::{
EnvFilter, Layer, filter::LevelFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt,
};
#[derive(Clone, Copy, Debug, Default)]
pub struct OtelConfig {
pub enabled: bool,
}
// 统一解析工作区日志过滤器，优先环境变量，其次回落到调用方传入的默认值。
pub fn resolve_env_filter(default_filter: &str) -> EnvFilter {
@@ -10,14 +27,196 @@ pub fn resolve_env_filter(default_filter: &str) -> EnvFilter {
}
// 统一初始化 tracing subscriber，避免各入口重复散落相同配置。
pub fn init_tracing(default_filter: &str) -> Result<(), io::Error> {
pub fn init_tracing(default_filter: &str, otel_config: OtelConfig) -> Result<(), io::Error> {
let env_filter = resolve_env_filter(default_filter);
let fmt_layer = fmt::layer().with_target(true).with_ansi(false).compact();
fmt()
.with_env_filter(env_filter)
.with_target(true)
.with_ansi(false)
.compact()
if !otel_config.enabled {
return tracing_subscriber::registry()
.with(env_filter)
.with(fmt_layer)
.try_init()
.map_err(|error| io::Error::other(format!("初始化 tracing subscriber 失败：{error}")));
}
let Some(otel) = build_otel_pipeline() else {
return tracing_subscriber::registry()
.with(env_filter)
.with(fmt_layer)
.try_init()
.map_err(|error| io::Error::other(format!("初始化 tracing subscriber 失败：{error}")));
};
tracing_subscriber::registry()
.with(env_filter)
.with(fmt_layer)
.with(
tracing_opentelemetry::layer()
.with_tracer(otel.tracer_provider.tracer("genarrative-api")),
)
.with(
OpenTelemetryTracingBridge::new(&otel.logger_provider).with_filter(LevelFilter::INFO),
)
.try_init()
.map_err(|error| io::Error::other(format!("初始化 tracing subscriber 失败：{error}")))
}
struct OtelPipeline {
tracer_provider: SdkTracerProvider,
_meter_provider: SdkMeterProvider,
logger_provider: SdkLoggerProvider,
}
fn build_otel_pipeline() -> Option<OtelPipeline> {
let resource = Resource::builder()
.with_service_name(read_env_or_default("OTEL_SERVICE_NAME", "genarrative-api"))
.with_attribute(KeyValue::new("service.namespace", "genarrative"))
.build();
let span_exporter = match opentelemetry_otlp::SpanExporter::builder()
.with_http()
.with_endpoint(resolve_otlp_http_signal_endpoint(
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"/v1/traces",
))
.build()
{
Ok(exporter) => exporter,
Err(error) => {
warn!(%error, "OpenTelemetry span exporter 初始化失败，已回退为本地日志");
return None;
}
};
let metric_exporter = match opentelemetry_otlp::MetricExporter::builder()
.with_http()
.with_endpoint(resolve_otlp_http_signal_endpoint(
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
"/v1/metrics",
))
.build()
{
Ok(exporter) => exporter,
Err(error) => {
warn!(%error, "OpenTelemetry metric exporter 初始化失败，已回退为本地日志");
return None;
}
};
let log_exporter = match opentelemetry_otlp::LogExporter::builder()
.with_http()
.with_endpoint(resolve_otlp_http_signal_endpoint(
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
"/v1/logs",
))
.build()
{
Ok(exporter) => exporter,
Err(error) => {
warn!(%error, "OpenTelemetry log exporter 初始化失败，已回退为本地日志");
return None;
}
};
let tracer_provider = SdkTracerProvider::builder()
.with_resource(resource.clone())
.with_batch_exporter(span_exporter)
.build();
let meter_provider = SdkMeterProvider::builder()
.with_resource(resource)
.with_periodic_exporter(metric_exporter)
.build();
let logger_provider = SdkLoggerProvider::builder()
.with_resource(Resource::builder()
.with_service_name(read_env_or_default("OTEL_SERVICE_NAME", "genarrative-api"))
.with_attribute(KeyValue::new("service.namespace", "genarrative"))
.build())
.with_batch_exporter(log_exporter)
.build();
global::set_tracer_provider(tracer_provider.clone());
global::set_meter_provider(meter_provider.clone());
Some(OtelPipeline {
tracer_provider,
_meter_provider: meter_provider,
logger_provider,
})
}
fn read_env_or_default(key: &str, default_value: &str) -> String {
std::env::var(key)
.ok()
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| default_value.to_string())
}
fn resolve_otlp_http_signal_endpoint(signal_key: &str, signal_path: &str) -> String {
if let Ok(value) = std::env::var(signal_key)
&& !value.trim().is_empty()
{
return value;
}
append_otlp_signal_path(
&read_env_or_default("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318"),
signal_path,
)
}
fn append_otlp_signal_path(base_endpoint: &str, signal_path: &str) -> String {
let base_endpoint = base_endpoint.trim_end_matches('/');
let signal_path = signal_path.trim_start_matches('/');
format!("{base_endpoint}/{signal_path}")
}
#[cfg(test)]
mod tests {
use std::sync::{Mutex, OnceLock};
use super::resolve_otlp_http_signal_endpoint;
const OTEL_ENDPOINT_ENV_KEYS: [&str; 4] = [
"OTEL_EXPORTER_OTLP_ENDPOINT",
"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT",
"OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
];
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: OnceLock<Mutex<()>> = OnceLock::new();
LOCK.get_or_init(|| Mutex::new(())).lock().unwrap()
}
fn clear_otel_endpoint_env() {
unsafe {
for key in OTEL_ENDPOINT_ENV_KEYS {
std::env::remove_var(key);
}
}
}
#[test]
fn generic_otlp_http_endpoint_expands_to_signal_paths() {
let _guard = env_lock();
clear_otel_endpoint_env();
unsafe {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:4318");
}
assert_eq!(
resolve_otlp_http_signal_endpoint("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "/v1/traces"),
"http://127.0.0.1:4318/v1/traces"
);
assert_eq!(
resolve_otlp_http_signal_endpoint("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "/v1/metrics"),
"http://127.0.0.1:4318/v1/metrics"
);
assert_eq!(
resolve_otlp_http_signal_endpoint("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "/v1/logs"),
"http://127.0.0.1:4318/v1/logs"
);
clear_otel_endpoint_env();
}
}