[model-gateway] add back router worker health metric and fix init state (#15622)

This commit is contained in:
fzyzcjy
2025-12-23 18:50:38 -08:00
committed by GitHub
parent 4d64f15086
commit 99d3bcdfed
3 changed files with 19 additions and 2 deletions
+1
View File
@@ -588,6 +588,7 @@ impl Worker for BasicWorker {
fn set_healthy(&self, healthy: bool) { fn set_healthy(&self, healthy: bool) {
self.healthy.store(healthy, Ordering::Release); self.healthy.store(healthy, Ordering::Release);
Metrics::set_worker_health(self.url(), healthy);
} }
async fn check_health_async(&self) -> WorkerResult<()> { async fn check_health_async(&self) -> WorkerResult<()> {
+5 -2
View File
@@ -9,7 +9,7 @@ use super::{
WorkerType, WorkerType,
}, },
}; };
use crate::routers::grpc::client::GrpcClient; use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient};
/// Builder for creating BasicWorker instances with fluent API /// Builder for creating BasicWorker instances with fluent API
pub struct BasicWorkerBuilder { pub struct BasicWorkerBuilder {
@@ -187,11 +187,14 @@ impl BasicWorkerBuilder {
None => OnceCell::new(), None => OnceCell::new(),
}); });
let healthy = true;
Metrics::set_worker_health(&self.url, healthy);
BasicWorker { BasicWorker {
metadata, metadata,
load_counter: Arc::new(AtomicUsize::new(0)), load_counter: Arc::new(AtomicUsize::new(0)),
processed_counter: Arc::new(AtomicUsize::new(0)), processed_counter: Arc::new(AtomicUsize::new(0)),
healthy: Arc::new(AtomicBool::new(true)), healthy: Arc::new(AtomicBool::new(healthy)),
consecutive_failures: Arc::new(AtomicUsize::new(0)), consecutive_failures: Arc::new(AtomicUsize::new(0)),
consecutive_successes: Arc::new(AtomicUsize::new(0)), consecutive_successes: Arc::new(AtomicUsize::new(0)),
circuit_breaker: CircuitBreaker::with_config_and_label( circuit_breaker: CircuitBreaker::with_config_and_label(
@@ -174,6 +174,10 @@ pub fn init_metrics() {
"smg_worker_requests_active", "smg_worker_requests_active",
"Currently running requests per worker" "Currently running requests per worker"
); );
describe_gauge!(
"smg_worker_health",
"Worker health status (1=healthy, 0=unhealthy)"
);
describe_counter!( describe_counter!(
"smg_worker_health_checks_total", "smg_worker_health_checks_total",
"Health check results by worker_type and result" "Health check results by worker_type and result"
@@ -807,6 +811,15 @@ impl Metrics {
.set(count as f64); .set(count as f64);
} }
/// Set worker health status
pub fn set_worker_health(worker_url: &str, healthy: bool) {
gauge!(
"smg_worker_health",
"worker" => worker_url.to_string()
)
.set(if healthy { 1.0 } else { 0.0 });
}
// ======================================================================== // ========================================================================
// Layer 3: Worker resilience metrics (circuit breaker) // Layer 3: Worker resilience metrics (circuit breaker)
// ======================================================================== // ========================================================================