[model-gateway] add back router worker health metric and fix init state (#15622)
This commit is contained in:
@@ -588,6 +588,7 @@ impl Worker for BasicWorker {
|
|||||||
|
|
||||||
fn set_healthy(&self, healthy: bool) {
|
fn set_healthy(&self, healthy: bool) {
|
||||||
self.healthy.store(healthy, Ordering::Release);
|
self.healthy.store(healthy, Ordering::Release);
|
||||||
|
Metrics::set_worker_health(self.url(), healthy);
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn check_health_async(&self) -> WorkerResult<()> {
|
async fn check_health_async(&self) -> WorkerResult<()> {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ use super::{
|
|||||||
WorkerType,
|
WorkerType,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
use crate::routers::grpc::client::GrpcClient;
|
use crate::{observability::metrics::Metrics, routers::grpc::client::GrpcClient};
|
||||||
|
|
||||||
/// Builder for creating BasicWorker instances with fluent API
|
/// Builder for creating BasicWorker instances with fluent API
|
||||||
pub struct BasicWorkerBuilder {
|
pub struct BasicWorkerBuilder {
|
||||||
@@ -187,11 +187,14 @@ impl BasicWorkerBuilder {
|
|||||||
None => OnceCell::new(),
|
None => OnceCell::new(),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let healthy = true;
|
||||||
|
Metrics::set_worker_health(&self.url, healthy);
|
||||||
|
|
||||||
BasicWorker {
|
BasicWorker {
|
||||||
metadata,
|
metadata,
|
||||||
load_counter: Arc::new(AtomicUsize::new(0)),
|
load_counter: Arc::new(AtomicUsize::new(0)),
|
||||||
processed_counter: Arc::new(AtomicUsize::new(0)),
|
processed_counter: Arc::new(AtomicUsize::new(0)),
|
||||||
healthy: Arc::new(AtomicBool::new(true)),
|
healthy: Arc::new(AtomicBool::new(healthy)),
|
||||||
consecutive_failures: Arc::new(AtomicUsize::new(0)),
|
consecutive_failures: Arc::new(AtomicUsize::new(0)),
|
||||||
consecutive_successes: Arc::new(AtomicUsize::new(0)),
|
consecutive_successes: Arc::new(AtomicUsize::new(0)),
|
||||||
circuit_breaker: CircuitBreaker::with_config_and_label(
|
circuit_breaker: CircuitBreaker::with_config_and_label(
|
||||||
|
|||||||
@@ -174,6 +174,10 @@ pub fn init_metrics() {
|
|||||||
"smg_worker_requests_active",
|
"smg_worker_requests_active",
|
||||||
"Currently running requests per worker"
|
"Currently running requests per worker"
|
||||||
);
|
);
|
||||||
|
describe_gauge!(
|
||||||
|
"smg_worker_health",
|
||||||
|
"Worker health status (1=healthy, 0=unhealthy)"
|
||||||
|
);
|
||||||
describe_counter!(
|
describe_counter!(
|
||||||
"smg_worker_health_checks_total",
|
"smg_worker_health_checks_total",
|
||||||
"Health check results by worker_type and result"
|
"Health check results by worker_type and result"
|
||||||
@@ -807,6 +811,15 @@ impl Metrics {
|
|||||||
.set(count as f64);
|
.set(count as f64);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set worker health status
|
||||||
|
pub fn set_worker_health(worker_url: &str, healthy: bool) {
|
||||||
|
gauge!(
|
||||||
|
"smg_worker_health",
|
||||||
|
"worker" => worker_url.to_string()
|
||||||
|
)
|
||||||
|
.set(if healthy { 1.0 } else { 0.0 });
|
||||||
|
}
|
||||||
|
|
||||||
// ========================================================================
|
// ========================================================================
|
||||||
// Layer 3: Worker resilience metrics (circuit breaker)
|
// Layer 3: Worker resilience metrics (circuit breaker)
|
||||||
// ========================================================================
|
// ========================================================================
|
||||||
|
|||||||
Reference in New Issue
Block a user