[model-gateway] Add circuit breaker and discovery watcher metrics (#15094)

sgl-model-gateway/src/core/worker.rs

@@ -156,6 +156,16 @@ pub trait Worker: Send + Sync + fmt::Debug {
             CircuitState::HalfOpen => 2u8,
         };
         RouterMetrics::set_cb_state(self.url(), state_code);
+
+        // Update consecutive failures/successes gauges
+        RouterMetrics::set_cb_consecutive_failures(
+            self.url(),
+            self.circuit_breaker().failure_count(),
+        );
+        RouterMetrics::set_cb_consecutive_successes(
+            self.url(),
+            self.circuit_breaker().success_count(),
+        );
     }
 
     /// Check if this worker is DP-aware

sgl-model-gateway/src/observability/metrics.rs

@@ -65,6 +65,23 @@ pub fn init_metrics() {
         "sgl_router_cb_outcomes_total",
         "Total number of circuit breaker outcomes by worker and outcome type (success/failure)"
     );
+    describe_gauge!(
+        "sgl_router_cb_consecutive_failures",
+        "Current consecutive failure count per worker circuit breaker"
+    );
+    describe_gauge!(
+        "sgl_router_cb_consecutive_successes",
+        "Current consecutive success count per worker circuit breaker"
+    );
+
+    describe_counter!(
+        "sgl_router_discovery_watcher_errors_total",
+        "Total number of Kubernetes watcher errors"
+    );
+    describe_counter!(
+        "sgl_router_discovery_watcher_restarts_total",
+        "Total number of Kubernetes watcher restarts"
+    );
 
     describe_gauge!(
         "sgl_router_active_workers",
@@ -461,9 +478,33 @@ impl RouterMetrics {
         .increment(1);
     }
 
+    pub fn set_cb_consecutive_failures(worker: &str, count: u32) {
+        gauge!("sgl_router_cb_consecutive_failures",
+            "worker" => worker.to_string()
+        )
+        .set(count as f64);
+    }
+
+    pub fn set_cb_consecutive_successes(worker: &str, count: u32) {
+        gauge!("sgl_router_cb_consecutive_successes",
+            "worker" => worker.to_string()
+        )
+        .set(count as f64);
+    }
+
+    pub fn record_discovery_watcher_error() {
+        counter!("sgl_router_discovery_watcher_errors_total").increment(1);
+    }
+
+    pub fn record_discovery_watcher_restart() {
+        counter!("sgl_router_discovery_watcher_restarts_total").increment(1);
+    }
+
     // TODO delete the metrics (instead of setting them to zero)
     pub fn remove_worker_metrics(worker_url: &str) {
         gauge!("sgl_router_cb_state","worker" => worker_url.to_string()).set(0.0);
+        gauge!("sgl_router_cb_consecutive_failures","worker" => worker_url.to_string()).set(0.0);
+        gauge!("sgl_router_cb_consecutive_successes","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_worker_health","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_running_requests","worker" => worker_url.to_string()).set(0.0);
         gauge!("sgl_router_tree_size","worker" => worker_url.to_string()).set(0.0);

sgl-model-gateway/src/service_discovery.rs

@@ -300,6 +300,7 @@ pub async fn start_service_discovery(
                 }
                 Err(err) => {
                     error!("Error in Kubernetes watcher: {}", err);
+                    RouterMetrics::record_discovery_watcher_error();
                     warn!(
                         "Retrying in {} seconds with exponential backoff",
                         retry_delay.as_secs()
@@ -314,6 +315,7 @@ pub async fn start_service_discovery(
                 "Kubernetes watcher exited, restarting in {} seconds",
                 config_arc.check_interval.as_secs()
             );
+            RouterMetrics::record_discovery_watcher_restart();
             time::sleep(config_arc.check_interval).await;
         }
     });