[Router] Add bucket-aware policy domains and native cache indexing (#38108)

Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com>
Co-authored-by: inkcherry <mingzhi.liu@amd.com>
Co-authored-by: yangbodong22011 <13137470+yangbodong22011@users.noreply.github.com>

experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs

@@ -32,7 +32,7 @@ use sgl_router::server::app_context::AppContext;
 use sgl_router::tokenizer::TokenizerRegistry;
 use sgl_router::workers::WorkerRegistry;
 use std::sync::Arc;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 use tower::ServiceExt;
 
 fn config() -> Config {
@@ -46,6 +46,8 @@ fn config() -> Config {
             id: "tiny".into(),
             tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(),
             policy: PolicyKind::RoundRobin,
+            decode_policy: Default::default(),
+            bucket_config: None,
             circuit_breaker: None,
             cache_aware: None,
             sticky: None,
@@ -98,7 +100,7 @@ async fn await_captured_body(
     timeout: Duration,
     label: &str,
 ) -> Bytes {
-    let start = std::time::Instant::now();
+    let start = Instant::now();
     loop {
         // Release the `std::sync::Mutex` guard before the sleep.await
         // (clippy: await_holding_lock).
@@ -189,6 +191,43 @@ async fn pd_mode_chat_injects_bootstrap_fields_into_both_bodies() {
     assert_eq!(bootstrap_port(&dj), Some(8997));
 }
 
+#[tokio::test]
+async fn round_robin_pd_prefill_does_not_track_dispatch_timestamps() {
+    let prefill =
+        crate::common::mock_worker::MockWorker::start_hanging(Duration::from_millis(200)).await;
+    let decode = crate::common::mock_worker::MockWorker::start(vec![]).await;
+    let ctx = build_ctx(vec![
+        WorkerSpec {
+            id: WorkerId("p1".into()),
+            url: prefill.url.clone(),
+            mode: WorkerMode::Prefill,
+            model_ids: vec![ModelId("tiny".into())],
+            bootstrap_port: Some(8997),
+        },
+        WorkerSpec {
+            id: WorkerId("d1".into()),
+            url: decode.url.clone(),
+            mode: WorkerMode::Decode,
+            model_ids: vec![ModelId("tiny".into())],
+            bootstrap_port: None,
+        },
+    ]);
+    let prefill_worker = ctx
+        .registry
+        .workers_for(&ModelId("tiny".into()))
+        .into_iter()
+        .find(|worker| worker.id.0 == "p1")
+        .expect("prefill worker is registered");
+    let cutoff = Instant::now() - Duration::from_secs(1);
+    let request = tokio::spawn(build_router(Arc::clone(&ctx)).oneshot(chat_request()));
+
+    await_captured_body(&prefill, Duration::from_secs(2), "prefill").await;
+    assert_eq!(prefill_worker.active_load(), 1);
+    assert_eq!(prefill_worker.slots_acquired_since(cutoff), 0);
+
+    assert_eq!(request.await.unwrap().unwrap().status(), StatusCode::OK);
+}
+
 /// Plain-mode (non-PD) requests do NOT carry any `bootstrap_*` field.
 /// The injection step is gated on `worker.mode() == Prefill`; plain
 /// workers serve the chat route directly without disagg bootstrapping.