[Router] Add load-aware prefill admission and bounded policy proposals (#37843)

Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com>
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>

experimental/sgl-router/tests/e2e/k8s_integration/conftest.py

@@ -11,6 +11,7 @@ Teardown: ./tests/e2e/k8s_integration/setup.sh teardown
 
 from __future__ import annotations
 
+import json
 import logging
 import socket
 import subprocess
@@ -97,6 +98,60 @@ def _wait_for_pod_ready(
     )
 
 
+def _wait_for_replacement_pod_ready(
+    old_pod: str,
+    selector: str,
+    namespace: str = NAMESPACE,
+    timeout: int = 120,
+    interval: float = 0.5,
+) -> str:
+    deadline = time.time() + timeout
+    last_observed = "no pods"
+
+    while time.time() < deadline:
+        result = _kubectl(
+            "get",
+            "pods",
+            "-n",
+            namespace,
+            "-l",
+            selector,
+            "-o",
+            "json",
+            check=False,
+        )
+        if getattr(result, "returncode", 0) == 0:
+            pods = json.loads(result.stdout or "{}").get("items", [])
+            names = [pod.get("metadata", {}).get("name", "") for pod in pods]
+            last_observed = ", ".join(filter(None, names)) or "no pods"
+
+            if old_pod not in names:
+                for pod in sorted(
+                    pods, key=lambda item: item.get("metadata", {}).get("name", "")
+                ):
+                    metadata = pod.get("metadata", {})
+                    status = pod.get("status", {})
+                    ready = any(
+                        condition.get("type") == "Ready"
+                        and condition.get("status") == "True"
+                        for condition in status.get("conditions", [])
+                    )
+                    if (
+                        metadata.get("name") != old_pod
+                        and not metadata.get("deletionTimestamp")
+                        and status.get("phase") == "Running"
+                        and ready
+                    ):
+                        return metadata["name"]
+
+        time.sleep(interval)
+
+    raise TimeoutError(
+        f"No ready replacement for pod {old_pod!r} with selector {selector!r} "
+        f"after {timeout}s; last observed: {last_observed}"
+    )
+
+
 def _wait_for_port(port: int, proc: subprocess.Popen, timeout: int = 15) -> None:
     """Poll until a TCP connection to localhost:port succeeds."""
     deadline = time.time() + timeout

experimental/sgl-router/tests/e2e/k8s_integration/test_conftest_helpers.py

@@ -0,0 +1,54 @@
+import json
+from types import SimpleNamespace
+
+import conftest as k8s_conftest
+
+
+def _pod(name: str, phase: str, ready: bool) -> dict:
+    return {
+        "metadata": {"name": name},
+        "status": {
+            "phase": phase,
+            "conditions": [
+                {
+                    "type": "Ready",
+                    "status": "True" if ready else "False",
+                }
+            ],
+        },
+    }
+
+
+def test_wait_for_replacement_pod_ignores_old_and_pending_pods(monkeypatch):
+    old_pod = "sgl-router-old"
+    new_pod = "sgl-router-new"
+    responses = iter(
+        [
+            [_pod(old_pod, "Running", True)],
+            [
+                _pod(old_pod, "Running", True),
+                _pod(new_pod, "Running", True),
+            ],
+            [_pod(new_pod, "Pending", False)],
+            [_pod(new_pod, "Running", True)],
+        ]
+    )
+    calls = []
+
+    def fake_kubectl(*args, **kwargs):
+        calls.append((args, kwargs))
+        return SimpleNamespace(stdout=json.dumps({"items": next(responses)}))
+
+    monkeypatch.setattr(k8s_conftest, "_kubectl", fake_kubectl)
+    monkeypatch.setattr(k8s_conftest.time, "sleep", lambda _: None)
+
+    replacement = k8s_conftest._wait_for_replacement_pod_ready(
+        old_pod,
+        "app=sgl-router",
+        timeout=5,
+        interval=0,
+    )
+
+    assert replacement == new_pod
+    assert len(calls) == 4
+    assert all("-o" in args and "json" in args for args, _ in calls)

experimental/sgl-router/tests/e2e/k8s_integration/test_lifecycle.py

@@ -13,10 +13,7 @@ by driving the deployment scale.
 
 from __future__ import annotations
 
-import logging
-
 import httpx
-import pytest
 from conftest import (
     NAMESPACE,
     _cleanup_port_forward,
@@ -24,7 +21,7 @@ from conftest import (
     _poll_until,
     _port_forward_start,
     _wait_for_deployment_ready,
-    logger,
+    _wait_for_replacement_pod_ready,
 )
 
 ROUTER_RESTART_PORT = 8092
@@ -132,7 +129,10 @@ class TestRouterRestart:
                 _cleanup_port_forward("router-restart-pre-kill", pf_holder[0])
                 pf_holder[0] = None
 
-            _wait_for_deployment_ready("sgl-router")
+            if old_pod:
+                _wait_for_replacement_pod_ready(old_pod, "app=sgl-router")
+            else:
+                _wait_for_deployment_ready("sgl-router")
 
             pf_holder[0] = _port_forward_start(
                 NAMESPACE, "sgl-router", ROUTER_RESTART_PORT, 8090