[Router] Add load-aware prefill admission and bounded policy proposals (#37843)

Signed-off-by: Vincent Gao <vincentbo@linux.alibaba.com>
Co-authored-by: Kangyan Zhou <zky314343421@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>

experimental/sgl-router/tests/e2e/infra/model_pool.py

@@ -34,6 +34,18 @@ from .model_specs import get_model_spec
 logger = logging.getLogger(__name__)
 
 
+def _wait_for_process_group_exit(pgid: int, timeout: float) -> bool:
+    deadline = time.monotonic() + timeout
+    while True:
+        try:
+            os.killpg(pgid, 0)
+        except ProcessLookupError:
+            return True
+        if time.monotonic() >= deadline:
+            return False
+        time.sleep(0.1)
+
+
 def _get_open_port() -> int:
     """Allocate an ephemeral TCP port in the range [20000, 55535].
 
@@ -72,6 +84,7 @@ class ModelInstance:
     model_id: str
     gpu_ids: list[int] = field(default_factory=list)
     kv_events_endpoint: str | None = None
+    _shutdown_started: bool = field(default=False, init=False, repr=False)
 
     def __enter__(self) -> "ModelInstance":
         return self
@@ -80,16 +93,31 @@ class ModelInstance:
         self.shutdown()
 
     def shutdown(self) -> None:
-        if self.process is not None and self.process.poll() is None:
-            try:
-                self.process.send_signal(signal.SIGTERM)
-                try:
-                    self.process.wait(timeout=60)
-                except subprocess.TimeoutExpired:
-                    self.process.kill()
-                    self.process.wait()
-            except ProcessLookupError:
-                pass
+        if self.process is None or self._shutdown_started:
+            return
+        self._shutdown_started = True
+        pgid = self.process.pid
+
+        try:
+            os.killpg(pgid, signal.SIGTERM)
+        except ProcessLookupError:
+            return
+
+        try:
+            self.process.wait(timeout=60)
+        except subprocess.TimeoutExpired:
+            pass
+
+        if _wait_for_process_group_exit(pgid, timeout=30):
+            return
+
+        try:
+            os.killpg(pgid, signal.SIGKILL)
+        except ProcessLookupError:
+            return
+        self.process.wait()
+        if not _wait_for_process_group_exit(pgid, timeout=5):
+            raise RuntimeError(f"worker process group {pgid} did not exit")
 
 
 def spawn_worker(

experimental/sgl-router/tests/e2e/infra/test_model_pool.py

@@ -0,0 +1,50 @@
+import signal
+
+from infra import model_pool
+
+
+class _Process:
+    pid = 1234
+
+    def __init__(self):
+        self.wait_timeouts = []
+
+    def poll(self):
+        return None
+
+    def send_signal(self, sig):
+        raise AssertionError(f"signaled only the parent process: {sig}")
+
+    def wait(self, timeout=None):
+        self.wait_timeouts.append(timeout)
+        return 0
+
+
+def test_shutdown_waits_for_the_worker_process_group(monkeypatch):
+    process = _Process()
+    signals = []
+    probes = iter([True, True, False])
+
+    def killpg(pgid, sig):
+        signals.append((pgid, sig))
+        if sig == 0 and not next(probes):
+            raise ProcessLookupError
+
+    monkeypatch.setattr(model_pool.os, "killpg", killpg)
+    monkeypatch.setattr(model_pool.time, "sleep", lambda _: None)
+
+    instance = model_pool.ModelInstance(
+        url="http://127.0.0.1:30000",
+        port=30000,
+        process=process,
+        model_id="qwen3-0.6b",
+    )
+    instance.shutdown()
+
+    assert signals == [
+        (process.pid, signal.SIGTERM),
+        (process.pid, 0),
+        (process.pid, 0),
+        (process.pid, 0),
+    ]
+    assert process.wait_timeouts == [60]