[Router] Real-GPU e2e coverage for storage-tier-aware cache routing (3/4) (#39110)

Co-authored-by: Kangyan Zhou <kangyan.zhou@radixark.ai>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

experimental/sgl-router/tests/e2e/infra/model_pool.py

@@ -24,8 +24,10 @@ import os
 import signal
 import socket
 import subprocess
+import tempfile
 import time
 from dataclasses import dataclass, field
+from pathlib import Path
 
 import httpx
 
@@ -84,8 +86,20 @@ class ModelInstance:
     model_id: str
     gpu_ids: list[int] = field(default_factory=list)
     kv_events_endpoint: str | None = None
+    log_path: Path | None = None
     _shutdown_started: bool = field(default=False, init=False, repr=False)
 
+    def log_tail(self, lines: int = 200) -> str:
+        """Last `lines` of the worker's log, for failure diagnostics."""
+        if self.log_path is None:
+            return "(no log file)"
+        try:
+            return "\n".join(
+                self.log_path.read_text(errors="replace").splitlines()[-lines:]
+            )
+        except OSError:
+            return f"({self.log_path} unreadable)"
+
     def __enter__(self) -> "ModelInstance":
         return self
 
@@ -198,13 +212,30 @@ def spawn_worker(
         disagg_mode,
     )
 
-    proc = subprocess.Popen(
-        cmd,
-        env=env,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        start_new_session=True,
-    )
+    # Stream the worker's output to a file rather than an unread
+    # subprocess.PIPE. Nothing in this process drains that pipe, so once its
+    # ~64 KB OS buffer fills the engine blocks on write and stops serving —
+    # requests then hang until the client timeout with no log to explain it.
+    # Startup alone (weight load, memory pool, CUDA-graph capture) can
+    # approach that, and a long test's per-request logging goes past it.
+    # `conftest.py`'s session-scoped fixture already learned this; this is the
+    # same fix for the per-test workers.
+    log_path = Path(tempfile.gettempdir()) / f"sglang-worker-{port}.log"
+    log_handle = open(log_path, "w", buffering=1)  # line-buffered
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            env=env,
+            stdout=log_handle,
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+    finally:
+        # The child keeps its own descriptor, so the parent's copy is done
+        # with. Holding it would leak one fd per worker for the session, and
+        # leave the file open with nothing writing through it. Failures read
+        # the log back from `log_path`, not from this handle.
+        log_handle.close()
 
     inst = ModelInstance(
         url=base_url,
@@ -213,6 +244,7 @@ def spawn_worker(
         model_id=model_id,
         gpu_ids=list(gpu_ids),
         kv_events_endpoint=kv_events_endpoint,
+        log_path=log_path,
     )
 
     # Wait for /health. Cold-start on H200 with weights uncached can take
@@ -220,15 +252,9 @@ def spawn_worker(
     deadline = time.time() + timeout
     while time.time() < deadline:
         if proc.poll() is not None:
-            out = b""
-            try:
-                if proc.stdout is not None:
-                    out = proc.stdout.read() or b""
-            except Exception:  # noqa: BLE001
-                pass
             raise RuntimeError(
                 f"sglang worker exited during startup with code {proc.returncode}; "
-                f"cmd: {' '.join(cmd)}\noutput:\n{out.decode(errors='replace')}",
+                f"cmd: {' '.join(cmd)}\noutput:\n{inst.log_tail()}",
             )
         try:
             resp = httpx.get(f"{base_url}/health", timeout=2.0)
@@ -241,5 +267,6 @@ def spawn_worker(
 
     inst.shutdown()
     raise TimeoutError(
-        f"sglang worker did not become healthy at {base_url} within {timeout}s",
+        f"sglang worker did not become healthy at {base_url} within {timeout}s; "
+        f"last log lines:\n{inst.log_tail()}",
     )