[CI] Restore SMG e2e on 2-gpu-h100 / 4-gpu-h100 runners (#24222)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

sgl-model-gateway/e2e_test/infra/gpu_allocator.py

@@ -74,12 +74,25 @@ class GPUSlot:
 
 
 def get_open_port() -> int:
-    """Get an available port by binding to port 0 and reading the assigned port."""
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("", 0))
-        s.listen(1)
-        port = s.getsockname()[1]
-    return port
+    """Get an available port by binding to port 0 and reading the assigned port.
+
+    Capped below 55536 so sglang's `--grpc-mode` default
+    `grpc_port = port + 10000` (in srt/server_args.py) cannot overflow
+    16-bit port range. The kernel's ephemeral range is typically
+    32768-60999, so a cap > 32768 keeps reasonable headroom while
+    avoiding the rare overflow that crashes worker startup.
+    """
+    for _ in range(20):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(("", 0))
+            s.listen(1)
+            port = s.getsockname()[1]
+        if port < 55536:
+            return port
+    raise RuntimeError(
+        "Failed to allocate an open port below 55536 after 20 attempts; "
+        "kernel ephemeral range may be unusually high"
+    )
 
 
 def get_physical_device_indices(devices: list[int]) -> list[int]:

sgl-model-gateway/e2e_test/infra/process_utils.py

@@ -127,11 +127,41 @@ def wait_for_workers_ready(
 
 
 def detect_ib_device() -> str | None:
-    """Detect first active InfiniBand device (e.g., mlx5_0).
+    """Detect first active InfiniBand device usable for PD KV transfer.
+
+    Enumerates `/sys/class/infiniband/` (the canonical device list that
+    sglang's `_validate_ib_devices` checks against) and returns the first
+    PORT_ACTIVE port. Prioritizes ``mlx5_ib*`` (native InfiniBand) over
+    ``mlx5_eth*`` (Ethernet/RoCE) — on the production CI runners both
+    families show up under /sys/class/infiniband, but ``mlx5_eth*`` are
+    plain Ethernet ports that don't carry the RDMA traffic mooncake
+    needs for prefill/decode KV transfer. Picking an eth port led to
+    decode-side 500s on every PD MMLU request (worker comes up healthy
+    but KV reads fail at request time).
+
+    Avoids the legacy `mlx5_<N>` alias form: on these runners
+    ``ibv_devinfo mlx5_0`` resolves to ``mlx5_ib0`` and reports active,
+    but sglang's `_validate_ib_devices` walks /sys/class/infiniband and
+    rejects the alias name.
 
     Returns:
-        Device name if found (e.g., "mlx5_0"), None otherwise.
+        Device name (e.g., ``mlx5_ib0``), or None if nothing usable.
     """
+    ib_dir = "/sys/class/infiniband"
+    try:
+        all_devs = os.listdir(ib_dir)
+    except FileNotFoundError:
+        return None
+
+    if not all_devs:
+        return None
+
+    # Prefer native IB ports over Ethernet/RoCE. Within each family keep
+    # /sys ordering stable.
+    ib_first = sorted(d for d in all_devs if "_ib" in d)
+    eth_last = sorted(d for d in all_devs if "_ib" not in d)
+    candidates = ib_first + eth_last
+
     try:
         subprocess.run(
             ["ibv_devinfo", "-l"],
@@ -142,8 +172,7 @@ def detect_ib_device() -> str | None:
     except (FileNotFoundError, subprocess.TimeoutExpired):
         return None
 
-    for i in range(12):
-        dev = f"mlx5_{i}"
+    for dev in candidates:
         try:
             res = subprocess.run(
                 ["ibv_devinfo", dev],
@@ -151,11 +180,12 @@ def detect_ib_device() -> str | None:
                 text=True,
                 timeout=2,
             )
-            if res.returncode == 0 and "state:" in res.stdout:
-                for line in res.stdout.splitlines():
-                    if "state:" in line and "PORT_ACTIVE" in line:
-                        logger.info("Detected IB device: %s", dev)
-                        return dev
+            if res.returncode != 0:
+                continue
+            for line in res.stdout.splitlines():
+                if "state:" in line and "PORT_ACTIVE" in line:
+                    logger.info("Detected IB device: %s", dev)
+                    return dev
         except Exception:
             pass
     return None