[misc] Improve benchmark determinism and dataset API coverage (#33255)

python/sglang/benchmark/datasets/common.py

@@ -67,11 +67,13 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int
 @lru_cache(maxsize=1)
 def get_available_tokens(tokenizer):
     """Get valid token ids from the tokenizer vocabulary."""
-    return [
+    # Canonical order: vocab dict iteration order varies across tokenizers
+    # versions, which would break --seed reproducibility.
+    return sorted(
         token_id
         for token_id in tokenizer.get_vocab().values()
         if isinstance(token_id, int)
-    ]
+    )
 
 
 def gen_prompt(tokenizer, token_num):

python/sglang/benchmark/datasets/image.py

@@ -1,4 +1,5 @@
 import io
+import random
 import warnings
 from argparse import Namespace
 from dataclasses import dataclass
@@ -30,6 +31,7 @@ class ImageDataset(BaseDataset):
     image_resolution: str
     backend: str
     random_image_count: bool
+    seed: int
 
     @classmethod
     def from_args(cls, args: Namespace) -> "ImageDataset":
@@ -44,10 +46,15 @@ class ImageDataset(BaseDataset):
             image_resolution=args.image_resolution,
             backend=args.backend,
             random_image_count=args.random_image_count,
+            seed=args.seed,
         )
 
     def load(self, tokenizer=None, model_id=None) -> List[DatasetRow]:
         processor = get_processor(model_id)
+        # Processor initialization may consume global RNG state. Reset it here so
+        # --seed fixes the generated prompts, image sizes, and image contents.
+        random.seed(self.seed)
+        np.random.seed(self.seed)
         return sample_image_requests(
             num_requests=self.num_requests,
             image_count=self.image_count,
@@ -148,7 +155,7 @@ def create_mm_data_row(
             prompt_str = f"<image>{text_prompt}"
 
     # Calculate total tokens (text + vision)
-    if type(processor).__name__ == "KimiK25Processor":
+    if type(processor).__name__ in ("KimiK25Processor", "KimiK3Processor"):
         medias = [{"type": "image", "image": img} for img in images]
         prompt_len = processor(
             text=prompt_str,

python/sglang/benchmark/one_batch_server.py

@@ -1002,10 +1002,21 @@ def run_benchmark_internal(
                 "token_capacity", 1000000000
             )
 
-        assert (
-            max_running_requests_per_dp > 0
-        ), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}"
-        skip_max_running_requests_threshold = max_running_requests_per_dp * dp_size
+        # Router /get_server_info responses carry "router_manager"; worker
+        # responses never do, so its presence confirms a router by design.
+        if not internal_states and server_info.get("router_manager"):
+            print(
+                "WARNING: base_url points at a PD router; worker internal "
+                "states are unavailable, so the max-running-requests and "
+                "token-capacity skip guards are disabled."
+            )
+            skip_max_running_requests_threshold = float("inf")
+            skip_token_capacity_threshold = float("inf")
+        else:
+            assert (
+                max_running_requests_per_dp > 0
+            ), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}"
+            skip_max_running_requests_threshold = max_running_requests_per_dp * dp_size
 
         print(f"{max_running_requests_per_dp=}")
         print(f"{dp_size=}")