[misc] Improve benchmark determinism and dataset API coverage (#33255)

benchmark/gsm8k/bench_sglang.py

@@ -89,7 +89,7 @@ def main(args):
                 messages,
                 tokenize=False,
                 add_generation_prompt=True,
-                enable_thinking=True,
+                **args.chat_template_kwargs,
             )
         questions.append(raw_question)
         labels.append(get_answer_value(lines[i]["answer"]))
@@ -184,6 +184,14 @@ if __name__ == "__main__":
         action="store_true",
         help="Enable thinking mode by wrapping prompts with chat template",
     )
+    parser.add_argument(
+        "--chat-template-kwargs",
+        type=json.loads,
+        default='{"enable_thinking": true}',
+        help="JSON dict passed through to tokenizer.apply_chat_template. "
+        "The thinking-toggle kwarg name is model-specific, e.g. "
+        "'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).",
+    )
     parser.add_argument(
         "--tokenizer-path",
         type=str,

python/sglang/benchmark/datasets/common.py

@@ -67,11 +67,13 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int
 @lru_cache(maxsize=1)
 def get_available_tokens(tokenizer):
     """Get valid token ids from the tokenizer vocabulary."""
-    return [
+    # Canonical order: vocab dict iteration order varies across tokenizers
+    # versions, which would break --seed reproducibility.
+    return sorted(
         token_id
         for token_id in tokenizer.get_vocab().values()
         if isinstance(token_id, int)
-    ]
+    )
 
 
 def gen_prompt(tokenizer, token_num):

python/sglang/benchmark/datasets/image.py

@@ -1,4 +1,5 @@
 import io
+import random
 import warnings
 from argparse import Namespace
 from dataclasses import dataclass
@@ -30,6 +31,7 @@ class ImageDataset(BaseDataset):
     image_resolution: str
     backend: str
     random_image_count: bool
+    seed: int
 
     @classmethod
     def from_args(cls, args: Namespace) -> "ImageDataset":
@@ -44,10 +46,15 @@ class ImageDataset(BaseDataset):
             image_resolution=args.image_resolution,
             backend=args.backend,
             random_image_count=args.random_image_count,
+            seed=args.seed,
         )
 
     def load(self, tokenizer=None, model_id=None) -> List[DatasetRow]:
         processor = get_processor(model_id)
+        # Processor initialization may consume global RNG state. Reset it here so
+        # --seed fixes the generated prompts, image sizes, and image contents.
+        random.seed(self.seed)
+        np.random.seed(self.seed)
         return sample_image_requests(
             num_requests=self.num_requests,
             image_count=self.image_count,
@@ -148,7 +155,7 @@ def create_mm_data_row(
             prompt_str = f"<image>{text_prompt}"
 
     # Calculate total tokens (text + vision)
-    if type(processor).__name__ == "KimiK25Processor":
+    if type(processor).__name__ in ("KimiK25Processor", "KimiK3Processor"):
         medias = [{"type": "image", "image": img} for img in images]
         prompt_len = processor(
             text=prompt_str,

python/sglang/benchmark/one_batch_server.py

@@ -1002,6 +1002,17 @@ def run_benchmark_internal(
                 "token_capacity", 1000000000
             )
 
+        # Router /get_server_info responses carry "router_manager"; worker
+        # responses never do, so its presence confirms a router by design.
+        if not internal_states and server_info.get("router_manager"):
+            print(
+                "WARNING: base_url points at a PD router; worker internal "
+                "states are unavailable, so the max-running-requests and "
+                "token-capacity skip guards are disabled."
+            )
+            skip_max_running_requests_threshold = float("inf")
+            skip_token_capacity_threshold = float("inf")
+        else:
             assert (
                 max_running_requests_per_dp > 0
             ), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}"

python/sglang/test/vlm_utils.py

@@ -215,14 +215,78 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase):
         with ThreadPoolExecutor(4) as executor:
             list(executor.map(self.run_decode_with_image, image_ids))
 
+    def test_image_prefix_cache_reuse(self):
+        """Image prefix (radix) cache correctness across requests.
+
+        Repeating an identical image must reuse the multimodal prefix without
+        changing the output, and a different image must NOT reuse the first
+        image's KV. This guards against image-token pad_value / feature-hash
+        regressions that would silently serve a cached *wrong* image's KV
+        (a correctness bug invisible to single-request tests). Pure greedy
+        request-level checks: no extra server flags, radix cache is on by
+        default.
+        """
+        client = openai.Client(api_key=self.api_key, base_url=self.base_url)
+
+        def describe(url: str) -> str:
+            response = client.chat.completions.create(
+                model="default",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "image_url", "image_url": {"url": url}},
+                            {
+                                "type": "text",
+                                "text": "Describe this image in one sentence.",
+                            },
+                        ],
+                    },
+                ],
+                temperature=0,
+                max_tokens=32,
+                **(self.get_vision_request_kwargs()),
+            )
+            assert response.usage.prompt_tokens > 0
+            content = response.choices[0].message.content
+            assert isinstance(content, str) and content
+            return content
+
+        # miss -> compute, then hit -> reuse the identical image's prefix
+        first = describe(IMAGE_MAN_IRONING_URL)
+        repeat = describe(IMAGE_MAN_IRONING_URL)
+        # a different image must be computed on its own, not reuse `first`'s KV
+        other = describe(IMAGE_SGL_LOGO_URL)
+        # the original image again, after a different one occupied the cache
+        first_again = describe(IMAGE_MAN_IRONING_URL)
+
+        self.assertEqual(
+            first,
+            repeat,
+            "Repeating an identical image changed the output; image prefix "
+            "reuse broke greedy determinism.",
+        )
+        self.assertEqual(
+            first,
+            first_again,
+            "The identical image after a different one changed the output; "
+            "image KV was cross-contaminated across requests.",
+        )
+        self.assertNotEqual(
+            first,
+            other,
+            "A different image produced an identical description; a wrong "
+            "image's KV may have been reused from the prefix cache.",
+        )
+
     def verify_single_image_response(self, response):
         assert response.choices[0].message.role == "assistant"
         text = response.choices[0].message.content
         assert isinstance(text, str)
 
         # `driver` is for gemma-3-it
-        assert (
+        assert any(
-            "man" in text or "person" or "driver" in text
+            keyword in text for keyword in ("man", "person", "driver")
         ), f"text: {text}, should contain man, person or driver"
         assert (
             "cab" in text

test/registered/bench_fn/test_benchmark_datasets_api.py

@@ -34,6 +34,7 @@ from sglang.benchmark.datasets.generated_shared_prefix import (
     sample_generated_shared_prefix_requests,
 )
 from sglang.benchmark.datasets.image import (
+    ImageDataset,
     parse_random_image_resolution,
     sample_image_requests,
 )
@@ -140,6 +141,21 @@ class DummyProcessor:
         return {"input_ids": _DummyTokenTensor(text_len + image_tokens)}
 
 
+class KimiK3Processor(DummyProcessor):
+    """Mimics the Kimi K3 HF processor's media-kwargs interface (#32541)."""
+
+    def __init__(self, tokenizer: PreTrainedTokenizerFast):
+        super().__init__(tokenizer)
+        self.media_call_count = 0
+
+    def __call__(self, text, medias=None, **kwargs):
+        if medias is None:
+            raise ValueError("Kimi K3 requires medias with text")
+        self.media_call_count += 1
+        text_len = len(self.tokenizer.encode(text))
+        return {"input_ids": _DummyTokenTensor(text_len + 4 * len(medias))}
+
+
 class _FakeMMMUDataset:
     def __init__(self, records):
         self.records = records
@@ -481,6 +497,26 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
                 for marker in ("user:", "assistant:", "[IMAGE]"):
                     self.assertNotIn(marker, rows[0].prompt)
 
+    def test_image_sampler_uses_kimi_k3_media_contract(self):
+        processor = KimiK3Processor(self.tokenizer)
+        rows = sample_image_requests(
+            num_requests=1,
+            image_count=1,
+            input_len=8,
+            output_len=4,
+            range_ratio=0.0,
+            processor=processor,
+            image_content="blank",
+            image_format="png",
+            image_resolution="8x8",
+            backend="sglang-oai-chat",
+            random_image_count=False,
+        )
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(processor.media_call_count, 1)
+        self.assertTrue(rows[0].image_data)
+
     def test_image_sampler_random_resolution(self):
         state = np.random.get_state()
         np.random.seed(20260711)
@@ -513,6 +549,43 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
             self.assertGreaterEqual(height, 8)
             self.assertLessEqual(height, 16)
 
+    def test_image_dataset_seed_is_independent_of_processor_initialization(self):
+        dataset = ImageDataset.from_args(
+            make_args(
+                num_prompts=3,
+                image_resolution="random:8x16-16x32",
+                seed=20260717,
+            )
+        )
+        processor_init_count = 0
+
+        def get_processor_with_rng_side_effects(_model_id):
+            nonlocal processor_init_count
+            processor_init_count += 1
+            random.random()
+            np.random.random(processor_init_count)
+            return self.processor
+
+        with patch(
+            "sglang.benchmark.datasets.image.get_processor",
+            side_effect=get_processor_with_rng_side_effects,
+        ):
+            first = dataset.load(model_id="test-model")
+            random.seed(999)
+            np.random.seed(999)
+            second = dataset.load(model_id="test-model")
+
+        self.assertEqual(
+            [
+                (row.prompt, row.prompt_len, row.output_len, row.image_data)
+                for row in first
+            ],
+            [
+                (row.prompt, row.prompt_len, row.output_len, row.image_data)
+                for row in second
+            ],
+        )
+
     def test_parse_random_image_resolution(self):
         self.assertEqual(
             parse_random_image_resolution("random:256x384-1024x1536"),
@@ -554,6 +627,30 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
         self.assertFalse(special_token_ids & sampled_pool)
         self.assertTrue(sampled_pool)
 
+    def test_gen_mm_prompt_is_independent_of_vocab_order(self):
+        class OrderedVocabTokenizer:
+            all_special_ids = []
+
+            def __init__(self, items):
+                self.vocab = dict(items)
+
+            def get_vocab(self):
+                return self.vocab
+
+            def decode(self, token_ids):
+                return " ".join(map(str, token_ids))
+
+        items = [(f"token_{token_id}", token_id) for token_id in range(32)]
+        first = OrderedVocabTokenizer(items)
+        second = OrderedVocabTokenizer(reversed(items))
+
+        random.seed(20260717)
+        first_prompt = gen_mm_prompt(first, image_pad_id=None, token_num=16)
+        random.seed(20260717)
+        second_prompt = gen_mm_prompt(second, image_pad_id=None, token_num=16)
+
+        self.assertEqual(first_prompt, second_prompt)
+
     def test_mmmu_sampler(self):
         fake_records = [
             {"image_1": Image.new("RGB", (4, 4), color="white"), "question": "q1"},