[misc] Improve benchmark determinism and dataset API coverage (#33255)

test/registered/bench_fn/test_benchmark_datasets_api.py

@@ -34,6 +34,7 @@ from sglang.benchmark.datasets.generated_shared_prefix import (
     sample_generated_shared_prefix_requests,
 )
 from sglang.benchmark.datasets.image import (
+    ImageDataset,
     parse_random_image_resolution,
     sample_image_requests,
 )
@@ -140,6 +141,21 @@ class DummyProcessor:
         return {"input_ids": _DummyTokenTensor(text_len + image_tokens)}
 
 
+class KimiK3Processor(DummyProcessor):
+    """Mimics the Kimi K3 HF processor's media-kwargs interface (#32541)."""
+
+    def __init__(self, tokenizer: PreTrainedTokenizerFast):
+        super().__init__(tokenizer)
+        self.media_call_count = 0
+
+    def __call__(self, text, medias=None, **kwargs):
+        if medias is None:
+            raise ValueError("Kimi K3 requires medias with text")
+        self.media_call_count += 1
+        text_len = len(self.tokenizer.encode(text))
+        return {"input_ids": _DummyTokenTensor(text_len + 4 * len(medias))}
+
+
 class _FakeMMMUDataset:
     def __init__(self, records):
         self.records = records
@@ -481,6 +497,26 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
                 for marker in ("user:", "assistant:", "[IMAGE]"):
                     self.assertNotIn(marker, rows[0].prompt)
 
+    def test_image_sampler_uses_kimi_k3_media_contract(self):
+        processor = KimiK3Processor(self.tokenizer)
+        rows = sample_image_requests(
+            num_requests=1,
+            image_count=1,
+            input_len=8,
+            output_len=4,
+            range_ratio=0.0,
+            processor=processor,
+            image_content="blank",
+            image_format="png",
+            image_resolution="8x8",
+            backend="sglang-oai-chat",
+            random_image_count=False,
+        )
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(processor.media_call_count, 1)
+        self.assertTrue(rows[0].image_data)
+
     def test_image_sampler_random_resolution(self):
         state = np.random.get_state()
         np.random.seed(20260711)
@@ -513,6 +549,43 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
             self.assertGreaterEqual(height, 8)
             self.assertLessEqual(height, 16)
 
+    def test_image_dataset_seed_is_independent_of_processor_initialization(self):
+        dataset = ImageDataset.from_args(
+            make_args(
+                num_prompts=3,
+                image_resolution="random:8x16-16x32",
+                seed=20260717,
+            )
+        )
+        processor_init_count = 0
+
+        def get_processor_with_rng_side_effects(_model_id):
+            nonlocal processor_init_count
+            processor_init_count += 1
+            random.random()
+            np.random.random(processor_init_count)
+            return self.processor
+
+        with patch(
+            "sglang.benchmark.datasets.image.get_processor",
+            side_effect=get_processor_with_rng_side_effects,
+        ):
+            first = dataset.load(model_id="test-model")
+            random.seed(999)
+            np.random.seed(999)
+            second = dataset.load(model_id="test-model")
+
+        self.assertEqual(
+            [
+                (row.prompt, row.prompt_len, row.output_len, row.image_data)
+                for row in first
+            ],
+            [
+                (row.prompt, row.prompt_len, row.output_len, row.image_data)
+                for row in second
+            ],
+        )
+
     def test_parse_random_image_resolution(self):
         self.assertEqual(
             parse_random_image_resolution("random:256x384-1024x1536"),
@@ -554,6 +627,30 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
         self.assertFalse(special_token_ids & sampled_pool)
         self.assertTrue(sampled_pool)
 
+    def test_gen_mm_prompt_is_independent_of_vocab_order(self):
+        class OrderedVocabTokenizer:
+            all_special_ids = []
+
+            def __init__(self, items):
+                self.vocab = dict(items)
+
+            def get_vocab(self):
+                return self.vocab
+
+            def decode(self, token_ids):
+                return " ".join(map(str, token_ids))
+
+        items = [(f"token_{token_id}", token_id) for token_id in range(32)]
+        first = OrderedVocabTokenizer(items)
+        second = OrderedVocabTokenizer(reversed(items))
+
+        random.seed(20260717)
+        first_prompt = gen_mm_prompt(first, image_pad_id=None, token_num=16)
+        random.seed(20260717)
+        second_prompt = gen_mm_prompt(second, image_pad_id=None, token_num=16)
+
+        self.assertEqual(first_prompt, second_prompt)
+
     def test_mmmu_sampler(self):
         fake_records = [
             {"image_1": Image.new("RGB", (4, 4), color="white"), "question": "q1"},