bench: support random image resolutions (#30879)

test/registered/bench_fn/test_bench_serving_reasoning_stream.py

@@ -1,10 +1,11 @@
-"""Unit tests for bench_serving streaming with reasoning_content chunks.
+"""Unit tests for bench_serving streaming with reasoning chunks.
 
 Reasoning models (DeepSeek-R1, MiMo, Qwen3 reasoning, Kimi-K2, ...) stream their
-chain-of-thought via OpenAI's `delta.reasoning_content` field. Without explicit
-support, bench_serving only inspects `delta.content` and silently reports zero
-TTFT / ITL and an empty `generated_text`, which then retokenizes to 0 tokens
-even though the backend completed real work.
+chain-of-thought via fields such as OpenAI's `delta.reasoning_content` and
+vLLM Kimi's `delta.reasoning`. Without explicit support, bench_serving only
+inspects `delta.content` and silently reports zero TTFT / ITL and an empty
+`generated_text`, which then retokenizes to 0 tokens even though the backend
+completed real work.
 """
 
 import asyncio
@@ -77,12 +78,16 @@ class _JSONHandler(BaseHTTPRequestHandler):
         return
 
 
-def _make_chunk(content=None, reasoning_content=None, completion_tokens=None):
+def _make_chunk(
+    content=None, reasoning_content=None, reasoning=None, completion_tokens=None
+):
     delta = {}
     if content is not None:
         delta["content"] = content
     if reasoning_content is not None:
         delta["reasoning_content"] = reasoning_content
+    if reasoning is not None:
+        delta["reasoning"] = reasoning
     chunk = {"choices": [{"index": 0, "delta": delta}]}
     if completion_tokens is not None:
         chunk["usage"] = {"completion_tokens": completion_tokens}
@@ -163,6 +168,21 @@ class TestBenchServingReasoningStream(CustomTestCase):
         self.assertEqual(out.text_chunks, ["me ", "think."])
         self.assertEqual(out.output_len, 3)
 
+    def test_vllm_kimi_reasoning_stream_populates_metrics(self):
+        chunks = [
+            _make_chunk(reasoning="Let "),
+            _make_chunk(reasoning="me "),
+            _make_chunk(reasoning="think."),
+            _make_chunk(completion_tokens=3),
+        ]
+        out = self._run(chunks)
+
+        self.assertTrue(out.success, msg=f"request failed: {out.error}")
+        self.assertEqual(out.generated_text, "Let me think.")
+        self.assertGreater(out.ttft, 0.0)
+        self.assertEqual(len(out.itl), 2, msg="should record ITL for chunks 2..N")
+        self.assertEqual(out.output_len, 3)
+
     def test_reasoning_then_content_accounts_both(self):
         chunks = [
             _make_chunk(reasoning_content="step1 "),

test/registered/bench_fn/test_benchmark_datasets_api.py

@@ -1,4 +1,6 @@
 import asyncio
+import base64
+import io
 import json
 import pickle
 import random
@@ -31,7 +33,10 @@ from sglang.benchmark.datasets.generated_shared_prefix import (
     get_gen_prefix_cache_path,
     sample_generated_shared_prefix_requests,
 )
-from sglang.benchmark.datasets.image import sample_image_requests
+from sglang.benchmark.datasets.image import (
+    parse_random_image_resolution,
+    sample_image_requests,
+)
 from sglang.benchmark.datasets.mmmu import sample_mmmu_requests
 from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time
 from sglang.benchmark.datasets.openai_dataset import sample_openai_requests
@@ -421,6 +426,66 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
         self.assertTrue(all(isinstance(row, DatasetRow) for row in rows))
         self.assertTrue(all(row.image_data for row in rows))
 
+    def test_image_sampler_vllm_chat(self):
+        rows = sample_image_requests(
+            num_requests=2,
+            image_count=1,
+            input_len=8,
+            output_len=4,
+            range_ratio=0.0,
+            processor=self.processor,
+            image_content="blank",
+            image_format="png",
+            image_resolution="8x8",
+            backend="vllm-chat",
+            random_image_count=False,
+        )
+        self.assertEqual(len(rows), 2)
+        self.assertTrue(all(isinstance(row, DatasetRow) for row in rows))
+        self.assertTrue(all(row.image_data for row in rows))
+        self.assertTrue(all("[IMAGE]" not in row.prompt for row in rows))
+
+    def test_image_sampler_random_resolution(self):
+        state = np.random.get_state()
+        np.random.seed(20260711)
+        try:
+            rows = sample_image_requests(
+                num_requests=4,
+                image_count=1,
+                input_len=8,
+                output_len=4,
+                range_ratio=0.0,
+                processor=self.processor,
+                image_content="blank",
+                image_format="png",
+                image_resolution="random:8x16-16x32",
+                backend="sglang",
+            )
+        finally:
+            np.random.set_state(state)
+
+        image_sizes = []
+        for row in rows:
+            encoded = row.image_data[0].split(",", maxsplit=1)[1]
+            with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:
+                image_sizes.append(image.size)
+
+        self.assertGreater(len(set(image_sizes)), 1)
+        for width, height in image_sizes:
+            self.assertGreaterEqual(width, 16)
+            self.assertLessEqual(width, 32)
+            self.assertGreaterEqual(height, 8)
+            self.assertLessEqual(height, 16)
+
+    def test_parse_random_image_resolution(self):
+        self.assertEqual(
+            parse_random_image_resolution("random:256x384-1024x1536"),
+            ((384, 256), (1536, 1024)),
+        )
+        self.assertIsNone(parse_random_image_resolution("256x384"))
+        with self.assertRaisesRegex(ValueError, "minimum cannot exceed"):
+            parse_random_image_resolution("random:1024x1024-256x256")
+
     def test_gen_mm_prompt_excludes_special_tokens(self):
         tokenizer = create_lightweight_tokenizer()
         multimodal_special_tokens = [