bench: support random image resolutions (#30879)

python/sglang/benchmark/datasets/image.py

@@ -2,7 +2,7 @@ import io
 import warnings
 from argparse import Namespace
 from dataclasses import dataclass
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 import numpy as np
 import pybase64
@@ -93,6 +93,33 @@ def parse_image_resolution(image_resolution: str) -> Tuple[int, int]:
     )
 
 
+def parse_random_image_resolution(
+    image_resolution: str,
+) -> Optional[Tuple[Tuple[int, int], Tuple[int, int]]]:
+    """Parse ``random:<min_h>x<min_w>-<max_h>x<max_w>`` image bounds.
+
+    Returns ``None`` for fixed resolutions. The returned dimensions are
+    ``(width, height)`` pairs, matching :func:`parse_image_resolution`.
+    """
+
+    prefix = "random:"
+    if not image_resolution.strip().lower().startswith(prefix):
+        return None
+
+    bounds = image_resolution.strip()[len(prefix) :].split("-", maxsplit=1)
+    if len(bounds) != 2:
+        raise ValueError(
+            "Random image resolution must be 'random:<min_h>x<min_w>-"
+            "<max_h>x<max_w>', for example 'random:256x256-1024x1024'."
+        )
+
+    min_width, min_height = parse_image_resolution(bounds[0])
+    max_width, max_height = parse_image_resolution(bounds[1])
+    if min_width > max_width or min_height > max_height:
+        raise ValueError("Random image resolution minimum cannot exceed maximum.")
+    return (min_width, min_height), (max_width, max_height)
+
+
 def create_mm_data_row(
     text_prompt, images: list, images_base64, output_len, processor, backend
 ):
@@ -172,17 +199,22 @@ def create_mm_data_row(
     # Vision tokens = total tokens - text tokens
     vision_prompt_len = prompt_len - text_prompt_len
 
-    supported_backends = ["sglang", "sglang-native", "sglang-oai-chat"]
+    supported_backends = [
+        "sglang",
+        "sglang-native",
+        "sglang-oai-chat",
+        "vllm-chat",
+    ]
     if backend not in supported_backends:
         raise ValueError(
             f"Image dataset only supports backends: {supported_backends}, "
             f"got '{backend}'."
         )
 
-    # sglang-oai-chat: server's chat handler applies chat template, so send raw text.
+    # OpenAI chat handlers apply the chat template and receive images separately, so
-    # sglang/sglang-native: /generate does not apply chat template, so send prompt_str
+    # send the raw text. /generate does not apply a chat template, so it needs
-    #         which contains image placeholder tokens needed by the multimodal processor.
+    # prompt_str, which contains the multimodal processor's image placeholders.
-    use_raw_prompt = backend == "sglang-oai-chat"
+    use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat")
 
     return DatasetRow(
         prompt=text_prompt if use_raw_prompt else prompt_str,
@@ -211,14 +243,20 @@ def sample_image_requests(
 
     - If ``random_image_count`` is True, each request includes a random number of images between 1 and ``image_count``.
     - If ``random_image_count`` is False, each request includes exactly ``image_count`` images.
-    - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p (1280x720), 360p (640x360),
+    - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p
-      or custom 'heightxwidth' (e.g., 1080x1920).
+      (1280x720), 360p (640x360), custom ``heightxwidth`` (e.g.,
+      1080x1920), or ``random:<min_h>x<min_w>-<max_h>x<max_w>``.
     - Text lengths follow the 'random' dataset sampling rule. ``prompt_len``
       only counts text tokens and excludes image data.
     """
 
-    # Parse resolution (supports presets and 'heightxwidth')
+    random_resolution_bounds = parse_random_image_resolution(image_resolution)
+    if random_resolution_bounds is None:
         width, height = parse_image_resolution(image_resolution)
+        min_width = max_width = width
+        min_height = max_height = height
+    else:
+        (min_width, min_height), (max_width, max_height) = random_resolution_bounds
 
     # Determine image counts for each request
     if random_image_count:
@@ -231,9 +269,9 @@ def sample_image_requests(
         total_images = image_count * num_requests
 
     # Check for potentially problematic combinations and warn user
-    if width * height >= 1920 * 1080 and total_images >= 100:
+    if max_width * max_height >= 1920 * 1080 and total_images >= 100:
         warnings.warn(
-            f"High resolution ({width}x{height}) with {total_images} total images "
+            f"High resolution (up to {max_width}x{max_height}) with {total_images} total images "
             f"may take a long time. Consider reducing resolution or image count.",
             UserWarning,
             stacklevel=2,
@@ -251,9 +289,12 @@ def sample_image_requests(
         num=num_requests,
     )
 
-    def _gen_random_image_data_uri(
+    def _gen_random_image_data_uri() -> Tuple[Image.Image, str, int, Tuple[int, int]]:
-        width: int = width, height: int = height
+        if random_resolution_bounds is None:
-    ) -> Tuple[Image.Image, str, int]:
+            width, height = min_width, min_height
+        else:
+            width = np.random.randint(min_width, max_width + 1)
+            height = np.random.randint(min_height, max_height + 1)
         if image_content == "blank":
             # Generate blank white image
             arr = np.full((height, width, 3), 255, dtype=np.uint8)
@@ -266,10 +307,11 @@ def sample_image_requests(
         encoded = pybase64.b64encode(buf.getvalue()).decode("utf-8")
         image_data = f"data:image/{image_format};base64,{encoded}"
         image_bytes = len(image_data.encode("utf-8"))
-        return img, image_data, image_bytes
+        return img, image_data, image_bytes, (width, height)
 
     dataset: List[DatasetRow] = []
     total_image_bytes = 0
+    all_image_sizes: list[Tuple[int, int]] = []
     for i in range(num_requests):
         # Get the number of images for this request
         request_image_count = int(image_counts[i])
@@ -282,10 +324,11 @@ def sample_image_requests(
         )
 
         # Generate image list
-        images, images_base64, images_bytes = zip(
+        images, images_base64, images_bytes, image_sizes = zip(
             *[_gen_random_image_data_uri() for _ in range(request_image_count)]
         )
         total_image_bytes += sum(images_bytes)
+        all_image_sizes.extend(image_sizes)
 
         data_row = create_mm_data_row(
             text_prompt,
@@ -309,6 +352,15 @@ def sample_image_requests(
     else:
         print(f"#Images per request: {image_count} (fixed)")
 
+    if random_resolution_bounds is not None:
+        widths, heights = zip(*all_image_sizes)
+        print(
+            "#Image resolution: "
+            f"min={min(widths)}x{min(heights)}, "
+            f"max={max(widths)}x{max(heights)}, "
+            f"mean={np.mean(widths):.1f}x{np.mean(heights):.1f}"
+        )
+
     # Detailed token breakdown (derived from dataset + input_lens)
     text_prompt_lens = np.array([r.text_prompt_len for r in dataset])
     vision_prompt_lens = np.array([r.vision_prompt_len for r in dataset])

python/sglang/benchmark/serving.py

@@ -136,7 +136,12 @@ def get_request_headers() -> Dict[str, str]:
 
 
 def _combine_openai_chat_content(message: Dict[str, Any]) -> str:
-    return (message.get("reasoning_content") or "") + (message.get("content") or "")
+    # Most OpenAI-compatible servers use ``reasoning_content``. vLLM's Kimi
+    # parser instead streams its reasoning in ``reasoning``. Prefer the
+    # standard field when both are present to avoid counting the same tokens
+    # twice on servers that expose aliases.
+    reasoning = message.get("reasoning_content") or message.get("reasoning") or ""
+    return reasoning + (message.get("content") or "")
 
 
 def wait_for_endpoint(url: str, timeout_sec: int = 60) -> bool:
@@ -2281,7 +2286,9 @@ def cli_main():
         default="1080p",
         help=(
             "Resolution of images for image dataset. "
-            "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)."
+            "Supports presets 4k/1080p/720p/360p, custom 'heightxwidth' "
+            "(e.g., 1080x1920), or random 'random:<min_h>x<min_w>-<max_h>x<max_w>' "
+            "bounds (e.g., random:256x256-1024x1024)."
         ),
     )
     parser.add_argument(

test/registered/bench_fn/test_bench_serving_reasoning_stream.py

@@ -1,10 +1,11 @@
-"""Unit tests for bench_serving streaming with reasoning_content chunks.
+"""Unit tests for bench_serving streaming with reasoning chunks.
 
 Reasoning models (DeepSeek-R1, MiMo, Qwen3 reasoning, Kimi-K2, ...) stream their
-chain-of-thought via OpenAI's `delta.reasoning_content` field. Without explicit
+chain-of-thought via fields such as OpenAI's `delta.reasoning_content` and
-support, bench_serving only inspects `delta.content` and silently reports zero
+vLLM Kimi's `delta.reasoning`. Without explicit support, bench_serving only
-TTFT / ITL and an empty `generated_text`, which then retokenizes to 0 tokens
+inspects `delta.content` and silently reports zero TTFT / ITL and an empty
-even though the backend completed real work.
+`generated_text`, which then retokenizes to 0 tokens even though the backend
+completed real work.
 """
 
 import asyncio
@@ -77,12 +78,16 @@ class _JSONHandler(BaseHTTPRequestHandler):
         return
 
 
-def _make_chunk(content=None, reasoning_content=None, completion_tokens=None):
+def _make_chunk(
+    content=None, reasoning_content=None, reasoning=None, completion_tokens=None
+):
     delta = {}
     if content is not None:
         delta["content"] = content
     if reasoning_content is not None:
         delta["reasoning_content"] = reasoning_content
+    if reasoning is not None:
+        delta["reasoning"] = reasoning
     chunk = {"choices": [{"index": 0, "delta": delta}]}
     if completion_tokens is not None:
         chunk["usage"] = {"completion_tokens": completion_tokens}
@@ -163,6 +168,21 @@ class TestBenchServingReasoningStream(CustomTestCase):
         self.assertEqual(out.text_chunks, ["me ", "think."])
         self.assertEqual(out.output_len, 3)
 
+    def test_vllm_kimi_reasoning_stream_populates_metrics(self):
+        chunks = [
+            _make_chunk(reasoning="Let "),
+            _make_chunk(reasoning="me "),
+            _make_chunk(reasoning="think."),
+            _make_chunk(completion_tokens=3),
+        ]
+        out = self._run(chunks)
+
+        self.assertTrue(out.success, msg=f"request failed: {out.error}")
+        self.assertEqual(out.generated_text, "Let me think.")
+        self.assertGreater(out.ttft, 0.0)
+        self.assertEqual(len(out.itl), 2, msg="should record ITL for chunks 2..N")
+        self.assertEqual(out.output_len, 3)
+
     def test_reasoning_then_content_accounts_both(self):
         chunks = [
             _make_chunk(reasoning_content="step1 "),

test/registered/bench_fn/test_benchmark_datasets_api.py

@@ -1,4 +1,6 @@
 import asyncio
+import base64
+import io
 import json
 import pickle
 import random
@@ -31,7 +33,10 @@ from sglang.benchmark.datasets.generated_shared_prefix import (
     get_gen_prefix_cache_path,
     sample_generated_shared_prefix_requests,
 )
-from sglang.benchmark.datasets.image import sample_image_requests
+from sglang.benchmark.datasets.image import (
+    parse_random_image_resolution,
+    sample_image_requests,
+)
 from sglang.benchmark.datasets.mmmu import sample_mmmu_requests
 from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time
 from sglang.benchmark.datasets.openai_dataset import sample_openai_requests
@@ -421,6 +426,66 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase):
         self.assertTrue(all(isinstance(row, DatasetRow) for row in rows))
         self.assertTrue(all(row.image_data for row in rows))
 
+    def test_image_sampler_vllm_chat(self):
+        rows = sample_image_requests(
+            num_requests=2,
+            image_count=1,
+            input_len=8,
+            output_len=4,
+            range_ratio=0.0,
+            processor=self.processor,
+            image_content="blank",
+            image_format="png",
+            image_resolution="8x8",
+            backend="vllm-chat",
+            random_image_count=False,
+        )
+        self.assertEqual(len(rows), 2)
+        self.assertTrue(all(isinstance(row, DatasetRow) for row in rows))
+        self.assertTrue(all(row.image_data for row in rows))
+        self.assertTrue(all("[IMAGE]" not in row.prompt for row in rows))
+
+    def test_image_sampler_random_resolution(self):
+        state = np.random.get_state()
+        np.random.seed(20260711)
+        try:
+            rows = sample_image_requests(
+                num_requests=4,
+                image_count=1,
+                input_len=8,
+                output_len=4,
+                range_ratio=0.0,
+                processor=self.processor,
+                image_content="blank",
+                image_format="png",
+                image_resolution="random:8x16-16x32",
+                backend="sglang",
+            )
+        finally:
+            np.random.set_state(state)
+
+        image_sizes = []
+        for row in rows:
+            encoded = row.image_data[0].split(",", maxsplit=1)[1]
+            with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:
+                image_sizes.append(image.size)
+
+        self.assertGreater(len(set(image_sizes)), 1)
+        for width, height in image_sizes:
+            self.assertGreaterEqual(width, 16)
+            self.assertLessEqual(width, 32)
+            self.assertGreaterEqual(height, 8)
+            self.assertLessEqual(height, 16)
+
+    def test_parse_random_image_resolution(self):
+        self.assertEqual(
+            parse_random_image_resolution("random:256x384-1024x1536"),
+            ((384, 256), (1536, 1024)),
+        )
+        self.assertIsNone(parse_random_image_resolution("256x384"))
+        with self.assertRaisesRegex(ValueError, "minimum cannot exceed"):
+            parse_random_image_resolution("random:1024x1024-256x256")
+
     def test_gen_mm_prompt_excludes_special_tokens(self):
         tokenizer = create_lightweight_tokenizer()
         multimodal_special_tokens = [