diff --git a/python/sglang/benchmark/datasets/image.py b/python/sglang/benchmark/datasets/image.py index 6958ea3bb..e55194f2e 100644 --- a/python/sglang/benchmark/datasets/image.py +++ b/python/sglang/benchmark/datasets/image.py @@ -2,7 +2,7 @@ import io import warnings from argparse import Namespace from dataclasses import dataclass -from typing import List, Tuple +from typing import List, Optional, Tuple import numpy as np import pybase64 @@ -93,6 +93,33 @@ def parse_image_resolution(image_resolution: str) -> Tuple[int, int]: ) +def parse_random_image_resolution( + image_resolution: str, +) -> Optional[Tuple[Tuple[int, int], Tuple[int, int]]]: + """Parse ``random:x-x`` image bounds. + + Returns ``None`` for fixed resolutions. The returned dimensions are + ``(width, height)`` pairs, matching :func:`parse_image_resolution`. + """ + + prefix = "random:" + if not image_resolution.strip().lower().startswith(prefix): + return None + + bounds = image_resolution.strip()[len(prefix) :].split("-", maxsplit=1) + if len(bounds) != 2: + raise ValueError( + "Random image resolution must be 'random:x-" + "x', for example 'random:256x256-1024x1024'." + ) + + min_width, min_height = parse_image_resolution(bounds[0]) + max_width, max_height = parse_image_resolution(bounds[1]) + if min_width > max_width or min_height > max_height: + raise ValueError("Random image resolution minimum cannot exceed maximum.") + return (min_width, min_height), (max_width, max_height) + + def create_mm_data_row( text_prompt, images: list, images_base64, output_len, processor, backend ): @@ -172,17 +199,22 @@ def create_mm_data_row( # Vision tokens = total tokens - text tokens vision_prompt_len = prompt_len - text_prompt_len - supported_backends = ["sglang", "sglang-native", "sglang-oai-chat"] + supported_backends = [ + "sglang", + "sglang-native", + "sglang-oai-chat", + "vllm-chat", + ] if backend not in supported_backends: raise ValueError( f"Image dataset only supports backends: {supported_backends}, " f"got '{backend}'." ) - # sglang-oai-chat: server's chat handler applies chat template, so send raw text. - # sglang/sglang-native: /generate does not apply chat template, so send prompt_str - # which contains image placeholder tokens needed by the multimodal processor. - use_raw_prompt = backend == "sglang-oai-chat" + # OpenAI chat handlers apply the chat template and receive images separately, so + # send the raw text. /generate does not apply a chat template, so it needs + # prompt_str, which contains the multimodal processor's image placeholders. + use_raw_prompt = backend in ("sglang-oai-chat", "vllm-chat") return DatasetRow( prompt=text_prompt if use_raw_prompt else prompt_str, @@ -211,14 +243,20 @@ def sample_image_requests( - If ``random_image_count`` is True, each request includes a random number of images between 1 and ``image_count``. - If ``random_image_count`` is False, each request includes exactly ``image_count`` images. - - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p (1280x720), 360p (640x360), - or custom 'heightxwidth' (e.g., 1080x1920). + - Supported resolutions: 4k (3840x2160), 1080p (1920x1080), 720p + (1280x720), 360p (640x360), custom ``heightxwidth`` (e.g., + 1080x1920), or ``random:x-x``. - Text lengths follow the 'random' dataset sampling rule. ``prompt_len`` only counts text tokens and excludes image data. """ - # Parse resolution (supports presets and 'heightxwidth') - width, height = parse_image_resolution(image_resolution) + random_resolution_bounds = parse_random_image_resolution(image_resolution) + if random_resolution_bounds is None: + width, height = parse_image_resolution(image_resolution) + min_width = max_width = width + min_height = max_height = height + else: + (min_width, min_height), (max_width, max_height) = random_resolution_bounds # Determine image counts for each request if random_image_count: @@ -231,9 +269,9 @@ def sample_image_requests( total_images = image_count * num_requests # Check for potentially problematic combinations and warn user - if width * height >= 1920 * 1080 and total_images >= 100: + if max_width * max_height >= 1920 * 1080 and total_images >= 100: warnings.warn( - f"High resolution ({width}x{height}) with {total_images} total images " + f"High resolution (up to {max_width}x{max_height}) with {total_images} total images " f"may take a long time. Consider reducing resolution or image count.", UserWarning, stacklevel=2, @@ -251,9 +289,12 @@ def sample_image_requests( num=num_requests, ) - def _gen_random_image_data_uri( - width: int = width, height: int = height - ) -> Tuple[Image.Image, str, int]: + def _gen_random_image_data_uri() -> Tuple[Image.Image, str, int, Tuple[int, int]]: + if random_resolution_bounds is None: + width, height = min_width, min_height + else: + width = np.random.randint(min_width, max_width + 1) + height = np.random.randint(min_height, max_height + 1) if image_content == "blank": # Generate blank white image arr = np.full((height, width, 3), 255, dtype=np.uint8) @@ -266,10 +307,11 @@ def sample_image_requests( encoded = pybase64.b64encode(buf.getvalue()).decode("utf-8") image_data = f"data:image/{image_format};base64,{encoded}" image_bytes = len(image_data.encode("utf-8")) - return img, image_data, image_bytes + return img, image_data, image_bytes, (width, height) dataset: List[DatasetRow] = [] total_image_bytes = 0 + all_image_sizes: list[Tuple[int, int]] = [] for i in range(num_requests): # Get the number of images for this request request_image_count = int(image_counts[i]) @@ -282,10 +324,11 @@ def sample_image_requests( ) # Generate image list - images, images_base64, images_bytes = zip( + images, images_base64, images_bytes, image_sizes = zip( *[_gen_random_image_data_uri() for _ in range(request_image_count)] ) total_image_bytes += sum(images_bytes) + all_image_sizes.extend(image_sizes) data_row = create_mm_data_row( text_prompt, @@ -309,6 +352,15 @@ def sample_image_requests( else: print(f"#Images per request: {image_count} (fixed)") + if random_resolution_bounds is not None: + widths, heights = zip(*all_image_sizes) + print( + "#Image resolution: " + f"min={min(widths)}x{min(heights)}, " + f"max={max(widths)}x{max(heights)}, " + f"mean={np.mean(widths):.1f}x{np.mean(heights):.1f}" + ) + # Detailed token breakdown (derived from dataset + input_lens) text_prompt_lens = np.array([r.text_prompt_len for r in dataset]) vision_prompt_lens = np.array([r.vision_prompt_len for r in dataset]) diff --git a/python/sglang/benchmark/serving.py b/python/sglang/benchmark/serving.py index 2a26b6809..9bebe908c 100644 --- a/python/sglang/benchmark/serving.py +++ b/python/sglang/benchmark/serving.py @@ -136,7 +136,12 @@ def get_request_headers() -> Dict[str, str]: def _combine_openai_chat_content(message: Dict[str, Any]) -> str: - return (message.get("reasoning_content") or "") + (message.get("content") or "") + # Most OpenAI-compatible servers use ``reasoning_content``. vLLM's Kimi + # parser instead streams its reasoning in ``reasoning``. Prefer the + # standard field when both are present to avoid counting the same tokens + # twice on servers that expose aliases. + reasoning = message.get("reasoning_content") or message.get("reasoning") or "" + return reasoning + (message.get("content") or "") def wait_for_endpoint(url: str, timeout_sec: int = 60) -> bool: @@ -2281,7 +2286,9 @@ def cli_main(): default="1080p", help=( "Resolution of images for image dataset. " - "Supports presets 4k/1080p/720p/360p or custom 'heightxwidth' (e.g., 1080x1920)." + "Supports presets 4k/1080p/720p/360p, custom 'heightxwidth' " + "(e.g., 1080x1920), or random 'random:x-x' " + "bounds (e.g., random:256x256-1024x1024)." ), ) parser.add_argument( diff --git a/test/registered/bench_fn/test_bench_serving_reasoning_stream.py b/test/registered/bench_fn/test_bench_serving_reasoning_stream.py index ec00e3b9b..1c46a913a 100644 --- a/test/registered/bench_fn/test_bench_serving_reasoning_stream.py +++ b/test/registered/bench_fn/test_bench_serving_reasoning_stream.py @@ -1,10 +1,11 @@ -"""Unit tests for bench_serving streaming with reasoning_content chunks. +"""Unit tests for bench_serving streaming with reasoning chunks. Reasoning models (DeepSeek-R1, MiMo, Qwen3 reasoning, Kimi-K2, ...) stream their -chain-of-thought via OpenAI's `delta.reasoning_content` field. Without explicit -support, bench_serving only inspects `delta.content` and silently reports zero -TTFT / ITL and an empty `generated_text`, which then retokenizes to 0 tokens -even though the backend completed real work. +chain-of-thought via fields such as OpenAI's `delta.reasoning_content` and +vLLM Kimi's `delta.reasoning`. Without explicit support, bench_serving only +inspects `delta.content` and silently reports zero TTFT / ITL and an empty +`generated_text`, which then retokenizes to 0 tokens even though the backend +completed real work. """ import asyncio @@ -77,12 +78,16 @@ class _JSONHandler(BaseHTTPRequestHandler): return -def _make_chunk(content=None, reasoning_content=None, completion_tokens=None): +def _make_chunk( + content=None, reasoning_content=None, reasoning=None, completion_tokens=None +): delta = {} if content is not None: delta["content"] = content if reasoning_content is not None: delta["reasoning_content"] = reasoning_content + if reasoning is not None: + delta["reasoning"] = reasoning chunk = {"choices": [{"index": 0, "delta": delta}]} if completion_tokens is not None: chunk["usage"] = {"completion_tokens": completion_tokens} @@ -163,6 +168,21 @@ class TestBenchServingReasoningStream(CustomTestCase): self.assertEqual(out.text_chunks, ["me ", "think."]) self.assertEqual(out.output_len, 3) + def test_vllm_kimi_reasoning_stream_populates_metrics(self): + chunks = [ + _make_chunk(reasoning="Let "), + _make_chunk(reasoning="me "), + _make_chunk(reasoning="think."), + _make_chunk(completion_tokens=3), + ] + out = self._run(chunks) + + self.assertTrue(out.success, msg=f"request failed: {out.error}") + self.assertEqual(out.generated_text, "Let me think.") + self.assertGreater(out.ttft, 0.0) + self.assertEqual(len(out.itl), 2, msg="should record ITL for chunks 2..N") + self.assertEqual(out.output_len, 3) + def test_reasoning_then_content_accounts_both(self): chunks = [ _make_chunk(reasoning_content="step1 "), diff --git a/test/registered/bench_fn/test_benchmark_datasets_api.py b/test/registered/bench_fn/test_benchmark_datasets_api.py index 0594ae465..2f7ca24d5 100644 --- a/test/registered/bench_fn/test_benchmark_datasets_api.py +++ b/test/registered/bench_fn/test_benchmark_datasets_api.py @@ -1,4 +1,6 @@ import asyncio +import base64 +import io import json import pickle import random @@ -31,7 +33,10 @@ from sglang.benchmark.datasets.generated_shared_prefix import ( get_gen_prefix_cache_path, sample_generated_shared_prefix_requests, ) -from sglang.benchmark.datasets.image import sample_image_requests +from sglang.benchmark.datasets.image import ( + parse_random_image_resolution, + sample_image_requests, +) from sglang.benchmark.datasets.mmmu import sample_mmmu_requests from sglang.benchmark.datasets.mooncake import get_mooncake_request_over_time from sglang.benchmark.datasets.openai_dataset import sample_openai_requests @@ -421,6 +426,66 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase): self.assertTrue(all(isinstance(row, DatasetRow) for row in rows)) self.assertTrue(all(row.image_data for row in rows)) + def test_image_sampler_vllm_chat(self): + rows = sample_image_requests( + num_requests=2, + image_count=1, + input_len=8, + output_len=4, + range_ratio=0.0, + processor=self.processor, + image_content="blank", + image_format="png", + image_resolution="8x8", + backend="vllm-chat", + random_image_count=False, + ) + self.assertEqual(len(rows), 2) + self.assertTrue(all(isinstance(row, DatasetRow) for row in rows)) + self.assertTrue(all(row.image_data for row in rows)) + self.assertTrue(all("[IMAGE]" not in row.prompt for row in rows)) + + def test_image_sampler_random_resolution(self): + state = np.random.get_state() + np.random.seed(20260711) + try: + rows = sample_image_requests( + num_requests=4, + image_count=1, + input_len=8, + output_len=4, + range_ratio=0.0, + processor=self.processor, + image_content="blank", + image_format="png", + image_resolution="random:8x16-16x32", + backend="sglang", + ) + finally: + np.random.set_state(state) + + image_sizes = [] + for row in rows: + encoded = row.image_data[0].split(",", maxsplit=1)[1] + with Image.open(io.BytesIO(base64.b64decode(encoded))) as image: + image_sizes.append(image.size) + + self.assertGreater(len(set(image_sizes)), 1) + for width, height in image_sizes: + self.assertGreaterEqual(width, 16) + self.assertLessEqual(width, 32) + self.assertGreaterEqual(height, 8) + self.assertLessEqual(height, 16) + + def test_parse_random_image_resolution(self): + self.assertEqual( + parse_random_image_resolution("random:256x384-1024x1536"), + ((384, 256), (1536, 1024)), + ) + self.assertIsNone(parse_random_image_resolution("256x384")) + with self.assertRaisesRegex(ValueError, "minimum cannot exceed"): + parse_random_image_resolution("random:1024x1024-256x256") + def test_gen_mm_prompt_excludes_special_tokens(self): tokenizer = create_lightweight_tokenizer() multimodal_special_tokens = [