From 558c9bdcc2edd4e1fe9345165acc62c885b02f10 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 2 Aug 2026 01:39:50 -0700 Subject: [PATCH] [misc] Improve benchmark determinism and dataset API coverage (#33255) --- benchmark/gsm8k/bench_sglang.py | 10 +- python/sglang/benchmark/datasets/common.py | 6 +- python/sglang/benchmark/datasets/image.py | 9 +- python/sglang/benchmark/one_batch_server.py | 19 +++- python/sglang/test/vlm_utils.py | 68 ++++++++++++- .../bench_fn/test_benchmark_datasets_api.py | 97 +++++++++++++++++++ 6 files changed, 199 insertions(+), 10 deletions(-) diff --git a/benchmark/gsm8k/bench_sglang.py b/benchmark/gsm8k/bench_sglang.py index be766cd9a..a0e09a39c 100644 --- a/benchmark/gsm8k/bench_sglang.py +++ b/benchmark/gsm8k/bench_sglang.py @@ -89,7 +89,7 @@ def main(args): messages, tokenize=False, add_generation_prompt=True, - enable_thinking=True, + **args.chat_template_kwargs, ) questions.append(raw_question) labels.append(get_answer_value(lines[i]["answer"])) @@ -184,6 +184,14 @@ if __name__ == "__main__": action="store_true", help="Enable thinking mode by wrapping prompts with chat template", ) + parser.add_argument( + "--chat-template-kwargs", + type=json.loads, + default='{"enable_thinking": true}', + help="JSON dict passed through to tokenizer.apply_chat_template. " + "The thinking-toggle kwarg name is model-specific, e.g. " + "'{\"enable_thinking\": true}' (Qwen) or '{\"thinking\": true}' (Kimi).", + ) parser.add_argument( "--tokenizer-path", type=str, diff --git a/python/sglang/benchmark/datasets/common.py b/python/sglang/benchmark/datasets/common.py index 29bafff6b..7dd567f83 100644 --- a/python/sglang/benchmark/datasets/common.py +++ b/python/sglang/benchmark/datasets/common.py @@ -67,11 +67,13 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int @lru_cache(maxsize=1) def get_available_tokens(tokenizer): """Get valid token ids from the tokenizer vocabulary.""" - return [ + # Canonical order: vocab dict iteration order varies across tokenizers + # versions, which would break --seed reproducibility. + return sorted( token_id for token_id in tokenizer.get_vocab().values() if isinstance(token_id, int) - ] + ) def gen_prompt(tokenizer, token_num): diff --git a/python/sglang/benchmark/datasets/image.py b/python/sglang/benchmark/datasets/image.py index 5deffe0a4..cbaac601c 100644 --- a/python/sglang/benchmark/datasets/image.py +++ b/python/sglang/benchmark/datasets/image.py @@ -1,4 +1,5 @@ import io +import random import warnings from argparse import Namespace from dataclasses import dataclass @@ -30,6 +31,7 @@ class ImageDataset(BaseDataset): image_resolution: str backend: str random_image_count: bool + seed: int @classmethod def from_args(cls, args: Namespace) -> "ImageDataset": @@ -44,10 +46,15 @@ class ImageDataset(BaseDataset): image_resolution=args.image_resolution, backend=args.backend, random_image_count=args.random_image_count, + seed=args.seed, ) def load(self, tokenizer=None, model_id=None) -> List[DatasetRow]: processor = get_processor(model_id) + # Processor initialization may consume global RNG state. Reset it here so + # --seed fixes the generated prompts, image sizes, and image contents. + random.seed(self.seed) + np.random.seed(self.seed) return sample_image_requests( num_requests=self.num_requests, image_count=self.image_count, @@ -148,7 +155,7 @@ def create_mm_data_row( prompt_str = f"{text_prompt}" # Calculate total tokens (text + vision) - if type(processor).__name__ == "KimiK25Processor": + if type(processor).__name__ in ("KimiK25Processor", "KimiK3Processor"): medias = [{"type": "image", "image": img} for img in images] prompt_len = processor( text=prompt_str, diff --git a/python/sglang/benchmark/one_batch_server.py b/python/sglang/benchmark/one_batch_server.py index b2fdd4dfd..b6eddb541 100644 --- a/python/sglang/benchmark/one_batch_server.py +++ b/python/sglang/benchmark/one_batch_server.py @@ -1002,10 +1002,21 @@ def run_benchmark_internal( "token_capacity", 1000000000 ) - assert ( - max_running_requests_per_dp > 0 - ), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}" - skip_max_running_requests_threshold = max_running_requests_per_dp * dp_size + # Router /get_server_info responses carry "router_manager"; worker + # responses never do, so its presence confirms a router by design. + if not internal_states and server_info.get("router_manager"): + print( + "WARNING: base_url points at a PD router; worker internal " + "states are unavailable, so the max-running-requests and " + "token-capacity skip guards are disabled." + ) + skip_max_running_requests_threshold = float("inf") + skip_token_capacity_threshold = float("inf") + else: + assert ( + max_running_requests_per_dp > 0 + ), f"effective_max_running_requests_per_dp is not set, {max_running_requests_per_dp=}" + skip_max_running_requests_threshold = max_running_requests_per_dp * dp_size print(f"{max_running_requests_per_dp=}") print(f"{dp_size=}") diff --git a/python/sglang/test/vlm_utils.py b/python/sglang/test/vlm_utils.py index b476e3381..4dfa8417f 100644 --- a/python/sglang/test/vlm_utils.py +++ b/python/sglang/test/vlm_utils.py @@ -215,14 +215,78 @@ class ImageOpenAITestMixin(TestOpenAIMLLMServerBase): with ThreadPoolExecutor(4) as executor: list(executor.map(self.run_decode_with_image, image_ids)) + def test_image_prefix_cache_reuse(self): + """Image prefix (radix) cache correctness across requests. + + Repeating an identical image must reuse the multimodal prefix without + changing the output, and a different image must NOT reuse the first + image's KV. This guards against image-token pad_value / feature-hash + regressions that would silently serve a cached *wrong* image's KV + (a correctness bug invisible to single-request tests). Pure greedy + request-level checks: no extra server flags, radix cache is on by + default. + """ + client = openai.Client(api_key=self.api_key, base_url=self.base_url) + + def describe(url: str) -> str: + response = client.chat.completions.create( + model="default", + messages=[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": url}}, + { + "type": "text", + "text": "Describe this image in one sentence.", + }, + ], + }, + ], + temperature=0, + max_tokens=32, + **(self.get_vision_request_kwargs()), + ) + assert response.usage.prompt_tokens > 0 + content = response.choices[0].message.content + assert isinstance(content, str) and content + return content + + # miss -> compute, then hit -> reuse the identical image's prefix + first = describe(IMAGE_MAN_IRONING_URL) + repeat = describe(IMAGE_MAN_IRONING_URL) + # a different image must be computed on its own, not reuse `first`'s KV + other = describe(IMAGE_SGL_LOGO_URL) + # the original image again, after a different one occupied the cache + first_again = describe(IMAGE_MAN_IRONING_URL) + + self.assertEqual( + first, + repeat, + "Repeating an identical image changed the output; image prefix " + "reuse broke greedy determinism.", + ) + self.assertEqual( + first, + first_again, + "The identical image after a different one changed the output; " + "image KV was cross-contaminated across requests.", + ) + self.assertNotEqual( + first, + other, + "A different image produced an identical description; a wrong " + "image's KV may have been reused from the prefix cache.", + ) + def verify_single_image_response(self, response): assert response.choices[0].message.role == "assistant" text = response.choices[0].message.content assert isinstance(text, str) # `driver` is for gemma-3-it - assert ( - "man" in text or "person" or "driver" in text + assert any( + keyword in text for keyword in ("man", "person", "driver") ), f"text: {text}, should contain man, person or driver" assert ( "cab" in text diff --git a/test/registered/bench_fn/test_benchmark_datasets_api.py b/test/registered/bench_fn/test_benchmark_datasets_api.py index 09ff64404..612753f4c 100644 --- a/test/registered/bench_fn/test_benchmark_datasets_api.py +++ b/test/registered/bench_fn/test_benchmark_datasets_api.py @@ -34,6 +34,7 @@ from sglang.benchmark.datasets.generated_shared_prefix import ( sample_generated_shared_prefix_requests, ) from sglang.benchmark.datasets.image import ( + ImageDataset, parse_random_image_resolution, sample_image_requests, ) @@ -140,6 +141,21 @@ class DummyProcessor: return {"input_ids": _DummyTokenTensor(text_len + image_tokens)} +class KimiK3Processor(DummyProcessor): + """Mimics the Kimi K3 HF processor's media-kwargs interface (#32541).""" + + def __init__(self, tokenizer: PreTrainedTokenizerFast): + super().__init__(tokenizer) + self.media_call_count = 0 + + def __call__(self, text, medias=None, **kwargs): + if medias is None: + raise ValueError("Kimi K3 requires medias with text") + self.media_call_count += 1 + text_len = len(self.tokenizer.encode(text)) + return {"input_ids": _DummyTokenTensor(text_len + 4 * len(medias))} + + class _FakeMMMUDataset: def __init__(self, records): self.records = records @@ -481,6 +497,26 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase): for marker in ("user:", "assistant:", "[IMAGE]"): self.assertNotIn(marker, rows[0].prompt) + def test_image_sampler_uses_kimi_k3_media_contract(self): + processor = KimiK3Processor(self.tokenizer) + rows = sample_image_requests( + num_requests=1, + image_count=1, + input_len=8, + output_len=4, + range_ratio=0.0, + processor=processor, + image_content="blank", + image_format="png", + image_resolution="8x8", + backend="sglang-oai-chat", + random_image_count=False, + ) + + self.assertEqual(len(rows), 1) + self.assertEqual(processor.media_call_count, 1) + self.assertTrue(rows[0].image_data) + def test_image_sampler_random_resolution(self): state = np.random.get_state() np.random.seed(20260711) @@ -513,6 +549,43 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase): self.assertGreaterEqual(height, 8) self.assertLessEqual(height, 16) + def test_image_dataset_seed_is_independent_of_processor_initialization(self): + dataset = ImageDataset.from_args( + make_args( + num_prompts=3, + image_resolution="random:8x16-16x32", + seed=20260717, + ) + ) + processor_init_count = 0 + + def get_processor_with_rng_side_effects(_model_id): + nonlocal processor_init_count + processor_init_count += 1 + random.random() + np.random.random(processor_init_count) + return self.processor + + with patch( + "sglang.benchmark.datasets.image.get_processor", + side_effect=get_processor_with_rng_side_effects, + ): + first = dataset.load(model_id="test-model") + random.seed(999) + np.random.seed(999) + second = dataset.load(model_id="test-model") + + self.assertEqual( + [ + (row.prompt, row.prompt_len, row.output_len, row.image_data) + for row in first + ], + [ + (row.prompt, row.prompt_len, row.output_len, row.image_data) + for row in second + ], + ) + def test_parse_random_image_resolution(self): self.assertEqual( parse_random_image_resolution("random:256x384-1024x1536"), @@ -554,6 +627,30 @@ class TestBenchmarkDatasetsAPI(unittest.TestCase): self.assertFalse(special_token_ids & sampled_pool) self.assertTrue(sampled_pool) + def test_gen_mm_prompt_is_independent_of_vocab_order(self): + class OrderedVocabTokenizer: + all_special_ids = [] + + def __init__(self, items): + self.vocab = dict(items) + + def get_vocab(self): + return self.vocab + + def decode(self, token_ids): + return " ".join(map(str, token_ids)) + + items = [(f"token_{token_id}", token_id) for token_id in range(32)] + first = OrderedVocabTokenizer(items) + second = OrderedVocabTokenizer(reversed(items)) + + random.seed(20260717) + first_prompt = gen_mm_prompt(first, image_pad_id=None, token_num=16) + random.seed(20260717) + second_prompt = gen_mm_prompt(second, image_pad_id=None, token_num=16) + + self.assertEqual(first_prompt, second_prompt) + def test_mmmu_sampler(self): fake_records = [ {"image_1": Image.new("RGB", (4, 4), color="white"), "question": "q1"},