From eacd0a07d7fc8da3453dc9c5b0d4963f883540a9 Mon Sep 17 00:00:00 2001 From: chaoyujiang <35489248+chaoyujiang@users.noreply.github.com> Date: Thu, 22 Oct 2026 08:17:15 +0800 Subject: [PATCH] [Feat] bench: support --tokenize-prompt for generated-shared-prefix (#40900) Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Co-authored-by: chaoyujiang Co-authored-by: Shangming Cai --- python/sglang/benchmark/datasets/common.py | 10 ++- .../datasets/generated_shared_prefix.py | 55 ++++++++++++++--- python/sglang/benchmark/serving.py | 9 ++- .../bench_fn/test_benchmark_datasets_api.py | 61 +++++++++++++++++++ 4 files changed, 121 insertions(+), 14 deletions(-) diff --git a/python/sglang/benchmark/datasets/common.py b/python/sglang/benchmark/datasets/common.py index 7dd567f830..a8da30c8ce 100644 --- a/python/sglang/benchmark/datasets/common.py +++ b/python/sglang/benchmark/datasets/common.py @@ -76,11 +76,15 @@ def get_available_tokens(tokenizer): ) +def gen_prompt_ids(tokenizer, token_num): + """Generate random token ids of specified length using tokenizer vocabulary.""" + all_available_tokens = get_available_tokens(tokenizer) + return random.choices(all_available_tokens, k=token_num) + + def gen_prompt(tokenizer, token_num): """Generate a random prompt of specified token length using tokenizer vocabulary.""" - all_available_tokens = get_available_tokens(tokenizer) - selected_tokens = random.choices(all_available_tokens, k=token_num) - return tokenizer.decode(selected_tokens) + return tokenizer.decode(gen_prompt_ids(tokenizer, token_num)) @lru_cache(maxsize=1) diff --git a/python/sglang/benchmark/datasets/generated_shared_prefix.py b/python/sglang/benchmark/datasets/generated_shared_prefix.py index f238990edf..283f7d727c 100644 --- a/python/sglang/benchmark/datasets/generated_shared_prefix.py +++ b/python/sglang/benchmark/datasets/generated_shared_prefix.py @@ -17,6 +17,7 @@ from sglang.benchmark.datasets.common import ( DatasetRow, compute_random_lens, gen_prompt, + gen_prompt_ids, ) @@ -51,10 +52,14 @@ class GeneratedSharedPrefixDataset(BaseDataset): ordered: bool group_distribution: str = "uniform" zipf_alpha: Optional[float] = None + input_ids: bool = False @classmethod def from_args(cls, args: Namespace) -> "GeneratedSharedPrefixDataset": - assert not getattr(args, "tokenize_prompt", False) + input_ids = getattr(args, "tokenize_prompt", False) + num_turns = getattr(args, "gsp_num_turns", 1) + if input_ids and num_turns > 1: + raise ValueError("--tokenize-prompt requires --gsp-num-turns=1") group_distribution = getattr(args, "gsp_group_distribution", "uniform") zipf_alpha = getattr(args, "gsp_zipf_alpha", None) @@ -93,10 +98,11 @@ class GeneratedSharedPrefixDataset(BaseDataset): seed=args.seed, fast_prepare=getattr(args, "gsp_fast_prepare", False), send_routing_key=getattr(args, "gsp_send_routing_key", False), - num_turns=getattr(args, "gsp_num_turns", 1), + num_turns=num_turns, ordered=getattr(args, "gsp_ordered", False), group_distribution=group_distribution, zipf_alpha=zipf_alpha, + input_ids=input_ids, ) def load( @@ -117,6 +123,7 @@ class GeneratedSharedPrefixDataset(BaseDataset): ordered=self.ordered, group_distribution=self.group_distribution, zipf_alpha=self.zipf_alpha, + input_ids=self.input_ids, ) @@ -130,6 +137,7 @@ def get_gen_prefix_cache_path( tokenizer, group_distribution: str = "uniform", zipf_alpha: Optional[float] = None, + input_ids: bool = False, ): """Create cache directory under ~/.cache/sglang/benchmark. @@ -142,6 +150,10 @@ def get_gen_prefix_cache_path( suffix = "" if group_distribution != "uniform": suffix = f"_{group_distribution}_{zipf_alpha}" + # input_ids payloads are raw token-id lists, not decoded text; keep them + # under a distinct key so a text-mode cache is never loaded as ids. + if input_ids: + suffix += "_ids" cache_key = ( f"gen_shared_prefix_{seed}_{num_groups}_{prompts_per_group}_" @@ -166,6 +178,7 @@ def sample_generated_shared_prefix_requests( ordered: bool = False, group_distribution: str = "uniform", zipf_alpha: Optional[float] = None, + input_ids: bool = False, ) -> List[DatasetRow]: """Generate benchmark requests with shared system prompts using random tokens and caching. @@ -180,6 +193,9 @@ def sample_generated_shared_prefix_requests( Zipf mode is cached on disk under a distinct key per (group_distribution, zipf_alpha) value. """ + if input_ids and num_turns > 1: + raise ValueError("Token-id prompts require num_turns=1") + cache_path = get_gen_prefix_cache_path( seed, num_groups, @@ -190,6 +206,7 @@ def sample_generated_shared_prefix_requests( tokenizer, group_distribution=group_distribution, zipf_alpha=zipf_alpha, + input_ids=input_ids, ) # range_ratio != 1 / num_turns > 1 perturb the payload but are not in the # cache key; send_routing_key embeds a per-run uuid + timestamp that is @@ -233,15 +250,17 @@ def sample_generated_shared_prefix_requests( ).reshape(num_groups, prompts_per_group) del system_prompt_len, question_len, output_len + generate_prompt = gen_prompt_ids if input_ids else gen_prompt system_prompts = [ - gen_prompt(tokenizer, system_prompt_lens[i]) for i in range(num_groups) + generate_prompt(tokenizer, int(system_prompt_lens[i])) + for i in range(num_groups) ] # shape: (num_groups, prompts_per_group, num_turns) questions = [ [ [ - gen_prompt(tokenizer, int(question_lens[g, p, t])) + generate_prompt(tokenizer, int(question_lens[g, p, t])) for t in range(num_turns) ] for p in range(prompts_per_group) @@ -281,9 +300,19 @@ def sample_generated_shared_prefix_requests( else None ) turn_questions = questions[src_g][src_p] - turn_prompts = [f"{system_prompt}\n\n{turn_questions[0]}"] + turn_questions[1:] + if input_ids: + turn_prompts = [system_prompt + turn_questions[0]] + else: + turn_prompts = [f"{system_prompt}\n\n{turn_questions[0]}"] + turn_questions[ + 1: + ] full_prompt = turn_prompts[0] if num_turns == 1 else turn_prompts - prompt_len = 1 if fast_prepare else len(tokenizer.encode(turn_prompts[0])) + if input_ids: + prompt_len = len(turn_prompts[0]) + elif fast_prepare: + prompt_len = 1 + else: + prompt_len = len(tokenizer.encode(turn_prompts[0])) output_len_val = int(output_lens[src_g, src_p]) input_requests.append( @@ -308,15 +337,21 @@ def sample_generated_shared_prefix_requests( if group_distribution == "zipf": print(f"Zipf alpha: {zipf_alpha}") print(f"Total prompts: {len(input_requests)}") - if not fast_prepare: + if not fast_prepare or input_ids: + system_token_lens = [ + len(sp) if input_ids else len(tokenizer.encode(sp)) for sp in system_prompts + ] + all_questions = [q for group in questions for conv in group for q in conv] + question_token_lens = [ + len(q) if input_ids else len(tokenizer.encode(q)) for q in all_questions + ] print(f"Total input tokens: {total_input_tokens}") print(f"Total output tokens: {total_output_tokens}") print( - f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens" + f"Average system prompt length: {sum(system_token_lens) / len(system_prompts):.1f} tokens" ) - all_questions = [q for group in questions for conv in group for q in conv] print( - f"Average question length: {sum(len(tokenizer.encode(q)) for q in all_questions) / len(all_questions):.1f} tokens\n" + f"Average question length: {sum(question_token_lens) / len(all_questions):.1f} tokens\n" ) if should_cache: diff --git a/python/sglang/benchmark/serving.py b/python/sglang/benchmark/serving.py index d14856b267..e6b4797da9 100644 --- a/python/sglang/benchmark/serving.py +++ b/python/sglang/benchmark/serving.py @@ -2259,13 +2259,20 @@ def _goodput_slo(pair: str) -> Tuple[str, float]: def _validate_parsed_gsp_args( parser: argparse.ArgumentParser, args: argparse.Namespace ) -> None: - """Reject malformed GSP distribution/alpha combinations at parse time. + """Reject unsupported GSP argument combinations at parse time. Invoked from the CLI entry point right after ``parser.parse_args()`` so users see a clear argparse-style error before any server, model, or tokenizer setup runs and masks the real cause with an unrelated network failure. """ + if ( + getattr(args, "dataset_name", None) == "generated-shared-prefix" + and getattr(args, "tokenize_prompt", False) + and getattr(args, "gsp_num_turns", 1) > 1 + ): + parser.error("--tokenize-prompt requires --gsp-num-turns=1") + distribution = getattr(args, "gsp_group_distribution", None) alpha = getattr(args, "gsp_zipf_alpha", None) if distribution == "zipf" and alpha is None: diff --git a/test/registered/bench_fn/test_benchmark_datasets_api.py b/test/registered/bench_fn/test_benchmark_datasets_api.py index 6c55933897..dec4e8c81e 100644 --- a/test/registered/bench_fn/test_benchmark_datasets_api.py +++ b/test/registered/bench_fn/test_benchmark_datasets_api.py @@ -54,6 +54,7 @@ from sglang.benchmark.serving import ( _EMBEDDING_BACKENDS, ASYNC_REQUEST_FUNCS, _finite_positive_float, + _validate_parsed_gsp_args, async_request_openai_embeddings, flush_server_cache, ) @@ -1034,6 +1035,66 @@ class TestBenchmarkDatasetsAPI(CustomTestCase): self.assertEqual(len(agentic_rows), 2) self.assertTrue(all(isinstance(row, DatasetRow) for row in agentic_rows)) + def test_gsp_token_ids_have_exact_lengths_and_statistics(self): + args = make_args( + tokenize_prompt=True, + gsp_system_prompt_len=8, + gsp_question_len=4, + gsp_range_ratio=1.0, + gsp_fast_prepare=True, + gsp_send_routing_key=True, + ) + dataset = GeneratedSharedPrefixDataset.from_args(args) + with ( + patch.object(self.tokenizer, "encode", side_effect=AssertionError), + patch.object(self.tokenizer, "decode", side_effect=AssertionError), + patch("builtins.print") as print_mock, + ): + rows = dataset.load(self.tokenizer) + self.assertTrue(rows) + for row in rows: + self.assertEqual(len(row.prompt), 12) + self.assertEqual(row.prompt_len, 12) + self.assertTrue(all(isinstance(token, int) for token in row.prompt)) + printed = "\n".join(str(call.args[0]) for call in print_mock.call_args_list) + self.assertIn(f"Total input tokens: {12 * len(rows)}", printed) + self.assertIn("Average system prompt length: 8.0 tokens", printed) + self.assertIn("Average question length: 4.0 tokens", printed) + + def test_gsp_token_ids_reject_multiple_turns(self): + parser = argparse.ArgumentParser() + with ( + patch("sys.stderr", new_callable=io.StringIO) as stderr, + self.assertRaises(SystemExit) as error, + ): + _validate_parsed_gsp_args( + parser, + make_args( + dataset_name="generated-shared-prefix", + tokenize_prompt=True, + gsp_num_turns=2, + ), + ) + self.assertEqual(error.exception.code, 2) + self.assertIn("--gsp-num-turns=1", stderr.getvalue()) + with self.assertRaisesRegex(ValueError, "--gsp-num-turns=1"): + GeneratedSharedPrefixDataset.from_args( + make_args(tokenize_prompt=True, gsp_num_turns=2) + ) + with self.assertRaisesRegex(ValueError, "num_turns=1"): + sample_generated_shared_prefix_requests( + num_groups=2, + prompts_per_group=2, + system_prompt_len=8, + question_len=4, + output_len=2, + range_ratio=1.0, + tokenizer=self.tokenizer, + seed=42, + input_ids=True, + num_turns=2, + ) + def test_get_dataset_unknown_dataset(self): args = make_args(dataset_name="not-a-dataset") with self.assertRaises(ValueError):