[Feat] bench: support --tokenize-prompt for generated-shared-prefix (#40900)
Co-authored-by: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Co-authored-by: chaoyujiang <chaoyujiang@hygon.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Yihao Wang
chaoyujiang
Shangming Cai
parent
ed89693c4a
commit
eacd0a07d7
@@ -76,11 +76,15 @@ def get_available_tokens(tokenizer):
|
||||
)
|
||||
|
||||
|
||||
def gen_prompt_ids(tokenizer, token_num):
|
||||
"""Generate random token ids of specified length using tokenizer vocabulary."""
|
||||
all_available_tokens = get_available_tokens(tokenizer)
|
||||
return random.choices(all_available_tokens, k=token_num)
|
||||
|
||||
|
||||
def gen_prompt(tokenizer, token_num):
|
||||
"""Generate a random prompt of specified token length using tokenizer vocabulary."""
|
||||
all_available_tokens = get_available_tokens(tokenizer)
|
||||
selected_tokens = random.choices(all_available_tokens, k=token_num)
|
||||
return tokenizer.decode(selected_tokens)
|
||||
return tokenizer.decode(gen_prompt_ids(tokenizer, token_num))
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.benchmark.datasets.common import (
|
||||
DatasetRow,
|
||||
compute_random_lens,
|
||||
gen_prompt,
|
||||
gen_prompt_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -51,10 +52,14 @@ class GeneratedSharedPrefixDataset(BaseDataset):
|
||||
ordered: bool
|
||||
group_distribution: str = "uniform"
|
||||
zipf_alpha: Optional[float] = None
|
||||
input_ids: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_args(cls, args: Namespace) -> "GeneratedSharedPrefixDataset":
|
||||
assert not getattr(args, "tokenize_prompt", False)
|
||||
input_ids = getattr(args, "tokenize_prompt", False)
|
||||
num_turns = getattr(args, "gsp_num_turns", 1)
|
||||
if input_ids and num_turns > 1:
|
||||
raise ValueError("--tokenize-prompt requires --gsp-num-turns=1")
|
||||
group_distribution = getattr(args, "gsp_group_distribution", "uniform")
|
||||
zipf_alpha = getattr(args, "gsp_zipf_alpha", None)
|
||||
|
||||
@@ -93,10 +98,11 @@ class GeneratedSharedPrefixDataset(BaseDataset):
|
||||
seed=args.seed,
|
||||
fast_prepare=getattr(args, "gsp_fast_prepare", False),
|
||||
send_routing_key=getattr(args, "gsp_send_routing_key", False),
|
||||
num_turns=getattr(args, "gsp_num_turns", 1),
|
||||
num_turns=num_turns,
|
||||
ordered=getattr(args, "gsp_ordered", False),
|
||||
group_distribution=group_distribution,
|
||||
zipf_alpha=zipf_alpha,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
|
||||
def load(
|
||||
@@ -117,6 +123,7 @@ class GeneratedSharedPrefixDataset(BaseDataset):
|
||||
ordered=self.ordered,
|
||||
group_distribution=self.group_distribution,
|
||||
zipf_alpha=self.zipf_alpha,
|
||||
input_ids=self.input_ids,
|
||||
)
|
||||
|
||||
|
||||
@@ -130,6 +137,7 @@ def get_gen_prefix_cache_path(
|
||||
tokenizer,
|
||||
group_distribution: str = "uniform",
|
||||
zipf_alpha: Optional[float] = None,
|
||||
input_ids: bool = False,
|
||||
):
|
||||
"""Create cache directory under ~/.cache/sglang/benchmark.
|
||||
|
||||
@@ -142,6 +150,10 @@ def get_gen_prefix_cache_path(
|
||||
suffix = ""
|
||||
if group_distribution != "uniform":
|
||||
suffix = f"_{group_distribution}_{zipf_alpha}"
|
||||
# input_ids payloads are raw token-id lists, not decoded text; keep them
|
||||
# under a distinct key so a text-mode cache is never loaded as ids.
|
||||
if input_ids:
|
||||
suffix += "_ids"
|
||||
|
||||
cache_key = (
|
||||
f"gen_shared_prefix_{seed}_{num_groups}_{prompts_per_group}_"
|
||||
@@ -166,6 +178,7 @@ def sample_generated_shared_prefix_requests(
|
||||
ordered: bool = False,
|
||||
group_distribution: str = "uniform",
|
||||
zipf_alpha: Optional[float] = None,
|
||||
input_ids: bool = False,
|
||||
) -> List[DatasetRow]:
|
||||
"""Generate benchmark requests with shared system prompts using random tokens and caching.
|
||||
|
||||
@@ -180,6 +193,9 @@ def sample_generated_shared_prefix_requests(
|
||||
Zipf mode is cached on disk under a distinct key per (group_distribution,
|
||||
zipf_alpha) value.
|
||||
"""
|
||||
if input_ids and num_turns > 1:
|
||||
raise ValueError("Token-id prompts require num_turns=1")
|
||||
|
||||
cache_path = get_gen_prefix_cache_path(
|
||||
seed,
|
||||
num_groups,
|
||||
@@ -190,6 +206,7 @@ def sample_generated_shared_prefix_requests(
|
||||
tokenizer,
|
||||
group_distribution=group_distribution,
|
||||
zipf_alpha=zipf_alpha,
|
||||
input_ids=input_ids,
|
||||
)
|
||||
# range_ratio != 1 / num_turns > 1 perturb the payload but are not in the
|
||||
# cache key; send_routing_key embeds a per-run uuid + timestamp that is
|
||||
@@ -233,15 +250,17 @@ def sample_generated_shared_prefix_requests(
|
||||
).reshape(num_groups, prompts_per_group)
|
||||
del system_prompt_len, question_len, output_len
|
||||
|
||||
generate_prompt = gen_prompt_ids if input_ids else gen_prompt
|
||||
system_prompts = [
|
||||
gen_prompt(tokenizer, system_prompt_lens[i]) for i in range(num_groups)
|
||||
generate_prompt(tokenizer, int(system_prompt_lens[i]))
|
||||
for i in range(num_groups)
|
||||
]
|
||||
|
||||
# shape: (num_groups, prompts_per_group, num_turns)
|
||||
questions = [
|
||||
[
|
||||
[
|
||||
gen_prompt(tokenizer, int(question_lens[g, p, t]))
|
||||
generate_prompt(tokenizer, int(question_lens[g, p, t]))
|
||||
for t in range(num_turns)
|
||||
]
|
||||
for p in range(prompts_per_group)
|
||||
@@ -281,9 +300,19 @@ def sample_generated_shared_prefix_requests(
|
||||
else None
|
||||
)
|
||||
turn_questions = questions[src_g][src_p]
|
||||
turn_prompts = [f"{system_prompt}\n\n{turn_questions[0]}"] + turn_questions[1:]
|
||||
if input_ids:
|
||||
turn_prompts = [system_prompt + turn_questions[0]]
|
||||
else:
|
||||
turn_prompts = [f"{system_prompt}\n\n{turn_questions[0]}"] + turn_questions[
|
||||
1:
|
||||
]
|
||||
full_prompt = turn_prompts[0] if num_turns == 1 else turn_prompts
|
||||
prompt_len = 1 if fast_prepare else len(tokenizer.encode(turn_prompts[0]))
|
||||
if input_ids:
|
||||
prompt_len = len(turn_prompts[0])
|
||||
elif fast_prepare:
|
||||
prompt_len = 1
|
||||
else:
|
||||
prompt_len = len(tokenizer.encode(turn_prompts[0]))
|
||||
output_len_val = int(output_lens[src_g, src_p])
|
||||
|
||||
input_requests.append(
|
||||
@@ -308,15 +337,21 @@ def sample_generated_shared_prefix_requests(
|
||||
if group_distribution == "zipf":
|
||||
print(f"Zipf alpha: {zipf_alpha}")
|
||||
print(f"Total prompts: {len(input_requests)}")
|
||||
if not fast_prepare:
|
||||
if not fast_prepare or input_ids:
|
||||
system_token_lens = [
|
||||
len(sp) if input_ids else len(tokenizer.encode(sp)) for sp in system_prompts
|
||||
]
|
||||
all_questions = [q for group in questions for conv in group for q in conv]
|
||||
question_token_lens = [
|
||||
len(q) if input_ids else len(tokenizer.encode(q)) for q in all_questions
|
||||
]
|
||||
print(f"Total input tokens: {total_input_tokens}")
|
||||
print(f"Total output tokens: {total_output_tokens}")
|
||||
print(
|
||||
f"Average system prompt length: {sum(len(tokenizer.encode(sp)) for sp in system_prompts) / len(system_prompts):.1f} tokens"
|
||||
f"Average system prompt length: {sum(system_token_lens) / len(system_prompts):.1f} tokens"
|
||||
)
|
||||
all_questions = [q for group in questions for conv in group for q in conv]
|
||||
print(
|
||||
f"Average question length: {sum(len(tokenizer.encode(q)) for q in all_questions) / len(all_questions):.1f} tokens\n"
|
||||
f"Average question length: {sum(question_token_lens) / len(all_questions):.1f} tokens\n"
|
||||
)
|
||||
|
||||
if should_cache:
|
||||
|
||||
@@ -2259,13 +2259,20 @@ def _goodput_slo(pair: str) -> Tuple[str, float]:
|
||||
def _validate_parsed_gsp_args(
|
||||
parser: argparse.ArgumentParser, args: argparse.Namespace
|
||||
) -> None:
|
||||
"""Reject malformed GSP distribution/alpha combinations at parse time.
|
||||
"""Reject unsupported GSP argument combinations at parse time.
|
||||
|
||||
Invoked from the CLI entry point right after ``parser.parse_args()`` so
|
||||
users see a clear argparse-style error before any server, model, or
|
||||
tokenizer setup runs and masks the real cause with an unrelated network
|
||||
failure.
|
||||
"""
|
||||
if (
|
||||
getattr(args, "dataset_name", None) == "generated-shared-prefix"
|
||||
and getattr(args, "tokenize_prompt", False)
|
||||
and getattr(args, "gsp_num_turns", 1) > 1
|
||||
):
|
||||
parser.error("--tokenize-prompt requires --gsp-num-turns=1")
|
||||
|
||||
distribution = getattr(args, "gsp_group_distribution", None)
|
||||
alpha = getattr(args, "gsp_zipf_alpha", None)
|
||||
if distribution == "zipf" and alpha is None:
|
||||
|
||||
Reference in New Issue
Block a user