diff --git a/python/sglang/benchmark/datasets/common.py b/python/sglang/benchmark/datasets/common.py index 648b6140d..d6eaad4ef 100644 --- a/python/sglang/benchmark/datasets/common.py +++ b/python/sglang/benchmark/datasets/common.py @@ -66,8 +66,12 @@ def compute_random_lens(full_len: int, range_ratio: float, num: int) -> List[int @lru_cache(maxsize=1) def get_available_tokens(tokenizer): - """Get all available token ids from the tokenizer vocabulary.""" - return list(tokenizer.get_vocab().values()) + """Get valid token ids from the tokenizer vocabulary.""" + return [ + token_id + for token_id in tokenizer.get_vocab().values() + if isinstance(token_id, int) + ] def gen_prompt(tokenizer, token_num): diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index 22c5d06e9..151205db8 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -936,6 +936,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): ) self.trace_slice(stage, self.decode_prealloc_queue_entry_time, ts) + if self.enable_metrics and self.bootstrap_done_time > 0: + bootstrap_ms = ( + self.bootstrap_done_time - self.decode_prealloc_queue_entry_time + ) * 1000 + alloc_ms = (ts - self.bootstrap_done_time) * 1000 + self.metrics_collector.observe_kv_transfer_bootstrap( + bootstrap_ms=bootstrap_ms, + alloc_ms=alloc_ms, + ) + def set_bootstrap_done_time(self, ts=None): ts = ts or time.perf_counter() if self.bootstrap_done_time == 0.0: diff --git a/scripts/playground/replay_request_dump.py b/scripts/playground/replay_request_dump.py index 5e42e80d6..7a5f6ef0d 100644 --- a/scripts/playground/replay_request_dump.py +++ b/scripts/playground/replay_request_dump.py @@ -1,10 +1,10 @@ """ Usage: # replay from a folder -python3 replay_request_dump.py --file-number 100 --parallel 512 --input-folder /data/lianmin/sglang_request_dump/grok-mini-0220-engine-5756f8f94-28bm6/ +python3 replay_request_dump.py --file-number 100 --parallel 512 --input-folder /data/lianmin/sglang_request_dump/engine-34xd1/ # replay from a single file -python3 replay_request_dump.py --parallel 512 --input-file /data/sglang_crash_dump/memx-cti-34-sr1.xpop.twttr.net/crash_dump_2025-06-04_20-13-18.pkl +python3 replay_request_dump.py --parallel 512 --input-file /data/sglang_crash_dump/crash_dump_2025-06-04_20-13-18.pkl """ import argparse