minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Deprecate --collect-tokens-histogram, auto-collect with --enable-metrics (#23595)

Browse Source
This commit is contained in:
Lianmin Zheng
2026-04-24 12:00:16 -07:00
committed by GitHub
parent 05696527ea
commit 6344b546c8
3 changed files with 55 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/tokenizer_manager.py
-1
View File
@@ -470,7 +470,6 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
bucket_time_to_first_token=self.server_args.bucket_time_to_first_token, bucket_time_to_first_token=self.server_args.bucket_time_to_first_token,
bucket_e2e_request_latency=self.server_args.bucket_e2e_request_latency, bucket_e2e_request_latency=self.server_args.bucket_e2e_request_latency,
bucket_inter_token_latency=self.server_args.bucket_inter_token_latency, bucket_inter_token_latency=self.server_args.bucket_inter_token_latency,
collect_tokens_histogram=self.server_args.collect_tokens_histogram,
) )
start_cpu_monitor_thread("tokenizer") start_cpu_monitor_thread("tokenizer")
python/sglang/srt/observability/metrics_collector.py
+53 -57
View File
@@ -1151,13 +1151,11 @@ class TokenizerMetricsCollector:
bucket_time_to_first_token: Optional[List[float]] = None, bucket_time_to_first_token: Optional[List[float]] = None,
bucket_inter_token_latency: Optional[List[float]] = None, bucket_inter_token_latency: Optional[List[float]] = None,
bucket_e2e_request_latency: Optional[List[float]] = None, bucket_e2e_request_latency: Optional[List[float]] = None,
collect_tokens_histogram: bool = False,
) -> None: ) -> None:
# We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR` # We need to import prometheus_client after setting the env variable `PROMETHEUS_MULTIPROC_DIR`
from prometheus_client import Counter, Histogram from prometheus_client import Counter, Histogram
self.labels = labels or {} self.labels = labels or {}
self.collect_tokens_histogram = collect_tokens_histogram
self.prompt_tokens_total = Counter( self.prompt_tokens_total = Counter(
name="sglang:prompt_tokens_total", name="sglang:prompt_tokens_total",
@@ -1171,56 +1169,55 @@ class TokenizerMetricsCollector:
labelnames=labels.keys(), labelnames=labels.keys(),
) )
if collect_tokens_histogram: default_bucket_prompt_tokens = [
default_bucket_prompt_tokens = [ 100,
100, 300,
300, 500,
500, 700,
700, 1000,
1000, 1500,
1500, 2000,
2000, 3000,
3000, 4000,
4000, 5000,
5000, 6000,
6000, 7000,
7000, 8000,
8000, 9000,
9000, 10000,
10000, 12000,
12000, 15000,
15000, 20000,
20000, 22000,
22000, 25000,
25000, 30000,
30000, 35000,
35000, 40000,
40000, 66000,
66000, 99000,
99000, 132000,
132000, 300000,
300000, 600000,
600000, 900000,
900000, 1100000,
1100000, ]
] self.prompt_tokens_histogram = Histogram(
self.prompt_tokens_histogram = Histogram( name="sglang:prompt_tokens_histogram",
name="sglang:prompt_tokens_histogram", documentation="Histogram of prompt token length.",
documentation="Histogram of prompt token length.", labelnames=labels.keys(),
labelnames=labels.keys(), buckets=generate_buckets(
buckets=generate_buckets( server_args.prompt_tokens_buckets, default_bucket_prompt_tokens
server_args.prompt_tokens_buckets, default_bucket_prompt_tokens ),
), )
) self.generation_tokens_histogram = Histogram(
self.generation_tokens_histogram = Histogram( name="sglang:generation_tokens_histogram",
name="sglang:generation_tokens_histogram", documentation="Histogram of generation token length.",
documentation="Histogram of generation token length.", labelnames=labels.keys(),
labelnames=labels.keys(), buckets=generate_buckets(
buckets=generate_buckets( server_args.generation_tokens_buckets,
server_args.generation_tokens_buckets, default_bucket_prompt_tokens,
default_bucket_prompt_tokens, ),
), )
)
self.cached_tokens_total = Counter( self.cached_tokens_total = Counter(
name="sglang:cached_tokens_total", name="sglang:cached_tokens_total",
@@ -1413,11 +1410,10 @@ class TokenizerMetricsCollector:
if has_grammar: if has_grammar:
self.num_so_requests_total.labels(**labels).inc(1) self.num_so_requests_total.labels(**labels).inc(1)
self.histogram_e2e_request_latency.labels(**labels).observe(float(e2e_latency)) self.histogram_e2e_request_latency.labels(**labels).observe(float(e2e_latency))
if self.collect_tokens_histogram: self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens))
self.prompt_tokens_histogram.labels(**labels).observe(float(prompt_tokens)) self.generation_tokens_histogram.labels(**labels).observe(
self.generation_tokens_histogram.labels(**labels).observe( float(generation_tokens)
float(generation_tokens) )
)
self.num_retractions.labels(**labels).observe(retraction_count) self.num_retractions.labels(**labels).observe(retraction_count)
def observe_time_to_first_token(self, labels: Dict[str, str], value: float): def observe_time_to_first_token(self, labels: Dict[str, str], value: float):
python/sglang/srt/server_args.py
+2 -4
View File
@@ -420,7 +420,6 @@ class ServerArgs:
bucket_time_to_first_token: Optional[List[float]] = None bucket_time_to_first_token: Optional[List[float]] = None
bucket_inter_token_latency: Optional[List[float]] = None bucket_inter_token_latency: Optional[List[float]] = None
bucket_e2e_request_latency: Optional[List[float]] = None bucket_e2e_request_latency: Optional[List[float]] = None
collect_tokens_histogram: bool = False
prompt_tokens_buckets: Optional[List[str]] = None prompt_tokens_buckets: Optional[List[str]] = None
generation_tokens_buckets: Optional[List[str]] = None generation_tokens_buckets: Optional[List[str]] = None
gc_warning_threshold_secs: float = 0.0 gc_warning_threshold_secs: float = 0.0
@@ -4803,9 +4802,8 @@ class ServerArgs:
) )
parser.add_argument( parser.add_argument(
"--collect-tokens-histogram", "--collect-tokens-histogram",
action="store_true", action=DeprecatedAction,
default=ServerArgs.collect_tokens_histogram, help="Deprecated. Token histograms are now automatically collected when --enable-metrics is set.",
help="Collect prompt/generation tokens histogram.",
) )
bucket_rule = ( bucket_rule = (
"Supports 3 rule types: 'default' uses predefined buckets; 'tse <middle> <base> <count>' " "Supports 3 rule types: 'default' uses predefined buckets; 'tse <middle> <base> <count>' "