From c32ee488863a3950e98750320b7ddfcb07b99783 Mon Sep 17 00:00:00 2001 From: Aishwarya Ramasethu <56765596+aramasethu@users.noreply.github.com> Date: Mon, 30 Mar 2026 02:40:06 -0400 Subject: [PATCH] MFU metrics in Prometheus (#19395) --- docs/advanced_features/server_arguments.md | 1 + docs/references/production_metrics.md | 38 +++- .../srt/observability/metrics_collector.py | 43 ++++ .../observability/scheduler_metrics_mixin.py | 187 +++++++++++++++++- python/sglang/srt/server_args.py | 6 + test/registered/metrics/test_metrics.py | 70 ++++++- 6 files changed, 338 insertions(+), 7 deletions(-) diff --git a/docs/advanced_features/server_arguments.md b/docs/advanced_features/server_arguments.md index 7de47a535..82fd7010e 100644 --- a/docs/advanced_features/server_arguments.md +++ b/docs/advanced_features/server_arguments.md @@ -185,6 +185,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s | `--crash-dump-folder` | Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled. | `None` | Type: str | | `--show-time-cost` | Show time cost of custom marks. | `False` | bool flag (set to enable) | | `--enable-metrics` | Enable log prometheus metrics. | `False` | bool flag (set to enable) | +| `--enable-mfu-metrics` | Enable estimated MFU-related prometheus metrics. | `False` | bool flag (set to enable) | | `--enable-metrics-for-all-schedulers` | Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. | `False` | bool flag (set to enable) | | `--tokenizer-metrics-custom-labels-header` | Specify the HTTP header for passing custom labels for tokenizer metrics. | `x-custom-labels` | Type: str | | `--tokenizer-metrics-allowed-custom-labels` | The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set. | `None` | List[str] | diff --git a/docs/references/production_metrics.md b/docs/references/production_metrics.md index 85a6ff8a6..d104584ee 100644 --- a/docs/references/production_metrics.md +++ b/docs/references/production_metrics.md @@ -142,7 +142,8 @@ This section describes how to set up the monitoring stack (Prometheus + Grafana) python -m sglang.launch_server \ --model-path \ --port 30000 \ - --enable-metrics + --enable-metrics \ + --enable-mfu-metrics ``` Replace `` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://:30000/metrics`. @@ -229,3 +230,38 @@ python3 -m sglang.bench_serving \ to generate some requests. Then you should be able to see the metrics in the Grafana dashboard. + +## Estimated Performance Metrics (MFU-related) + +SGLang exports the following estimated per-GPU counters that can be used to derive +Model FLOPs Utilization (MFU)-related signals: + +- `sglang:estimated_flops_per_gpu_total`: Estimated floating-point operations. +- `sglang:estimated_read_bytes_per_gpu_total`: Estimated bytes read from memory. +- `sglang:estimated_write_bytes_per_gpu_total`: Estimated bytes written to memory. + +These metrics are available when both `--enable-metrics` and +`--enable-mfu-metrics` are enabled. + +These are cumulative counters. Use Prometheus `rate(...)` to get per-second values. + +### PromQL examples + +Average TFLOPS per GPU: + +```promql +rate(sglang:estimated_flops_per_gpu_total[1m]) / 1e12 +``` + +Average estimated memory bandwidth in GB/s: + +```promql +(rate(sglang:estimated_read_bytes_per_gpu_total[1m]) + + rate(sglang:estimated_write_bytes_per_gpu_total[1m])) / 1e9 +``` + +### Notes + +- These metrics are estimates intended for observability and trend analysis. +- Estimated memory bytes reflect modeled traffic and are not a direct hardware + counter from GPU profilers. diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 16da400d5..490d755ed 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -702,6 +702,30 @@ class SchedulerMetricsCollector: ), labelnames=list(labels.keys()) + ["category"], ) + self.estimated_flops_per_gpu_total = Counter( + name="sglang:estimated_flops_per_gpu_total", + documentation=( + "Estimated number of floating point operations per GPU " + "(for Model FLOPs Utilization calculations)." + ), + labelnames=labels.keys(), + ) + self.estimated_read_bytes_per_gpu_total = Counter( + name="sglang:estimated_read_bytes_per_gpu_total", + documentation=( + "Estimated number of bytes read from memory per GPU " + "(for Model FLOPs Utilization calculations)." + ), + labelnames=labels.keys(), + ) + self.estimated_write_bytes_per_gpu_total = Counter( + name="sglang:estimated_write_bytes_per_gpu_total", + documentation=( + "Estimated number of bytes written to memory per GPU " + "(for Model FLOPs Utilization calculations)." + ), + labelnames=labels.keys(), + ) self.dp_cooperation_realtime_tokens_total = Counter( name="sglang:dp_cooperation_realtime_tokens_total", @@ -928,6 +952,25 @@ class SchedulerMetricsCollector: **dp_cooperation_info.to_labels(), ).inc(t) + def increment_estimated_perf( + self, + num_flops_per_gpu: float = 0.0, + num_read_bytes_per_gpu: float = 0.0, + num_write_bytes_per_gpu: float = 0.0, + ) -> None: + if num_flops_per_gpu > 0: + self.estimated_flops_per_gpu_total.labels(**self.labels).inc( + num_flops_per_gpu + ) + if num_read_bytes_per_gpu > 0: + self.estimated_read_bytes_per_gpu_total.labels(**self.labels).inc( + num_read_bytes_per_gpu + ) + if num_write_bytes_per_gpu > 0: + self.estimated_write_bytes_per_gpu_total.labels(**self.labels).inc( + num_write_bytes_per_gpu + ) + def log_stats(self, stats: SchedulerStats) -> None: self._log_gauge_queue_count(self.num_running_reqs, stats.num_running_reqs) self._log_gauge(self.num_used_tokens, stats.num_used_tokens) diff --git a/python/sglang/srt/observability/scheduler_metrics_mixin.py b/python/sglang/srt/observability/scheduler_metrics_mixin.py index 052c23645..ff5695ce2 100644 --- a/python/sglang/srt/observability/scheduler_metrics_mixin.py +++ b/python/sglang/srt/observability/scheduler_metrics_mixin.py @@ -5,7 +5,7 @@ import logging import time from collections import defaultdict from contextlib import contextmanager -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Tuple, Union from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch from sglang.srt.disaggregation.utils import DisaggregationMode @@ -114,6 +114,7 @@ class SchedulerMetricsMixin: self.stats = SchedulerStats() # Metrics + self.enable_mfu_metrics = False self.enable_metrics = self.server_args.enable_metrics self.is_stats_logging_rank = self.attn_tp_rank == 0 self.current_scheduler_metrics_enabled = self.enable_metrics and ( @@ -148,6 +149,12 @@ class SchedulerMetricsMixin: enable_hierarchical_cache=self.enable_hierarchical_cache, server_args=self.server_args, ) + self.enable_mfu_metrics = bool(self.server_args.enable_mfu_metrics) + if self.enable_mfu_metrics: + self._init_estimated_perf_constants() + self._mfu_log_flops = 0.0 + self._mfu_log_read_bytes = 0.0 + self._mfu_log_write_bytes = 0.0 if ENABLE_METRICS_DEVICE_TIMER: self.forward_pass_device_timer = DeviceTimer( @@ -175,6 +182,139 @@ class SchedulerMetricsMixin: self.spec_num_forward_ct += bs self.num_generated_tokens += num_accepted_tokens + def _init_estimated_perf_constants(self: Scheduler) -> None: + model_config = self.model_config + hf_text_config = model_config.hf_text_config + + hidden_size = float(model_config.hidden_size) + num_layers = float(getattr(model_config, "num_attention_layers", 0)) + head_dim = float(getattr(model_config, "head_dim", 0)) + num_attn_heads = float(model_config.get_num_attention_heads(self.tp_size)) + num_kv_heads = float(model_config.get_num_kv_heads(self.tp_size)) + intermediate_size = getattr(hf_text_config, "intermediate_size", None) + if intermediate_size is None: + intermediate_size = getattr(hf_text_config, "ffn_hidden_size", 0) + intermediate_size = float(intermediate_size) + + dtype_num_bytes = getattr(model_config.dtype, "itemsize", None) + if dtype_num_bytes is None: + dtype_num_bytes = 2 + # Keep this estimator lightweight and consistent with current server dtype. + # KV cache quantization-aware bytes can be added in a follow-up. + act_bytes = float(dtype_num_bytes) + w_bytes = float(dtype_num_bytes) + cache_bytes = float(dtype_num_bytes) + + # Linear-layer FLOPs per token on one GPU. + attn_linear_flops = ( + 2.0 * hidden_size * head_dim * (num_attn_heads + 2.0 * num_kv_heads) + + 2.0 * hidden_size * head_dim * num_attn_heads + ) + mlp_flops = ( + 6.0 * hidden_size * intermediate_size if intermediate_size > 0 else 0.0 + ) + self._linear_flops_per_token = max( + 0.0, (attn_linear_flops + mlp_flops) * num_layers + ) + + # Attention dot-product FLOPs coefficient to multiply token-context product. + # attn_qk + attn_av = 4 * q * TC * d * L + self._attn_dot_flops_coeff = 4.0 * num_attn_heads * head_dim * num_layers + + # KV cache bytes (write one K and one V vector per generated token). + self._kv_cache_bytes_per_token = ( + 2.0 * num_layers * num_kv_heads * head_dim * cache_bytes + ) + + # Weight read bytes per token. + self._weight_read_bytes_per_token = ( + hidden_size + * head_dim + * (num_attn_heads + 2.0 * num_kv_heads) + * w_bytes + * num_layers + + hidden_size * head_dim * num_attn_heads * w_bytes * num_layers + + ( + 3.0 * hidden_size * intermediate_size * w_bytes * num_layers + if intermediate_size > 0 + else 0.0 + ) + ) + + # Activation movement bytes per token (coarse approximation). + self._qkv_act_bytes_per_token = ( + hidden_size * act_bytes * num_layers + + (num_attn_heads + 2.0 * num_kv_heads) * head_dim * act_bytes * num_layers + + head_dim * num_attn_heads * act_bytes * num_layers + + hidden_size * act_bytes * num_layers + ) + self._ffn_act_bytes_per_token = ( + 3.0 * intermediate_size * act_bytes * num_layers + if intermediate_size > 0 + else 0.0 + ) + + # Prefill reads Q/K/V activations from on-device memory. + self._prefill_attn_act_read_per_token = ( + (num_attn_heads + 2.0 * num_kv_heads) * head_dim * act_bytes * num_layers + ) + + # Decode reads Q from activation memory; K/V reads are from KV cache. + self._decode_q_read_bytes_per_token = ( + num_attn_heads * head_dim * act_bytes * num_layers + ) + + def _estimate_prefill_perf( + self: Scheduler, num_tokens: int + ) -> Tuple[float, float, float]: + tokens = max(0, int(num_tokens)) + if tokens == 0: + return 0.0, 0.0, 0.0 + + # Causal prefill token-context product. + context_product = tokens * (tokens + 1) / 2.0 + flops = ( + tokens * self._linear_flops_per_token + + self._attn_dot_flops_coeff * context_product + ) + + read_bytes = ( + tokens * self._weight_read_bytes_per_token + + tokens * self._qkv_act_bytes_per_token + + tokens * self._prefill_attn_act_read_per_token + ) + write_bytes = ( + tokens * self._kv_cache_bytes_per_token + + tokens * self._qkv_act_bytes_per_token + + tokens * self._ffn_act_bytes_per_token + ) + return flops, read_bytes, write_bytes + + def _estimate_decode_perf( + self: Scheduler, batch: ScheduleBatch, num_tokens: int + ) -> Tuple[float, float, float]: + tokens = max(0, int(num_tokens)) + if tokens == 0: + return 0.0, 0.0, 0.0 + + total_context = float(batch.seq_lens_cpu.sum().item()) + flops = ( + tokens * self._linear_flops_per_token + + self._attn_dot_flops_coeff * total_context + ) + read_bytes = ( + tokens * self._weight_read_bytes_per_token + + tokens * self._qkv_act_bytes_per_token + + tokens * self._decode_q_read_bytes_per_token + + total_context * self._kv_cache_bytes_per_token + ) + write_bytes = ( + tokens * self._kv_cache_bytes_per_token + + tokens * self._qkv_act_bytes_per_token + + tokens * self._ffn_act_bytes_per_token + ) + return flops, read_bytes, write_bytes + def reset_metrics(self: Scheduler): self.forward_ct_decode = 0 self.num_generated_tokens = 0 @@ -275,6 +415,11 @@ class SchedulerMetricsMixin: msg += f"{graph_backend[self.device]}: {can_run_cuda_graph}, " msg += f"input throughput (token/s): {self.last_input_throughput:.2f}" + if self.enable_mfu_metrics and gap_latency > 0: + flops, _, _ = self._estimate_prefill_perf(prefill_stats.log_input_tokens) + tflops_per_s = flops / gap_latency / 1e12 + msg += f", est. prefill TFLOPS/s (per GPU): {tflops_per_s:.2f}" + if self.is_stats_logging_rank: logger.info(msg) @@ -287,6 +432,15 @@ class SchedulerMetricsMixin: prefill_cache_tokens=prefill_stats.log_hit_tokens, dp_cooperation_info=dp_cooperation_info, ) + if self.enable_mfu_metrics: + flops, read_bytes, write_bytes = self._estimate_prefill_perf( + prefill_stats.log_input_tokens + ) + self.metrics_collector.increment_estimated_perf( + num_flops_per_gpu=flops, + num_read_bytes_per_gpu=read_bytes, + num_write_bytes_per_gpu=write_bytes, + ) # Basics total_tokens = prefill_stats.log_input_tokens + prefill_stats.log_hit_tokens @@ -354,11 +508,24 @@ class SchedulerMetricsMixin: # Every-iteration work: realtime token counting + status logger if self.current_scheduler_metrics_enabled: + decode_tokens = batch.batch_size() + num_accepted_tokens self.metrics_collector.increment_realtime_tokens( # TODO unify this w/ the bumping logic in `Scheduler.num_generated_tokens` accumulator - decode_tokens=batch.batch_size() + num_accepted_tokens, + decode_tokens=decode_tokens, dp_cooperation_info=batch.dp_cooperation_info, ) + if self.enable_mfu_metrics: + flops, read_bytes, write_bytes = self._estimate_decode_perf( + batch, decode_tokens + ) + self.metrics_collector.increment_estimated_perf( + num_flops_per_gpu=flops, + num_read_bytes_per_gpu=read_bytes, + num_write_bytes_per_gpu=write_bytes, + ) + self._mfu_log_flops += flops + self._mfu_log_read_bytes += read_bytes + self._mfu_log_write_bytes += write_bytes if x := self.scheduler_status_logger: x.maybe_dump(batch, self.waiting_queue) @@ -490,6 +657,22 @@ class SchedulerMetricsMixin: f"#queue-req: {len(self.waiting_queue)}" ) + if self.enable_mfu_metrics and gap_latency > 0: + flops_per_s = self._mfu_log_flops / gap_latency + read_bytes_per_s = self._mfu_log_read_bytes / gap_latency + write_bytes_per_s = self._mfu_log_write_bytes / gap_latency + tflops_per_s = flops_per_s / 1e12 + read_gb_per_s = read_bytes_per_s / 1e9 + write_gb_per_s = write_bytes_per_s / 1e9 + msg += ( + f", est. decode TFLOPS/s (per GPU): {tflops_per_s:.2f}, " + f"est. read BW (GB/s per GPU): {read_gb_per_s:.2f}, " + f"est. write BW (GB/s per GPU): {write_gb_per_s:.2f}" + ) + self._mfu_log_flops = 0.0 + self._mfu_log_read_bytes = 0.0 + self._mfu_log_write_bytes = 0.0 + if self.is_stats_logging_rank: logger.info(msg) if self.current_scheduler_metrics_enabled: diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 04a365927..c10626913 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -398,6 +398,7 @@ class ServerArgs: crash_dump_folder: Optional[str] = None show_time_cost: bool = False enable_metrics: bool = False + enable_mfu_metrics: bool = False enable_metrics_for_all_schedulers: bool = False tokenizer_metrics_custom_labels_header: str = "x-custom-labels" tokenizer_metrics_allowed_custom_labels: Optional[List[str]] = None @@ -4220,6 +4221,11 @@ class ServerArgs: action="store_true", help="Enable log prometheus metrics.", ) + parser.add_argument( + "--enable-mfu-metrics", + action="store_true", + help="Enable estimated MFU-related prometheus metrics.", + ) parser.add_argument( "--enable-metrics-for-all-schedulers", action="store_true", diff --git a/test/registered/metrics/test_metrics.py b/test/registered/metrics/test_metrics.py index 4aafbabae..012bad626 100644 --- a/test/registered/metrics/test_metrics.py +++ b/test/registered/metrics/test_metrics.py @@ -32,6 +32,17 @@ class TestEnableMetrics(CustomTestCase): self._execute_core( other_args=[], verify_metrics_extra=None, + expect_mfu_metrics=True, + enable_mfu_metrics=True, + ) + + def test_mfu_metrics_gate_disabled(self): + """MFU metrics should not be emitted when the gate is disabled.""" + self._execute_core( + other_args=[], + verify_metrics_extra=None, + expect_mfu_metrics=False, + enable_mfu_metrics=False, ) def test_metrics_2gpu(self): @@ -71,19 +82,30 @@ class TestEnableMetrics(CustomTestCase): self._execute_core( other_args=["--tp", "2", "--dp", "2", "--enable-dp-attention"], verify_metrics_extra=_verify_metrics_extra, + expect_mfu_metrics=True, + enable_mfu_metrics=True, ) - def _execute_core(self, other_args, verify_metrics_extra): + def _execute_core( + self, + other_args, + verify_metrics_extra, + expect_mfu_metrics: bool, + enable_mfu_metrics: bool, + ): with ( envs.SGLANG_ENABLE_METRICS_DP_ATTENTION.override(True), envs.SGLANG_ENABLE_METRICS_DEVICE_TIMER.override(True), envs.SGLANG_TEST_RETRACT.override(True), ): + launch_args = ["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args] + if enable_mfu_metrics: + launch_args.insert(1, "--enable-mfu-metrics") process = popen_launch_server( _MODEL_NAME, DEFAULT_URL_FOR_TEST, timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args], + other_args=launch_args, ) try: @@ -125,13 +147,13 @@ class TestEnableMetrics(CustomTestCase): print(f"metrics_text=\n{metrics_text}") metrics = _parse_prometheus_metrics(metrics_text) - self._verify_metrics_common(metrics_text, metrics) + self._verify_metrics_common(metrics_text, metrics, expect_mfu_metrics) if verify_metrics_extra is not None: verify_metrics_extra(metrics) finally: kill_process_tree(process.pid) - def _verify_metrics_common(self, metrics_text, metrics): + def _verify_metrics_common(self, metrics_text, metrics, expect_mfu_metrics: bool): essential_metrics = [ "sglang:num_running_reqs", "sglang:num_used_tokens", @@ -154,6 +176,13 @@ class TestEnableMetrics(CustomTestCase): "sglang:routing_key_running_req_count", "sglang:routing_key_all_req_count", ] + mfu_metrics = [ + "sglang:estimated_flops_per_gpu_total", + "sglang:estimated_read_bytes_per_gpu_total", + "sglang:estimated_write_bytes_per_gpu_total", + ] + if expect_mfu_metrics: + essential_metrics.extend(mfu_metrics) for metric in essential_metrics: self.assertIn(metric, metrics_text, f"Missing metric: {metric}") @@ -186,6 +215,39 @@ class TestEnableMetrics(CustomTestCase): ] _check_metrics_positive(self, metrics, metrics_to_check) + if expect_mfu_metrics: + # Estimated perf metrics may have multiple series (e.g., by rank). Ensure + # that at least one series for this model has a positive accumulated value. + for metric_name in mfu_metrics: + values = [ + sample.value + for sample in metrics.get(metric_name, []) + if sample.labels.get("model_name") == _MODEL_NAME + ] + self.assertTrue( + values, f"{metric_name}: no samples for model {_MODEL_NAME}" + ) + self.assertGreater( + sum(values), + 0, + f"{metric_name}: expected positive total for model {_MODEL_NAME}", + ) + else: + # With only --enable-metrics (without --enable-mfu-metrics), MFU + # counters should not emit positive values. + for metric_name in mfu_metrics: + values = [ + sample.value + for sample in metrics.get(metric_name, []) + if sample.labels.get("model_name") == _MODEL_NAME + ] + if values: + self.assertEqual( + sum(values), + 0, + f"{metric_name}: expected no positive samples with MFU metrics gate disabled", + ) + def _parse_prometheus_metrics(metrics_text: str) -> Dict[str, List[Sample]]: result = {}