MFU metrics in Prometheus (#19395)

docs/advanced_features/server_arguments.md

@@ -185,6 +185,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
 | `--crash-dump-folder` | Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled. | `None` | Type: str |
 | `--show-time-cost` | Show time cost of custom marks. | `False` | bool flag (set to enable) |
 | `--enable-metrics` | Enable log prometheus metrics. | `False` | bool flag (set to enable) |
+| `--enable-mfu-metrics` | Enable estimated MFU-related prometheus metrics. | `False` | bool flag (set to enable) |
 | `--enable-metrics-for-all-schedulers` | Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0. | `False` | bool flag (set to enable) |
 | `--tokenizer-metrics-custom-labels-header` | Specify the HTTP header for passing custom labels for tokenizer metrics. | `x-custom-labels` | Type: str |
 | `--tokenizer-metrics-allowed-custom-labels` | The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set. | `None` | List[str] |

docs/references/production_metrics.md

@@ -142,7 +142,8 @@ This section describes how to set up the monitoring stack (Prometheus + Grafana)
     python -m sglang.launch_server \
       --model-path <your_model_path> \
       --port 30000 \
-      --enable-metrics
+      --enable-metrics \
+      --enable-mfu-metrics
     ```
     Replace `<your_model_path>` with the actual path to your model (e.g., `meta-llama/Meta-Llama-3.1-8B-Instruct`). Ensure the server is accessible from the monitoring stack (you might need `--host 0.0.0.0` if running in Docker). By default, the metrics endpoint will be available at `http://<sglang_server_host>:30000/metrics`.
 
@@ -229,3 +230,38 @@ python3 -m sglang.bench_serving \
 to generate some requests.
 
 Then you should be able to see the metrics in the Grafana dashboard.
+
+## Estimated Performance Metrics (MFU-related)
+
+SGLang exports the following estimated per-GPU counters that can be used to derive
+Model FLOPs Utilization (MFU)-related signals:
+
+- `sglang:estimated_flops_per_gpu_total`: Estimated floating-point operations.
+- `sglang:estimated_read_bytes_per_gpu_total`: Estimated bytes read from memory.
+- `sglang:estimated_write_bytes_per_gpu_total`: Estimated bytes written to memory.
+
+These metrics are available when both `--enable-metrics` and
+`--enable-mfu-metrics` are enabled.
+
+These are cumulative counters. Use Prometheus `rate(...)` to get per-second values.
+
+### PromQL examples
+
+Average TFLOPS per GPU:
+
+```promql
+rate(sglang:estimated_flops_per_gpu_total[1m]) / 1e12
+```
+
+Average estimated memory bandwidth in GB/s:
+
+```promql
+(rate(sglang:estimated_read_bytes_per_gpu_total[1m]) +
+ rate(sglang:estimated_write_bytes_per_gpu_total[1m])) / 1e9
+```
+
+### Notes
+
+- These metrics are estimates intended for observability and trend analysis.
+- Estimated memory bytes reflect modeled traffic and are not a direct hardware
+  counter from GPU profilers.

python/sglang/srt/observability/metrics_collector.py

@@ -702,6 +702,30 @@ class SchedulerMetricsCollector:
             ),
             labelnames=list(labels.keys()) + ["category"],
         )
+        self.estimated_flops_per_gpu_total = Counter(
+            name="sglang:estimated_flops_per_gpu_total",
+            documentation=(
+                "Estimated number of floating point operations per GPU "
+                "(for Model FLOPs Utilization calculations)."
+            ),
+            labelnames=labels.keys(),
+        )
+        self.estimated_read_bytes_per_gpu_total = Counter(
+            name="sglang:estimated_read_bytes_per_gpu_total",
+            documentation=(
+                "Estimated number of bytes read from memory per GPU "
+                "(for Model FLOPs Utilization calculations)."
+            ),
+            labelnames=labels.keys(),
+        )
+        self.estimated_write_bytes_per_gpu_total = Counter(
+            name="sglang:estimated_write_bytes_per_gpu_total",
+            documentation=(
+                "Estimated number of bytes written to memory per GPU "
+                "(for Model FLOPs Utilization calculations)."
+            ),
+            labelnames=labels.keys(),
+        )
 
         self.dp_cooperation_realtime_tokens_total = Counter(
             name="sglang:dp_cooperation_realtime_tokens_total",
@@ -928,6 +952,25 @@ class SchedulerMetricsCollector:
                 **dp_cooperation_info.to_labels(),
             ).inc(t)
 
+    def increment_estimated_perf(
+        self,
+        num_flops_per_gpu: float = 0.0,
+        num_read_bytes_per_gpu: float = 0.0,
+        num_write_bytes_per_gpu: float = 0.0,
+    ) -> None:
+        if num_flops_per_gpu > 0:
+            self.estimated_flops_per_gpu_total.labels(**self.labels).inc(
+                num_flops_per_gpu
+            )
+        if num_read_bytes_per_gpu > 0:
+            self.estimated_read_bytes_per_gpu_total.labels(**self.labels).inc(
+                num_read_bytes_per_gpu
+            )
+        if num_write_bytes_per_gpu > 0:
+            self.estimated_write_bytes_per_gpu_total.labels(**self.labels).inc(
+                num_write_bytes_per_gpu
+            )
+
     def log_stats(self, stats: SchedulerStats) -> None:
         self._log_gauge_queue_count(self.num_running_reqs, stats.num_running_reqs)
         self._log_gauge(self.num_used_tokens, stats.num_used_tokens)

python/sglang/srt/observability/scheduler_metrics_mixin.py

@@ -5,7 +5,7 @@ import logging
 import time
 from collections import defaultdict
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 from sglang.srt.disaggregation.kv_events import EventPublisherFactory, KVEventBatch
 from sglang.srt.disaggregation.utils import DisaggregationMode
@@ -114,6 +114,7 @@ class SchedulerMetricsMixin:
         self.stats = SchedulerStats()
 
         # Metrics
+        self.enable_mfu_metrics = False
         self.enable_metrics = self.server_args.enable_metrics
         self.is_stats_logging_rank = self.attn_tp_rank == 0
         self.current_scheduler_metrics_enabled = self.enable_metrics and (
@@ -148,6 +149,12 @@ class SchedulerMetricsMixin:
                 enable_hierarchical_cache=self.enable_hierarchical_cache,
                 server_args=self.server_args,
             )
+            self.enable_mfu_metrics = bool(self.server_args.enable_mfu_metrics)
+            if self.enable_mfu_metrics:
+                self._init_estimated_perf_constants()
+                self._mfu_log_flops = 0.0
+                self._mfu_log_read_bytes = 0.0
+                self._mfu_log_write_bytes = 0.0
 
             if ENABLE_METRICS_DEVICE_TIMER:
                 self.forward_pass_device_timer = DeviceTimer(
@@ -175,6 +182,139 @@ class SchedulerMetricsMixin:
         self.spec_num_forward_ct += bs
         self.num_generated_tokens += num_accepted_tokens
 
+    def _init_estimated_perf_constants(self: Scheduler) -> None:
+        model_config = self.model_config
+        hf_text_config = model_config.hf_text_config
+
+        hidden_size = float(model_config.hidden_size)
+        num_layers = float(getattr(model_config, "num_attention_layers", 0))
+        head_dim = float(getattr(model_config, "head_dim", 0))
+        num_attn_heads = float(model_config.get_num_attention_heads(self.tp_size))
+        num_kv_heads = float(model_config.get_num_kv_heads(self.tp_size))
+        intermediate_size = getattr(hf_text_config, "intermediate_size", None)
+        if intermediate_size is None:
+            intermediate_size = getattr(hf_text_config, "ffn_hidden_size", 0)
+        intermediate_size = float(intermediate_size)
+
+        dtype_num_bytes = getattr(model_config.dtype, "itemsize", None)
+        if dtype_num_bytes is None:
+            dtype_num_bytes = 2
+        # Keep this estimator lightweight and consistent with current server dtype.
+        # KV cache quantization-aware bytes can be added in a follow-up.
+        act_bytes = float(dtype_num_bytes)
+        w_bytes = float(dtype_num_bytes)
+        cache_bytes = float(dtype_num_bytes)
+
+        # Linear-layer FLOPs per token on one GPU.
+        attn_linear_flops = (
+            2.0 * hidden_size * head_dim * (num_attn_heads + 2.0 * num_kv_heads)
+            + 2.0 * hidden_size * head_dim * num_attn_heads
+        )
+        mlp_flops = (
+            6.0 * hidden_size * intermediate_size if intermediate_size > 0 else 0.0
+        )
+        self._linear_flops_per_token = max(
+            0.0, (attn_linear_flops + mlp_flops) * num_layers
+        )
+
+        # Attention dot-product FLOPs coefficient to multiply token-context product.
+        # attn_qk + attn_av = 4 * q * TC * d * L
+        self._attn_dot_flops_coeff = 4.0 * num_attn_heads * head_dim * num_layers
+
+        # KV cache bytes (write one K and one V vector per generated token).
+        self._kv_cache_bytes_per_token = (
+            2.0 * num_layers * num_kv_heads * head_dim * cache_bytes
+        )
+
+        # Weight read bytes per token.
+        self._weight_read_bytes_per_token = (
+            hidden_size
+            * head_dim
+            * (num_attn_heads + 2.0 * num_kv_heads)
+            * w_bytes
+            * num_layers
+            + hidden_size * head_dim * num_attn_heads * w_bytes * num_layers
+            + (
+                3.0 * hidden_size * intermediate_size * w_bytes * num_layers
+                if intermediate_size > 0
+                else 0.0
+            )
+        )
+
+        # Activation movement bytes per token (coarse approximation).
+        self._qkv_act_bytes_per_token = (
+            hidden_size * act_bytes * num_layers
+            + (num_attn_heads + 2.0 * num_kv_heads) * head_dim * act_bytes * num_layers
+            + head_dim * num_attn_heads * act_bytes * num_layers
+            + hidden_size * act_bytes * num_layers
+        )
+        self._ffn_act_bytes_per_token = (
+            3.0 * intermediate_size * act_bytes * num_layers
+            if intermediate_size > 0
+            else 0.0
+        )
+
+        # Prefill reads Q/K/V activations from on-device memory.
+        self._prefill_attn_act_read_per_token = (
+            (num_attn_heads + 2.0 * num_kv_heads) * head_dim * act_bytes * num_layers
+        )
+
+        # Decode reads Q from activation memory; K/V reads are from KV cache.
+        self._decode_q_read_bytes_per_token = (
+            num_attn_heads * head_dim * act_bytes * num_layers
+        )
+
+    def _estimate_prefill_perf(
+        self: Scheduler, num_tokens: int
+    ) -> Tuple[float, float, float]:
+        tokens = max(0, int(num_tokens))
+        if tokens == 0:
+            return 0.0, 0.0, 0.0
+
+        # Causal prefill token-context product.
+        context_product = tokens * (tokens + 1) / 2.0
+        flops = (
+            tokens * self._linear_flops_per_token
+            + self._attn_dot_flops_coeff * context_product
+        )
+
+        read_bytes = (
+            tokens * self._weight_read_bytes_per_token
+            + tokens * self._qkv_act_bytes_per_token
+            + tokens * self._prefill_attn_act_read_per_token
+        )
+        write_bytes = (
+            tokens * self._kv_cache_bytes_per_token
+            + tokens * self._qkv_act_bytes_per_token
+            + tokens * self._ffn_act_bytes_per_token
+        )
+        return flops, read_bytes, write_bytes
+
+    def _estimate_decode_perf(
+        self: Scheduler, batch: ScheduleBatch, num_tokens: int
+    ) -> Tuple[float, float, float]:
+        tokens = max(0, int(num_tokens))
+        if tokens == 0:
+            return 0.0, 0.0, 0.0
+
+        total_context = float(batch.seq_lens_cpu.sum().item())
+        flops = (
+            tokens * self._linear_flops_per_token
+            + self._attn_dot_flops_coeff * total_context
+        )
+        read_bytes = (
+            tokens * self._weight_read_bytes_per_token
+            + tokens * self._qkv_act_bytes_per_token
+            + tokens * self._decode_q_read_bytes_per_token
+            + total_context * self._kv_cache_bytes_per_token
+        )
+        write_bytes = (
+            tokens * self._kv_cache_bytes_per_token
+            + tokens * self._qkv_act_bytes_per_token
+            + tokens * self._ffn_act_bytes_per_token
+        )
+        return flops, read_bytes, write_bytes
+
     def reset_metrics(self: Scheduler):
         self.forward_ct_decode = 0
         self.num_generated_tokens = 0
@@ -275,6 +415,11 @@ class SchedulerMetricsMixin:
         msg += f"{graph_backend[self.device]}: {can_run_cuda_graph}, "
         msg += f"input throughput (token/s): {self.last_input_throughput:.2f}"
 
+        if self.enable_mfu_metrics and gap_latency > 0:
+            flops, _, _ = self._estimate_prefill_perf(prefill_stats.log_input_tokens)
+            tflops_per_s = flops / gap_latency / 1e12
+            msg += f", est. prefill TFLOPS/s (per GPU): {tflops_per_s:.2f}"
+
         if self.is_stats_logging_rank:
             logger.info(msg)
 
@@ -287,6 +432,15 @@ class SchedulerMetricsMixin:
                 prefill_cache_tokens=prefill_stats.log_hit_tokens,
                 dp_cooperation_info=dp_cooperation_info,
             )
+            if self.enable_mfu_metrics:
+                flops, read_bytes, write_bytes = self._estimate_prefill_perf(
+                    prefill_stats.log_input_tokens
+                )
+                self.metrics_collector.increment_estimated_perf(
+                    num_flops_per_gpu=flops,
+                    num_read_bytes_per_gpu=read_bytes,
+                    num_write_bytes_per_gpu=write_bytes,
+                )
 
             # Basics
             total_tokens = prefill_stats.log_input_tokens + prefill_stats.log_hit_tokens
@@ -354,11 +508,24 @@ class SchedulerMetricsMixin:
 
         # Every-iteration work: realtime token counting + status logger
         if self.current_scheduler_metrics_enabled:
+            decode_tokens = batch.batch_size() + num_accepted_tokens
             self.metrics_collector.increment_realtime_tokens(
                 # TODO unify this w/ the bumping logic in `Scheduler.num_generated_tokens` accumulator
-                decode_tokens=batch.batch_size() + num_accepted_tokens,
+                decode_tokens=decode_tokens,
                 dp_cooperation_info=batch.dp_cooperation_info,
             )
+            if self.enable_mfu_metrics:
+                flops, read_bytes, write_bytes = self._estimate_decode_perf(
+                    batch, decode_tokens
+                )
+                self.metrics_collector.increment_estimated_perf(
+                    num_flops_per_gpu=flops,
+                    num_read_bytes_per_gpu=read_bytes,
+                    num_write_bytes_per_gpu=write_bytes,
+                )
+                self._mfu_log_flops += flops
+                self._mfu_log_read_bytes += read_bytes
+                self._mfu_log_write_bytes += write_bytes
 
             if x := self.scheduler_status_logger:
                 x.maybe_dump(batch, self.waiting_queue)
@@ -490,6 +657,22 @@ class SchedulerMetricsMixin:
             f"#queue-req: {len(self.waiting_queue)}"
         )
 
+        if self.enable_mfu_metrics and gap_latency > 0:
+            flops_per_s = self._mfu_log_flops / gap_latency
+            read_bytes_per_s = self._mfu_log_read_bytes / gap_latency
+            write_bytes_per_s = self._mfu_log_write_bytes / gap_latency
+            tflops_per_s = flops_per_s / 1e12
+            read_gb_per_s = read_bytes_per_s / 1e9
+            write_gb_per_s = write_bytes_per_s / 1e9
+            msg += (
+                f", est. decode TFLOPS/s (per GPU): {tflops_per_s:.2f}, "
+                f"est. read BW (GB/s per GPU): {read_gb_per_s:.2f}, "
+                f"est. write BW (GB/s per GPU): {write_gb_per_s:.2f}"
+            )
+            self._mfu_log_flops = 0.0
+            self._mfu_log_read_bytes = 0.0
+            self._mfu_log_write_bytes = 0.0
+
         if self.is_stats_logging_rank:
             logger.info(msg)
         if self.current_scheduler_metrics_enabled:

python/sglang/srt/server_args.py

@@ -398,6 +398,7 @@ class ServerArgs:
     crash_dump_folder: Optional[str] = None
     show_time_cost: bool = False
     enable_metrics: bool = False
+    enable_mfu_metrics: bool = False
     enable_metrics_for_all_schedulers: bool = False
     tokenizer_metrics_custom_labels_header: str = "x-custom-labels"
     tokenizer_metrics_allowed_custom_labels: Optional[List[str]] = None
@@ -4220,6 +4221,11 @@ class ServerArgs:
             action="store_true",
             help="Enable log prometheus metrics.",
         )
+        parser.add_argument(
+            "--enable-mfu-metrics",
+            action="store_true",
+            help="Enable estimated MFU-related prometheus metrics.",
+        )
         parser.add_argument(
             "--enable-metrics-for-all-schedulers",
             action="store_true",

test/registered/metrics/test_metrics.py

@@ -32,6 +32,17 @@ class TestEnableMetrics(CustomTestCase):
         self._execute_core(
             other_args=[],
             verify_metrics_extra=None,
+            expect_mfu_metrics=True,
+            enable_mfu_metrics=True,
+        )
+
+    def test_mfu_metrics_gate_disabled(self):
+        """MFU metrics should not be emitted when the gate is disabled."""
+        self._execute_core(
+            other_args=[],
+            verify_metrics_extra=None,
+            expect_mfu_metrics=False,
+            enable_mfu_metrics=False,
         )
 
     def test_metrics_2gpu(self):
@@ -71,19 +82,30 @@ class TestEnableMetrics(CustomTestCase):
         self._execute_core(
             other_args=["--tp", "2", "--dp", "2", "--enable-dp-attention"],
             verify_metrics_extra=_verify_metrics_extra,
+            expect_mfu_metrics=True,
+            enable_mfu_metrics=True,
         )
 
-    def _execute_core(self, other_args, verify_metrics_extra):
+    def _execute_core(
+        self,
+        other_args,
+        verify_metrics_extra,
+        expect_mfu_metrics: bool,
+        enable_mfu_metrics: bool,
+    ):
         with (
             envs.SGLANG_ENABLE_METRICS_DP_ATTENTION.override(True),
             envs.SGLANG_ENABLE_METRICS_DEVICE_TIMER.override(True),
             envs.SGLANG_TEST_RETRACT.override(True),
         ):
+            launch_args = ["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args]
+            if enable_mfu_metrics:
+                launch_args.insert(1, "--enable-mfu-metrics")
             process = popen_launch_server(
                 _MODEL_NAME,
                 DEFAULT_URL_FOR_TEST,
                 timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                other_args=["--enable-metrics", "--cuda-graph-max-bs", 2, *other_args],
+                other_args=launch_args,
             )
 
         try:
@@ -125,13 +147,13 @@ class TestEnableMetrics(CustomTestCase):
             print(f"metrics_text=\n{metrics_text}")
 
             metrics = _parse_prometheus_metrics(metrics_text)
-            self._verify_metrics_common(metrics_text, metrics)
+            self._verify_metrics_common(metrics_text, metrics, expect_mfu_metrics)
             if verify_metrics_extra is not None:
                 verify_metrics_extra(metrics)
         finally:
             kill_process_tree(process.pid)
 
-    def _verify_metrics_common(self, metrics_text, metrics):
+    def _verify_metrics_common(self, metrics_text, metrics, expect_mfu_metrics: bool):
         essential_metrics = [
             "sglang:num_running_reqs",
             "sglang:num_used_tokens",
@@ -154,6 +176,13 @@ class TestEnableMetrics(CustomTestCase):
             "sglang:routing_key_running_req_count",
             "sglang:routing_key_all_req_count",
         ]
+        mfu_metrics = [
+            "sglang:estimated_flops_per_gpu_total",
+            "sglang:estimated_read_bytes_per_gpu_total",
+            "sglang:estimated_write_bytes_per_gpu_total",
+        ]
+        if expect_mfu_metrics:
+            essential_metrics.extend(mfu_metrics)
         for metric in essential_metrics:
             self.assertIn(metric, metrics_text, f"Missing metric: {metric}")
 
@@ -186,6 +215,39 @@ class TestEnableMetrics(CustomTestCase):
         ]
         _check_metrics_positive(self, metrics, metrics_to_check)
 
+        if expect_mfu_metrics:
+            # Estimated perf metrics may have multiple series (e.g., by rank). Ensure
+            # that at least one series for this model has a positive accumulated value.
+            for metric_name in mfu_metrics:
+                values = [
+                    sample.value
+                    for sample in metrics.get(metric_name, [])
+                    if sample.labels.get("model_name") == _MODEL_NAME
+                ]
+                self.assertTrue(
+                    values, f"{metric_name}: no samples for model {_MODEL_NAME}"
+                )
+                self.assertGreater(
+                    sum(values),
+                    0,
+                    f"{metric_name}: expected positive total for model {_MODEL_NAME}",
+                )
+        else:
+            # With only --enable-metrics (without --enable-mfu-metrics), MFU
+            # counters should not emit positive values.
+            for metric_name in mfu_metrics:
+                values = [
+                    sample.value
+                    for sample in metrics.get(metric_name, [])
+                    if sample.labels.get("model_name") == _MODEL_NAME
+                ]
+                if values:
+                    self.assertEqual(
+                        sum(values),
+                        0,
+                        f"{metric_name}: expected no positive samples with MFU metrics gate disabled",
+                    )
+
 
 def _parse_prometheus_metrics(metrics_text: str) -> Dict[str, List[Sample]]:
     result = {}