From 2abb761639d39fedf40b187ec5ff676cbce36e6c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 8 Oct 2026 15:01:37 -0700 Subject: [PATCH] [metrics] Count per-rank DP attention pairs and their imbalance (#43180) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: hanming-lu --- .../managers/scheduler_components/dp_attn.py | 32 +++++ .../scheduler_components/metrics_reporter.py | 21 +-- .../srt/observability/metrics_collector.py | 49 ++++++- test/registered/observability/test_metrics.py | 5 +- .../scheduler_components/test_dp_attn.py | 130 +++++++++++++++++- .../test_forward_pass_metrics.py | 50 ++++++- 6 files changed, 269 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 91347fe84b..7101103d5f 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -14,6 +14,10 @@ from sglang.srt.layers.cp.utils import get_cp_strategy from sglang.srt.layers.dp_attention import dp_gather_width, world_dp_gather_enabled from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.managers.scheduler_components.metrics_reporter import ( + _decode_total_seq_lens, + _prefill_attention_pairs, +) from sglang.srt.managers.scheduler_components.recv_skipper import ( SchedulerRecvSkipper, ) @@ -125,6 +129,8 @@ class MLPSyncBatchInfo: prefill_cuda_graph_max_prefix_len: int = 0 # Every request in the local batch is a health-check probe. is_health_check: bool = False + # Causal (query, key) pairs this rank attends this step. + attention_pairs: int = 0 # some gathered elements tp0_info_cpu: torch.Tensor = None @@ -132,6 +138,7 @@ class MLPSyncBatchInfo: sync_wait_seconds: float = 0.0 global_num_tokens: list[int] = None global_num_tokens_for_logprob: list[int] = None + global_attention_pairs: list[int] = None tbo_split_seq_index: torch.Tensor = None global_forward_mode: int = None dp_cooperation_info: Optional[DPCooperationInfo] = None @@ -149,6 +156,7 @@ class MLPSyncBatchInfo: self.prefill_cuda_graph_max_prefix_len, int(self.can_run_draft_cuda_graph), int(self.is_health_check), + self.attention_pairs, ], device=device, dtype=dtype, @@ -167,6 +175,7 @@ class MLPSyncBatchInfo: 0, # prefill_cuda_graph_max_prefix_len 1, # can_run_draft_cuda_graph 0, # is_health_check + 0, # attention_pairs ], device=device, dtype=dtype, @@ -177,6 +186,7 @@ class MLPSyncBatchInfo: self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1) self.global_num_tokens = [self.num_tokens] self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob] + self.global_attention_pairs = [self.attention_pairs] self.any_health_check = self.is_health_check if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( @@ -253,6 +263,7 @@ class MLPSyncBatchInfo: self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max()) self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min()) self.any_health_check = bool(tp0_info_cpu[:, 9].max()) + self.global_attention_pairs = tp0_info_cpu[:, 10].tolist() self.sync_wait_seconds = time.perf_counter() - sync_start if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( @@ -428,6 +439,22 @@ def _local_prefill_cuda_graph_vote( ) +def _local_attention_pairs(local_batch: Optional[ScheduleBatch]) -> int: + """Causal (query, key) pairs the batch attends, pricing every layer as full + attention: one query per decode row over its committed length (spec + batches have no CPU seq_lens at schedule time), the chunk formula for + prefill.""" + if ( + local_batch is None + or local_batch.forward_mode.is_prebuilt() + or local_batch.forward_mode.is_idle() + ): + return 0 + if local_batch.forward_mode.is_decode(): + return _decode_total_seq_lens(local_batch) + return _prefill_attention_pairs(local_batch) + + class DPSyncWaitCarry: """MLP-sync wait of batch-less gathers, folded into the next recorded step. @@ -563,6 +590,9 @@ def prepare_mlp_sync_batch_raw( local_forward_mode=local_forward_mode, prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len, is_health_check=is_health_check, + # Gathered unconditionally like the other columns, so a per-process + # metrics flag cannot make ranks disagree on what the tensor holds. + attention_pairs=_local_attention_pairs(local_batch), ) if num_dp_ranks == 1: @@ -628,6 +658,8 @@ def prepare_mlp_sync_batch_raw( local_batch.dp_balance_stats = DPBalanceStats.create( local_tokens=num_tokens, global_num_tokens=mlp_sync_info.global_num_tokens, + local_attention_pairs=mlp_sync_info.attention_pairs, + global_attention_pairs=mlp_sync_info.global_attention_pairs, sync_wait_seconds=sync_wait_seconds, ) elif local_batch is None and sync_wait_carry is not None: diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index adfd129410..f66f53949a 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -90,6 +90,17 @@ def _decode_total_seq_lens(batch: ScheduleBatch) -> int: return sum(req.seqlen for req in batch.reqs) +def _prefill_attention_pairs(batch: ScheduleBatch) -> int: + """Causal query-key pairs: each chunk against its cached prefix, plus + the causal pairs within the chunk itself.""" + return sum( + extend_len * prefix_len + extend_len * (extend_len + 1) // 2 + for prefix_len, extend_len in zip( + batch.prefix_lens, batch.extend_lens, strict=True + ) + ) + + @dataclasses.dataclass class PrefillStats: """Stats for logging prefill batch metrics.""" @@ -604,14 +615,6 @@ class SchedulerMetricsReporter: num_attn_heads * head_dim * act_bytes * num_layers ) - @staticmethod - def _prefill_attention_pairs(batch) -> float: - """Causal query-key pairs: each chunk against its cached prefix, plus - the causal pairs within the chunk itself.""" - prefix_pairs = sum(c * p for c, p in zip(batch.extend_lens, batch.prefix_lens)) - within_chunk_pairs = sum(c * (c + 1) / 2.0 for c in batch.extend_lens) - return float(prefix_pairs + within_chunk_pairs) - def _estimate_prefill_perf(self, batch) -> Tuple[float, float, float]: if batch is None or batch.extend_lens is None: return 0.0, 0.0, 0.0 @@ -619,7 +622,7 @@ class SchedulerMetricsReporter: if tokens == 0: return 0.0, 0.0, 0.0 - context_product = self._prefill_attention_pairs(batch) + context_product = float(_prefill_attention_pairs(batch)) flops = ( tokens * self._linear_flops_per_token + self._attn_dot_flops_coeff * context_product diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 4ea3ad7d99..7f74a5a203 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -215,11 +215,18 @@ class DPBalanceStats(msgspec.Struct, frozen=True): the gathered MLP-sync values, so on speculative-decode steps they are batch rows (requests); the verify width is uniform across ranks, which leaves the ratios unchanged. + + ``*_attention_pairs`` are the causal (query, key) pairs the step attends + (``dp_attn._local_attention_pairs``). Tokens price the MoE and the + projections, which run on the gathered padded rows; pairs price attention, + which runs on local rows, so pair skew is what shows up as sync wait. """ local_tokens: int max_tokens: int sum_tokens: int + local_attention_pairs: int + max_attention_pairs: int num_ranks: int # Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path # (the default with overlap scheduling) this is waiting for the slowest @@ -228,15 +235,28 @@ class DPBalanceStats(msgspec.Struct, frozen=True): @classmethod def create( - cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float + cls, + *, + local_tokens: int, + global_num_tokens: List[int], + local_attention_pairs: int, + global_attention_pairs: List[int], + sync_wait_seconds: float, ) -> DPBalanceStats: max_tokens = max(global_num_tokens) if max_tokens == 0: raise ValueError("DP balance stats need a rank with tokens") + max_attention_pairs = max(global_attention_pairs) + if max_attention_pairs == 0: + # Every row attends at least one key, so this is a gather or + # pair-count bug, not a quiet step. + raise ValueError("A rank with tokens attends at least one pair") return cls( local_tokens=local_tokens, max_tokens=max_tokens, sum_tokens=sum(global_num_tokens), + local_attention_pairs=local_attention_pairs, + max_attention_pairs=max_attention_pairs, num_ranks=len(global_num_tokens), sync_wait_seconds=sync_wait_seconds, ) @@ -245,6 +265,10 @@ class DPBalanceStats(msgspec.Struct, frozen=True): def imbalance_tokens(self) -> int: return self.max_tokens - self.local_tokens + @property + def imbalance_attention_pairs(self) -> int: + return self.max_attention_pairs - self.local_attention_pairs + @property def max_over_mean(self) -> float: """Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced.""" @@ -1093,6 +1117,21 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): ), labelnames=list(labels.keys()) + ["rank_state"], ) + self.dp_attention_pairs_total = Counter( + name="sglang:dp_attention_pairs_total", + documentation=( + "Causal (query, key) pairs attended by MLP-sync steps on this DP " + "rank, pricing every layer as full attention: a decode row reads " + "its sequence, a prefill chunk its prefix plus the pairs within " + "the chunk. kind=scheduled: this rank; kind=imbalance: the busiest " + "rank's excess over this rank. The context-length counterpart of " + "sglang:dp_attention_tokens_total; like it, speculative-decode " + "steps count one query per row. Sliding-window layers execute " + "fewer pairs than counted, so on hybrid-SWA models long rows are " + "overstated." + ), + labelnames=list(labels.keys()) + ["kind"], + ) self.dp_attention_token_imbalance_ratio = Histogram( name="sglang:dp_attention_token_imbalance_ratio", documentation=( @@ -1155,6 +1194,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): state: self.dp_attention_steps_total.labels(**labels, rank_state=state) for state in ("active", "idle") } + self._dp_attention_pairs = { + kind: self.dp_attention_pairs_total.labels(**labels, kind=kind) + for kind in ("scheduled", "imbalance") + } self._dp_attention_token_imbalance_ratio = ( self.dp_attention_token_imbalance_ratio.labels(**labels) ) @@ -1467,6 +1510,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): self._dp_attention_tokens["scheduled"].inc(stats.local_tokens) if stats.imbalance_tokens: self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens) + if stats.local_attention_pairs: + self._dp_attention_pairs["scheduled"].inc(stats.local_attention_pairs) + if stats.imbalance_attention_pairs: + self._dp_attention_pairs["imbalance"].inc(stats.imbalance_attention_pairs) rank_state = "idle" if stats.local_tokens == 0 else "active" self._dp_attention_steps[rank_state].inc(1) self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds) diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index 1aa4fe6da7..220e03ea76 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -78,8 +78,11 @@ class TestEnableMetrics(CustomTestCase): {"mode": "decode"}, ), ("sglang:dp_attention_tokens_total", {"kind": "scheduled"}), + ("sglang:dp_attention_pairs_total", {"kind": "scheduled"}), ("sglang:dp_attention_steps_total", {"rank_state": "active"}), - ("sglang:dp_attention_token_imbalance_ratio_count", {}), + # Only the attention-rank-0 scheduler observes the ratio; the + # other rank's pre-seeded empty histogram must not be picked. + ("sglang:dp_attention_token_imbalance_ratio_count", {"tp_rank": "0"}), ("sglang:dp_attention_sync_wait_seconds_count", {}), ] _check_metrics_positive(self, metrics, metrics_to_check) diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index dacc869c27..c50fca132d 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -1,5 +1,6 @@ import unittest from types import SimpleNamespace +from typing import Optional from unittest.mock import Mock, patch import torch @@ -33,6 +34,7 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): batch_size=lambda: 4, spec_info=None, reqs=[], + seq_lens_cpu=torch.ones(4, dtype=torch.int64), ) tbo_preparer = Mock() tbo_preparer.prepare_all_gather.return_value = ( @@ -94,17 +96,25 @@ class TestDPBalanceStats(CustomTestCase): local_tokens: int, peer_tokens: int, *, - reqs=(), + reqs: Optional[list] = None, + peer_attention_pairs: Optional[int] = None, peer_health_check: bool = False, get_idle_batch=None, sync_wait_carry=None, wait_seconds=None, ): + if reqs is None: + # One-token rows, the shortest a decode row can be. + reqs = [ + SimpleNamespace(rid=f"req-{i}", seqlen=1) for i in range(local_tokens) + ] batch = SimpleNamespace( forward_mode=forward_mode, batch_size=lambda: local_tokens, spec_info=None, reqs=list(reqs), + # Spec batches carry no CPU seq_lens at schedule time. + seq_lens_cpu=None, dp_balance_stats=None, ) gathered_mode = forward_mode or ForwardMode.IDLE @@ -116,6 +126,10 @@ class TestDPBalanceStats(CustomTestCase): peer = local.clone() peer[0] = peer[1] = peer_tokens # num_tokens, num_tokens_for_logprob peer[9] = int(peer_health_check) + # attention_pairs; one-token rows unless the test says otherwise. + peer[10] = ( + peer_tokens if peer_attention_pairs is None else peer_attention_pairs + ) output.copy_(torch.stack([local, peer]).flatten()) # all_gather reads the clock once before and once after the collective. @@ -183,6 +197,99 @@ class TestDPBalanceStats(CustomTestCase): self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 12) self.assertGreater(stats.sync_wait_seconds, 0) + def test_attention_pairs_follow_context_not_rows(self): + """Fewer rows with longer contexts is the busier rank in pairs. + + Rows and pairs are gathered side by side, so the two series can + disagree on which rank is the busiest; this is the case the pair + series exists for. + """ + reqs = [SimpleNamespace(rid=f"req-{n}", seqlen=n) for n in (1000, 3000)] + result = self._prepare_dp2_batch( + ForwardMode.DECODE, + local_tokens=2, + peer_tokens=8, + reqs=reqs, + peer_attention_pairs=800, + ) + + stats = result.dp_balance_stats + self.assertEqual((stats.local_tokens, stats.imbalance_tokens), (2, 6)) + self.assertEqual( + (stats.local_attention_pairs, stats.max_attention_pairs), (4000, 4000) + ) + self.assertEqual(stats.imbalance_attention_pairs, 0) + self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 10) + + def test_attention_pairs_gathered_without_the_metrics_flag(self): + """The pair column must not depend on a per-process flag: a rank with + the flag off would gather zeros and trip the pair-count guard on its + flag-on peers.""" + with patch.object(dp_attn, "_ENABLE_METRICS_DP_ATTENTION", False): + info = dp_attn.MLPSyncBatchInfo( + num_dp_ranks=2, + tp_size=1, + cp_size=1, + num_tokens=2, + num_tokens_for_logprob=2, + can_run_decode_cuda_graph=True, + can_run_draft_cuda_graph=True, + can_run_prefill_cuda_graph=False, + is_extend_in_batch=False, + local_can_run_tbo=False, + local_forward_mode=ForwardMode.DECODE.value, + attention_pairs=dp_attn._local_attention_pairs( + SimpleNamespace( + forward_mode=ForwardMode.DECODE, + seq_lens_cpu=None, + reqs=[SimpleNamespace(seqlen=5), SimpleNamespace(seqlen=7)], + ) + ), + ) + self.assertEqual(info._get_local_tensor(device="cpu")[10].item(), 12) + + def test_prefill_chunk_pairs_are_causal(self): + """A chunk after prefix p reads p keys per query plus the causal + triangle inside the chunk; a one-token chunk after prefix p therefore + costs the same as a decode row of length p + 1.""" + extend = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, prefix_lens=[10, 6], extend_lens=[4, 1] + ) + decode = SimpleNamespace( + forward_mode=ForwardMode.DECODE, + seq_lens_cpu=None, + reqs=[SimpleNamespace(seqlen=7)], + ) + self.assertEqual(dp_attn._local_attention_pairs(extend), (10 * 4 + 10) + 7) + self.assertEqual(dp_attn._local_attention_pairs(decode), 7) + self.assertEqual(dp_attn._local_attention_pairs(None), 0) + self.assertEqual( + dp_attn._local_attention_pairs( + SimpleNamespace(forward_mode=ForwardMode.IDLE) + ), + 0, + ) + + def test_decode_pairs_from_seq_lens_cpu_match_request_lengths(self): + """The prepared seq_lens_cpu fast path and the committed-length + fallback count the same pairs, so the series has no discontinuity when + a batch switches between them.""" + lens = [3, 5, 9] + fast = SimpleNamespace( + forward_mode=ForwardMode.DECODE, + seq_lens_cpu=torch.tensor(lens), + reqs=[], + ) + slow = SimpleNamespace( + forward_mode=ForwardMode.DECODE, + seq_lens_cpu=None, + reqs=[SimpleNamespace(seqlen=n) for n in lens], + ) + self.assertEqual( + dp_attn._local_attention_pairs(fast), dp_attn._local_attention_pairs(slow) + ) + self.assertEqual(dp_attn._local_attention_pairs(fast), 17) + def _adapter(self): return dp_attn.SchedulerDPAttnAdapter( model_runner=object(), @@ -247,7 +354,7 @@ class TestDPBalanceStats(CustomTestCase): self.assertEqual(carry.seconds, 0.0) def test_health_check_probe_steps_not_recorded(self): - probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0") + probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0", seqlen=1) result = self._prepare_dp2_batch( ForwardMode.DECODE, local_tokens=1, peer_tokens=0, reqs=[probe] ) @@ -267,9 +374,24 @@ class TestDPBalanceStats(CustomTestCase): self.assertIs(result, idle_batch) self.assertIsNone(result.dp_balance_stats) - def test_create_rejects_steps_without_tokens(self): + def test_create_rejects_steps_without_tokens_or_pairs(self): with self.assertRaises(ValueError): - DPBalanceStats.create(0, [0, 0], 0.0) + DPBalanceStats.create( + local_tokens=0, + global_num_tokens=[0, 0], + local_attention_pairs=0, + global_attention_pairs=[0, 0], + sync_wait_seconds=0.0, + ) + # Rows without pairs cannot come from a correct gather. + with self.assertRaises(ValueError): + DPBalanceStats.create( + local_tokens=0, + global_num_tokens=[0, 8], + local_attention_pairs=0, + global_attention_pairs=[0, 0], + sync_wait_seconds=0.0, + ) class TestDecodeToExtendConversionVote(CustomTestCase): diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 95f80447cb..9ecb5cbcce 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -348,6 +348,30 @@ class TestForwardPassMetrics(unittest.TestCase): self.assertFalse(scheduler.enable_fpm) +def _dp_balance_stats( + *, + local_tokens: int, + global_num_tokens: list, + sync_wait_seconds: float, + local_attention_pairs: int | None = None, + global_attention_pairs: list | None = None, +): + """One-token rows unless the pairs are given explicitly.""" + return DPBalanceStats.create( + local_tokens=local_tokens, + global_num_tokens=global_num_tokens, + local_attention_pairs=( + local_tokens if local_attention_pairs is None else local_attention_pairs + ), + global_attention_pairs=( + global_num_tokens + if global_attention_pairs is None + else global_attention_pairs + ), + sync_wait_seconds=sync_wait_seconds, + ) + + class TestDPBalanceMetrics(CustomTestCase): def _reporter_with_collector(self, attn_dp_rank: int): scheduler = types.SimpleNamespace( @@ -401,10 +425,18 @@ class TestDPBalanceMetrics(CustomTestCase): ) batch = types.SimpleNamespace( - dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002) + dp_balance_stats=_dp_balance_stats( + local_tokens=4, + global_num_tokens=[8, 4], + sync_wait_seconds=0.002, + local_attention_pairs=4000, + global_attention_pairs=[800, 4000], + ) ) reporter.log_batch_result_stats(batch, result=object()) - batch.dp_balance_stats = DPBalanceStats.create(0, [8, 0], 0.004) + batch.dp_balance_stats = _dp_balance_stats( + local_tokens=0, global_num_tokens=[8, 0], sync_wait_seconds=0.004 + ) reporter.log_batch_result_stats(batch, result=object()) batch.dp_balance_stats = None reporter.log_batch_result_stats(batch, result=object()) @@ -430,6 +462,16 @@ class TestDPBalanceMetrics(CustomTestCase): get("sglang:dp_attention_token_imbalance_ratio_sum", labels), 8 * 2 / 12 + 2.0, ) + # Pairs: the first step's long-context rank is the pair-busiest + # despite having fewer rows; the idle step adds the peer's 8. + self.assertEqual( + get("sglang:dp_attention_pairs_total", {**labels, "kind": "scheduled"}), + 4000, + ) + self.assertEqual( + get("sglang:dp_attention_pairs_total", {**labels, "kind": "imbalance"}), + 8, + ) self.assertAlmostEqual( get("sglang:dp_attention_sync_wait_seconds_sum", labels), 0.006 ) @@ -473,7 +515,9 @@ class TestDPBalanceMetrics(CustomTestCase): def test_engine_ratio_reported_by_dp_rank_zero_only(self): reporter, registry, labels = self._reporter_with_collector(attn_dp_rank=1) batch = types.SimpleNamespace( - dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002) + dp_balance_stats=_dp_balance_stats( + local_tokens=4, global_num_tokens=[8, 4], sync_wait_seconds=0.002 + ) ) reporter.log_batch_result_stats(batch, result=object())