[metrics] Count per-rank DP attention pairs and their imbalance (#43180)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: hanming-lu <hanming-lu@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-08 15:01:37 -07:00
committed by GitHub
co-authored by Hanming Lu hanming-lu
parent 0b658c12ff
commit 2abb761639
6 changed files with 269 additions and 18 deletions
@@ -14,6 +14,10 @@ from sglang.srt.layers.cp.utils import get_cp_strategy
from sglang.srt.layers.dp_attention import dp_gather_width, world_dp_gather_enabled
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler_components.metrics_reporter import (
_decode_total_seq_lens,
_prefill_attention_pairs,
)
from sglang.srt.managers.scheduler_components.recv_skipper import (
SchedulerRecvSkipper,
)
@@ -125,6 +129,8 @@ class MLPSyncBatchInfo:
prefill_cuda_graph_max_prefix_len: int = 0
# Every request in the local batch is a health-check probe.
is_health_check: bool = False
# Causal (query, key) pairs this rank attends this step.
attention_pairs: int = 0
# some gathered elements
tp0_info_cpu: torch.Tensor = None
@@ -132,6 +138,7 @@ class MLPSyncBatchInfo:
sync_wait_seconds: float = 0.0
global_num_tokens: list[int] = None
global_num_tokens_for_logprob: list[int] = None
global_attention_pairs: list[int] = None
tbo_split_seq_index: torch.Tensor = None
global_forward_mode: int = None
dp_cooperation_info: Optional[DPCooperationInfo] = None
@@ -149,6 +156,7 @@ class MLPSyncBatchInfo:
self.prefill_cuda_graph_max_prefix_len,
int(self.can_run_draft_cuda_graph),
int(self.is_health_check),
self.attention_pairs,
],
device=device,
dtype=dtype,
@@ -167,6 +175,7 @@ class MLPSyncBatchInfo:
0, # prefill_cuda_graph_max_prefix_len
1, # can_run_draft_cuda_graph
0, # is_health_check
0, # attention_pairs
],
device=device,
dtype=dtype,
@@ -177,6 +186,7 @@ class MLPSyncBatchInfo:
self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1)
self.global_num_tokens = [self.num_tokens]
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
self.global_attention_pairs = [self.attention_pairs]
self.any_health_check = self.is_health_check
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -253,6 +263,7 @@ class MLPSyncBatchInfo:
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
self.sync_wait_seconds = time.perf_counter() - sync_start
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -428,6 +439,22 @@ def _local_prefill_cuda_graph_vote(
)
def _local_attention_pairs(local_batch: Optional[ScheduleBatch]) -> int:
"""Causal (query, key) pairs the batch attends, pricing every layer as full
attention: one query per decode row over its committed length (spec
batches have no CPU seq_lens at schedule time), the chunk formula for
prefill."""
if (
local_batch is None
or local_batch.forward_mode.is_prebuilt()
or local_batch.forward_mode.is_idle()
):
return 0
if local_batch.forward_mode.is_decode():
return _decode_total_seq_lens(local_batch)
return _prefill_attention_pairs(local_batch)
class DPSyncWaitCarry:
"""MLP-sync wait of batch-less gathers, folded into the next recorded step.
@@ -563,6 +590,9 @@ def prepare_mlp_sync_batch_raw(
local_forward_mode=local_forward_mode,
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
is_health_check=is_health_check,
# Gathered unconditionally like the other columns, so a per-process
# metrics flag cannot make ranks disagree on what the tensor holds.
attention_pairs=_local_attention_pairs(local_batch),
)
if num_dp_ranks == 1:
@@ -628,6 +658,8 @@ def prepare_mlp_sync_batch_raw(
local_batch.dp_balance_stats = DPBalanceStats.create(
local_tokens=num_tokens,
global_num_tokens=mlp_sync_info.global_num_tokens,
local_attention_pairs=mlp_sync_info.attention_pairs,
global_attention_pairs=mlp_sync_info.global_attention_pairs,
sync_wait_seconds=sync_wait_seconds,
)
elif local_batch is None and sync_wait_carry is not None:
@@ -90,6 +90,17 @@ def _decode_total_seq_lens(batch: ScheduleBatch) -> int:
return sum(req.seqlen for req in batch.reqs)
def _prefill_attention_pairs(batch: ScheduleBatch) -> int:
"""Causal query-key pairs: each chunk against its cached prefix, plus
the causal pairs within the chunk itself."""
return sum(
extend_len * prefix_len + extend_len * (extend_len + 1) // 2
for prefix_len, extend_len in zip(
batch.prefix_lens, batch.extend_lens, strict=True
)
)
@dataclasses.dataclass
class PrefillStats:
"""Stats for logging prefill batch metrics."""
@@ -604,14 +615,6 @@ class SchedulerMetricsReporter:
num_attn_heads * head_dim * act_bytes * num_layers
)
@staticmethod
def _prefill_attention_pairs(batch) -> float:
"""Causal query-key pairs: each chunk against its cached prefix, plus
the causal pairs within the chunk itself."""
prefix_pairs = sum(c * p for c, p in zip(batch.extend_lens, batch.prefix_lens))
within_chunk_pairs = sum(c * (c + 1) / 2.0 for c in batch.extend_lens)
return float(prefix_pairs + within_chunk_pairs)
def _estimate_prefill_perf(self, batch) -> Tuple[float, float, float]:
if batch is None or batch.extend_lens is None:
return 0.0, 0.0, 0.0
@@ -619,7 +622,7 @@ class SchedulerMetricsReporter:
if tokens == 0:
return 0.0, 0.0, 0.0
context_product = self._prefill_attention_pairs(batch)
context_product = float(_prefill_attention_pairs(batch))
flops = (
tokens * self._linear_flops_per_token
+ self._attn_dot_flops_coeff * context_product
@@ -215,11 +215,18 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
the gathered MLP-sync values, so on speculative-decode steps they are
batch rows (requests); the verify width is uniform across ranks, which
leaves the ratios unchanged.
``*_attention_pairs`` are the causal (query, key) pairs the step attends
(``dp_attn._local_attention_pairs``). Tokens price the MoE and the
projections, which run on the gathered padded rows; pairs price attention,
which runs on local rows, so pair skew is what shows up as sync wait.
"""
local_tokens: int
max_tokens: int
sum_tokens: int
local_attention_pairs: int
max_attention_pairs: int
num_ranks: int
# Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path
# (the default with overlap scheduling) this is waiting for the slowest
@@ -228,15 +235,28 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
@classmethod
def create(
cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float
cls,
*,
local_tokens: int,
global_num_tokens: List[int],
local_attention_pairs: int,
global_attention_pairs: List[int],
sync_wait_seconds: float,
) -> DPBalanceStats:
max_tokens = max(global_num_tokens)
if max_tokens == 0:
raise ValueError("DP balance stats need a rank with tokens")
max_attention_pairs = max(global_attention_pairs)
if max_attention_pairs == 0:
# Every row attends at least one key, so this is a gather or
# pair-count bug, not a quiet step.
raise ValueError("A rank with tokens attends at least one pair")
return cls(
local_tokens=local_tokens,
max_tokens=max_tokens,
sum_tokens=sum(global_num_tokens),
local_attention_pairs=local_attention_pairs,
max_attention_pairs=max_attention_pairs,
num_ranks=len(global_num_tokens),
sync_wait_seconds=sync_wait_seconds,
)
@@ -245,6 +265,10 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
def imbalance_tokens(self) -> int:
return self.max_tokens - self.local_tokens
@property
def imbalance_attention_pairs(self) -> int:
return self.max_attention_pairs - self.local_attention_pairs
@property
def max_over_mean(self) -> float:
"""Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced."""
@@ -1093,6 +1117,21 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
),
labelnames=list(labels.keys()) + ["rank_state"],
)
self.dp_attention_pairs_total = Counter(
name="sglang:dp_attention_pairs_total",
documentation=(
"Causal (query, key) pairs attended by MLP-sync steps on this DP "
"rank, pricing every layer as full attention: a decode row reads "
"its sequence, a prefill chunk its prefix plus the pairs within "
"the chunk. kind=scheduled: this rank; kind=imbalance: the busiest "
"rank's excess over this rank. The context-length counterpart of "
"sglang:dp_attention_tokens_total; like it, speculative-decode "
"steps count one query per row. Sliding-window layers execute "
"fewer pairs than counted, so on hybrid-SWA models long rows are "
"overstated."
),
labelnames=list(labels.keys()) + ["kind"],
)
self.dp_attention_token_imbalance_ratio = Histogram(
name="sglang:dp_attention_token_imbalance_ratio",
documentation=(
@@ -1155,6 +1194,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
state: self.dp_attention_steps_total.labels(**labels, rank_state=state)
for state in ("active", "idle")
}
self._dp_attention_pairs = {
kind: self.dp_attention_pairs_total.labels(**labels, kind=kind)
for kind in ("scheduled", "imbalance")
}
self._dp_attention_token_imbalance_ratio = (
self.dp_attention_token_imbalance_ratio.labels(**labels)
)
@@ -1467,6 +1510,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self._dp_attention_tokens["scheduled"].inc(stats.local_tokens)
if stats.imbalance_tokens:
self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens)
if stats.local_attention_pairs:
self._dp_attention_pairs["scheduled"].inc(stats.local_attention_pairs)
if stats.imbalance_attention_pairs:
self._dp_attention_pairs["imbalance"].inc(stats.imbalance_attention_pairs)
rank_state = "idle" if stats.local_tokens == 0 else "active"
self._dp_attention_steps[rank_state].inc(1)
self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds)
@@ -78,8 +78,11 @@ class TestEnableMetrics(CustomTestCase):
{"mode": "decode"},
),
("sglang:dp_attention_tokens_total", {"kind": "scheduled"}),
("sglang:dp_attention_pairs_total", {"kind": "scheduled"}),
("sglang:dp_attention_steps_total", {"rank_state": "active"}),
("sglang:dp_attention_token_imbalance_ratio_count", {}),
# Only the attention-rank-0 scheduler observes the ratio; the
# other rank's pre-seeded empty histogram must not be picked.
("sglang:dp_attention_token_imbalance_ratio_count", {"tp_rank": "0"}),
("sglang:dp_attention_sync_wait_seconds_count", {}),
]
_check_metrics_positive(self, metrics, metrics_to_check)
@@ -1,5 +1,6 @@
import unittest
from types import SimpleNamespace
from typing import Optional
from unittest.mock import Mock, patch
import torch
@@ -33,6 +34,7 @@ class TestDPAttnSchedulerMetadata(CustomTestCase):
batch_size=lambda: 4,
spec_info=None,
reqs=[],
seq_lens_cpu=torch.ones(4, dtype=torch.int64),
)
tbo_preparer = Mock()
tbo_preparer.prepare_all_gather.return_value = (
@@ -94,17 +96,25 @@ class TestDPBalanceStats(CustomTestCase):
local_tokens: int,
peer_tokens: int,
*,
reqs=(),
reqs: Optional[list] = None,
peer_attention_pairs: Optional[int] = None,
peer_health_check: bool = False,
get_idle_batch=None,
sync_wait_carry=None,
wait_seconds=None,
):
if reqs is None:
# One-token rows, the shortest a decode row can be.
reqs = [
SimpleNamespace(rid=f"req-{i}", seqlen=1) for i in range(local_tokens)
]
batch = SimpleNamespace(
forward_mode=forward_mode,
batch_size=lambda: local_tokens,
spec_info=None,
reqs=list(reqs),
# Spec batches carry no CPU seq_lens at schedule time.
seq_lens_cpu=None,
dp_balance_stats=None,
)
gathered_mode = forward_mode or ForwardMode.IDLE
@@ -116,6 +126,10 @@ class TestDPBalanceStats(CustomTestCase):
peer = local.clone()
peer[0] = peer[1] = peer_tokens # num_tokens, num_tokens_for_logprob
peer[9] = int(peer_health_check)
# attention_pairs; one-token rows unless the test says otherwise.
peer[10] = (
peer_tokens if peer_attention_pairs is None else peer_attention_pairs
)
output.copy_(torch.stack([local, peer]).flatten())
# all_gather reads the clock once before and once after the collective.
@@ -183,6 +197,99 @@ class TestDPBalanceStats(CustomTestCase):
self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 12)
self.assertGreater(stats.sync_wait_seconds, 0)
def test_attention_pairs_follow_context_not_rows(self):
"""Fewer rows with longer contexts is the busier rank in pairs.
Rows and pairs are gathered side by side, so the two series can
disagree on which rank is the busiest; this is the case the pair
series exists for.
"""
reqs = [SimpleNamespace(rid=f"req-{n}", seqlen=n) for n in (1000, 3000)]
result = self._prepare_dp2_batch(
ForwardMode.DECODE,
local_tokens=2,
peer_tokens=8,
reqs=reqs,
peer_attention_pairs=800,
)
stats = result.dp_balance_stats
self.assertEqual((stats.local_tokens, stats.imbalance_tokens), (2, 6))
self.assertEqual(
(stats.local_attention_pairs, stats.max_attention_pairs), (4000, 4000)
)
self.assertEqual(stats.imbalance_attention_pairs, 0)
self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 10)
def test_attention_pairs_gathered_without_the_metrics_flag(self):
"""The pair column must not depend on a per-process flag: a rank with
the flag off would gather zeros and trip the pair-count guard on its
flag-on peers."""
with patch.object(dp_attn, "_ENABLE_METRICS_DP_ATTENTION", False):
info = dp_attn.MLPSyncBatchInfo(
num_dp_ranks=2,
tp_size=1,
cp_size=1,
num_tokens=2,
num_tokens_for_logprob=2,
can_run_decode_cuda_graph=True,
can_run_draft_cuda_graph=True,
can_run_prefill_cuda_graph=False,
is_extend_in_batch=False,
local_can_run_tbo=False,
local_forward_mode=ForwardMode.DECODE.value,
attention_pairs=dp_attn._local_attention_pairs(
SimpleNamespace(
forward_mode=ForwardMode.DECODE,
seq_lens_cpu=None,
reqs=[SimpleNamespace(seqlen=5), SimpleNamespace(seqlen=7)],
)
),
)
self.assertEqual(info._get_local_tensor(device="cpu")[10].item(), 12)
def test_prefill_chunk_pairs_are_causal(self):
"""A chunk after prefix p reads p keys per query plus the causal
triangle inside the chunk; a one-token chunk after prefix p therefore
costs the same as a decode row of length p + 1."""
extend = SimpleNamespace(
forward_mode=ForwardMode.EXTEND, prefix_lens=[10, 6], extend_lens=[4, 1]
)
decode = SimpleNamespace(
forward_mode=ForwardMode.DECODE,
seq_lens_cpu=None,
reqs=[SimpleNamespace(seqlen=7)],
)
self.assertEqual(dp_attn._local_attention_pairs(extend), (10 * 4 + 10) + 7)
self.assertEqual(dp_attn._local_attention_pairs(decode), 7)
self.assertEqual(dp_attn._local_attention_pairs(None), 0)
self.assertEqual(
dp_attn._local_attention_pairs(
SimpleNamespace(forward_mode=ForwardMode.IDLE)
),
0,
)
def test_decode_pairs_from_seq_lens_cpu_match_request_lengths(self):
"""The prepared seq_lens_cpu fast path and the committed-length
fallback count the same pairs, so the series has no discontinuity when
a batch switches between them."""
lens = [3, 5, 9]
fast = SimpleNamespace(
forward_mode=ForwardMode.DECODE,
seq_lens_cpu=torch.tensor(lens),
reqs=[],
)
slow = SimpleNamespace(
forward_mode=ForwardMode.DECODE,
seq_lens_cpu=None,
reqs=[SimpleNamespace(seqlen=n) for n in lens],
)
self.assertEqual(
dp_attn._local_attention_pairs(fast), dp_attn._local_attention_pairs(slow)
)
self.assertEqual(dp_attn._local_attention_pairs(fast), 17)
def _adapter(self):
return dp_attn.SchedulerDPAttnAdapter(
model_runner=object(),
@@ -247,7 +354,7 @@ class TestDPBalanceStats(CustomTestCase):
self.assertEqual(carry.seconds, 0.0)
def test_health_check_probe_steps_not_recorded(self):
probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0")
probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0", seqlen=1)
result = self._prepare_dp2_batch(
ForwardMode.DECODE, local_tokens=1, peer_tokens=0, reqs=[probe]
)
@@ -267,9 +374,24 @@ class TestDPBalanceStats(CustomTestCase):
self.assertIs(result, idle_batch)
self.assertIsNone(result.dp_balance_stats)
def test_create_rejects_steps_without_tokens(self):
def test_create_rejects_steps_without_tokens_or_pairs(self):
with self.assertRaises(ValueError):
DPBalanceStats.create(0, [0, 0], 0.0)
DPBalanceStats.create(
local_tokens=0,
global_num_tokens=[0, 0],
local_attention_pairs=0,
global_attention_pairs=[0, 0],
sync_wait_seconds=0.0,
)
# Rows without pairs cannot come from a correct gather.
with self.assertRaises(ValueError):
DPBalanceStats.create(
local_tokens=0,
global_num_tokens=[0, 8],
local_attention_pairs=0,
global_attention_pairs=[0, 0],
sync_wait_seconds=0.0,
)
class TestDecodeToExtendConversionVote(CustomTestCase):
@@ -348,6 +348,30 @@ class TestForwardPassMetrics(unittest.TestCase):
self.assertFalse(scheduler.enable_fpm)
def _dp_balance_stats(
*,
local_tokens: int,
global_num_tokens: list,
sync_wait_seconds: float,
local_attention_pairs: int | None = None,
global_attention_pairs: list | None = None,
):
"""One-token rows unless the pairs are given explicitly."""
return DPBalanceStats.create(
local_tokens=local_tokens,
global_num_tokens=global_num_tokens,
local_attention_pairs=(
local_tokens if local_attention_pairs is None else local_attention_pairs
),
global_attention_pairs=(
global_num_tokens
if global_attention_pairs is None
else global_attention_pairs
),
sync_wait_seconds=sync_wait_seconds,
)
class TestDPBalanceMetrics(CustomTestCase):
def _reporter_with_collector(self, attn_dp_rank: int):
scheduler = types.SimpleNamespace(
@@ -401,10 +425,18 @@ class TestDPBalanceMetrics(CustomTestCase):
)
batch = types.SimpleNamespace(
dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002)
dp_balance_stats=_dp_balance_stats(
local_tokens=4,
global_num_tokens=[8, 4],
sync_wait_seconds=0.002,
local_attention_pairs=4000,
global_attention_pairs=[800, 4000],
)
)
reporter.log_batch_result_stats(batch, result=object())
batch.dp_balance_stats = DPBalanceStats.create(0, [8, 0], 0.004)
batch.dp_balance_stats = _dp_balance_stats(
local_tokens=0, global_num_tokens=[8, 0], sync_wait_seconds=0.004
)
reporter.log_batch_result_stats(batch, result=object())
batch.dp_balance_stats = None
reporter.log_batch_result_stats(batch, result=object())
@@ -430,6 +462,16 @@ class TestDPBalanceMetrics(CustomTestCase):
get("sglang:dp_attention_token_imbalance_ratio_sum", labels),
8 * 2 / 12 + 2.0,
)
# Pairs: the first step's long-context rank is the pair-busiest
# despite having fewer rows; the idle step adds the peer's 8.
self.assertEqual(
get("sglang:dp_attention_pairs_total", {**labels, "kind": "scheduled"}),
4000,
)
self.assertEqual(
get("sglang:dp_attention_pairs_total", {**labels, "kind": "imbalance"}),
8,
)
self.assertAlmostEqual(
get("sglang:dp_attention_sync_wait_seconds_sum", labels), 0.006
)
@@ -473,7 +515,9 @@ class TestDPBalanceMetrics(CustomTestCase):
def test_engine_ratio_reported_by_dp_rank_zero_only(self):
reporter, registry, labels = self._reporter_with_collector(attn_dp_rank=1)
batch = types.SimpleNamespace(
dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002)
dp_balance_stats=_dp_balance_stats(
local_tokens=4, global_num_tokens=[8, 4], sync_wait_seconds=0.002
)
)
reporter.log_batch_result_stats(batch, result=object())