[metrics] Count per-rank DP attention pairs and their imbalance (#43180)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: hanming-lu <hanming-lu@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-08 15:01:37 -07:00
committed by GitHub
co-authored by Hanming Lu hanming-lu
parent 0b658c12ff
commit 2abb761639
6 changed files with 269 additions and 18 deletions
@@ -14,6 +14,10 @@ from sglang.srt.layers.cp.utils import get_cp_strategy
from sglang.srt.layers.dp_attention import dp_gather_width, world_dp_gather_enabled
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler_components.metrics_reporter import (
_decode_total_seq_lens,
_prefill_attention_pairs,
)
from sglang.srt.managers.scheduler_components.recv_skipper import (
SchedulerRecvSkipper,
)
@@ -125,6 +129,8 @@ class MLPSyncBatchInfo:
prefill_cuda_graph_max_prefix_len: int = 0
# Every request in the local batch is a health-check probe.
is_health_check: bool = False
# Causal (query, key) pairs this rank attends this step.
attention_pairs: int = 0
# some gathered elements
tp0_info_cpu: torch.Tensor = None
@@ -132,6 +138,7 @@ class MLPSyncBatchInfo:
sync_wait_seconds: float = 0.0
global_num_tokens: list[int] = None
global_num_tokens_for_logprob: list[int] = None
global_attention_pairs: list[int] = None
tbo_split_seq_index: torch.Tensor = None
global_forward_mode: int = None
dp_cooperation_info: Optional[DPCooperationInfo] = None
@@ -149,6 +156,7 @@ class MLPSyncBatchInfo:
self.prefill_cuda_graph_max_prefix_len,
int(self.can_run_draft_cuda_graph),
int(self.is_health_check),
self.attention_pairs,
],
device=device,
dtype=dtype,
@@ -167,6 +175,7 @@ class MLPSyncBatchInfo:
0, # prefill_cuda_graph_max_prefix_len
1, # can_run_draft_cuda_graph
0, # is_health_check
0, # attention_pairs
],
device=device,
dtype=dtype,
@@ -177,6 +186,7 @@ class MLPSyncBatchInfo:
self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1)
self.global_num_tokens = [self.num_tokens]
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
self.global_attention_pairs = [self.attention_pairs]
self.any_health_check = self.is_health_check
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -253,6 +263,7 @@ class MLPSyncBatchInfo:
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
self.sync_wait_seconds = time.perf_counter() - sync_start
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
@@ -428,6 +439,22 @@ def _local_prefill_cuda_graph_vote(
)
def _local_attention_pairs(local_batch: Optional[ScheduleBatch]) -> int:
"""Causal (query, key) pairs the batch attends, pricing every layer as full
attention: one query per decode row over its committed length (spec
batches have no CPU seq_lens at schedule time), the chunk formula for
prefill."""
if (
local_batch is None
or local_batch.forward_mode.is_prebuilt()
or local_batch.forward_mode.is_idle()
):
return 0
if local_batch.forward_mode.is_decode():
return _decode_total_seq_lens(local_batch)
return _prefill_attention_pairs(local_batch)
class DPSyncWaitCarry:
"""MLP-sync wait of batch-less gathers, folded into the next recorded step.
@@ -563,6 +590,9 @@ def prepare_mlp_sync_batch_raw(
local_forward_mode=local_forward_mode,
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
is_health_check=is_health_check,
# Gathered unconditionally like the other columns, so a per-process
# metrics flag cannot make ranks disagree on what the tensor holds.
attention_pairs=_local_attention_pairs(local_batch),
)
if num_dp_ranks == 1:
@@ -628,6 +658,8 @@ def prepare_mlp_sync_batch_raw(
local_batch.dp_balance_stats = DPBalanceStats.create(
local_tokens=num_tokens,
global_num_tokens=mlp_sync_info.global_num_tokens,
local_attention_pairs=mlp_sync_info.attention_pairs,
global_attention_pairs=mlp_sync_info.global_attention_pairs,
sync_wait_seconds=sync_wait_seconds,
)
elif local_batch is None and sync_wait_carry is not None:
@@ -90,6 +90,17 @@ def _decode_total_seq_lens(batch: ScheduleBatch) -> int:
return sum(req.seqlen for req in batch.reqs)
def _prefill_attention_pairs(batch: ScheduleBatch) -> int:
"""Causal query-key pairs: each chunk against its cached prefix, plus
the causal pairs within the chunk itself."""
return sum(
extend_len * prefix_len + extend_len * (extend_len + 1) // 2
for prefix_len, extend_len in zip(
batch.prefix_lens, batch.extend_lens, strict=True
)
)
@dataclasses.dataclass
class PrefillStats:
"""Stats for logging prefill batch metrics."""
@@ -604,14 +615,6 @@ class SchedulerMetricsReporter:
num_attn_heads * head_dim * act_bytes * num_layers
)
@staticmethod
def _prefill_attention_pairs(batch) -> float:
"""Causal query-key pairs: each chunk against its cached prefix, plus
the causal pairs within the chunk itself."""
prefix_pairs = sum(c * p for c, p in zip(batch.extend_lens, batch.prefix_lens))
within_chunk_pairs = sum(c * (c + 1) / 2.0 for c in batch.extend_lens)
return float(prefix_pairs + within_chunk_pairs)
def _estimate_prefill_perf(self, batch) -> Tuple[float, float, float]:
if batch is None or batch.extend_lens is None:
return 0.0, 0.0, 0.0
@@ -619,7 +622,7 @@ class SchedulerMetricsReporter:
if tokens == 0:
return 0.0, 0.0, 0.0
context_product = self._prefill_attention_pairs(batch)
context_product = float(_prefill_attention_pairs(batch))
flops = (
tokens * self._linear_flops_per_token
+ self._attn_dot_flops_coeff * context_product
@@ -215,11 +215,18 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
the gathered MLP-sync values, so on speculative-decode steps they are
batch rows (requests); the verify width is uniform across ranks, which
leaves the ratios unchanged.
``*_attention_pairs`` are the causal (query, key) pairs the step attends
(``dp_attn._local_attention_pairs``). Tokens price the MoE and the
projections, which run on the gathered padded rows; pairs price attention,
which runs on local rows, so pair skew is what shows up as sync wait.
"""
local_tokens: int
max_tokens: int
sum_tokens: int
local_attention_pairs: int
max_attention_pairs: int
num_ranks: int
# Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path
# (the default with overlap scheduling) this is waiting for the slowest
@@ -228,15 +235,28 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
@classmethod
def create(
cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float
cls,
*,
local_tokens: int,
global_num_tokens: List[int],
local_attention_pairs: int,
global_attention_pairs: List[int],
sync_wait_seconds: float,
) -> DPBalanceStats:
max_tokens = max(global_num_tokens)
if max_tokens == 0:
raise ValueError("DP balance stats need a rank with tokens")
max_attention_pairs = max(global_attention_pairs)
if max_attention_pairs == 0:
# Every row attends at least one key, so this is a gather or
# pair-count bug, not a quiet step.
raise ValueError("A rank with tokens attends at least one pair")
return cls(
local_tokens=local_tokens,
max_tokens=max_tokens,
sum_tokens=sum(global_num_tokens),
local_attention_pairs=local_attention_pairs,
max_attention_pairs=max_attention_pairs,
num_ranks=len(global_num_tokens),
sync_wait_seconds=sync_wait_seconds,
)
@@ -245,6 +265,10 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
def imbalance_tokens(self) -> int:
return self.max_tokens - self.local_tokens
@property
def imbalance_attention_pairs(self) -> int:
return self.max_attention_pairs - self.local_attention_pairs
@property
def max_over_mean(self) -> float:
"""Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced."""
@@ -1093,6 +1117,21 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
),
labelnames=list(labels.keys()) + ["rank_state"],
)
self.dp_attention_pairs_total = Counter(
name="sglang:dp_attention_pairs_total",
documentation=(
"Causal (query, key) pairs attended by MLP-sync steps on this DP "
"rank, pricing every layer as full attention: a decode row reads "
"its sequence, a prefill chunk its prefix plus the pairs within "
"the chunk. kind=scheduled: this rank; kind=imbalance: the busiest "
"rank's excess over this rank. The context-length counterpart of "
"sglang:dp_attention_tokens_total; like it, speculative-decode "
"steps count one query per row. Sliding-window layers execute "
"fewer pairs than counted, so on hybrid-SWA models long rows are "
"overstated."
),
labelnames=list(labels.keys()) + ["kind"],
)
self.dp_attention_token_imbalance_ratio = Histogram(
name="sglang:dp_attention_token_imbalance_ratio",
documentation=(
@@ -1155,6 +1194,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
state: self.dp_attention_steps_total.labels(**labels, rank_state=state)
for state in ("active", "idle")
}
self._dp_attention_pairs = {
kind: self.dp_attention_pairs_total.labels(**labels, kind=kind)
for kind in ("scheduled", "imbalance")
}
self._dp_attention_token_imbalance_ratio = (
self.dp_attention_token_imbalance_ratio.labels(**labels)
)
@@ -1467,6 +1510,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
self._dp_attention_tokens["scheduled"].inc(stats.local_tokens)
if stats.imbalance_tokens:
self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens)
if stats.local_attention_pairs:
self._dp_attention_pairs["scheduled"].inc(stats.local_attention_pairs)
if stats.imbalance_attention_pairs:
self._dp_attention_pairs["imbalance"].inc(stats.imbalance_attention_pairs)
rank_state = "idle" if stats.local_tokens == 0 else "active"
self._dp_attention_steps[rank_state].inc(1)
self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds)