[metrics] Count per-rank DP attention pairs and their imbalance (#43180)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: hanming-lu <hanming-lu@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
hanming-lu
parent
0b658c12ff
commit
2abb761639
@@ -14,6 +14,10 @@ from sglang.srt.layers.cp.utils import get_cp_strategy
|
||||
from sglang.srt.layers.dp_attention import dp_gather_width, world_dp_gather_enabled
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler_components.metrics_reporter import (
|
||||
_decode_total_seq_lens,
|
||||
_prefill_attention_pairs,
|
||||
)
|
||||
from sglang.srt.managers.scheduler_components.recv_skipper import (
|
||||
SchedulerRecvSkipper,
|
||||
)
|
||||
@@ -125,6 +129,8 @@ class MLPSyncBatchInfo:
|
||||
prefill_cuda_graph_max_prefix_len: int = 0
|
||||
# Every request in the local batch is a health-check probe.
|
||||
is_health_check: bool = False
|
||||
# Causal (query, key) pairs this rank attends this step.
|
||||
attention_pairs: int = 0
|
||||
|
||||
# some gathered elements
|
||||
tp0_info_cpu: torch.Tensor = None
|
||||
@@ -132,6 +138,7 @@ class MLPSyncBatchInfo:
|
||||
sync_wait_seconds: float = 0.0
|
||||
global_num_tokens: list[int] = None
|
||||
global_num_tokens_for_logprob: list[int] = None
|
||||
global_attention_pairs: list[int] = None
|
||||
tbo_split_seq_index: torch.Tensor = None
|
||||
global_forward_mode: int = None
|
||||
dp_cooperation_info: Optional[DPCooperationInfo] = None
|
||||
@@ -149,6 +156,7 @@ class MLPSyncBatchInfo:
|
||||
self.prefill_cuda_graph_max_prefix_len,
|
||||
int(self.can_run_draft_cuda_graph),
|
||||
int(self.is_health_check),
|
||||
self.attention_pairs,
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -167,6 +175,7 @@ class MLPSyncBatchInfo:
|
||||
0, # prefill_cuda_graph_max_prefix_len
|
||||
1, # can_run_draft_cuda_graph
|
||||
0, # is_health_check
|
||||
0, # attention_pairs
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -177,6 +186,7 @@ class MLPSyncBatchInfo:
|
||||
self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1)
|
||||
self.global_num_tokens = [self.num_tokens]
|
||||
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
|
||||
self.global_attention_pairs = [self.attention_pairs]
|
||||
self.any_health_check = self.is_health_check
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
@@ -253,6 +263,7 @@ class MLPSyncBatchInfo:
|
||||
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
|
||||
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
|
||||
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
|
||||
self.global_attention_pairs = tp0_info_cpu[:, 10].tolist()
|
||||
self.sync_wait_seconds = time.perf_counter() - sync_start
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
@@ -428,6 +439,22 @@ def _local_prefill_cuda_graph_vote(
|
||||
)
|
||||
|
||||
|
||||
def _local_attention_pairs(local_batch: Optional[ScheduleBatch]) -> int:
|
||||
"""Causal (query, key) pairs the batch attends, pricing every layer as full
|
||||
attention: one query per decode row over its committed length (spec
|
||||
batches have no CPU seq_lens at schedule time), the chunk formula for
|
||||
prefill."""
|
||||
if (
|
||||
local_batch is None
|
||||
or local_batch.forward_mode.is_prebuilt()
|
||||
or local_batch.forward_mode.is_idle()
|
||||
):
|
||||
return 0
|
||||
if local_batch.forward_mode.is_decode():
|
||||
return _decode_total_seq_lens(local_batch)
|
||||
return _prefill_attention_pairs(local_batch)
|
||||
|
||||
|
||||
class DPSyncWaitCarry:
|
||||
"""MLP-sync wait of batch-less gathers, folded into the next recorded step.
|
||||
|
||||
@@ -563,6 +590,9 @@ def prepare_mlp_sync_batch_raw(
|
||||
local_forward_mode=local_forward_mode,
|
||||
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
|
||||
is_health_check=is_health_check,
|
||||
# Gathered unconditionally like the other columns, so a per-process
|
||||
# metrics flag cannot make ranks disagree on what the tensor holds.
|
||||
attention_pairs=_local_attention_pairs(local_batch),
|
||||
)
|
||||
|
||||
if num_dp_ranks == 1:
|
||||
@@ -628,6 +658,8 @@ def prepare_mlp_sync_batch_raw(
|
||||
local_batch.dp_balance_stats = DPBalanceStats.create(
|
||||
local_tokens=num_tokens,
|
||||
global_num_tokens=mlp_sync_info.global_num_tokens,
|
||||
local_attention_pairs=mlp_sync_info.attention_pairs,
|
||||
global_attention_pairs=mlp_sync_info.global_attention_pairs,
|
||||
sync_wait_seconds=sync_wait_seconds,
|
||||
)
|
||||
elif local_batch is None and sync_wait_carry is not None:
|
||||
|
||||
@@ -90,6 +90,17 @@ def _decode_total_seq_lens(batch: ScheduleBatch) -> int:
|
||||
return sum(req.seqlen for req in batch.reqs)
|
||||
|
||||
|
||||
def _prefill_attention_pairs(batch: ScheduleBatch) -> int:
|
||||
"""Causal query-key pairs: each chunk against its cached prefix, plus
|
||||
the causal pairs within the chunk itself."""
|
||||
return sum(
|
||||
extend_len * prefix_len + extend_len * (extend_len + 1) // 2
|
||||
for prefix_len, extend_len in zip(
|
||||
batch.prefix_lens, batch.extend_lens, strict=True
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PrefillStats:
|
||||
"""Stats for logging prefill batch metrics."""
|
||||
@@ -604,14 +615,6 @@ class SchedulerMetricsReporter:
|
||||
num_attn_heads * head_dim * act_bytes * num_layers
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _prefill_attention_pairs(batch) -> float:
|
||||
"""Causal query-key pairs: each chunk against its cached prefix, plus
|
||||
the causal pairs within the chunk itself."""
|
||||
prefix_pairs = sum(c * p for c, p in zip(batch.extend_lens, batch.prefix_lens))
|
||||
within_chunk_pairs = sum(c * (c + 1) / 2.0 for c in batch.extend_lens)
|
||||
return float(prefix_pairs + within_chunk_pairs)
|
||||
|
||||
def _estimate_prefill_perf(self, batch) -> Tuple[float, float, float]:
|
||||
if batch is None or batch.extend_lens is None:
|
||||
return 0.0, 0.0, 0.0
|
||||
@@ -619,7 +622,7 @@ class SchedulerMetricsReporter:
|
||||
if tokens == 0:
|
||||
return 0.0, 0.0, 0.0
|
||||
|
||||
context_product = self._prefill_attention_pairs(batch)
|
||||
context_product = float(_prefill_attention_pairs(batch))
|
||||
flops = (
|
||||
tokens * self._linear_flops_per_token
|
||||
+ self._attn_dot_flops_coeff * context_product
|
||||
|
||||
@@ -215,11 +215,18 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
|
||||
the gathered MLP-sync values, so on speculative-decode steps they are
|
||||
batch rows (requests); the verify width is uniform across ranks, which
|
||||
leaves the ratios unchanged.
|
||||
|
||||
``*_attention_pairs`` are the causal (query, key) pairs the step attends
|
||||
(``dp_attn._local_attention_pairs``). Tokens price the MoE and the
|
||||
projections, which run on the gathered padded rows; pairs price attention,
|
||||
which runs on local rows, so pair skew is what shows up as sync wait.
|
||||
"""
|
||||
|
||||
local_tokens: int
|
||||
max_tokens: int
|
||||
sum_tokens: int
|
||||
local_attention_pairs: int
|
||||
max_attention_pairs: int
|
||||
num_ranks: int
|
||||
# Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path
|
||||
# (the default with overlap scheduling) this is waiting for the slowest
|
||||
@@ -228,15 +235,28 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float
|
||||
cls,
|
||||
*,
|
||||
local_tokens: int,
|
||||
global_num_tokens: List[int],
|
||||
local_attention_pairs: int,
|
||||
global_attention_pairs: List[int],
|
||||
sync_wait_seconds: float,
|
||||
) -> DPBalanceStats:
|
||||
max_tokens = max(global_num_tokens)
|
||||
if max_tokens == 0:
|
||||
raise ValueError("DP balance stats need a rank with tokens")
|
||||
max_attention_pairs = max(global_attention_pairs)
|
||||
if max_attention_pairs == 0:
|
||||
# Every row attends at least one key, so this is a gather or
|
||||
# pair-count bug, not a quiet step.
|
||||
raise ValueError("A rank with tokens attends at least one pair")
|
||||
return cls(
|
||||
local_tokens=local_tokens,
|
||||
max_tokens=max_tokens,
|
||||
sum_tokens=sum(global_num_tokens),
|
||||
local_attention_pairs=local_attention_pairs,
|
||||
max_attention_pairs=max_attention_pairs,
|
||||
num_ranks=len(global_num_tokens),
|
||||
sync_wait_seconds=sync_wait_seconds,
|
||||
)
|
||||
@@ -245,6 +265,10 @@ class DPBalanceStats(msgspec.Struct, frozen=True):
|
||||
def imbalance_tokens(self) -> int:
|
||||
return self.max_tokens - self.local_tokens
|
||||
|
||||
@property
|
||||
def imbalance_attention_pairs(self) -> int:
|
||||
return self.max_attention_pairs - self.local_attention_pairs
|
||||
|
||||
@property
|
||||
def max_over_mean(self) -> float:
|
||||
"""Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced."""
|
||||
@@ -1093,6 +1117,21 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["rank_state"],
|
||||
)
|
||||
self.dp_attention_pairs_total = Counter(
|
||||
name="sglang:dp_attention_pairs_total",
|
||||
documentation=(
|
||||
"Causal (query, key) pairs attended by MLP-sync steps on this DP "
|
||||
"rank, pricing every layer as full attention: a decode row reads "
|
||||
"its sequence, a prefill chunk its prefix plus the pairs within "
|
||||
"the chunk. kind=scheduled: this rank; kind=imbalance: the busiest "
|
||||
"rank's excess over this rank. The context-length counterpart of "
|
||||
"sglang:dp_attention_tokens_total; like it, speculative-decode "
|
||||
"steps count one query per row. Sliding-window layers execute "
|
||||
"fewer pairs than counted, so on hybrid-SWA models long rows are "
|
||||
"overstated."
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["kind"],
|
||||
)
|
||||
self.dp_attention_token_imbalance_ratio = Histogram(
|
||||
name="sglang:dp_attention_token_imbalance_ratio",
|
||||
documentation=(
|
||||
@@ -1155,6 +1194,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
state: self.dp_attention_steps_total.labels(**labels, rank_state=state)
|
||||
for state in ("active", "idle")
|
||||
}
|
||||
self._dp_attention_pairs = {
|
||||
kind: self.dp_attention_pairs_total.labels(**labels, kind=kind)
|
||||
for kind in ("scheduled", "imbalance")
|
||||
}
|
||||
self._dp_attention_token_imbalance_ratio = (
|
||||
self.dp_attention_token_imbalance_ratio.labels(**labels)
|
||||
)
|
||||
@@ -1467,6 +1510,10 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
self._dp_attention_tokens["scheduled"].inc(stats.local_tokens)
|
||||
if stats.imbalance_tokens:
|
||||
self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens)
|
||||
if stats.local_attention_pairs:
|
||||
self._dp_attention_pairs["scheduled"].inc(stats.local_attention_pairs)
|
||||
if stats.imbalance_attention_pairs:
|
||||
self._dp_attention_pairs["imbalance"].inc(stats.imbalance_attention_pairs)
|
||||
rank_state = "idle" if stats.local_tokens == 0 else "active"
|
||||
self._dp_attention_steps[rank_state].inc(1)
|
||||
self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds)
|
||||
|
||||
@@ -78,8 +78,11 @@ class TestEnableMetrics(CustomTestCase):
|
||||
{"mode": "decode"},
|
||||
),
|
||||
("sglang:dp_attention_tokens_total", {"kind": "scheduled"}),
|
||||
("sglang:dp_attention_pairs_total", {"kind": "scheduled"}),
|
||||
("sglang:dp_attention_steps_total", {"rank_state": "active"}),
|
||||
("sglang:dp_attention_token_imbalance_ratio_count", {}),
|
||||
# Only the attention-rank-0 scheduler observes the ratio; the
|
||||
# other rank's pre-seeded empty histogram must not be picked.
|
||||
("sglang:dp_attention_token_imbalance_ratio_count", {"tp_rank": "0"}),
|
||||
("sglang:dp_attention_sync_wait_seconds_count", {}),
|
||||
]
|
||||
_check_metrics_positive(self, metrics, metrics_to_check)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import torch
|
||||
@@ -33,6 +34,7 @@ class TestDPAttnSchedulerMetadata(CustomTestCase):
|
||||
batch_size=lambda: 4,
|
||||
spec_info=None,
|
||||
reqs=[],
|
||||
seq_lens_cpu=torch.ones(4, dtype=torch.int64),
|
||||
)
|
||||
tbo_preparer = Mock()
|
||||
tbo_preparer.prepare_all_gather.return_value = (
|
||||
@@ -94,17 +96,25 @@ class TestDPBalanceStats(CustomTestCase):
|
||||
local_tokens: int,
|
||||
peer_tokens: int,
|
||||
*,
|
||||
reqs=(),
|
||||
reqs: Optional[list] = None,
|
||||
peer_attention_pairs: Optional[int] = None,
|
||||
peer_health_check: bool = False,
|
||||
get_idle_batch=None,
|
||||
sync_wait_carry=None,
|
||||
wait_seconds=None,
|
||||
):
|
||||
if reqs is None:
|
||||
# One-token rows, the shortest a decode row can be.
|
||||
reqs = [
|
||||
SimpleNamespace(rid=f"req-{i}", seqlen=1) for i in range(local_tokens)
|
||||
]
|
||||
batch = SimpleNamespace(
|
||||
forward_mode=forward_mode,
|
||||
batch_size=lambda: local_tokens,
|
||||
spec_info=None,
|
||||
reqs=list(reqs),
|
||||
# Spec batches carry no CPU seq_lens at schedule time.
|
||||
seq_lens_cpu=None,
|
||||
dp_balance_stats=None,
|
||||
)
|
||||
gathered_mode = forward_mode or ForwardMode.IDLE
|
||||
@@ -116,6 +126,10 @@ class TestDPBalanceStats(CustomTestCase):
|
||||
peer = local.clone()
|
||||
peer[0] = peer[1] = peer_tokens # num_tokens, num_tokens_for_logprob
|
||||
peer[9] = int(peer_health_check)
|
||||
# attention_pairs; one-token rows unless the test says otherwise.
|
||||
peer[10] = (
|
||||
peer_tokens if peer_attention_pairs is None else peer_attention_pairs
|
||||
)
|
||||
output.copy_(torch.stack([local, peer]).flatten())
|
||||
|
||||
# all_gather reads the clock once before and once after the collective.
|
||||
@@ -183,6 +197,99 @@ class TestDPBalanceStats(CustomTestCase):
|
||||
self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 12)
|
||||
self.assertGreater(stats.sync_wait_seconds, 0)
|
||||
|
||||
def test_attention_pairs_follow_context_not_rows(self):
|
||||
"""Fewer rows with longer contexts is the busier rank in pairs.
|
||||
|
||||
Rows and pairs are gathered side by side, so the two series can
|
||||
disagree on which rank is the busiest; this is the case the pair
|
||||
series exists for.
|
||||
"""
|
||||
reqs = [SimpleNamespace(rid=f"req-{n}", seqlen=n) for n in (1000, 3000)]
|
||||
result = self._prepare_dp2_batch(
|
||||
ForwardMode.DECODE,
|
||||
local_tokens=2,
|
||||
peer_tokens=8,
|
||||
reqs=reqs,
|
||||
peer_attention_pairs=800,
|
||||
)
|
||||
|
||||
stats = result.dp_balance_stats
|
||||
self.assertEqual((stats.local_tokens, stats.imbalance_tokens), (2, 6))
|
||||
self.assertEqual(
|
||||
(stats.local_attention_pairs, stats.max_attention_pairs), (4000, 4000)
|
||||
)
|
||||
self.assertEqual(stats.imbalance_attention_pairs, 0)
|
||||
self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 10)
|
||||
|
||||
def test_attention_pairs_gathered_without_the_metrics_flag(self):
|
||||
"""The pair column must not depend on a per-process flag: a rank with
|
||||
the flag off would gather zeros and trip the pair-count guard on its
|
||||
flag-on peers."""
|
||||
with patch.object(dp_attn, "_ENABLE_METRICS_DP_ATTENTION", False):
|
||||
info = dp_attn.MLPSyncBatchInfo(
|
||||
num_dp_ranks=2,
|
||||
tp_size=1,
|
||||
cp_size=1,
|
||||
num_tokens=2,
|
||||
num_tokens_for_logprob=2,
|
||||
can_run_decode_cuda_graph=True,
|
||||
can_run_draft_cuda_graph=True,
|
||||
can_run_prefill_cuda_graph=False,
|
||||
is_extend_in_batch=False,
|
||||
local_can_run_tbo=False,
|
||||
local_forward_mode=ForwardMode.DECODE.value,
|
||||
attention_pairs=dp_attn._local_attention_pairs(
|
||||
SimpleNamespace(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
seq_lens_cpu=None,
|
||||
reqs=[SimpleNamespace(seqlen=5), SimpleNamespace(seqlen=7)],
|
||||
)
|
||||
),
|
||||
)
|
||||
self.assertEqual(info._get_local_tensor(device="cpu")[10].item(), 12)
|
||||
|
||||
def test_prefill_chunk_pairs_are_causal(self):
|
||||
"""A chunk after prefix p reads p keys per query plus the causal
|
||||
triangle inside the chunk; a one-token chunk after prefix p therefore
|
||||
costs the same as a decode row of length p + 1."""
|
||||
extend = SimpleNamespace(
|
||||
forward_mode=ForwardMode.EXTEND, prefix_lens=[10, 6], extend_lens=[4, 1]
|
||||
)
|
||||
decode = SimpleNamespace(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
seq_lens_cpu=None,
|
||||
reqs=[SimpleNamespace(seqlen=7)],
|
||||
)
|
||||
self.assertEqual(dp_attn._local_attention_pairs(extend), (10 * 4 + 10) + 7)
|
||||
self.assertEqual(dp_attn._local_attention_pairs(decode), 7)
|
||||
self.assertEqual(dp_attn._local_attention_pairs(None), 0)
|
||||
self.assertEqual(
|
||||
dp_attn._local_attention_pairs(
|
||||
SimpleNamespace(forward_mode=ForwardMode.IDLE)
|
||||
),
|
||||
0,
|
||||
)
|
||||
|
||||
def test_decode_pairs_from_seq_lens_cpu_match_request_lengths(self):
|
||||
"""The prepared seq_lens_cpu fast path and the committed-length
|
||||
fallback count the same pairs, so the series has no discontinuity when
|
||||
a batch switches between them."""
|
||||
lens = [3, 5, 9]
|
||||
fast = SimpleNamespace(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
seq_lens_cpu=torch.tensor(lens),
|
||||
reqs=[],
|
||||
)
|
||||
slow = SimpleNamespace(
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
seq_lens_cpu=None,
|
||||
reqs=[SimpleNamespace(seqlen=n) for n in lens],
|
||||
)
|
||||
self.assertEqual(
|
||||
dp_attn._local_attention_pairs(fast), dp_attn._local_attention_pairs(slow)
|
||||
)
|
||||
self.assertEqual(dp_attn._local_attention_pairs(fast), 17)
|
||||
|
||||
def _adapter(self):
|
||||
return dp_attn.SchedulerDPAttnAdapter(
|
||||
model_runner=object(),
|
||||
@@ -247,7 +354,7 @@ class TestDPBalanceStats(CustomTestCase):
|
||||
self.assertEqual(carry.seconds, 0.0)
|
||||
|
||||
def test_health_check_probe_steps_not_recorded(self):
|
||||
probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0")
|
||||
probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0", seqlen=1)
|
||||
result = self._prepare_dp2_batch(
|
||||
ForwardMode.DECODE, local_tokens=1, peer_tokens=0, reqs=[probe]
|
||||
)
|
||||
@@ -267,9 +374,24 @@ class TestDPBalanceStats(CustomTestCase):
|
||||
self.assertIs(result, idle_batch)
|
||||
self.assertIsNone(result.dp_balance_stats)
|
||||
|
||||
def test_create_rejects_steps_without_tokens(self):
|
||||
def test_create_rejects_steps_without_tokens_or_pairs(self):
|
||||
with self.assertRaises(ValueError):
|
||||
DPBalanceStats.create(0, [0, 0], 0.0)
|
||||
DPBalanceStats.create(
|
||||
local_tokens=0,
|
||||
global_num_tokens=[0, 0],
|
||||
local_attention_pairs=0,
|
||||
global_attention_pairs=[0, 0],
|
||||
sync_wait_seconds=0.0,
|
||||
)
|
||||
# Rows without pairs cannot come from a correct gather.
|
||||
with self.assertRaises(ValueError):
|
||||
DPBalanceStats.create(
|
||||
local_tokens=0,
|
||||
global_num_tokens=[0, 8],
|
||||
local_attention_pairs=0,
|
||||
global_attention_pairs=[0, 0],
|
||||
sync_wait_seconds=0.0,
|
||||
)
|
||||
|
||||
|
||||
class TestDecodeToExtendConversionVote(CustomTestCase):
|
||||
|
||||
@@ -348,6 +348,30 @@ class TestForwardPassMetrics(unittest.TestCase):
|
||||
self.assertFalse(scheduler.enable_fpm)
|
||||
|
||||
|
||||
def _dp_balance_stats(
|
||||
*,
|
||||
local_tokens: int,
|
||||
global_num_tokens: list,
|
||||
sync_wait_seconds: float,
|
||||
local_attention_pairs: int | None = None,
|
||||
global_attention_pairs: list | None = None,
|
||||
):
|
||||
"""One-token rows unless the pairs are given explicitly."""
|
||||
return DPBalanceStats.create(
|
||||
local_tokens=local_tokens,
|
||||
global_num_tokens=global_num_tokens,
|
||||
local_attention_pairs=(
|
||||
local_tokens if local_attention_pairs is None else local_attention_pairs
|
||||
),
|
||||
global_attention_pairs=(
|
||||
global_num_tokens
|
||||
if global_attention_pairs is None
|
||||
else global_attention_pairs
|
||||
),
|
||||
sync_wait_seconds=sync_wait_seconds,
|
||||
)
|
||||
|
||||
|
||||
class TestDPBalanceMetrics(CustomTestCase):
|
||||
def _reporter_with_collector(self, attn_dp_rank: int):
|
||||
scheduler = types.SimpleNamespace(
|
||||
@@ -401,10 +425,18 @@ class TestDPBalanceMetrics(CustomTestCase):
|
||||
)
|
||||
|
||||
batch = types.SimpleNamespace(
|
||||
dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002)
|
||||
dp_balance_stats=_dp_balance_stats(
|
||||
local_tokens=4,
|
||||
global_num_tokens=[8, 4],
|
||||
sync_wait_seconds=0.002,
|
||||
local_attention_pairs=4000,
|
||||
global_attention_pairs=[800, 4000],
|
||||
)
|
||||
)
|
||||
reporter.log_batch_result_stats(batch, result=object())
|
||||
batch.dp_balance_stats = DPBalanceStats.create(0, [8, 0], 0.004)
|
||||
batch.dp_balance_stats = _dp_balance_stats(
|
||||
local_tokens=0, global_num_tokens=[8, 0], sync_wait_seconds=0.004
|
||||
)
|
||||
reporter.log_batch_result_stats(batch, result=object())
|
||||
batch.dp_balance_stats = None
|
||||
reporter.log_batch_result_stats(batch, result=object())
|
||||
@@ -430,6 +462,16 @@ class TestDPBalanceMetrics(CustomTestCase):
|
||||
get("sglang:dp_attention_token_imbalance_ratio_sum", labels),
|
||||
8 * 2 / 12 + 2.0,
|
||||
)
|
||||
# Pairs: the first step's long-context rank is the pair-busiest
|
||||
# despite having fewer rows; the idle step adds the peer's 8.
|
||||
self.assertEqual(
|
||||
get("sglang:dp_attention_pairs_total", {**labels, "kind": "scheduled"}),
|
||||
4000,
|
||||
)
|
||||
self.assertEqual(
|
||||
get("sglang:dp_attention_pairs_total", {**labels, "kind": "imbalance"}),
|
||||
8,
|
||||
)
|
||||
self.assertAlmostEqual(
|
||||
get("sglang:dp_attention_sync_wait_seconds_sum", labels), 0.006
|
||||
)
|
||||
@@ -473,7 +515,9 @@ class TestDPBalanceMetrics(CustomTestCase):
|
||||
def test_engine_ratio_reported_by_dp_rank_zero_only(self):
|
||||
reporter, registry, labels = self._reporter_with_collector(attn_dp_rank=1)
|
||||
batch = types.SimpleNamespace(
|
||||
dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002)
|
||||
dp_balance_stats=_dp_balance_stats(
|
||||
local_tokens=4, global_num_tokens=[8, 4], sync_wait_seconds=0.002
|
||||
)
|
||||
)
|
||||
reporter.log_batch_result_stats(batch, result=object())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user