diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index eef421653b..ebccdb1629 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -132,6 +132,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import ( CudaIpcTensorTransportProxy, ) from sglang.srt.observability.metrics_collector import ( + DPBalanceStats, DPCooperationInfo, SchedulerMetricsCollector, ) @@ -2413,6 +2414,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): # Metrics dp_cooperation_info: Optional[DPCooperationInfo] = None + dp_balance_stats: Optional[DPBalanceStats] = None prefill_stats: Optional[PrefillStats] = None forward_iter: Optional[int] = None launch_ts: Optional[float] = None @@ -3880,6 +3882,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): mamba_decode_batch_idx_cpu=self.mamba_decode_batch_idx_cpu, mamba_lazy_spec_track_positions_cpu=self.mamba_lazy_spec_track_positions_cpu, dp_cooperation_info=self.dp_cooperation_info, + dp_balance_stats=self.dp_balance_stats, prefill_stats=self.prefill_stats, fpm_start_time=self.fpm_start_time, forward_iter=self.forward_iter, diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 0aafc6baa5..60f6308213 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4919,6 +4919,7 @@ class Scheduler( @scheduler_stage_method(SCHEDULER_STAGE_IDLE) def on_idle(self): """Idle housekeeping: guard, check, metrics, reset, sleep.""" + self.dp_attn_adapter.drop_sync_wait_carry() # Flush any health-check signal deferred while the engine was busy. self.maybe_send_health_check_signal() diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 3c650b67a7..91347fe84b 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -1,6 +1,7 @@ from __future__ import annotations -from dataclasses import dataclass +import time +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Callable, Optional import torch @@ -16,6 +17,7 @@ from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.managers.scheduler_components.recv_skipper import ( SchedulerRecvSkipper, ) +from sglang.srt.managers.utils import is_health_check_generate_req from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.kv_cache_builder import uses_ssm_state @@ -28,7 +30,10 @@ from sglang.srt.model_executor.cuda_graph_config import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.model_executor.runner import PrefillCudaGraphRunner -from sglang.srt.observability.metrics_collector import DPCooperationInfo +from sglang.srt.observability.metrics_collector import ( + DPBalanceStats, + DPCooperationInfo, +) from sglang.srt.runtime_context import ( get_exec, get_memory, @@ -118,9 +123,13 @@ class MLPSyncBatchInfo: local_can_run_tbo: bool local_forward_mode: int prefill_cuda_graph_max_prefix_len: int = 0 + # Every request in the local batch is a health-check probe. + is_health_check: bool = False # some gathered elements tp0_info_cpu: torch.Tensor = None + any_health_check: bool = False + sync_wait_seconds: float = 0.0 global_num_tokens: list[int] = None global_num_tokens_for_logprob: list[int] = None tbo_split_seq_index: torch.Tensor = None @@ -139,6 +148,7 @@ class MLPSyncBatchInfo: int(self.can_run_prefill_cuda_graph), self.prefill_cuda_graph_max_prefix_len, int(self.can_run_draft_cuda_graph), + int(self.is_health_check), ], device=device, dtype=dtype, @@ -156,6 +166,7 @@ class MLPSyncBatchInfo: 0, # can_run_prefill_cuda_graph 0, # prefill_cuda_graph_max_prefix_len 1, # can_run_draft_cuda_graph + 0, # is_health_check ], device=device, dtype=dtype, @@ -166,6 +177,7 @@ class MLPSyncBatchInfo: self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1) self.global_num_tokens = [self.num_tokens] self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob] + self.any_health_check = self.is_health_check if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( self.tp0_info_cpu[:, 5].tolist() @@ -177,6 +189,7 @@ class MLPSyncBatchInfo: group: torch.distributed.ProcessGroup, use_all_reduce: bool = False, ): + sync_start = time.perf_counter() local_info_tensor = self._get_local_tensor(device=device) fallback_tensor = self._get_fallback_tensor(device=device) info_width = local_info_tensor.numel() @@ -239,6 +252,8 @@ class MLPSyncBatchInfo: self.can_run_prefill_cuda_graph = bool(tp0_info_cpu[:, 6].min()) self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max()) self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min()) + self.any_health_check = bool(tp0_info_cpu[:, 9].max()) + self.sync_wait_seconds = time.perf_counter() - sync_start if _ENABLE_METRICS_DP_ATTENTION: self.dp_cooperation_info = DPCooperationInfo.create( tp0_info_cpu[:, 5].tolist() @@ -413,6 +428,24 @@ def _local_prefill_cuda_graph_vote( ) +class DPSyncWaitCarry: + """MLP-sync wait of batch-less gathers, folded into the next recorded step. + + One scheduler iteration can gather more than once (with speculative + decoding, get_next_batch_to_run gathers for the new prefill batch first, + then for the running batch); the first gather absorbs the cross-rank + arrival skew but yields no batch to record on. Only batch-less gathers carry: an iteration + that ends with no batch drops the carry in Scheduler.on_idle, and a step + that runs a batch without recording (prebuilt, health check) drops it too, + so an idle engine's gathers never pile onto the first busy step. + """ + + __slots__ = ("seconds",) + + def __init__(self) -> None: + self.seconds = 0.0 + + def prepare_mlp_sync_batch_raw( local_batch: ScheduleBatch, model_runner: ModelRunner, @@ -423,6 +456,7 @@ def prepare_mlp_sync_batch_raw( offload_tags: set[str], draft_require_mlp_tp_gather: Optional[bool] = None, dwdp: bool = False, + sync_wait_carry: Optional[DPSyncWaitCarry] = None, ): parallel = get_parallel() num_dp_ranks = parallel.num_dp_ranks @@ -481,6 +515,11 @@ def prepare_mlp_sync_batch_raw( ) is_extend_in_batch = local_batch.forward_mode.is_extend() if local_batch else False + is_health_check = bool( + local_batch is not None + and local_batch.reqs + and all(is_health_check_generate_req(req) for req in local_batch.reqs) + ) if local_batch is not None: local_batch.is_extend_in_batch = is_extend_in_batch @@ -523,6 +562,7 @@ def prepare_mlp_sync_batch_raw( local_can_run_tbo=local_can_run_tbo, local_forward_mode=local_forward_mode, prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len, + is_health_check=is_health_check, ) if num_dp_ranks == 1: @@ -576,6 +616,23 @@ def prepare_mlp_sync_batch_raw( if _ENABLE_METRICS_DP_ATTENTION and local_batch is not None: local_batch.dp_cooperation_info = mlp_sync_info.dp_cooperation_info + local_batch.dp_balance_stats = None + if _ENABLE_METRICS_DP_ATTENTION and num_dp_ranks > 1 and not skip_all_gather: + sync_wait_seconds = mlp_sync_info.sync_wait_seconds + if sync_wait_carry is not None: + sync_wait_seconds += sync_wait_carry.seconds + sync_wait_carry.seconds = 0.0 + # Health-check probes run on an otherwise idle engine; recording them + # would show maximal imbalance where there is no load. + if need_idle_batch and not mlp_sync_info.any_health_check: + local_batch.dp_balance_stats = DPBalanceStats.create( + local_tokens=num_tokens, + global_num_tokens=mlp_sync_info.global_num_tokens, + sync_wait_seconds=sync_wait_seconds, + ) + elif local_batch is None and sync_wait_carry is not None: + # No batch yet: this iteration may gather again, or end in on_idle. + sync_wait_carry.seconds = sync_wait_seconds return local_batch @@ -591,6 +648,11 @@ class SchedulerDPAttnAdapter: enable_overlap: bool spec_algorithm: SpeculativeAlgorithm get_require_mlp_sync: Callable[[], bool] + sync_wait_carry: DPSyncWaitCarry = field(default_factory=DPSyncWaitCarry) + + def drop_sync_wait_carry(self) -> None: + """The iteration produced no batch, so its gather waits have no step to land on.""" + self.sync_wait_carry.seconds = 0.0 def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch): draft_require_mlp_tp_gather = None @@ -612,6 +674,7 @@ class SchedulerDPAttnAdapter: disable_overlap_schedule=get_schedule().disable_overlap_schedule, offload_tags=self.offload_tags, dwdp=get_parallel().dwdp_size > 1, + sync_wait_carry=self.sync_wait_carry, ) def maybe_prepare_mlp_sync_batch( diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index b8d2fdc9ce..adfd129410 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -219,6 +219,13 @@ class SchedulerMetricsReporter: self.enable_kv_cache_events = ( self.metrics_collector_context.enable_kv_cache_events ) + parallel = get_parallel() + # The max/mean token ratio is engine-wide; one scheduler reports it. + self.reports_dp_token_imbalance_ratio = ( + parallel.attn_dp_rank == 0 + and parallel.attn_tp_rank == 0 + and parallel.attn_cp_rank == 0 + ) self._init_metrics() self._install_device_timer_on_runners() # Keep log history after the existing async result copy so reporting does @@ -1116,6 +1123,16 @@ class SchedulerMetricsReporter: batch: ScheduleBatch, result: Union[GenerationBatchResult, EmbeddingBatchResult], ): + if ( + self.current_scheduler_metrics_enabled + and (dp_balance_stats := batch.dp_balance_stats) is not None + ): + self.metrics_collector.observe_dp_balance(dp_balance_stats) + if self.reports_dp_token_imbalance_ratio: + self.metrics_collector.observe_dp_token_imbalance_ratio( + dp_balance_stats + ) + if not isinstance(result, GenerationBatchResult): return diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 270877c9e3..cae4225e50 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -23,6 +23,8 @@ from collections import Counter from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set, Union +import msgspec + from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.observability.scheduler_stage_metrics import ( @@ -203,6 +205,52 @@ class DPCooperationInfo: return dataclasses.asdict(self) +class DPBalanceStats(msgspec.Struct, frozen=True): + """Token balance across the attention-DP ranks of one MLP-sync step. + + DP attention runs the ranks in lockstep, so the step costs what the + busiest rank costs. ``max_tokens - local_tokens`` is the capacity this + rank gives up on the step: computed as padding rows under MAX_LEN + padding, or spent waiting for the busiest rank under SUM_LEN. Counts are + the gathered MLP-sync values, so on speculative-decode steps they are + batch rows (requests); the verify width is uniform across ranks, which + leaves the ratios unchanged. + """ + + local_tokens: int + max_tokens: int + sum_tokens: int + num_ranks: int + # Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path + # (the default with overlap scheduling) this is waiting for the slowest + # rank; on the device-group paths it also includes the gather's stream sync. + sync_wait_seconds: float + + @classmethod + def create( + cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float + ) -> DPBalanceStats: + max_tokens = max(global_num_tokens) + if max_tokens == 0: + raise ValueError("DP balance stats need a rank with tokens") + return cls( + local_tokens=local_tokens, + max_tokens=max_tokens, + sum_tokens=sum(global_num_tokens), + num_ranks=len(global_num_tokens), + sync_wait_seconds=sync_wait_seconds, + ) + + @property + def imbalance_tokens(self) -> int: + return self.max_tokens - self.local_tokens + + @property + def max_over_mean(self) -> float: + """Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced.""" + return self.max_tokens * self.num_ranks / self.sum_tokens + + # Role keys used by ServerArgs.stat_loggers to look up collector overrides. # Embedded-use callers (e.g. Ray Serve LLM) pass {"scheduler": MyClass, ...} on # ServerArgs and the five collector instantiation sites pick the right class. @@ -1027,6 +1075,92 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): ), labelnames=list(labels.keys()) + ["category", "num_prefill_ranks"], ) + self.dp_attention_tokens_total = Counter( + name="sglang:dp_attention_tokens_total", + documentation=( + "Tokens of MLP-sync steps on this DP rank. kind=scheduled: tokens " + "this rank ran; kind=imbalance: tokens the busiest rank ran beyond " + "this rank, i.e. capacity lost to DP imbalance on this rank. " + "On speculative-decode steps the unit is batch rows (requests)." + ), + labelnames=list(labels.keys()) + ["kind"], + ) + self.dp_attention_steps_total = Counter( + name="sglang:dp_attention_steps_total", + documentation=( + "MLP-sync steps on this DP rank. rank_state=idle: this rank had no " + "tokens and only ran to keep the DP collectives in lockstep." + ), + labelnames=list(labels.keys()) + ["rank_state"], + ) + self.dp_attention_token_imbalance_ratio = Histogram( + name="sglang:dp_attention_token_imbalance_ratio", + documentation=( + "Per MLP-sync step, max tokens over mean tokens across the DP " + "ranks (1.0 = balanced, num_dp_ranks = one busy rank). Reported " + "once per engine, by the scheduler of DP rank 0." + ), + labelnames=labels.keys(), + buckets=( + 1.0, + 1.05, + 1.1, + 1.25, + 1.5, + 2.0, + 3.0, + 4.0, + 6.0, + 8.0, + 12.0, + 16.0, + 24.0, + 32.0, + 48.0, + 64.0, + ), + ) + self.dp_attention_sync_wait_seconds = Histogram( + name="sglang:dp_attention_sync_wait_seconds", + documentation=( + "Wall time of this DP rank's scheduler MLP-sync all-gather per step. " + "On the gloo path (the default with overlap scheduling) this is " + "waiting for the slowest rank; on the device-group paths it also " + "includes the gather's stream sync." + ), + labelnames=labels.keys(), + buckets=( + 0.0001, + 0.00025, + 0.0005, + 0.001, + 0.0025, + 0.005, + 0.01, + 0.025, + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + ), + ) + # Pre-seed and cache the children: every step observes them, and a + # ratio chart needs both kinds / states present from the start. + self._dp_attention_tokens = { + kind: self.dp_attention_tokens_total.labels(**labels, kind=kind) + for kind in ("scheduled", "imbalance") + } + self._dp_attention_steps = { + state: self.dp_attention_steps_total.labels(**labels, rank_state=state) + for state in ("active", "idle") + } + self._dp_attention_token_imbalance_ratio = ( + self.dp_attention_token_imbalance_ratio.labels(**labels) + ) + self._dp_attention_sync_wait_seconds = ( + self.dp_attention_sync_wait_seconds.labels(**labels) + ) # ================================================================= # Prefill delayer @@ -1328,6 +1462,18 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): balancedness ) + def observe_dp_balance(self, stats: DPBalanceStats) -> None: + if stats.local_tokens: + self._dp_attention_tokens["scheduled"].inc(stats.local_tokens) + if stats.imbalance_tokens: + self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens) + rank_state = "idle" if stats.local_tokens == 0 else "active" + self._dp_attention_steps[rank_state].inc(1) + self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds) + + def observe_dp_token_imbalance_ratio(self, stats: DPBalanceStats) -> None: + self._dp_attention_token_imbalance_ratio.observe(stats.max_over_mean) + def increment_realtime_tokens( self, dp_cooperation_info: Optional[DPCooperationInfo], diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index feb1e77a11..1aa4fe6da7 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -77,6 +77,10 @@ class TestEnableMetrics(CustomTestCase): "sglang:dp_cooperation_realtime_tokens_total", {"mode": "decode"}, ), + ("sglang:dp_attention_tokens_total", {"kind": "scheduled"}), + ("sglang:dp_attention_steps_total", {"rank_state": "active"}), + ("sglang:dp_attention_token_imbalance_ratio_count", {}), + ("sglang:dp_attention_sync_wait_seconds_count", {}), ] _check_metrics_positive(self, metrics, metrics_to_check) diff --git a/test/registered/unit/managers/scheduler_components/test_dp_attn.py b/test/registered/unit/managers/scheduler_components/test_dp_attn.py index 8d855facbe..dacc869c27 100644 --- a/test/registered/unit/managers/scheduler_components/test_dp_attn.py +++ b/test/registered/unit/managers/scheduler_components/test_dp_attn.py @@ -9,9 +9,11 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() +from sglang.srt.constants import HEALTH_CHECK_RID_PREFIX # noqa: E402 from sglang.srt.environ import envs # noqa: E402 from sglang.srt.managers.scheduler_components import dp_attn # noqa: E402 from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402 +from sglang.srt.observability.metrics_collector import DPBalanceStats # noqa: E402 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm # noqa: E402 register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -30,6 +32,7 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): forward_mode=ForwardMode.DECODE, batch_size=lambda: 4, spec_info=None, + reqs=[], ) tbo_preparer = Mock() tbo_preparer.prepare_all_gather.return_value = ( @@ -84,6 +87,191 @@ class TestDPAttnSchedulerMetadata(CustomTestCase): ) +class TestDPBalanceStats(CustomTestCase): + def _prepare_dp2_batch( + self, + forward_mode, + local_tokens: int, + peer_tokens: int, + *, + reqs=(), + peer_health_check: bool = False, + get_idle_batch=None, + sync_wait_carry=None, + wait_seconds=None, + ): + batch = SimpleNamespace( + forward_mode=forward_mode, + batch_size=lambda: local_tokens, + spec_info=None, + reqs=list(reqs), + dp_balance_stats=None, + ) + gathered_mode = forward_mode or ForwardMode.IDLE + tbo_preparer = Mock() + tbo_preparer.prepare_all_gather.return_value = (True, gathered_mode.value) + tbo_preparer.compute_output.return_value = (None, gathered_mode) + + def fake_all_gather_single(output, local, group, **_): + peer = local.clone() + peer[0] = peer[1] = peer_tokens # num_tokens, num_tokens_for_logprob + peer[9] = int(peer_health_check) + output.copy_(torch.stack([local, peer]).flatten()) + + # all_gather reads the clock once before and once after the collective. + clock = ( + patch.object(dp_attn.time, "perf_counter", side_effect=[0.0, wait_seconds]) + if wait_seconds is not None + else patch.object( + dp_attn.time, "perf_counter", wraps=dp_attn.time.perf_counter + ) + ) + with ( + clock, + envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.override(False), + patch.object(dp_attn, "_ENABLE_METRICS_DP_ATTENTION", True), + patch.object(dp_attn, "TboDPAttentionPreparer", return_value=tbo_preparer), + patch.object(dp_attn, "world_dp_gather_enabled", return_value=False), + patch.object( + dp_attn, + "get_parallel", + return_value=SimpleNamespace( + num_dp_ranks=2, + attn_tp_size=1, + attn_cp_size=1, + tp_group=SimpleNamespace( + device_group=object(), + device="cpu", + cpu_group=object(), + active_ranks_cpu=torch.ones(2, dtype=torch.int64), + ), + ), + ), + patch.object(dp_attn, "check_cuda_graph_backend", return_value=False), + patch.object( + dp_attn, "all_gather_single", side_effect=fake_all_gather_single + ), + ): + return dp_attn.prepare_mlp_sync_batch_raw( + batch if forward_mode is not None else None, + model_runner=SimpleNamespace( + prefill_cuda_graph_runner=None, + spec_algorithm=SpeculativeAlgorithm.NONE, + model_config=object(), + ), + get_idle_batch=get_idle_batch + or Mock(side_effect=AssertionError("local batch must not be replaced")), + disable_cuda_graph=False, + require_mlp_tp_gather=True, + disable_overlap_schedule=True, + offload_tags=set(), + sync_wait_carry=sync_wait_carry, + ) + + def test_attached_to_batch_after_dp_gather(self): + result = self._prepare_dp2_batch( + ForwardMode.DECODE, local_tokens=4, peer_tokens=8 + ) + + self.assertEqual(result.global_num_tokens, [4, 8]) + stats = result.dp_balance_stats + self.assertEqual( + (stats.local_tokens, stats.max_tokens, stats.sum_tokens, stats.num_ranks), + (4, 8, 12, 2), + ) + self.assertEqual(stats.imbalance_tokens, 4) + self.assertAlmostEqual(stats.max_over_mean, 8 * 2 / 12) + self.assertGreater(stats.sync_wait_seconds, 0) + + def _adapter(self): + return dp_attn.SchedulerDPAttnAdapter( + model_runner=object(), + req_to_token_pool=object(), + token_to_kv_pool_allocator=object(), + tree_cache=object(), + offload_tags=set(), + model_config=object(), + enable_overlap=False, + spec_algorithm=SpeculativeAlgorithm.NONE, + get_require_mlp_sync=lambda: True, + ) + + def test_batchless_gathers_carry_wait_until_recorded_or_idle(self): + adapter = self._adapter() + carry = adapter.sync_wait_carry + idle_batch = SimpleNamespace( + forward_mode=ForwardMode.IDLE, reqs=[], dp_balance_stats=None + ) + + def batchless_gather(wait_seconds): + result = self._prepare_dp2_batch( + None, + local_tokens=0, + peer_tokens=0, + get_idle_batch=Mock(return_value=idle_batch), + sync_wait_carry=carry, + wait_seconds=wait_seconds, + ) + self.assertIsNone(result) + + batchless_gather(0.001) + batchless_gather(0.002) + self.assertAlmostEqual(carry.seconds, 0.003) + result = self._prepare_dp2_batch( + ForwardMode.DECODE, + local_tokens=4, + peer_tokens=8, + sync_wait_carry=carry, + wait_seconds=0.0005, + ) + self.assertAlmostEqual(result.dp_balance_stats.sync_wait_seconds, 0.0035) + self.assertEqual(carry.seconds, 0.0) + + # An iteration that ends without a batch drops what it carried. + batchless_gather(0.004) + adapter.drop_sync_wait_carry() + self.assertEqual(carry.seconds, 0.0) + + def test_unrecorded_batch_steps_do_not_carry_wait(self): + carry = self._adapter().sync_wait_carry + carry.seconds = 0.003 + # A prebuilt batch with no work anywhere runs a step but records nothing. + result = self._prepare_dp2_batch( + ForwardMode.PREBUILT, + local_tokens=0, + peer_tokens=0, + sync_wait_carry=carry, + wait_seconds=0.001, + ) + self.assertIsNone(result.dp_balance_stats) + self.assertEqual(carry.seconds, 0.0) + + def test_health_check_probe_steps_not_recorded(self): + probe = SimpleNamespace(rid=f"{HEALTH_CHECK_RID_PREFIX}-0") + result = self._prepare_dp2_batch( + ForwardMode.DECODE, local_tokens=1, peer_tokens=0, reqs=[probe] + ) + self.assertIsNone(result.dp_balance_stats) + + # The gathered flag skips the step on the ranks that only ran idle. + idle_batch = SimpleNamespace( + forward_mode=ForwardMode.IDLE, reqs=[], dp_balance_stats=None + ) + result = self._prepare_dp2_batch( + None, + local_tokens=0, + peer_tokens=1, + peer_health_check=True, + get_idle_batch=Mock(return_value=idle_batch), + ) + self.assertIs(result, idle_batch) + self.assertIsNone(result.dp_balance_stats) + + def test_create_rejects_steps_without_tokens(self): + with self.assertRaises(ValueError): + DPBalanceStats.create(0, [0, 0], 0.0) + + class TestDecodeToExtendConversionVote(CustomTestCase): """A decode batch votes for the prefill graph only when its 1-token extend view can represent every row. Beam requests cannot: the converted batch diff --git a/test/registered/unit/managers/test_scheduler_on_idle_load.py b/test/registered/unit/managers/test_scheduler_on_idle_load.py index 185dd30eff..e49cf21331 100644 --- a/test/registered/unit/managers/test_scheduler_on_idle_load.py +++ b/test/registered/unit/managers/test_scheduler_on_idle_load.py @@ -33,6 +33,7 @@ class TestOnIdleStallPublish(CustomTestCase): s.load_publisher = MagicMock() s.load_inquirer = MagicMock() s.metrics_reporter = MagicMock() + s.dp_attn_adapter = MagicMock() s._last_stall_publish_ts = float("-inf") return s diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index dbb27afbec..cc3b78a03e 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -17,7 +17,10 @@ from sglang.srt.managers.scheduler_components.metrics_reporter import ( SchedulerMetricsReporter, _CacheHitRateWindow, ) -from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector +from sglang.srt.observability.metrics_collector import ( + DPBalanceStats, + SchedulerMetricsCollector, +) from sglang.srt.observability.scheduler_stage_metrics import ( SCHEDULER_STAGE_RECV_REQUESTS, ) @@ -345,6 +348,109 @@ class TestForwardPassMetrics(unittest.TestCase): self.assertFalse(scheduler.enable_fpm) +class TestDPBalanceMetrics(CustomTestCase): + def _reporter_with_collector(self, attn_dp_rank: int): + scheduler = types.SimpleNamespace( + running_batch=types.SimpleNamespace(reqs=[]), + waiting_queue=[], + grammar_manager=[], + enable_priority_scheduling=False, + disaggregation_mode=DisaggregationMode.NULL, + ) + scheduler.server_args = _publish_server_args( + self, + enable_metrics=False, + enable_metrics_for_all_schedulers=False, + kv_events_config=None, + enable_mfu_metrics=False, + enable_forward_pass_metrics=False, + ) + enter_scope( + self, + get_parallel().override( + tp_size=2, + tp_rank=attn_dp_rank, + attn_dp_size=2, + attn_dp_rank=attn_dp_rank, + moe_tp_size=2, + ), + ) + reporter = _make_reporter(self, scheduler) + + registry = prometheus_client.CollectorRegistry() + labels = {"model_name": "test", "moe_ep_rank": 0, "dp_rank": attn_dp_rank} + with patch.multiple( + prometheus_client, + **{ + kind: partial(getattr(prometheus_client, kind), registry=registry) + for kind in ("Counter", "Gauge", "Histogram", "Summary") + }, + ): + reporter.metrics_collector = SchedulerMetricsCollector( + labels=labels, server_args=scheduler.server_args + ) + reporter.current_scheduler_metrics_enabled = True + return reporter, registry, {k: str(v) for k, v in labels.items()} + + def test_step_stats_exported_once_per_step(self): + reporter, registry, labels = self._reporter_with_collector(attn_dp_rank=0) + get = registry.get_sample_value + # Pre-seeded, so ratio charts have both operands before the first idle step. + self.assertEqual( + get("sglang:dp_attention_steps_total", {**labels, "rank_state": "idle"}), 0 + ) + + batch = types.SimpleNamespace( + dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002) + ) + reporter.log_batch_result_stats(batch, result=object()) + batch.dp_balance_stats = DPBalanceStats.create(0, [8, 0], 0.004) + reporter.log_batch_result_stats(batch, result=object()) + batch.dp_balance_stats = None + reporter.log_batch_result_stats(batch, result=object()) + + self.assertEqual( + get("sglang:dp_attention_tokens_total", {**labels, "kind": "scheduled"}), 4 + ) + self.assertEqual( + get("sglang:dp_attention_tokens_total", {**labels, "kind": "imbalance"}), + 12, + ) + self.assertEqual( + get("sglang:dp_attention_steps_total", {**labels, "rank_state": "active"}), + 1, + ) + self.assertEqual( + get("sglang:dp_attention_steps_total", {**labels, "rank_state": "idle"}), 1 + ) + self.assertEqual( + get("sglang:dp_attention_token_imbalance_ratio_count", labels), 2 + ) + self.assertAlmostEqual( + get("sglang:dp_attention_token_imbalance_ratio_sum", labels), + 8 * 2 / 12 + 2.0, + ) + self.assertAlmostEqual( + get("sglang:dp_attention_sync_wait_seconds_sum", labels), 0.006 + ) + + def test_engine_ratio_reported_by_dp_rank_zero_only(self): + reporter, registry, labels = self._reporter_with_collector(attn_dp_rank=1) + batch = types.SimpleNamespace( + dp_balance_stats=DPBalanceStats.create(4, [8, 4], 0.002) + ) + reporter.log_batch_result_stats(batch, result=object()) + + get = registry.get_sample_value + self.assertEqual( + get("sglang:dp_attention_steps_total", {**labels, "rank_state": "active"}), + 1, + ) + self.assertEqual( + get("sglang:dp_attention_token_imbalance_ratio_count", labels), 0 + ) + + class TestIdleMetrics(CustomTestCase): def setUp(self): self.scheduler = types.SimpleNamespace(