[Metrics] Add per-rank DP attention imbalance metrics (#42755)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: hanming-lu <hanming-lu@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
hanming-lu
parent
38bdf25f8d
commit
4506927635
@@ -132,6 +132,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import (
|
||||
CudaIpcTensorTransportProxy,
|
||||
)
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
DPBalanceStats,
|
||||
DPCooperationInfo,
|
||||
SchedulerMetricsCollector,
|
||||
)
|
||||
@@ -2413,6 +2414,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
# Metrics
|
||||
dp_cooperation_info: Optional[DPCooperationInfo] = None
|
||||
dp_balance_stats: Optional[DPBalanceStats] = None
|
||||
prefill_stats: Optional[PrefillStats] = None
|
||||
forward_iter: Optional[int] = None
|
||||
launch_ts: Optional[float] = None
|
||||
@@ -3880,6 +3882,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
mamba_decode_batch_idx_cpu=self.mamba_decode_batch_idx_cpu,
|
||||
mamba_lazy_spec_track_positions_cpu=self.mamba_lazy_spec_track_positions_cpu,
|
||||
dp_cooperation_info=self.dp_cooperation_info,
|
||||
dp_balance_stats=self.dp_balance_stats,
|
||||
prefill_stats=self.prefill_stats,
|
||||
fpm_start_time=self.fpm_start_time,
|
||||
forward_iter=self.forward_iter,
|
||||
|
||||
@@ -4919,6 +4919,7 @@ class Scheduler(
|
||||
@scheduler_stage_method(SCHEDULER_STAGE_IDLE)
|
||||
def on_idle(self):
|
||||
"""Idle housekeeping: guard, check, metrics, reset, sleep."""
|
||||
self.dp_attn_adapter.drop_sync_wait_carry()
|
||||
# Flush any health-check signal deferred while the engine was busy.
|
||||
self.maybe_send_health_check_signal()
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
import torch
|
||||
@@ -16,6 +17,7 @@ from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler_components.recv_skipper import (
|
||||
SchedulerRecvSkipper,
|
||||
)
|
||||
from sglang.srt.managers.utils import is_health_check_generate_req
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.kv_cache_builder import uses_ssm_state
|
||||
@@ -28,7 +30,10 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.model_executor.runner import PrefillCudaGraphRunner
|
||||
from sglang.srt.observability.metrics_collector import DPCooperationInfo
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
DPBalanceStats,
|
||||
DPCooperationInfo,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_memory,
|
||||
@@ -118,9 +123,13 @@ class MLPSyncBatchInfo:
|
||||
local_can_run_tbo: bool
|
||||
local_forward_mode: int
|
||||
prefill_cuda_graph_max_prefix_len: int = 0
|
||||
# Every request in the local batch is a health-check probe.
|
||||
is_health_check: bool = False
|
||||
|
||||
# some gathered elements
|
||||
tp0_info_cpu: torch.Tensor = None
|
||||
any_health_check: bool = False
|
||||
sync_wait_seconds: float = 0.0
|
||||
global_num_tokens: list[int] = None
|
||||
global_num_tokens_for_logprob: list[int] = None
|
||||
tbo_split_seq_index: torch.Tensor = None
|
||||
@@ -139,6 +148,7 @@ class MLPSyncBatchInfo:
|
||||
int(self.can_run_prefill_cuda_graph),
|
||||
self.prefill_cuda_graph_max_prefix_len,
|
||||
int(self.can_run_draft_cuda_graph),
|
||||
int(self.is_health_check),
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -156,6 +166,7 @@ class MLPSyncBatchInfo:
|
||||
0, # can_run_prefill_cuda_graph
|
||||
0, # prefill_cuda_graph_max_prefix_len
|
||||
1, # can_run_draft_cuda_graph
|
||||
0, # is_health_check
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -166,6 +177,7 @@ class MLPSyncBatchInfo:
|
||||
self.tp0_info_cpu = self._get_local_tensor(device="cpu").view(1, -1)
|
||||
self.global_num_tokens = [self.num_tokens]
|
||||
self.global_num_tokens_for_logprob = [self.num_tokens_for_logprob]
|
||||
self.any_health_check = self.is_health_check
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
self.tp0_info_cpu[:, 5].tolist()
|
||||
@@ -177,6 +189,7 @@ class MLPSyncBatchInfo:
|
||||
group: torch.distributed.ProcessGroup,
|
||||
use_all_reduce: bool = False,
|
||||
):
|
||||
sync_start = time.perf_counter()
|
||||
local_info_tensor = self._get_local_tensor(device=device)
|
||||
fallback_tensor = self._get_fallback_tensor(device=device)
|
||||
info_width = local_info_tensor.numel()
|
||||
@@ -239,6 +252,8 @@ class MLPSyncBatchInfo:
|
||||
self.can_run_prefill_cuda_graph = bool(tp0_info_cpu[:, 6].min())
|
||||
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
|
||||
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
|
||||
self.any_health_check = bool(tp0_info_cpu[:, 9].max())
|
||||
self.sync_wait_seconds = time.perf_counter() - sync_start
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
tp0_info_cpu[:, 5].tolist()
|
||||
@@ -413,6 +428,24 @@ def _local_prefill_cuda_graph_vote(
|
||||
)
|
||||
|
||||
|
||||
class DPSyncWaitCarry:
|
||||
"""MLP-sync wait of batch-less gathers, folded into the next recorded step.
|
||||
|
||||
One scheduler iteration can gather more than once (with speculative
|
||||
decoding, get_next_batch_to_run gathers for the new prefill batch first,
|
||||
then for the running batch); the first gather absorbs the cross-rank
|
||||
arrival skew but yields no batch to record on. Only batch-less gathers carry: an iteration
|
||||
that ends with no batch drops the carry in Scheduler.on_idle, and a step
|
||||
that runs a batch without recording (prebuilt, health check) drops it too,
|
||||
so an idle engine's gathers never pile onto the first busy step.
|
||||
"""
|
||||
|
||||
__slots__ = ("seconds",)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.seconds = 0.0
|
||||
|
||||
|
||||
def prepare_mlp_sync_batch_raw(
|
||||
local_batch: ScheduleBatch,
|
||||
model_runner: ModelRunner,
|
||||
@@ -423,6 +456,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
offload_tags: set[str],
|
||||
draft_require_mlp_tp_gather: Optional[bool] = None,
|
||||
dwdp: bool = False,
|
||||
sync_wait_carry: Optional[DPSyncWaitCarry] = None,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
num_dp_ranks = parallel.num_dp_ranks
|
||||
@@ -481,6 +515,11 @@ def prepare_mlp_sync_batch_raw(
|
||||
)
|
||||
|
||||
is_extend_in_batch = local_batch.forward_mode.is_extend() if local_batch else False
|
||||
is_health_check = bool(
|
||||
local_batch is not None
|
||||
and local_batch.reqs
|
||||
and all(is_health_check_generate_req(req) for req in local_batch.reqs)
|
||||
)
|
||||
if local_batch is not None:
|
||||
local_batch.is_extend_in_batch = is_extend_in_batch
|
||||
|
||||
@@ -523,6 +562,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
local_can_run_tbo=local_can_run_tbo,
|
||||
local_forward_mode=local_forward_mode,
|
||||
prefill_cuda_graph_max_prefix_len=prefill_cuda_graph_max_prefix_len,
|
||||
is_health_check=is_health_check,
|
||||
)
|
||||
|
||||
if num_dp_ranks == 1:
|
||||
@@ -576,6 +616,23 @@ def prepare_mlp_sync_batch_raw(
|
||||
|
||||
if _ENABLE_METRICS_DP_ATTENTION and local_batch is not None:
|
||||
local_batch.dp_cooperation_info = mlp_sync_info.dp_cooperation_info
|
||||
local_batch.dp_balance_stats = None
|
||||
if _ENABLE_METRICS_DP_ATTENTION and num_dp_ranks > 1 and not skip_all_gather:
|
||||
sync_wait_seconds = mlp_sync_info.sync_wait_seconds
|
||||
if sync_wait_carry is not None:
|
||||
sync_wait_seconds += sync_wait_carry.seconds
|
||||
sync_wait_carry.seconds = 0.0
|
||||
# Health-check probes run on an otherwise idle engine; recording them
|
||||
# would show maximal imbalance where there is no load.
|
||||
if need_idle_batch and not mlp_sync_info.any_health_check:
|
||||
local_batch.dp_balance_stats = DPBalanceStats.create(
|
||||
local_tokens=num_tokens,
|
||||
global_num_tokens=mlp_sync_info.global_num_tokens,
|
||||
sync_wait_seconds=sync_wait_seconds,
|
||||
)
|
||||
elif local_batch is None and sync_wait_carry is not None:
|
||||
# No batch yet: this iteration may gather again, or end in on_idle.
|
||||
sync_wait_carry.seconds = sync_wait_seconds
|
||||
|
||||
return local_batch
|
||||
|
||||
@@ -591,6 +648,11 @@ class SchedulerDPAttnAdapter:
|
||||
enable_overlap: bool
|
||||
spec_algorithm: SpeculativeAlgorithm
|
||||
get_require_mlp_sync: Callable[[], bool]
|
||||
sync_wait_carry: DPSyncWaitCarry = field(default_factory=DPSyncWaitCarry)
|
||||
|
||||
def drop_sync_wait_carry(self) -> None:
|
||||
"""The iteration produced no batch, so its gather waits have no step to land on."""
|
||||
self.sync_wait_carry.seconds = 0.0
|
||||
|
||||
def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
|
||||
draft_require_mlp_tp_gather = None
|
||||
@@ -612,6 +674,7 @@ class SchedulerDPAttnAdapter:
|
||||
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
|
||||
offload_tags=self.offload_tags,
|
||||
dwdp=get_parallel().dwdp_size > 1,
|
||||
sync_wait_carry=self.sync_wait_carry,
|
||||
)
|
||||
|
||||
def maybe_prepare_mlp_sync_batch(
|
||||
|
||||
@@ -219,6 +219,13 @@ class SchedulerMetricsReporter:
|
||||
self.enable_kv_cache_events = (
|
||||
self.metrics_collector_context.enable_kv_cache_events
|
||||
)
|
||||
parallel = get_parallel()
|
||||
# The max/mean token ratio is engine-wide; one scheduler reports it.
|
||||
self.reports_dp_token_imbalance_ratio = (
|
||||
parallel.attn_dp_rank == 0
|
||||
and parallel.attn_tp_rank == 0
|
||||
and parallel.attn_cp_rank == 0
|
||||
)
|
||||
self._init_metrics()
|
||||
self._install_device_timer_on_runners()
|
||||
# Keep log history after the existing async result copy so reporting does
|
||||
@@ -1116,6 +1123,16 @@ class SchedulerMetricsReporter:
|
||||
batch: ScheduleBatch,
|
||||
result: Union[GenerationBatchResult, EmbeddingBatchResult],
|
||||
):
|
||||
if (
|
||||
self.current_scheduler_metrics_enabled
|
||||
and (dp_balance_stats := batch.dp_balance_stats) is not None
|
||||
):
|
||||
self.metrics_collector.observe_dp_balance(dp_balance_stats)
|
||||
if self.reports_dp_token_imbalance_ratio:
|
||||
self.metrics_collector.observe_dp_token_imbalance_ratio(
|
||||
dp_balance_stats
|
||||
)
|
||||
|
||||
if not isinstance(result, GenerationBatchResult):
|
||||
return
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set, Union
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
@@ -203,6 +205,52 @@ class DPCooperationInfo:
|
||||
return dataclasses.asdict(self)
|
||||
|
||||
|
||||
class DPBalanceStats(msgspec.Struct, frozen=True):
|
||||
"""Token balance across the attention-DP ranks of one MLP-sync step.
|
||||
|
||||
DP attention runs the ranks in lockstep, so the step costs what the
|
||||
busiest rank costs. ``max_tokens - local_tokens`` is the capacity this
|
||||
rank gives up on the step: computed as padding rows under MAX_LEN
|
||||
padding, or spent waiting for the busiest rank under SUM_LEN. Counts are
|
||||
the gathered MLP-sync values, so on speculative-decode steps they are
|
||||
batch rows (requests); the verify width is uniform across ranks, which
|
||||
leaves the ratios unchanged.
|
||||
"""
|
||||
|
||||
local_tokens: int
|
||||
max_tokens: int
|
||||
sum_tokens: int
|
||||
num_ranks: int
|
||||
# Wall time of this rank's scheduler MLP-sync all-gather. On the gloo path
|
||||
# (the default with overlap scheduling) this is waiting for the slowest
|
||||
# rank; on the device-group paths it also includes the gather's stream sync.
|
||||
sync_wait_seconds: float
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls, local_tokens: int, global_num_tokens: List[int], sync_wait_seconds: float
|
||||
) -> DPBalanceStats:
|
||||
max_tokens = max(global_num_tokens)
|
||||
if max_tokens == 0:
|
||||
raise ValueError("DP balance stats need a rank with tokens")
|
||||
return cls(
|
||||
local_tokens=local_tokens,
|
||||
max_tokens=max_tokens,
|
||||
sum_tokens=sum(global_num_tokens),
|
||||
num_ranks=len(global_num_tokens),
|
||||
sync_wait_seconds=sync_wait_seconds,
|
||||
)
|
||||
|
||||
@property
|
||||
def imbalance_tokens(self) -> int:
|
||||
return self.max_tokens - self.local_tokens
|
||||
|
||||
@property
|
||||
def max_over_mean(self) -> float:
|
||||
"""Busiest rank's tokens over the per-rank mean; 1.0 is perfectly balanced."""
|
||||
return self.max_tokens * self.num_ranks / self.sum_tokens
|
||||
|
||||
|
||||
# Role keys used by ServerArgs.stat_loggers to look up collector overrides.
|
||||
# Embedded-use callers (e.g. Ray Serve LLM) pass {"scheduler": MyClass, ...} on
|
||||
# ServerArgs and the five collector instantiation sites pick the right class.
|
||||
@@ -1027,6 +1075,92 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["category", "num_prefill_ranks"],
|
||||
)
|
||||
self.dp_attention_tokens_total = Counter(
|
||||
name="sglang:dp_attention_tokens_total",
|
||||
documentation=(
|
||||
"Tokens of MLP-sync steps on this DP rank. kind=scheduled: tokens "
|
||||
"this rank ran; kind=imbalance: tokens the busiest rank ran beyond "
|
||||
"this rank, i.e. capacity lost to DP imbalance on this rank. "
|
||||
"On speculative-decode steps the unit is batch rows (requests)."
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["kind"],
|
||||
)
|
||||
self.dp_attention_steps_total = Counter(
|
||||
name="sglang:dp_attention_steps_total",
|
||||
documentation=(
|
||||
"MLP-sync steps on this DP rank. rank_state=idle: this rank had no "
|
||||
"tokens and only ran to keep the DP collectives in lockstep."
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["rank_state"],
|
||||
)
|
||||
self.dp_attention_token_imbalance_ratio = Histogram(
|
||||
name="sglang:dp_attention_token_imbalance_ratio",
|
||||
documentation=(
|
||||
"Per MLP-sync step, max tokens over mean tokens across the DP "
|
||||
"ranks (1.0 = balanced, num_dp_ranks = one busy rank). Reported "
|
||||
"once per engine, by the scheduler of DP rank 0."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
buckets=(
|
||||
1.0,
|
||||
1.05,
|
||||
1.1,
|
||||
1.25,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
8.0,
|
||||
12.0,
|
||||
16.0,
|
||||
24.0,
|
||||
32.0,
|
||||
48.0,
|
||||
64.0,
|
||||
),
|
||||
)
|
||||
self.dp_attention_sync_wait_seconds = Histogram(
|
||||
name="sglang:dp_attention_sync_wait_seconds",
|
||||
documentation=(
|
||||
"Wall time of this DP rank's scheduler MLP-sync all-gather per step. "
|
||||
"On the gloo path (the default with overlap scheduling) this is "
|
||||
"waiting for the slowest rank; on the device-group paths it also "
|
||||
"includes the gather's stream sync."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
buckets=(
|
||||
0.0001,
|
||||
0.00025,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.0025,
|
||||
0.005,
|
||||
0.01,
|
||||
0.025,
|
||||
0.05,
|
||||
0.1,
|
||||
0.25,
|
||||
0.5,
|
||||
1.0,
|
||||
),
|
||||
)
|
||||
# Pre-seed and cache the children: every step observes them, and a
|
||||
# ratio chart needs both kinds / states present from the start.
|
||||
self._dp_attention_tokens = {
|
||||
kind: self.dp_attention_tokens_total.labels(**labels, kind=kind)
|
||||
for kind in ("scheduled", "imbalance")
|
||||
}
|
||||
self._dp_attention_steps = {
|
||||
state: self.dp_attention_steps_total.labels(**labels, rank_state=state)
|
||||
for state in ("active", "idle")
|
||||
}
|
||||
self._dp_attention_token_imbalance_ratio = (
|
||||
self.dp_attention_token_imbalance_ratio.labels(**labels)
|
||||
)
|
||||
self._dp_attention_sync_wait_seconds = (
|
||||
self.dp_attention_sync_wait_seconds.labels(**labels)
|
||||
)
|
||||
|
||||
# =================================================================
|
||||
# Prefill delayer
|
||||
@@ -1328,6 +1462,18 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
balancedness
|
||||
)
|
||||
|
||||
def observe_dp_balance(self, stats: DPBalanceStats) -> None:
|
||||
if stats.local_tokens:
|
||||
self._dp_attention_tokens["scheduled"].inc(stats.local_tokens)
|
||||
if stats.imbalance_tokens:
|
||||
self._dp_attention_tokens["imbalance"].inc(stats.imbalance_tokens)
|
||||
rank_state = "idle" if stats.local_tokens == 0 else "active"
|
||||
self._dp_attention_steps[rank_state].inc(1)
|
||||
self._dp_attention_sync_wait_seconds.observe(stats.sync_wait_seconds)
|
||||
|
||||
def observe_dp_token_imbalance_ratio(self, stats: DPBalanceStats) -> None:
|
||||
self._dp_attention_token_imbalance_ratio.observe(stats.max_over_mean)
|
||||
|
||||
def increment_realtime_tokens(
|
||||
self,
|
||||
dp_cooperation_info: Optional[DPCooperationInfo],
|
||||
|
||||
Reference in New Issue
Block a user