[Metrics] Label scheduler stage wall time by sampled forward overlap (#42025)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-02 13:43:53 -07:00
committed by GitHub
co-authored by merrymercy Jialin
parent f6fcda8827
commit 4a4dad1cf1
9 changed files with 269 additions and 55 deletions
+6 -4
View File
@@ -2074,9 +2074,11 @@ class Scheduler(
and get_parallel().attn_cp_rank == 0
):
local_reqs = self._poll_timeout_aborts()
recv_start_ns = time.monotonic_ns()
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
if recv_reqs:
self.metrics_reporter.record_scheduler_active()
# Count successful receive/broadcast time as active; empty polls stay idle.
self.metrics_reporter.record_scheduler_active(recv_start_ns)
self.process_input_requests(recv_reqs)
return recv_reqs
@@ -4344,7 +4346,7 @@ class Scheduler(
pp_proxy_tensors: Optional[PPProxyTensors] = None,
) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
"""Run a batch."""
self.metrics_reporter.record_scheduler_active()
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
self.forward_ct += 1
batch.forward_iter = self.forward_ct
batch.launch_ts = time.monotonic()
@@ -4928,7 +4930,7 @@ class Scheduler(
# post-flush below.
fully_idle = self.is_fully_idle()
if not fully_idle:
self.metrics_reporter.record_scheduler_active()
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
now = time.monotonic()
if now - self._last_stall_publish_ts >= LOAD_STALL_REFRESH_S:
self._last_stall_publish_ts = now
@@ -5007,7 +5009,7 @@ class Scheduler(
if self.is_fully_idle():
self.metrics_reporter.record_scheduler_idle()
else:
self.metrics_reporter.record_scheduler_active()
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
# Health check piggybacks on running requests in process_output.
@@ -287,10 +287,6 @@ class SchedulerMetricsReporter:
self._scheduler_time_accounting: Optional[_SchedulerTimeAccountingSnapshot] = (
None
)
self.scheduler_stage_metrics = SchedulerStageMetricsRecorder(
enabled=self.enable_metrics
)
self.forward_pass_device_timer: Optional[DeviceTimer] = None
self._forward_occupancy_log_window: Optional[_ForwardOccupancyLogWindow] = None
@@ -312,6 +308,19 @@ class SchedulerMetricsReporter:
self.forward_pass_device_timer = DeviceTimer(
reporter=_wrap_execution_reporter,
)
# PDMux event polling across green-context streams is not validated;
# keep its overlap label at `none` without disabling forward timing.
self.scheduler_stage_metrics = SchedulerStageMetricsRecorder(
enabled=self.enable_metrics,
query_forward_active=(
self.forward_pass_device_timer.is_active
if self.enable_metrics
and self.forward_pass_device_timer is not None
and self.scheduler.device == "cuda"
and not get_disagg().enable_pdmux
else None
),
)
self._init_fpm()
@@ -1307,13 +1316,15 @@ class SchedulerMetricsReporter:
self._forward_occupancy_log_window.reset(now_wall_ns)
self.scheduler_stage_metrics.start(now_wall_ns)
def record_scheduler_active(self) -> None:
self._record_scheduler_time(is_idle=False)
def record_scheduler_active(self, event_ns: int) -> None:
"""End CPU-side idle at event_ns (monotonic nanoseconds)."""
self._record_scheduler_time(is_idle=False, event_ns=event_ns)
def record_scheduler_idle(self) -> None:
self._record_scheduler_time(is_idle=True)
"""Start CPU-side idle at the current monotonic time."""
self._record_scheduler_time(is_idle=True, event_ns=time.monotonic_ns())
def _record_scheduler_time(self, is_idle: bool) -> None:
def _record_scheduler_time(self, is_idle: bool, event_ns: int) -> None:
if not (self.enable_metrics or ENABLE_METRICS_DEVICE_TIMER):
return
@@ -1330,17 +1341,22 @@ class SchedulerMetricsReporter:
self.scheduler_stage_metrics.start(now_wall_ns)
return
if ENABLE_METRICS_DEVICE_TIMER:
window = self._forward_occupancy_log_window
if is_idle and not accounting.is_idle:
# Drain the completed burst once, then retain the new idle
# window across polls so the next request includes its idle gap.
self.forward_pass_device_timer._report()
window.reset(now_wall_ns)
elif accounting.is_idle:
window.idle_ns += now_wall_ns - accounting.last_sample_ns
# Clamp delayed events to the still-unaccounted wall-time range.
event_ns = max(accounting.last_sample_ns, min(event_ns, now_wall_ns))
if ENABLE_METRICS_DEVICE_TIMER and is_idle and not accounting.is_idle:
# Drain the completed burst once, then retain the new idle
# window across polls so the next request includes its idle gap.
self.forward_pass_device_timer._report()
self._forward_occupancy_log_window.reset(event_ns)
idle_ns_before = accounting.accumulate_idle_ns
accounting.sample(event_ns, is_idle)
# Reporting and stage drains stay at now, after any completed receive span.
accounting.sample(now_wall_ns, is_idle)
if ENABLE_METRICS_DEVICE_TIMER:
self._forward_occupancy_log_window.idle_ns += (
accounting.accumulate_idle_ns - idle_ns_before
)
if not self.enable_metrics or not accounting.should_record(now_wall_ns):
return
@@ -1353,11 +1369,11 @@ class SchedulerMetricsReporter:
self.metrics_collector.increment_scheduler_process_cpu_seconds(
elapsed_process_cpu_ns / 1e9
)
for stage, elapsed_wall_ns in self.scheduler_stage_metrics.drain(
for (stage, overlap), elapsed_wall_ns in self.scheduler_stage_metrics.drain(
now_wall_ns
).items():
self.metrics_collector.increment_scheduler_stage_seconds(
stage=stage, seconds=elapsed_wall_ns / 1e9
stage=stage, seconds=elapsed_wall_ns / 1e9, forward_overlap=overlap
)
accounting.reset(now_wall_ns, now_process_cpu_ns, is_idle)
@@ -26,6 +26,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set, Union
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.observability.scheduler_stage_metrics import (
FORWARD_OVERLAP_CATEGORIES,
FORWARD_OVERLAP_NONE,
SCHEDULER_STAGE_CATEGORIES,
)
from sglang.srt.observability.utils import exponential_buckets, generate_buckets
@@ -950,13 +952,18 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
name="sglang:scheduler_stage_seconds_total",
documentation=(
"Total scheduler-loop wall time exclusively attributed to each stage."
f" forward_overlap ({', '.join(FORWARD_OVERLAP_CATEGORIES)}):"
" full=both endpoints active, partial=one, none=no observed overlap"
" (including unavailable timing). Not exact overlapped/exposed seconds."
),
labelnames=list(labels.keys()) + ["category"],
labelnames=list(labels.keys()) + ["category", "forward_overlap"],
)
self.scheduler_idle_seconds_total.labels(**labels)
self.scheduler_process_cpu_seconds_total.labels(**labels)
for category in SCHEDULER_STAGE_CATEGORIES:
self.scheduler_stage_seconds_total.labels(**labels, category=category)
self.scheduler_stage_seconds_total.labels(
**labels, category=category, forward_overlap=FORWARD_OVERLAP_NONE
)
self.estimated_flops_per_gpu_total = Counter(
name="sglang:estimated_flops_per_gpu_total",
documentation=(
@@ -1352,10 +1359,12 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
def increment_scheduler_process_cpu_seconds(self, t: float) -> None:
self.scheduler_process_cpu_seconds_total.labels(**self.labels).inc(t)
def increment_scheduler_stage_seconds(self, stage: str, seconds: float) -> None:
self.scheduler_stage_seconds_total.labels(**self.labels, category=stage).inc(
seconds
)
def increment_scheduler_stage_seconds(
self, stage: str, seconds: float, forward_overlap: str = FORWARD_OVERLAP_NONE
) -> None:
self.scheduler_stage_seconds_total.labels(
**self.labels, category=stage, forward_overlap=forward_overlap
).inc(seconds)
def increment_estimated_perf(
self,
@@ -51,6 +51,15 @@ SCHEDULER_STAGE_CATEGORIES = (
)
_SCHEDULER_STAGE_CATEGORY_SET = frozenset(SCHEDULER_STAGE_CATEGORIES)
FORWARD_OVERLAP_FULL = "full"
FORWARD_OVERLAP_PARTIAL = "partial"
FORWARD_OVERLAP_NONE = "none"
FORWARD_OVERLAP_CATEGORIES = (
FORWARD_OVERLAP_FULL,
FORWARD_OVERLAP_PARTIAL,
FORWARD_OVERLAP_NONE,
)
@dataclass(slots=True)
class SchedulerStageMetricsRecorder:
@@ -60,19 +69,31 @@ class SchedulerStageMetricsRecorder:
assigned to ``other``. Summing all categories therefore recovers elapsed
scheduler-loop wall time. Active torch profilers receive matching
``scheduler.<stage>`` ranges without requiring Python stacks.
Forward overlap is sampled at each exclusive span's endpoints: ``full`` when
both observe active forward, ``partial`` when only one does, ``none`` otherwise
(including unavailable timing). ``full`` can span a GPU gap, and ``none`` can
span an entire forward; these labels do not measure overlapped/exposed seconds.
"""
enabled: bool
query_forward_active: Callable[[], bool] | None = None
_current_stage: str = SCHEDULER_STAGE_OTHER
_trace_stage: str | None = None
_last_wall_ns: int | None = None
_wall_ns: dict[str, int] = field(default_factory=lambda: defaultdict(int))
_last_forward_active: bool = False
_wall_ns: dict[tuple[str, str], int] = field(
default_factory=lambda: defaultdict(int)
)
def start(self, wall_ns: int) -> None:
if not self.enabled:
return
self._current_stage = SCHEDULER_STAGE_OTHER
self._last_wall_ns = wall_ns
self._last_forward_active = (
self.query_forward_active() if self.query_forward_active else False
)
self._wall_ns.clear()
def enter(self, stage: str) -> str | None:
@@ -115,7 +136,7 @@ class SchedulerStageMetricsRecorder:
self._trace_stage = previous_trace_stage
self.exit(previous_stage)
def drain(self, wall_ns: int) -> dict[str, int]:
def drain(self, wall_ns: int) -> dict[tuple[str, str], int]:
if not self.enabled or self._last_wall_ns is None:
return {}
self._sample(wall_ns)
@@ -125,8 +146,18 @@ class SchedulerStageMetricsRecorder:
def _sample(self, wall_ns: int) -> None:
assert self._last_wall_ns is not None
self._wall_ns[self._current_stage] += wall_ns - self._last_wall_ns
forward_active = (
self.query_forward_active() if self.query_forward_active else False
)
if self._last_forward_active and forward_active:
overlap = FORWARD_OVERLAP_FULL
elif self._last_forward_active or forward_active:
overlap = FORWARD_OVERLAP_PARTIAL
else:
overlap = FORWARD_OVERLAP_NONE
self._wall_ns[self._current_stage, overlap] += wall_ns - self._last_wall_ns
self._last_wall_ns = wall_ns
self._last_forward_active = forward_active
_F = TypeVar("_F", bound=Callable)
+28 -1
View File
@@ -26,6 +26,17 @@ class DeviceTimer:
def add_reporter(self, reporter: Callable):
self._reporters.append(reporter)
def is_active(self) -> bool:
"""Observe whether any timed interval has started but not completed.
This is a nonblocking snapshot, not a GPU completion timestamp or proof
of continuous kernel execution. Reuse the existing forward events.
"""
for interval in self._intervals:
if interval.is_active():
return True
return False
@contextmanager
def wrap(self, metadata: Dict):
# Not re-entrant: a nested wrap would end the wrong interval and leave
@@ -44,7 +55,7 @@ class DeviceTimer:
def _report(self):
while len(self._intervals) > 0:
interval = self._intervals[0]
if not interval.end_event.query():
if not interval.is_complete():
break
self._intervals.popleft()
@@ -87,6 +98,22 @@ class _TimingInterval:
start_event: torch.cuda.Event
end_event: Optional[torch.cuda.Event] = None
metadata: Optional[Dict] = None
_started: bool = False
_completed: bool = False
def is_complete(self) -> bool:
# Once observed, the end event stays complete; avoid polling it again.
if not self._completed and self.end_event is not None:
self._completed = self.end_event.query()
return self._completed
def is_active(self) -> bool:
if self.is_complete():
return False
# Once observed, an event stays complete; avoid polling its start again.
if not self._started:
self._started = self.start_event.query()
return self._started
@staticmethod
def create():