[Metrics] Label scheduler stage wall time by sampled forward overlap (#42025)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
Jialin
parent
f6fcda8827
commit
4a4dad1cf1
@@ -2074,9 +2074,11 @@ class Scheduler(
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
):
|
||||
local_reqs = self._poll_timeout_aborts()
|
||||
recv_start_ns = time.monotonic_ns()
|
||||
recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs)
|
||||
if recv_reqs:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
# Count successful receive/broadcast time as active; empty polls stay idle.
|
||||
self.metrics_reporter.record_scheduler_active(recv_start_ns)
|
||||
self.process_input_requests(recv_reqs)
|
||||
return recv_reqs
|
||||
|
||||
@@ -4344,7 +4346,7 @@ class Scheduler(
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[GenerationBatchResult, EmbeddingBatchResult]:
|
||||
"""Run a batch."""
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
|
||||
self.forward_ct += 1
|
||||
batch.forward_iter = self.forward_ct
|
||||
batch.launch_ts = time.monotonic()
|
||||
@@ -4928,7 +4930,7 @@ class Scheduler(
|
||||
# post-flush below.
|
||||
fully_idle = self.is_fully_idle()
|
||||
if not fully_idle:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
|
||||
now = time.monotonic()
|
||||
if now - self._last_stall_publish_ts >= LOAD_STALL_REFRESH_S:
|
||||
self._last_stall_publish_ts = now
|
||||
@@ -5007,7 +5009,7 @@ class Scheduler(
|
||||
if self.is_fully_idle():
|
||||
self.metrics_reporter.record_scheduler_idle()
|
||||
else:
|
||||
self.metrics_reporter.record_scheduler_active()
|
||||
self.metrics_reporter.record_scheduler_active(time.monotonic_ns())
|
||||
|
||||
def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool:
|
||||
# Health check piggybacks on running requests in process_output.
|
||||
|
||||
@@ -287,10 +287,6 @@ class SchedulerMetricsReporter:
|
||||
self._scheduler_time_accounting: Optional[_SchedulerTimeAccountingSnapshot] = (
|
||||
None
|
||||
)
|
||||
self.scheduler_stage_metrics = SchedulerStageMetricsRecorder(
|
||||
enabled=self.enable_metrics
|
||||
)
|
||||
|
||||
self.forward_pass_device_timer: Optional[DeviceTimer] = None
|
||||
self._forward_occupancy_log_window: Optional[_ForwardOccupancyLogWindow] = None
|
||||
|
||||
@@ -312,6 +308,19 @@ class SchedulerMetricsReporter:
|
||||
self.forward_pass_device_timer = DeviceTimer(
|
||||
reporter=_wrap_execution_reporter,
|
||||
)
|
||||
# PDMux event polling across green-context streams is not validated;
|
||||
# keep its overlap label at `none` without disabling forward timing.
|
||||
self.scheduler_stage_metrics = SchedulerStageMetricsRecorder(
|
||||
enabled=self.enable_metrics,
|
||||
query_forward_active=(
|
||||
self.forward_pass_device_timer.is_active
|
||||
if self.enable_metrics
|
||||
and self.forward_pass_device_timer is not None
|
||||
and self.scheduler.device == "cuda"
|
||||
and not get_disagg().enable_pdmux
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
self._init_fpm()
|
||||
|
||||
@@ -1307,13 +1316,15 @@ class SchedulerMetricsReporter:
|
||||
self._forward_occupancy_log_window.reset(now_wall_ns)
|
||||
self.scheduler_stage_metrics.start(now_wall_ns)
|
||||
|
||||
def record_scheduler_active(self) -> None:
|
||||
self._record_scheduler_time(is_idle=False)
|
||||
def record_scheduler_active(self, event_ns: int) -> None:
|
||||
"""End CPU-side idle at event_ns (monotonic nanoseconds)."""
|
||||
self._record_scheduler_time(is_idle=False, event_ns=event_ns)
|
||||
|
||||
def record_scheduler_idle(self) -> None:
|
||||
self._record_scheduler_time(is_idle=True)
|
||||
"""Start CPU-side idle at the current monotonic time."""
|
||||
self._record_scheduler_time(is_idle=True, event_ns=time.monotonic_ns())
|
||||
|
||||
def _record_scheduler_time(self, is_idle: bool) -> None:
|
||||
def _record_scheduler_time(self, is_idle: bool, event_ns: int) -> None:
|
||||
if not (self.enable_metrics or ENABLE_METRICS_DEVICE_TIMER):
|
||||
return
|
||||
|
||||
@@ -1330,17 +1341,22 @@ class SchedulerMetricsReporter:
|
||||
self.scheduler_stage_metrics.start(now_wall_ns)
|
||||
return
|
||||
|
||||
if ENABLE_METRICS_DEVICE_TIMER:
|
||||
window = self._forward_occupancy_log_window
|
||||
if is_idle and not accounting.is_idle:
|
||||
# Drain the completed burst once, then retain the new idle
|
||||
# window across polls so the next request includes its idle gap.
|
||||
self.forward_pass_device_timer._report()
|
||||
window.reset(now_wall_ns)
|
||||
elif accounting.is_idle:
|
||||
window.idle_ns += now_wall_ns - accounting.last_sample_ns
|
||||
# Clamp delayed events to the still-unaccounted wall-time range.
|
||||
event_ns = max(accounting.last_sample_ns, min(event_ns, now_wall_ns))
|
||||
if ENABLE_METRICS_DEVICE_TIMER and is_idle and not accounting.is_idle:
|
||||
# Drain the completed burst once, then retain the new idle
|
||||
# window across polls so the next request includes its idle gap.
|
||||
self.forward_pass_device_timer._report()
|
||||
self._forward_occupancy_log_window.reset(event_ns)
|
||||
|
||||
idle_ns_before = accounting.accumulate_idle_ns
|
||||
accounting.sample(event_ns, is_idle)
|
||||
# Reporting and stage drains stay at now, after any completed receive span.
|
||||
accounting.sample(now_wall_ns, is_idle)
|
||||
if ENABLE_METRICS_DEVICE_TIMER:
|
||||
self._forward_occupancy_log_window.idle_ns += (
|
||||
accounting.accumulate_idle_ns - idle_ns_before
|
||||
)
|
||||
if not self.enable_metrics or not accounting.should_record(now_wall_ns):
|
||||
return
|
||||
|
||||
@@ -1353,11 +1369,11 @@ class SchedulerMetricsReporter:
|
||||
self.metrics_collector.increment_scheduler_process_cpu_seconds(
|
||||
elapsed_process_cpu_ns / 1e9
|
||||
)
|
||||
for stage, elapsed_wall_ns in self.scheduler_stage_metrics.drain(
|
||||
for (stage, overlap), elapsed_wall_ns in self.scheduler_stage_metrics.drain(
|
||||
now_wall_ns
|
||||
).items():
|
||||
self.metrics_collector.increment_scheduler_stage_seconds(
|
||||
stage=stage, seconds=elapsed_wall_ns / 1e9
|
||||
stage=stage, seconds=elapsed_wall_ns / 1e9, forward_overlap=overlap
|
||||
)
|
||||
accounting.reset(now_wall_ns, now_process_cpu_ns, is_idle)
|
||||
|
||||
|
||||
@@ -26,6 +26,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set, Union
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
FORWARD_OVERLAP_CATEGORIES,
|
||||
FORWARD_OVERLAP_NONE,
|
||||
SCHEDULER_STAGE_CATEGORIES,
|
||||
)
|
||||
from sglang.srt.observability.utils import exponential_buckets, generate_buckets
|
||||
@@ -950,13 +952,18 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
name="sglang:scheduler_stage_seconds_total",
|
||||
documentation=(
|
||||
"Total scheduler-loop wall time exclusively attributed to each stage."
|
||||
f" forward_overlap ({', '.join(FORWARD_OVERLAP_CATEGORIES)}):"
|
||||
" full=both endpoints active, partial=one, none=no observed overlap"
|
||||
" (including unavailable timing). Not exact overlapped/exposed seconds."
|
||||
),
|
||||
labelnames=list(labels.keys()) + ["category"],
|
||||
labelnames=list(labels.keys()) + ["category", "forward_overlap"],
|
||||
)
|
||||
self.scheduler_idle_seconds_total.labels(**labels)
|
||||
self.scheduler_process_cpu_seconds_total.labels(**labels)
|
||||
for category in SCHEDULER_STAGE_CATEGORIES:
|
||||
self.scheduler_stage_seconds_total.labels(**labels, category=category)
|
||||
self.scheduler_stage_seconds_total.labels(
|
||||
**labels, category=category, forward_overlap=FORWARD_OVERLAP_NONE
|
||||
)
|
||||
self.estimated_flops_per_gpu_total = Counter(
|
||||
name="sglang:estimated_flops_per_gpu_total",
|
||||
documentation=(
|
||||
@@ -1352,10 +1359,12 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
def increment_scheduler_process_cpu_seconds(self, t: float) -> None:
|
||||
self.scheduler_process_cpu_seconds_total.labels(**self.labels).inc(t)
|
||||
|
||||
def increment_scheduler_stage_seconds(self, stage: str, seconds: float) -> None:
|
||||
self.scheduler_stage_seconds_total.labels(**self.labels, category=stage).inc(
|
||||
seconds
|
||||
)
|
||||
def increment_scheduler_stage_seconds(
|
||||
self, stage: str, seconds: float, forward_overlap: str = FORWARD_OVERLAP_NONE
|
||||
) -> None:
|
||||
self.scheduler_stage_seconds_total.labels(
|
||||
**self.labels, category=stage, forward_overlap=forward_overlap
|
||||
).inc(seconds)
|
||||
|
||||
def increment_estimated_perf(
|
||||
self,
|
||||
|
||||
@@ -51,6 +51,15 @@ SCHEDULER_STAGE_CATEGORIES = (
|
||||
)
|
||||
_SCHEDULER_STAGE_CATEGORY_SET = frozenset(SCHEDULER_STAGE_CATEGORIES)
|
||||
|
||||
FORWARD_OVERLAP_FULL = "full"
|
||||
FORWARD_OVERLAP_PARTIAL = "partial"
|
||||
FORWARD_OVERLAP_NONE = "none"
|
||||
FORWARD_OVERLAP_CATEGORIES = (
|
||||
FORWARD_OVERLAP_FULL,
|
||||
FORWARD_OVERLAP_PARTIAL,
|
||||
FORWARD_OVERLAP_NONE,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SchedulerStageMetricsRecorder:
|
||||
@@ -60,19 +69,31 @@ class SchedulerStageMetricsRecorder:
|
||||
assigned to ``other``. Summing all categories therefore recovers elapsed
|
||||
scheduler-loop wall time. Active torch profilers receive matching
|
||||
``scheduler.<stage>`` ranges without requiring Python stacks.
|
||||
|
||||
Forward overlap is sampled at each exclusive span's endpoints: ``full`` when
|
||||
both observe active forward, ``partial`` when only one does, ``none`` otherwise
|
||||
(including unavailable timing). ``full`` can span a GPU gap, and ``none`` can
|
||||
span an entire forward; these labels do not measure overlapped/exposed seconds.
|
||||
"""
|
||||
|
||||
enabled: bool
|
||||
query_forward_active: Callable[[], bool] | None = None
|
||||
_current_stage: str = SCHEDULER_STAGE_OTHER
|
||||
_trace_stage: str | None = None
|
||||
_last_wall_ns: int | None = None
|
||||
_wall_ns: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
||||
_last_forward_active: bool = False
|
||||
_wall_ns: dict[tuple[str, str], int] = field(
|
||||
default_factory=lambda: defaultdict(int)
|
||||
)
|
||||
|
||||
def start(self, wall_ns: int) -> None:
|
||||
if not self.enabled:
|
||||
return
|
||||
self._current_stage = SCHEDULER_STAGE_OTHER
|
||||
self._last_wall_ns = wall_ns
|
||||
self._last_forward_active = (
|
||||
self.query_forward_active() if self.query_forward_active else False
|
||||
)
|
||||
self._wall_ns.clear()
|
||||
|
||||
def enter(self, stage: str) -> str | None:
|
||||
@@ -115,7 +136,7 @@ class SchedulerStageMetricsRecorder:
|
||||
self._trace_stage = previous_trace_stage
|
||||
self.exit(previous_stage)
|
||||
|
||||
def drain(self, wall_ns: int) -> dict[str, int]:
|
||||
def drain(self, wall_ns: int) -> dict[tuple[str, str], int]:
|
||||
if not self.enabled or self._last_wall_ns is None:
|
||||
return {}
|
||||
self._sample(wall_ns)
|
||||
@@ -125,8 +146,18 @@ class SchedulerStageMetricsRecorder:
|
||||
|
||||
def _sample(self, wall_ns: int) -> None:
|
||||
assert self._last_wall_ns is not None
|
||||
self._wall_ns[self._current_stage] += wall_ns - self._last_wall_ns
|
||||
forward_active = (
|
||||
self.query_forward_active() if self.query_forward_active else False
|
||||
)
|
||||
if self._last_forward_active and forward_active:
|
||||
overlap = FORWARD_OVERLAP_FULL
|
||||
elif self._last_forward_active or forward_active:
|
||||
overlap = FORWARD_OVERLAP_PARTIAL
|
||||
else:
|
||||
overlap = FORWARD_OVERLAP_NONE
|
||||
self._wall_ns[self._current_stage, overlap] += wall_ns - self._last_wall_ns
|
||||
self._last_wall_ns = wall_ns
|
||||
self._last_forward_active = forward_active
|
||||
|
||||
|
||||
_F = TypeVar("_F", bound=Callable)
|
||||
|
||||
@@ -26,6 +26,17 @@ class DeviceTimer:
|
||||
def add_reporter(self, reporter: Callable):
|
||||
self._reporters.append(reporter)
|
||||
|
||||
def is_active(self) -> bool:
|
||||
"""Observe whether any timed interval has started but not completed.
|
||||
|
||||
This is a nonblocking snapshot, not a GPU completion timestamp or proof
|
||||
of continuous kernel execution. Reuse the existing forward events.
|
||||
"""
|
||||
for interval in self._intervals:
|
||||
if interval.is_active():
|
||||
return True
|
||||
return False
|
||||
|
||||
@contextmanager
|
||||
def wrap(self, metadata: Dict):
|
||||
# Not re-entrant: a nested wrap would end the wrong interval and leave
|
||||
@@ -44,7 +55,7 @@ class DeviceTimer:
|
||||
def _report(self):
|
||||
while len(self._intervals) > 0:
|
||||
interval = self._intervals[0]
|
||||
if not interval.end_event.query():
|
||||
if not interval.is_complete():
|
||||
break
|
||||
|
||||
self._intervals.popleft()
|
||||
@@ -87,6 +98,22 @@ class _TimingInterval:
|
||||
start_event: torch.cuda.Event
|
||||
end_event: Optional[torch.cuda.Event] = None
|
||||
metadata: Optional[Dict] = None
|
||||
_started: bool = False
|
||||
_completed: bool = False
|
||||
|
||||
def is_complete(self) -> bool:
|
||||
# Once observed, the end event stays complete; avoid polling it again.
|
||||
if not self._completed and self.end_event is not None:
|
||||
self._completed = self.end_event.query()
|
||||
return self._completed
|
||||
|
||||
def is_active(self) -> bool:
|
||||
if self.is_complete():
|
||||
return False
|
||||
# Once observed, an event stays complete; avoid polling its start again.
|
||||
if not self._started:
|
||||
self._started = self.start_event.query()
|
||||
return self._started
|
||||
|
||||
@staticmethod
|
||||
def create():
|
||||
|
||||
Reference in New Issue
Block a user