diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index eaa1944f8e..5c8dbd76b9 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2074,9 +2074,11 @@ class Scheduler( and get_parallel().attn_cp_rank == 0 ): local_reqs = self._poll_timeout_aborts() + recv_start_ns = time.monotonic_ns() recv_reqs = self.request_receiver.recv_requests(local_reqs=local_reqs) if recv_reqs: - self.metrics_reporter.record_scheduler_active() + # Count successful receive/broadcast time as active; empty polls stay idle. + self.metrics_reporter.record_scheduler_active(recv_start_ns) self.process_input_requests(recv_reqs) return recv_reqs @@ -4344,7 +4346,7 @@ class Scheduler( pp_proxy_tensors: Optional[PPProxyTensors] = None, ) -> Union[GenerationBatchResult, EmbeddingBatchResult]: """Run a batch.""" - self.metrics_reporter.record_scheduler_active() + self.metrics_reporter.record_scheduler_active(time.monotonic_ns()) self.forward_ct += 1 batch.forward_iter = self.forward_ct batch.launch_ts = time.monotonic() @@ -4928,7 +4930,7 @@ class Scheduler( # post-flush below. fully_idle = self.is_fully_idle() if not fully_idle: - self.metrics_reporter.record_scheduler_active() + self.metrics_reporter.record_scheduler_active(time.monotonic_ns()) now = time.monotonic() if now - self._last_stall_publish_ts >= LOAD_STALL_REFRESH_S: self._last_stall_publish_ts = now @@ -5007,7 +5009,7 @@ class Scheduler( if self.is_fully_idle(): self.metrics_reporter.record_scheduler_idle() else: - self.metrics_reporter.record_scheduler_active() + self.metrics_reporter.record_scheduler_active(time.monotonic_ns()) def is_fully_idle(self, for_health_check=False, ignore_waiting=False) -> bool: # Health check piggybacks on running requests in process_output. diff --git a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py index 8203e705cd..fb1b8b3197 100644 --- a/python/sglang/srt/managers/scheduler_components/metrics_reporter.py +++ b/python/sglang/srt/managers/scheduler_components/metrics_reporter.py @@ -287,10 +287,6 @@ class SchedulerMetricsReporter: self._scheduler_time_accounting: Optional[_SchedulerTimeAccountingSnapshot] = ( None ) - self.scheduler_stage_metrics = SchedulerStageMetricsRecorder( - enabled=self.enable_metrics - ) - self.forward_pass_device_timer: Optional[DeviceTimer] = None self._forward_occupancy_log_window: Optional[_ForwardOccupancyLogWindow] = None @@ -312,6 +308,19 @@ class SchedulerMetricsReporter: self.forward_pass_device_timer = DeviceTimer( reporter=_wrap_execution_reporter, ) + # PDMux event polling across green-context streams is not validated; + # keep its overlap label at `none` without disabling forward timing. + self.scheduler_stage_metrics = SchedulerStageMetricsRecorder( + enabled=self.enable_metrics, + query_forward_active=( + self.forward_pass_device_timer.is_active + if self.enable_metrics + and self.forward_pass_device_timer is not None + and self.scheduler.device == "cuda" + and not get_disagg().enable_pdmux + else None + ), + ) self._init_fpm() @@ -1307,13 +1316,15 @@ class SchedulerMetricsReporter: self._forward_occupancy_log_window.reset(now_wall_ns) self.scheduler_stage_metrics.start(now_wall_ns) - def record_scheduler_active(self) -> None: - self._record_scheduler_time(is_idle=False) + def record_scheduler_active(self, event_ns: int) -> None: + """End CPU-side idle at event_ns (monotonic nanoseconds).""" + self._record_scheduler_time(is_idle=False, event_ns=event_ns) def record_scheduler_idle(self) -> None: - self._record_scheduler_time(is_idle=True) + """Start CPU-side idle at the current monotonic time.""" + self._record_scheduler_time(is_idle=True, event_ns=time.monotonic_ns()) - def _record_scheduler_time(self, is_idle: bool) -> None: + def _record_scheduler_time(self, is_idle: bool, event_ns: int) -> None: if not (self.enable_metrics or ENABLE_METRICS_DEVICE_TIMER): return @@ -1330,17 +1341,22 @@ class SchedulerMetricsReporter: self.scheduler_stage_metrics.start(now_wall_ns) return - if ENABLE_METRICS_DEVICE_TIMER: - window = self._forward_occupancy_log_window - if is_idle and not accounting.is_idle: - # Drain the completed burst once, then retain the new idle - # window across polls so the next request includes its idle gap. - self.forward_pass_device_timer._report() - window.reset(now_wall_ns) - elif accounting.is_idle: - window.idle_ns += now_wall_ns - accounting.last_sample_ns + # Clamp delayed events to the still-unaccounted wall-time range. + event_ns = max(accounting.last_sample_ns, min(event_ns, now_wall_ns)) + if ENABLE_METRICS_DEVICE_TIMER and is_idle and not accounting.is_idle: + # Drain the completed burst once, then retain the new idle + # window across polls so the next request includes its idle gap. + self.forward_pass_device_timer._report() + self._forward_occupancy_log_window.reset(event_ns) + idle_ns_before = accounting.accumulate_idle_ns + accounting.sample(event_ns, is_idle) + # Reporting and stage drains stay at now, after any completed receive span. accounting.sample(now_wall_ns, is_idle) + if ENABLE_METRICS_DEVICE_TIMER: + self._forward_occupancy_log_window.idle_ns += ( + accounting.accumulate_idle_ns - idle_ns_before + ) if not self.enable_metrics or not accounting.should_record(now_wall_ns): return @@ -1353,11 +1369,11 @@ class SchedulerMetricsReporter: self.metrics_collector.increment_scheduler_process_cpu_seconds( elapsed_process_cpu_ns / 1e9 ) - for stage, elapsed_wall_ns in self.scheduler_stage_metrics.drain( + for (stage, overlap), elapsed_wall_ns in self.scheduler_stage_metrics.drain( now_wall_ns ).items(): self.metrics_collector.increment_scheduler_stage_seconds( - stage=stage, seconds=elapsed_wall_ns / 1e9 + stage=stage, seconds=elapsed_wall_ns / 1e9, forward_overlap=overlap ) accounting.reset(now_wall_ns, now_process_cpu_ns, is_idle) diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index 04e6e9c6c4..0650f65600 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -26,6 +26,8 @@ from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set, Union from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.observability.scheduler_stage_metrics import ( + FORWARD_OVERLAP_CATEGORIES, + FORWARD_OVERLAP_NONE, SCHEDULER_STAGE_CATEGORIES, ) from sglang.srt.observability.utils import exponential_buckets, generate_buckets @@ -950,13 +952,18 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): name="sglang:scheduler_stage_seconds_total", documentation=( "Total scheduler-loop wall time exclusively attributed to each stage." + f" forward_overlap ({', '.join(FORWARD_OVERLAP_CATEGORIES)}):" + " full=both endpoints active, partial=one, none=no observed overlap" + " (including unavailable timing). Not exact overlapped/exposed seconds." ), - labelnames=list(labels.keys()) + ["category"], + labelnames=list(labels.keys()) + ["category", "forward_overlap"], ) self.scheduler_idle_seconds_total.labels(**labels) self.scheduler_process_cpu_seconds_total.labels(**labels) for category in SCHEDULER_STAGE_CATEGORIES: - self.scheduler_stage_seconds_total.labels(**labels, category=category) + self.scheduler_stage_seconds_total.labels( + **labels, category=category, forward_overlap=FORWARD_OVERLAP_NONE + ) self.estimated_flops_per_gpu_total = Counter( name="sglang:estimated_flops_per_gpu_total", documentation=( @@ -1352,10 +1359,12 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): def increment_scheduler_process_cpu_seconds(self, t: float) -> None: self.scheduler_process_cpu_seconds_total.labels(**self.labels).inc(t) - def increment_scheduler_stage_seconds(self, stage: str, seconds: float) -> None: - self.scheduler_stage_seconds_total.labels(**self.labels, category=stage).inc( - seconds - ) + def increment_scheduler_stage_seconds( + self, stage: str, seconds: float, forward_overlap: str = FORWARD_OVERLAP_NONE + ) -> None: + self.scheduler_stage_seconds_total.labels( + **self.labels, category=stage, forward_overlap=forward_overlap + ).inc(seconds) def increment_estimated_perf( self, diff --git a/python/sglang/srt/observability/scheduler_stage_metrics.py b/python/sglang/srt/observability/scheduler_stage_metrics.py index 0d3bf13bbb..e9517fcb7e 100644 --- a/python/sglang/srt/observability/scheduler_stage_metrics.py +++ b/python/sglang/srt/observability/scheduler_stage_metrics.py @@ -51,6 +51,15 @@ SCHEDULER_STAGE_CATEGORIES = ( ) _SCHEDULER_STAGE_CATEGORY_SET = frozenset(SCHEDULER_STAGE_CATEGORIES) +FORWARD_OVERLAP_FULL = "full" +FORWARD_OVERLAP_PARTIAL = "partial" +FORWARD_OVERLAP_NONE = "none" +FORWARD_OVERLAP_CATEGORIES = ( + FORWARD_OVERLAP_FULL, + FORWARD_OVERLAP_PARTIAL, + FORWARD_OVERLAP_NONE, +) + @dataclass(slots=True) class SchedulerStageMetricsRecorder: @@ -60,19 +69,31 @@ class SchedulerStageMetricsRecorder: assigned to ``other``. Summing all categories therefore recovers elapsed scheduler-loop wall time. Active torch profilers receive matching ``scheduler.`` ranges without requiring Python stacks. + + Forward overlap is sampled at each exclusive span's endpoints: ``full`` when + both observe active forward, ``partial`` when only one does, ``none`` otherwise + (including unavailable timing). ``full`` can span a GPU gap, and ``none`` can + span an entire forward; these labels do not measure overlapped/exposed seconds. """ enabled: bool + query_forward_active: Callable[[], bool] | None = None _current_stage: str = SCHEDULER_STAGE_OTHER _trace_stage: str | None = None _last_wall_ns: int | None = None - _wall_ns: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + _last_forward_active: bool = False + _wall_ns: dict[tuple[str, str], int] = field( + default_factory=lambda: defaultdict(int) + ) def start(self, wall_ns: int) -> None: if not self.enabled: return self._current_stage = SCHEDULER_STAGE_OTHER self._last_wall_ns = wall_ns + self._last_forward_active = ( + self.query_forward_active() if self.query_forward_active else False + ) self._wall_ns.clear() def enter(self, stage: str) -> str | None: @@ -115,7 +136,7 @@ class SchedulerStageMetricsRecorder: self._trace_stage = previous_trace_stage self.exit(previous_stage) - def drain(self, wall_ns: int) -> dict[str, int]: + def drain(self, wall_ns: int) -> dict[tuple[str, str], int]: if not self.enabled or self._last_wall_ns is None: return {} self._sample(wall_ns) @@ -125,8 +146,18 @@ class SchedulerStageMetricsRecorder: def _sample(self, wall_ns: int) -> None: assert self._last_wall_ns is not None - self._wall_ns[self._current_stage] += wall_ns - self._last_wall_ns + forward_active = ( + self.query_forward_active() if self.query_forward_active else False + ) + if self._last_forward_active and forward_active: + overlap = FORWARD_OVERLAP_FULL + elif self._last_forward_active or forward_active: + overlap = FORWARD_OVERLAP_PARTIAL + else: + overlap = FORWARD_OVERLAP_NONE + self._wall_ns[self._current_stage, overlap] += wall_ns - self._last_wall_ns self._last_wall_ns = wall_ns + self._last_forward_active = forward_active _F = TypeVar("_F", bound=Callable) diff --git a/python/sglang/srt/utils/device_timer.py b/python/sglang/srt/utils/device_timer.py index 5ef5e14ad3..aa035bfbcf 100644 --- a/python/sglang/srt/utils/device_timer.py +++ b/python/sglang/srt/utils/device_timer.py @@ -26,6 +26,17 @@ class DeviceTimer: def add_reporter(self, reporter: Callable): self._reporters.append(reporter) + def is_active(self) -> bool: + """Observe whether any timed interval has started but not completed. + + This is a nonblocking snapshot, not a GPU completion timestamp or proof + of continuous kernel execution. Reuse the existing forward events. + """ + for interval in self._intervals: + if interval.is_active(): + return True + return False + @contextmanager def wrap(self, metadata: Dict): # Not re-entrant: a nested wrap would end the wrong interval and leave @@ -44,7 +55,7 @@ class DeviceTimer: def _report(self): while len(self._intervals) > 0: interval = self._intervals[0] - if not interval.end_event.query(): + if not interval.is_complete(): break self._intervals.popleft() @@ -87,6 +98,22 @@ class _TimingInterval: start_event: torch.cuda.Event end_event: Optional[torch.cuda.Event] = None metadata: Optional[Dict] = None + _started: bool = False + _completed: bool = False + + def is_complete(self) -> bool: + # Once observed, the end event stays complete; avoid polling it again. + if not self._completed and self.end_event is not None: + self._completed = self.end_event.query() + return self._completed + + def is_active(self) -> bool: + if self.is_complete(): + return False + # Once observed, an event stays complete; avoid polling its start again. + if not self._started: + self._started = self.start_event.query() + return self._started @staticmethod def create(): diff --git a/test/registered/observability/test_metrics.py b/test/registered/observability/test_metrics.py index cba73727b5..feb1e77a11 100644 --- a/test/registered/observability/test_metrics.py +++ b/test/registered/observability/test_metrics.py @@ -15,6 +15,9 @@ from sglang.srt.observability.metrics_collector import ( SchedulerMetricsCollector, compute_routing_key_stats, ) +from sglang.srt.observability.scheduler_stage_metrics import ( + FORWARD_OVERLAP_CATEGORIES, +) from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import ( @@ -233,7 +236,6 @@ class TestEnableMetrics(CustomTestCase): ("sglang:forward_execution_seconds_total", {"category": "extend"}), ("sglang:forward_execution_seconds_total", {"category": "decode"}), ("sglang:scheduler_process_cpu_seconds_total", {}), - ("sglang:scheduler_stage_seconds_total", {"category": "other"}), ("sglang:process_cpu_seconds_total", {"component": "tokenizer"}), ("sglang:weight_memory_usage_gb", {"model_name": _MODEL_NAME}), ("sglang:kv_cache_memory_usage_gb", {"model_name": _MODEL_NAME}), @@ -253,6 +255,22 @@ class TestEnableMetrics(CustomTestCase): ] _check_metrics_positive(self, metrics, metrics_to_check) + # Reuse this serving fixture; the overlap mix depends on the workload. + stage_samples = metrics["sglang:scheduler_stage_seconds_total"] + self.assertTrue( + all( + sample.labels.get("forward_overlap") in FORWARD_OVERLAP_CATEGORIES + for sample in stage_samples + ) + ) + self.assertGreater( + sum( + sample.value + for sample in stage_samples + if sample.labels["category"] == "other" + ), + 0, + ) for metric_name in ( "sglang:graph_memory_usage_gb", "sglang:startup_cuda_graph_time_seconds", diff --git a/test/registered/unit/managers/test_scheduler_pause_generation.py b/test/registered/unit/managers/test_scheduler_pause_generation.py index 752b4a5a60..2ada4535b1 100644 --- a/test/registered/unit/managers/test_scheduler_pause_generation.py +++ b/test/registered/unit/managers/test_scheduler_pause_generation.py @@ -275,7 +275,7 @@ class TestSchedulerPauseGeneration(CustomTestCase): scheduler.metrics_reporter.record_scheduler_idle.assert_called_once_with() scheduler.metrics_reporter.record_scheduler_active.assert_not_called() else: - scheduler.metrics_reporter.record_scheduler_active.assert_called_once_with() + scheduler.metrics_reporter.record_scheduler_active.assert_called_once() scheduler.metrics_reporter.record_scheduler_idle.assert_not_called() def test_inplace_does_not_drain_overlap_queue(self): diff --git a/test/registered/unit/observability/test_forward_pass_metrics.py b/test/registered/unit/observability/test_forward_pass_metrics.py index 2a6e20f33c..28c83b7471 100644 --- a/test/registered/unit/observability/test_forward_pass_metrics.py +++ b/test/registered/unit/observability/test_forward_pass_metrics.py @@ -18,6 +18,9 @@ from sglang.srt.managers.scheduler_components.metrics_reporter import ( _CacheHitRateWindow, ) from sglang.srt.observability.metrics_collector import SchedulerMetricsCollector +from sglang.srt.observability.scheduler_stage_metrics import ( + SCHEDULER_STAGE_RECV_REQUESTS, +) from sglang.test.test_utils import CustomTestCase, enter_scope @@ -466,7 +469,14 @@ class TestIdleMetrics(CustomTestCase): class TestSchedulerTimeAccounting(CustomTestCase): def setUp(self): - self.reporter = _make_reporter(self, types.SimpleNamespace()) + enter_scope( + self, + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER", + True, + ), + ) + self.reporter = _make_reporter(self, types.SimpleNamespace(device="cpu")) self.idle_seconds = [] self.process_cpu_seconds = [] self.stage_seconds = [] @@ -484,9 +494,12 @@ class TestSchedulerTimeAccounting(CustomTestCase): wall_timestamps = [ 0, 1_200_000_000, + 1_200_000_000, 1_500_000_000, 2_700_000_000, + 2_900_000_000, 3_000_000_000, + 4_000_000_000, 4_100_000_000, ] process_cpu_timestamps = [ @@ -507,12 +520,19 @@ class TestSchedulerTimeAccounting(CustomTestCase): ): self.reporter.start_scheduler_time_accounting() self.reporter.record_scheduler_idle() - self.reporter.record_scheduler_active() - self.reporter.record_scheduler_active() + self.reporter.record_scheduler_active(event_ns=1_300_000_000) + self.assertEqual( + self.reporter._forward_occupancy_log_window.idle_ns, 1_300_000_000 + ) + self.reporter.record_scheduler_active(event_ns=0) + # Time between idle capture and accounting still counts as idle. self.reporter.record_scheduler_idle() self.reporter.record_scheduler_idle() - self.assertAlmostEqual(sum(self.idle_seconds), 2.6) + self.assertAlmostEqual(sum(self.idle_seconds), 2.5) + self.assertEqual( + self.reporter._forward_occupancy_log_window.idle_ns, 1_200_000_000 + ) self.assertAlmostEqual(sum(self.process_cpu_seconds), 1.9) self.assertAlmostEqual( sum(sample["seconds"] for sample in self.stage_seconds), 4.1 @@ -521,9 +541,22 @@ class TestSchedulerTimeAccounting(CustomTestCase): def test_state_transitions_accumulate_until_periodic_update(self): with ( + patch( + "sglang.srt.managers.scheduler_components.metrics_reporter.ENABLE_METRICS_DEVICE_TIMER", + False, + ), patch( "sglang.srt.managers.scheduler_components.metrics_reporter.time.monotonic_ns", - side_effect=[0, 200_000_000, 400_000_000, 700_000_000, 1_100_000_000], + side_effect=[ + 0, + 200_000_000, + 400_000_000, + 400_000_000, + 700_000_000, + 1_100_000_000, + 1_200_000_000, + 1_300_000_000, + ], ), patch( "sglang.srt.managers.scheduler_components.metrics_reporter.time.process_time_ns", @@ -532,16 +565,34 @@ class TestSchedulerTimeAccounting(CustomTestCase): ): self.reporter.start_scheduler_time_accounting() accounting = self.reporter._scheduler_time_accounting - self.reporter.record_scheduler_active() + self.reporter.record_scheduler_active(event_ns=200_000_000) self.reporter.record_scheduler_idle() - self.reporter.record_scheduler_active() self.assertEqual(self.idle_seconds, []) self.assertEqual(self.process_cpu_seconds, []) - self.assertEqual( - self.reporter._scheduler_time_accounting.accumulate_idle_ns, - 500_000_000, + with self.reporter.scheduler_stage_metrics.record( + SCHEDULER_STAGE_RECV_REQUESTS + ): + pass + # Receive crossed the reporting deadline. Backdate only idle time, + # not the stage drain (which has already sampled receive completion). + self.reporter.record_scheduler_active(event_ns=700_000_000) + self.assertEqual(self.idle_seconds, [0.5]) + self.assertEqual(accounting.last_sample_ns, 1_200_000_000) + self.assertAlmostEqual( + sum(sample["seconds"] for sample in self.stage_seconds), 1.2 ) - self.reporter.record_scheduler_active() + self.assertTrue( + all(sample["seconds"] >= 0 for sample in self.stage_seconds) + ) + self.assertAlmostEqual( + sum( + sample["seconds"] + for sample in self.stage_seconds + if sample["stage"] == SCHEDULER_STAGE_RECV_REQUESTS + ), + 0.4, + ) + self.reporter.record_scheduler_active(event_ns=1_300_000_000) self.assertIs(self.reporter._scheduler_time_accounting, accounting) self.assertEqual(process_time.call_count, 2) @@ -560,8 +611,9 @@ class TestSchedulerTimeAccounting(CustomTestCase): ), ): self.reporter.start_scheduler_time_accounting() - self.reporter.record_scheduler_active() - self.reporter.record_scheduler_active() + self.reporter.record_scheduler_active(event_ns=-1) + self.assertEqual(self.reporter._forward_occupancy_log_window.idle_ns, 0) + self.reporter.record_scheduler_active(event_ns=1_000_000_000) self.assertEqual(self.idle_seconds, []) self.assertEqual(self.process_cpu_seconds, [0.0]) diff --git a/test/registered/unit/observability/test_scheduler_stage_metrics.py b/test/registered/unit/observability/test_scheduler_stage_metrics.py index b115acb956..c9632556bf 100644 --- a/test/registered/unit/observability/test_scheduler_stage_metrics.py +++ b/test/registered/unit/observability/test_scheduler_stage_metrics.py @@ -1,9 +1,12 @@ import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch import torch from sglang.srt.observability.scheduler_stage_metrics import ( + FORWARD_OVERLAP_FULL, + FORWARD_OVERLAP_NONE, + FORWARD_OVERLAP_PARTIAL, SCHEDULER_STAGE_CATEGORIES, SCHEDULER_STAGE_GET_NEXT_BATCH, SCHEDULER_STAGE_PROCESS_QUEUE, @@ -12,6 +15,7 @@ from sglang.srt.observability.scheduler_stage_metrics import ( SchedulerStageMetricsRecorder, scheduler_stage_method, ) +from sglang.srt.utils.device_timer import DeviceTimer from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -36,7 +40,11 @@ class TestSchedulerStageMetricsRecorder(CustomTestCase): ) def test_nested_stages_are_exclusive(self): - recorder = SchedulerStageMetricsRecorder(enabled=True) + # Each exclusive span uses its own endpoints, including the resumed parent. + active = iter([False, False, True, True, False, False]) + recorder = SchedulerStageMetricsRecorder( + enabled=True, query_forward_active=lambda: next(active) + ) recorder.start(wall_ns=0) with patch( @@ -53,9 +61,9 @@ class TestSchedulerStageMetricsRecorder(CustomTestCase): self.assertEqual( wall_ns, { - "other": 30, - "get_next_batch_to_run": 50, - "process_queue": 20, + ("other", FORWARD_OVERLAP_NONE): 30, + ("get_next_batch_to_run", FORWARD_OVERLAP_PARTIAL): 50, + ("process_queue", FORWARD_OVERLAP_FULL): 20, }, ) self.assertEqual(sum(wall_ns.values()), 100) @@ -81,7 +89,13 @@ class TestSchedulerStageMetricsRecorder(CustomTestCase): SchedulerLike().fail() wall_ns = recorder.drain(wall_ns=50) - self.assertEqual(wall_ns, {"other": 20, "run_batch": 30}) + self.assertEqual( + wall_ns, + { + ("other", FORWARD_OVERLAP_NONE): 20, + ("run_batch", FORWARD_OVERLAP_NONE): 30, + }, + ) def test_nested_same_stage_does_not_double_count(self): recorder = SchedulerStageMetricsRecorder(enabled=True) @@ -96,7 +110,13 @@ class TestSchedulerStageMetricsRecorder(CustomTestCase): pass wall_ns = recorder.drain(wall_ns=50) - self.assertEqual(wall_ns, {"other": 20, "run_batch": 30}) + self.assertEqual( + wall_ns, + { + ("other", FORWARD_OVERLAP_NONE): 20, + ("run_batch", FORWARD_OVERLAP_NONE): 30, + }, + ) def test_trace_spans_do_not_require_python_stacks(self): recorder = SchedulerStageMetricsRecorder(enabled=False) @@ -125,6 +145,45 @@ class TestSchedulerStageMetricsRecorder(CustomTestCase): ) self.assertTrue(all(not event.stack for event in stage_events)) + def test_forward_query_distinguishes_queued_active_and_completed_intervals(self): + samples = [] + timer = DeviceTimer(lambda **sample: samples.append(sample)) + start, end, next_start, next_end = ( + Mock(query=Mock(return_value=False)) for _ in range(4) + ) + start.elapsed_time.return_value = 2.0 + next_start.elapsed_time.return_value = 3.0 + with patch( + "sglang.srt.utils.device_timer.torch.cuda.Event", + side_effect=[start, end, next_start, next_end], + ): + self.assertFalse(timer.is_active()) + with timer.wrap(metadata={"category": "extend"}): + self.assertFalse(timer.is_active()) # Queued, not started. + start.query.return_value = True + self.assertTrue(timer.is_active()) # End not recorded yet. + self.assertEqual(samples, []) + + with timer.wrap(metadata={"category": "decode"}): + # A later queued forward must not hide the earlier active one. + self.assertTrue(timer.is_active()) + end.query.return_value = True + self.assertFalse(timer.is_active()) + next_start.query.return_value = True + self.assertTrue(timer.is_active()) + self.assertEqual(samples, [{"category": "extend", "t": 0.002}]) + next_end.query.return_value = True + self.assertFalse(timer.is_active()) + timer._report() + timer._report() + self.assertEqual( + samples, + [ + {"category": "extend", "t": 0.002}, + {"category": "decode", "t": 0.003}, + ], + ) + def test_decorator_preserves_existing_trace_names(self): recorder = SchedulerStageMetricsRecorder(enabled=False)