diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 97e19ff859..06514236cd 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4102,6 +4102,7 @@ class Scheduler( prefetch_done = self.tree_cache.check_prefetch_progress( req.cache_request_handle ) + req.time_stats.observe_storage_prefetch_ready(prefetch_done) if not prefetch_done: # skip staging requests that are ongoing prefetch continue @@ -4122,12 +4123,11 @@ class Scheduler( self._prefetch_after_device_hit_loss(req) ): continue - if ( - self.enable_hicache_storage - and buffer_pipeline is not None - and not buffer_pipeline.prepare_staged_prefetch(req) - ): - continue + if self.enable_hicache_storage and buffer_pipeline is not None: + staging_ready = buffer_pipeline.prepare_staged_prefetch(req) + req.time_stats.observe_storage_staging_ready(staging_ready) + if not staging_ready: + continue res = adder.add_one_req( req, has_chunked_req=(self.chunked_req is not None), diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index e79470e030..068059ecb2 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -664,6 +664,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): # re-enters it). Must not end in "time": __setstate__ clock-rebases those. queue_duration_s: float = 0.0 + # Poll-observed waits; staging can include a re-prefetch and overlap prefetch. + # None means unmeasured (including stats received from an older scheduler). + storage_prefetch_wait_s: Optional[float] = None + storage_staging_wait_s: Optional[float] = None + _storage_prefetch_wait_start: Optional[float] = None + _storage_staging_wait_start: Optional[float] = None + has_timing_data: bool = False def __getstate__(self) -> object: @@ -678,6 +685,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): "prefill_finished_time": self.prefill_finished_time, # Read by meta_info["queue_time"] on the tokenizer; timestamps cannot rebuild it. "queue_duration_s": self.queue_duration_s, + "storage_prefetch_wait_s": ( + self.storage_prefetch_wait_s + if self._storage_prefetch_wait_start is None + else None + ), + "storage_staging_wait_s": ( + self.storage_staging_wait_s + if self._storage_staging_wait_start is None + else None + ), "diff_realtime_monotonic": global_diff_realtime_monotonic, } return state @@ -737,10 +754,42 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): self.last_prefill_finished_time = 0.0 self.last_chunked_prefill_finish_time = 0.0 self.queue_duration_s = 0.0 + self.storage_prefetch_wait_s = None + self.storage_staging_wait_s = None + self._storage_prefetch_wait_start = None + self._storage_staging_wait_start = None + + def observe_storage_prefetch_ready(self, ready: bool): + """Accumulate first blocked poll to the subsequent ready poll, per episode.""" + if self.storage_prefetch_wait_s is None: + return + if ready: + if self._storage_prefetch_wait_start is not None: + self.storage_prefetch_wait_s += ( + time.perf_counter() - self._storage_prefetch_wait_start + ) + self._storage_prefetch_wait_start = None + elif self._storage_prefetch_wait_start is None: + self._storage_prefetch_wait_start = time.perf_counter() + + def observe_storage_staging_ready(self, ready: bool): + if self.storage_staging_wait_s is None: + return + if ready: + if self._storage_staging_wait_start is not None: + self.storage_staging_wait_s += ( + time.perf_counter() - self._storage_staging_wait_start + ) + self._storage_staging_wait_start = None + elif self._storage_staging_wait_start is None: + self._storage_staging_wait_start = time.perf_counter() def set_wait_queue_entry_time(self, ts=None): ts = ts or time.perf_counter() if self.wait_queue_entry_time == 0.0: + if self.enable_metrics and self.disagg_mode != DisaggregationMode.DECODE: + self.storage_prefetch_wait_s = 0.0 + self.storage_staging_wait_s = 0.0 if self.enable_metrics or self.trace_ctx.tracing_enable: if self.disagg_mode == DisaggregationMode.PREFILL: stage = RequestStage.PREFILL_BOOTSTRAP @@ -1172,6 +1221,16 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): def convert_to_output_meta_info(self): meta_data = {} + if ( + self.storage_prefetch_wait_s is not None + and self._storage_prefetch_wait_start is None + ): + meta_data["storage_prefetch_wait"] = self.storage_prefetch_wait_s + if ( + self.storage_staging_wait_s is not None + and self._storage_staging_wait_start is None + ): + meta_data["storage_staging_wait"] = self.storage_staging_wait_s if self.forward_entry_time > 0.0: meta_data["forward_entry_time"] = convert_time_to_realtime( self.forward_entry_time diff --git a/test/registered/unit/observability/test_req_time_stats.py b/test/registered/unit/observability/test_req_time_stats.py index 61fd94dd69..8dedf1be51 100644 --- a/test/registered/unit/observability/test_req_time_stats.py +++ b/test/registered/unit/observability/test_req_time_stats.py @@ -266,5 +266,117 @@ class TestQueueTimeAcrossRetraction(CustomTestCase): self.assertAlmostEqual(stats.get_queueing_time(), 0.2) +class TestStorageAdmissionWait(CustomTestCase): + def _stats(self): + stats = rts.SchedulerReqTimeStats( + enable_metrics=True, metrics_collector=mock.MagicMock() + ) + stats.set_wait_queue_entry_time(100.0) + return stats + + def test_blocked_polls_accumulate_wait_episodes(self): + stats = self._stats() + for observe, output_key in ( + (stats.observe_storage_prefetch_ready, "storage_prefetch_wait"), + (stats.observe_storage_staging_ready, "storage_staging_wait"), + ): + with ( + self.subTest(gate=output_key), + mock.patch.object(rts.time, "perf_counter") as clock, + ): + for ts, ready in ( + (100.0, True), + (100.1, False), + (100.15, False), + (100.2, False), + (100.3, True), + (100.4, True), + (100.5, False), + (100.6, True), + ): + clock.return_value = ts + observe(ready) + if not ready: + self.assertNotIn( + output_key, stats.convert_to_output_meta_info() + ) + self.assertAlmostEqual( + stats.convert_to_output_meta_info()[output_key], 0.3 + ) + + def test_zero_missing_and_open_intervals_survive_serialization(self): + stats = self._stats() + stats.observe_storage_prefetch_ready(True) + stats.observe_storage_staging_ready(True) + zero = pickle.loads(pickle.dumps(stats)).convert_to_output_meta_info() + self.assertEqual(zero["storage_prefetch_wait"], 0.0) + self.assertEqual(zero["storage_staging_wait"], 0.0) + + with mock.patch.object(rts.time, "perf_counter", return_value=100.1): + stats.observe_storage_prefetch_ready(False) + stats.observe_storage_staging_ready(False) + incomplete = pickle.loads(pickle.dumps(stats)).convert_to_output_meta_info() + self.assertNotIn("storage_prefetch_wait", incomplete) + self.assertNotIn("storage_staging_wait", incomplete) + + legacy = rts.SchedulerReqTimeStats() + legacy.__setstate__({"has_timing_data": True, "queue_duration_s": 1.0}) + missing = pickle.loads(pickle.dumps(legacy)).convert_to_output_meta_info() + self.assertNotIn("storage_prefetch_wait", missing) + self.assertNotIn("storage_staging_wait", missing) + + def test_overlap_retraction_clock_rebase_and_retry(self): + stats = self._stats() + with mock.patch.object( + rts.time, "perf_counter", side_effect=[100.1, 100.2, 100.4, 100.5] + ): + stats.observe_storage_staging_ready(False) + stats.observe_storage_prefetch_ready(False) + stats.observe_storage_prefetch_ready(True) + stats.observe_storage_staging_ready(True) + stats.set_forward_entry_time(100.6) + stats.set_wait_queue_entry_time(101.0) + with mock.patch.object(rts.time, "perf_counter", side_effect=[101.1, 101.4]): + stats.observe_storage_prefetch_ready(False) + stats.observe_storage_prefetch_ready(True) + stats.set_forward_entry_time(101.5) + + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_000.0): + blob = pickle.dumps(stats) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_005.0): + blob = pickle.dumps(pickle.loads(blob)) + with mock.patch.object(rts, "global_diff_realtime_monotonic", 1_000_009.0): + meta = pickle.loads(blob).convert_to_output_meta_info() + self.assertAlmostEqual(meta["storage_prefetch_wait"], 0.5) + self.assertAlmostEqual(meta["storage_staging_wait"], 0.4) + self.assertAlmostEqual(meta["queue_time"], 1.1) + + with mock.patch.object(rts.time, "perf_counter", return_value=102.0): + stats.observe_storage_staging_ready(False) + stats.reset_prefill_retry_time() + stats.set_wait_queue_entry_time(103.0) + meta = stats.convert_to_output_meta_info() + self.assertEqual(meta["storage_prefetch_wait"], 0.0) + self.assertEqual(meta["storage_staging_wait"], 0.0) + + def test_metrics_disabled_and_decode_do_not_record_local_prefetch(self): + for stats in ( + rts.SchedulerReqTimeStats(), + rts.SchedulerReqTimeStats( + enable_metrics=True, + metrics_collector=mock.MagicMock(), + disagg_mode=rts.DisaggregationMode.DECODE, + ), + ): + stats.set_wait_queue_entry_time(100.0) + stats.observe_storage_prefetch_ready(False) + stats.observe_storage_prefetch_ready(True) + stats.observe_storage_staging_ready(False) + stats.observe_storage_staging_ready(True) + meta = stats.convert_to_output_meta_info() + self.assertNotIn("storage_prefetch_wait", meta) + self.assertNotIn("storage_staging_wait", meta) + + if __name__ == "__main__": unittest.main()