diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index a7bf9904a..fbd9186b5 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -24,6 +24,8 @@ class StateType(str, enum.Enum): class KVTransferMetric: # Backends that cannot isolate transfer latency can leave this as None. transfer_latency_s: Optional[float] = None + # Backends that cannot isolate allocation wait latency can leave this as None. + alloc_latency_s: Optional[float] = None transfer_total_bytes: Optional[int] = None diff --git a/python/sglang/srt/observability/req_time_stats.py b/python/sglang/srt/observability/req_time_stats.py index 2de10730c..f672045ad 100644 --- a/python/sglang/srt/observability/req_time_stats.py +++ b/python/sglang/srt/observability/req_time_stats.py @@ -903,7 +903,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase): bootstrap_ms = ( self.bootstrap_done_time - self.prefill_bootstrap_queue_entry_time ) * 1000 - alloc_ms = (self.wait_queue_entry_time - self.bootstrap_done_time) * 1000 + if transfer_metric.alloc_latency_s is not None: + alloc_ms = transfer_metric.alloc_latency_s * 1000 + else: + alloc_ms = ( + max(0.0, self.wait_queue_entry_time - self.bootstrap_done_time) + * 1000 + ) result["bootstrap_ms"] = bootstrap_ms result["alloc_ms"] = alloc_ms