[PD] Fix negative prefill kv_transfer_alloc_ms under optimistic prefill (#27885)
Co-authored-by: cctry <cctry@fb.com>
This commit is contained in:
@@ -24,6 +24,8 @@ class StateType(str, enum.Enum):
|
||||
class KVTransferMetric:
|
||||
# Backends that cannot isolate transfer latency can leave this as None.
|
||||
transfer_latency_s: Optional[float] = None
|
||||
# Backends that cannot isolate allocation wait latency can leave this as None.
|
||||
alloc_latency_s: Optional[float] = None
|
||||
transfer_total_bytes: Optional[int] = None
|
||||
|
||||
|
||||
|
||||
@@ -903,7 +903,13 @@ class SchedulerReqTimeStats(ReqTimeStatsBase):
|
||||
bootstrap_ms = (
|
||||
self.bootstrap_done_time - self.prefill_bootstrap_queue_entry_time
|
||||
) * 1000
|
||||
alloc_ms = (self.wait_queue_entry_time - self.bootstrap_done_time) * 1000
|
||||
if transfer_metric.alloc_latency_s is not None:
|
||||
alloc_ms = transfer_metric.alloc_latency_s * 1000
|
||||
else:
|
||||
alloc_ms = (
|
||||
max(0.0, self.wait_queue_entry_time - self.bootstrap_done_time)
|
||||
* 1000
|
||||
)
|
||||
|
||||
result["bootstrap_ms"] = bootstrap_ms
|
||||
result["alloc_ms"] = alloc_ms
|
||||
|
||||
Reference in New Issue
Block a user