Log per-request time stats in a dedicated tail step (#25707)
This commit is contained in:
@@ -142,13 +142,7 @@ class SchedulerOutputStreamer:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
acc.accept(req=req)
|
acc.accept(req=req)
|
||||||
|
self._maybe_log_time_stats(req=req)
|
||||||
if (
|
|
||||||
req.finished()
|
|
||||||
and self.ps.attn_tp_rank == 0
|
|
||||||
and self.server_args.enable_request_time_stats_logging
|
|
||||||
):
|
|
||||||
req.log_time_stats()
|
|
||||||
|
|
||||||
dp_ranks = [self.ps.dp_rank] * len(acc.rids) if acc.rids else None
|
dp_ranks = [self.ps.dp_rank] * len(acc.rids) if acc.rids else None
|
||||||
|
|
||||||
@@ -200,6 +194,14 @@ class SchedulerOutputStreamer:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _maybe_log_time_stats(self, *, req: Req) -> None:
|
||||||
|
if (
|
||||||
|
req.finished()
|
||||||
|
and self.ps.attn_tp_rank == 0
|
||||||
|
and self.server_args.enable_request_time_stats_logging
|
||||||
|
):
|
||||||
|
req.log_time_stats()
|
||||||
|
|
||||||
def _stream_output_embedding(self, reqs: List[Req]):
|
def _stream_output_embedding(self, reqs: List[Req]):
|
||||||
rids = []
|
rids = []
|
||||||
http_worker_ipcs = []
|
http_worker_ipcs = []
|
||||||
|
|||||||
Reference in New Issue
Block a user