Add scheduler metrics extension hooks (#29207)
Co-authored-by: Yinghai Lu <yinghai@meta.com>
This commit is contained in:
co-authored by
Yinghai Lu
parent
e26bceb81e
commit
7e63feee6f
@@ -3052,13 +3052,18 @@ class ModelRunner(ModelRunnerKVCacheMixin):
|
||||
and self.prefill_cuda_graph_runner.can_run_graph(forward_batch)
|
||||
and get_cp_strategy() is None
|
||||
):
|
||||
category = (
|
||||
"target_verify"
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
else "extend"
|
||||
)
|
||||
# Prefill cuda graph (piecewise).
|
||||
kwargs = self._extend_forward_kwargs(forward_batch, pp_proxy_tensors)
|
||||
# TODO: device_timer.wrap is too broad here — it also includes
|
||||
# load_batch time. Move timing into the prefill cuda graph runner
|
||||
# to capture only the model.forward part.
|
||||
ctx = (
|
||||
self.device_timer.wrap(metadata={"category": "extend"})
|
||||
self.device_timer.wrap(metadata={"category": category})
|
||||
if self.device_timer
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
@@ -282,8 +282,13 @@ class EagerRunner(BaseRunner):
|
||||
kwargs["input_embeds"] = sharded_hidden_states
|
||||
forward_positions = sharded_positions
|
||||
|
||||
category = (
|
||||
"target_verify"
|
||||
if forward_batch.forward_mode.is_target_verify()
|
||||
else "extend"
|
||||
)
|
||||
ctx = (
|
||||
model_runner.device_timer.wrap(metadata={"category": "extend"})
|
||||
model_runner.device_timer.wrap(metadata={"category": category})
|
||||
if model_runner.device_timer
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user