[Fix] disaggregation: Reuse startup decode graphs across PD role switches (#43760)
This commit is contained in:
@@ -1189,6 +1189,11 @@ class ModelRunner:
|
||||
self.eager_runner = capture.eager_runner
|
||||
self.prefill_cuda_graph_runner = capture.prefill.runner
|
||||
self.decode_cuda_graph_runner = capture.decode.runner
|
||||
# Reuse startup decode graphs on a subsequent PD role switch.
|
||||
self.decode_cuda_graph_captured = self.decode_cuda_graph_runner is not None
|
||||
self.decode_cuda_graph_capture_bs = list(
|
||||
getattr(self.decode_cuda_graph_runner, "capture_bs", []) or []
|
||||
)
|
||||
self.graph_memory_usage = capture.memory_usage
|
||||
self.graph_time_usage = capture.time_usage
|
||||
|
||||
|
||||
Reference in New Issue
Block a user