[Piecewise] support disable decode cuda graph when enable piecewise cuda graph (#13965)
This commit is contained in:
@@ -297,6 +297,7 @@ class ModelRunner:
|
||||
self.req_to_token_pool = req_to_token_pool
|
||||
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
|
||||
self.is_hybrid = model_config.is_hybrid
|
||||
self.is_hybrid_swa = self.is_hybrid
|
||||
self.use_mla_backend = self.model_config.attention_arch == AttentionArch.MLA
|
||||
self.attention_chunk_size = model_config.attention_chunk_size
|
||||
self.forward_pass_id = 0
|
||||
@@ -1615,11 +1616,6 @@ class ModelRunner:
|
||||
)
|
||||
|
||||
def can_run_piecewise_cuda_graph(self):
|
||||
if self.server_args.disable_cuda_graph:
|
||||
log_info_on_rank0(
|
||||
logger, "Disable piecewise CUDA graph because disable_cuda_graph is set"
|
||||
)
|
||||
return False
|
||||
if self.server_args.enable_torch_compile:
|
||||
log_info_on_rank0(
|
||||
logger,
|
||||
|
||||
Reference in New Issue
Block a user