fix is_arch_support_pdl function usage (#24600)
Signed-off-by: P V R K Jyothendra Varma <polisetty.v.r.k.jyothendra.varma@intel.com> Co-authored-by: Ma Mingfei <mingfei.ma@intel.com>
This commit is contained in:
co-authored by
Ma Mingfei
parent
1613bae412
commit
50ed01674e
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, List, Optional
|
|||||||
import torch
|
import torch
|
||||||
import triton
|
import triton
|
||||||
import triton.language as tl
|
import triton.language as tl
|
||||||
from sgl_kernel.utils import is_arch_support_pdl
|
|
||||||
|
|
||||||
from sglang.srt.configs.model_config import AttentionArch
|
from sglang.srt.configs.model_config import AttentionArch
|
||||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||||
@@ -20,9 +19,15 @@ from sglang.srt.utils import (
|
|||||||
get_bool_env_var,
|
get_bool_env_var,
|
||||||
get_device_core_count,
|
get_device_core_count,
|
||||||
get_int_env_var,
|
get_int_env_var,
|
||||||
|
is_cuda,
|
||||||
next_power_of_2,
|
next_power_of_2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_is_cuda = is_cuda()
|
||||||
|
|
||||||
|
if _is_cuda:
|
||||||
|
from sgl_kernel.utils import is_arch_support_pdl
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from sglang.srt.layers.radix_attention import RadixAttention
|
from sglang.srt.layers.radix_attention import RadixAttention
|
||||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||||
@@ -148,7 +153,10 @@ class TritonAttnBackend(AttentionBackend):
|
|||||||
self.device_core_count,
|
self.device_core_count,
|
||||||
self.max_context_len,
|
self.max_context_len,
|
||||||
)
|
)
|
||||||
self.use_pdl = is_arch_support_pdl()
|
if _is_cuda:
|
||||||
|
self.use_pdl = is_arch_support_pdl()
|
||||||
|
else:
|
||||||
|
self.use_pdl = False
|
||||||
|
|
||||||
self.allow_bidirectional_attention_in_extend = (
|
self.allow_bidirectional_attention_in_extend = (
|
||||||
model_runner.server_args.disable_cuda_graph
|
model_runner.server_args.disable_cuda_graph
|
||||||
|
|||||||
Reference in New Issue
Block a user