diff --git a/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py b/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py index 548977a580..dd03f26d51 100644 --- a/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py +++ b/python/sglang/srt/layers/attention/dsv4/v41_indexer/__init__.py @@ -9,6 +9,8 @@ from typing import TYPE_CHECKING, Tuple import torch +from sglang.srt.utils import is_cuda + from .full_topk import FullTopKIndexer from .types import ( CandidateMetadata, @@ -39,8 +41,9 @@ __all__ = [ @functools.cache def is_sm100_or_newer() -> bool: - # DeepGEMM's fp8_fp4 mqa-logits kernels need SM100+; Hopper takes the torch indexer. - return torch.cuda.get_device_capability()[0] >= 10 + # DeepGEMM's fp8_fp4 mqa-logits kernels need SM100+; Hopper and non-CUDA devices + # take the torch indexer. + return is_cuda() and torch.cuda.get_device_capability()[0] >= 10 @functools.cache