diff --git a/docs_new/docs/advanced_features/server_arguments.mdx b/docs_new/docs/advanced_features/server_arguments.mdx
index 2fe313a5f..ffd820701 100644
--- a/docs_new/docs/advanced_features/server_arguments.mdx
+++ b/docs_new/docs/advanced_features/server_arguments.mdx
@@ -1439,6 +1439,12 @@ Please consult the documentation below and [server_args.py](https://github.com/s
Choose the runner backend for NVFP4 GEMM operations. Options: 'auto' (default; selects flashinfer_cutedsl on SM100, marlin on SM80-SM90, flashinfer_cutlass otherwise (including SM120)), 'flashinfer_cutlass' (FlashInfer CUTLASS backend), 'flashinfer_cudnn' (FlashInfer cuDNN backend, optimal on CUDA 13+ with cuDNN 9.15+), 'flashinfer_cutedsl' (FlashInfer CuTe DSL backend), 'flashinfer_trtllm' (FlashInfer TensorRT-LLM backend, requires different weight preparation with shuffling), 'marlin' (weight-only W4A16 fallback for SM80-SM90). Requires FlashInfer to be installed. |
`auto` |
auto, flashinfer_cudnn, flashinfer_cutedsl, flashinfer_cutlass, flashinfer_trtllm, marlin |
+
+
+ | `--bf16-gemm-backend` |
+ Choose the backend for unquantized BF16 GEMM operations. Options: 'auto' (default; selects cutedsl on SM100/SM103 (Blackwell), otherwise uses cuBLAS via `torch.nn.functional.linear`), 'cutedsl' (SGLang JIT CuTe DSL TGV BF16 GEMM on SM10X; dispatches between the CuTe DSL kernel and cuBLAS). |
+ `auto` |
+ auto, cutedsl |
| `--disable-flashinfer-autotune` |
diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py
index 282b1ced1..f6942218f 100644
--- a/python/sglang/srt/layers/quantization/unquant.py
+++ b/python/sglang/srt/layers/quantization/unquant.py
@@ -67,6 +67,7 @@ if _use_aiter:
class Bf16GemmBackend(Enum):
AUTO = "auto"
CUTEDSL = "cutedsl"
+ TORCH = "torch"
def is_auto(self) -> bool:
return self == Bf16GemmBackend.AUTO
@@ -83,11 +84,15 @@ _use_cutedsl_bf16_gemm = None
def initialize_bf16_gemm_config(server_args: ServerArgs) -> None:
global _BF16_GEMM_BACKEND, _cutedsl_bf16_gemm, _use_cutedsl_bf16_gemm
- backend = Bf16GemmBackend(server_args.bf16_gemm_backend)
+ from sglang.srt.utils import is_sm100_supported
+
+ backend_str = server_args.bf16_gemm_backend
+ if backend_str == "auto" and is_sm100_supported():
+ backend_str = "cutedsl"
+
+ backend = Bf16GemmBackend(backend_str)
if backend.is_cutedsl():
- from sglang.srt.utils import is_sm100_supported
-
if not is_sm100_supported():
raise ValueError(
"--bf16-gemm-backend cutedsl requires SM100/SM103 (Blackwell)"
@@ -207,6 +212,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
and x.dtype == torch.bfloat16
and layer.weight.dtype == torch.bfloat16
and (bias is None or bias.dtype == torch.bfloat16)
+ and not layer.weight.requires_grad
+ and (bias is None or not bias.requires_grad)
and _use_cutedsl_bf16_gemm(
x.numel() // x.shape[-1],
layer.weight.shape[0],
diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py
index c0ca0679f..4eac6e339 100644
--- a/python/sglang/srt/server_args.py
+++ b/python/sglang/srt/server_args.py
@@ -306,7 +306,7 @@ FP4_GEMM_RUNNER_BACKEND_CHOICES = [
"marlin",
]
-BF16_GEMM_BACKEND_CHOICES = ["auto", "cutedsl"]
+BF16_GEMM_BACKEND_CHOICES = ["auto", "cutedsl", "torch"]
RADIX_EVICTION_POLICY_CHOICES = ["lru", "lfu", "slru", "priority"]
RETRACTION_POLICY_CHOICES = ["length", "priority"]
@@ -1662,7 +1662,7 @@ class ServerArgs:
bf16_gemm_backend: A[
str,
Arg(
- help="Choose the backend for unquantized BF16 GEMM operations. Options: 'auto' (default; uses cuBLAS via torch.nn.functional.linear), 'cutedsl' (SGLang JIT CuTe DSL TGV BF16 GEMM on SM10X; dispatches between the CuTe DSL kernel and cuBLAS).",
+ help="Choose the backend for unquantized BF16 GEMM operations. Options: 'auto' (default; selects 'cutedsl' on SM100/SM103 (Blackwell), otherwise uses cuBLAS via torch.nn.functional.linear), 'cutedsl' (SGLang JIT CuTe DSL TGV BF16 GEMM on SM10X; dispatches between the CuTe DSL kernel and cuBLAS), 'torch' (always uses cuBLAS via torch.nn.functional.linear, even on SM100/SM103).",
cli_name="--bf16-gemm-backend",
choices=BF16_GEMM_BACKEND_CHOICES,
),