diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 7c1752a72..a194ac05e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -619,7 +619,7 @@ class Envs: SGLANG_OPT_USE_TOPK_V2 = EnvBool(False) # GEMM / kernel fusion - SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(False) + SGLANG_OPT_FP8_WO_A_GEMM = EnvBool(True) SGLANG_OPT_BF16_FP32_GEMM_ALGO = EnvStr("cublas") SGLANG_OPT_USE_JIT_EP_ACTIVATION = EnvBool(True) SGLANG_OPT_USE_JIT_NORM = EnvBool(False) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 3ad06d8c2..f4128884a 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1095,7 +1095,10 @@ class DeepseekV4ForCausalLM(nn.Module): def _setup_fp8_wo_a_scales(self, is_nextn: bool) -> None: from deep_gemm import transform_sf_into_required_layout - layers = self.model.layers + if is_nextn: + layers = [self.model.decoder] + else: + layers = self.model.layers for layer in layers: attn = layer.self_attn G = attn.n_local_groups diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 4ff4826de..856945dbc 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -4259,6 +4259,15 @@ class ServerArgs: "Debug mode for CUDA graph is enabled via breakable CUDA graph. " "All operations will run eagerly through the graph capture/replay path." ) + # FP8 W_o GEMM requires Blackwell (sm100+). Auto-disable on Hopper. + if is_cuda() and envs.SGLANG_OPT_FP8_WO_A_GEMM.get() and get_device_sm() < 100: + if envs.SGLANG_OPT_FP8_WO_A_GEMM.is_set(): + logger.warning( + "Disabling SGLANG_OPT_FP8_WO_A_GEMM: requires sm100+ (Blackwell), " + "detected sm%d.", + get_device_sm(), + ) + envs.SGLANG_OPT_FP8_WO_A_GEMM.set(False) def _handle_cache_compatibility(self): if self.enable_hierarchical_cache and self.disable_radix_cache: