diff --git a/docker/Dockerfile b/docker/Dockerfile index 6f9c62714..5eb4ea67f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ ARG PIP_DEFAULT_INDEX ARG UBUNTU_MIRROR ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 -ARG FLASHINFER_VERSION=0.6.15 +ARG FLASHINFER_VERSION=0.6.14 ARG MOONCAKE_VERSION=0.3.11.post1 ARG MSCCLPP_VERSION=sglang-v0.9.1 #if need other arg please add in MOONCAKE_COMPILE_ARG diff --git a/python/pyproject.toml b/python/pyproject.toml index 3095c03e1..49a4c9323 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "einops", "fastapi", "flash-attn-4==4.0.0b15", - "flashinfer_python[cu13]==0.6.15", # keep it aligned with jit-cache version in Dockerfile + "flashinfer_python[cu13]==0.6.14", # keep it aligned with jit-cache version in Dockerfile "gguf", "humming-kernels[cu13]==0.1.10", "interegular", diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index aa45593db..5c158535e 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1304,7 +1304,7 @@ def _set_envs_and_config(server_args: ServerArgs): if server_args.attention_backend == "flashinfer": assert_pkg_version( "flashinfer_python", - "0.6.15", + "0.6.14", "Please uninstall the old version and " "reinstall the latest version by following the instructions " "at https://docs.flashinfer.ai/installation.html.", diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index 02366f886..daa6ba2f9 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -222,17 +222,6 @@ def resolve_cutedsl_standard_scales( return w1_alpha, fc2_input_scale, w2_alpha, used_input_scale -def _cutedsl_wrapper_activation_type(activation: str, activation_type_cls: Any) -> Any: - if activation == "silu": - return activation_type_cls.Swiglu - if activation == "relu2": - return activation_type_cls.Relu2 - raise ValueError( - f"CuteDSL MoE wrapper supports 'silu' (gated) or 'relu2' (non-gated) " - f"activation, got {activation!r}." - ) - - def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: """Lazily create CuteDslMoEWrapper and resolve scales on first forward. @@ -248,7 +237,7 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: return try: - from flashinfer import ActivationType, CuteDslMoEWrapper + from flashinfer import CuteDslMoEWrapper except ImportError as e: raise ImportError( "flashinfer_cutedsl backend requires FlashInfer with CuteDSL support. " @@ -295,9 +284,7 @@ def ensure_cutedsl_wrapper(layer: torch.nn.Module) -> None: local_expert_offset=layer.moe_ep_rank * layer.num_local_experts, output_dtype=layer.moe_runner_config.params_dtype, device=str(layer.w13_weight.device), - activation_type=_cutedsl_wrapper_activation_type( - layer.moe_runner_config.activation, ActivationType - ), + activation=layer.moe_runner_config.activation, ) w1_alpha, fc2_input_scale, w2_alpha, used_input_scale = ( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 7428ce183..a086013be 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -459,6 +459,14 @@ class MoEGate(nn.Module): "quark", ): correction_bias_dtype = torch.bfloat16 + # NOTE(kpham-sgl): flashinfer trtllm routing requires a bf16 + # routing_bias; an fp32 bias yields NaN routing on exact ties. + # Mirror the fp8 path's cast. + if ( + quant_config.get_name() == "modelopt_fp4" + and get_moe_runner_backend().is_flashinfer_trtllm() + ): + correction_bias_dtype = torch.bfloat16 self.e_score_correction_bias = nn.Parameter( torch.empty((config.n_routed_experts), dtype=correction_bias_dtype) ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 608492653..578d55007 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -1907,7 +1907,7 @@ def check_pkg_version_at_least(pkg: str, min_version: str) -> bool: Args: pkg: Package name (distribution name, e.g., "flashinfer-python") - min_version: Minimum version required (e.g., "0.6.15") + min_version: Minimum version required (e.g., "0.6.14") Returns: True if package is installed and version >= min_version, False otherwise diff --git a/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py b/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py index 8f1c3d84d..210112a5a 100644 --- a/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py +++ b/test/registered/models_e2e/test_dsa_glm52_nvfp4_tp_mtp.py @@ -21,7 +21,7 @@ class TestGLM52NVFP4TPMTP( model = "nvidia/GLM-5.2-NVFP4" tp_size = 4 mem_fraction_static = 0.8 - bs_1_speed_thres = 280 + bs_1_speed_thres = 250 extra_server_args = [ "--moe-runner-backend", "flashinfer_trtllm",