From 3ed6367d3f28f11674f72fd298c355f72aee540f Mon Sep 17 00:00:00 2001 From: DarkSharpness <76582120+DarkSharpness@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:02:18 +0800 Subject: [PATCH] [DSV4/DSA] Name the FlashMLA KV format and drop the V4.1 support probe (#41337) --- docker/Dockerfile | 2 +- python/pyproject.toml | 2 +- python/sglang/srt/entrypoints/engine.py | 2 +- python/sglang/srt/environ.py | 2 +- .../srt/layers/attention/dsa_backend.py | 3 +++ .../srt/mem_cache/deepseek_v4_memory_pool.py | 20 +------------------ 6 files changed, 8 insertions(+), 23 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a35b6c76e..6176ead24e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base ARG TARGETARCH ARG BUILD_TYPE=all ARG BRANCH_TYPE=remote -ARG SGL_KERNEL_VERSION=0.4.7 +ARG SGL_KERNEL_VERSION=0.4.8 ARG SGL_VERSION ARG USE_LATEST_SGLANG=0 ARG GDRCOPY_VERSION=2.5.1 diff --git a/python/pyproject.toml b/python/pyproject.toml index 0f18e0fc83..1e53fcfd62 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -78,7 +78,7 @@ dependencies = [ "setproctitle", "sgl-deep-ep==0.1.2", "sgl-deep-gemm==0.2.0", - "sglang-kernel==0.4.7", + "sglang-kernel==0.4.8", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", "tiktoken", diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index d167a2d93d..85a23dc6e1 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs): if _is_cuda: assert_pkg_version( "sglang-kernel", - "0.4.7", + "0.4.8", "Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`", ) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 207940e525..82de552fa5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1538,7 +1538,7 @@ class Envs: SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False) # Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every # GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or - # fp4 compressed caches) or "auto" (v41 on SM100 when FlashMLA supports it). + # fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere). SGLANG_DSV4_KV_LAYOUT = EnvStr("v4") # Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded # ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all. diff --git a/python/sglang/srt/layers/attention/dsa_backend.py b/python/sglang/srt/layers/attention/dsa_backend.py index 6611f00237..265563c5a9 100644 --- a/python/sglang/srt/layers/attention/dsa_backend.py +++ b/python/sglang/srt/layers/attention/dsa_backend.py @@ -394,6 +394,8 @@ class DeepseekSparseAttnBackend( self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim self.kv_lora_rank = model_runner.model_config.kv_lora_rank self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim + # FlashMLA cannot tell the 528 B/token zero-RoPE cache from V4.1 by shape. + self.flashmla_kv_format = "V32_NO_ROPE" if self.qk_rope_head_dim == 0 else "V32" assert model_runner.req_to_token_pool is not None self.req_to_token_pool = model_runner.req_to_token_pool @@ -2984,6 +2986,7 @@ class DeepseekSparseAttnBackend( (q_all.shape[0], 0), dtype=torch.int32, device=q_all.device ), is_fp8_kvcache=True, + kv_format=self.flashmla_kv_format, ) if target_q_heads != num_q_heads: diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index f364e89d06..8e6640ad0b 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -159,16 +159,6 @@ def resolve_compressed_kv_layout( return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41 -def flashmla_supports_v41_kv_layouts() -> bool: - """Whether the installed FlashMLA decode kernel reads the V41 / V41_FP4 - formats; its docstring lists the bytes-per-token it detects.""" - try: - from sgl_kernel.flash_mla import flash_mla_with_kvcache - except Exception: - return False - return "528" in (flash_mla_with_kvcache.__doc__ or "") - - def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: """The (main-cache layout, compressed-cache option) for a new DeepSeek-V4 family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950 @@ -194,17 +184,9 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: "V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention" ) return KVLayout.V41, option - supported = flashmla_supports_v41_kv_layouts() - if mode == "auto": - if is_sm100 and supported: - return KVLayout.V41, option + if mode == "auto" and not is_sm100: return KVLayout.V4, None assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU" - if not supported: - logger.warning( - "SGLANG_DSV4_KV_LAYOUT=v41 but the installed FlashMLA does not advertise " - "the V4.1 KV cache formats; the attention kernel will reject the cache." - ) return KVLayout.V41, option