[DSV4/DSA] Name the FlashMLA KV format and drop the V4.1 support probe (#41337)

This commit is contained in:
DarkSharpness
2026-10-01 00:02:18 -07:00
committed by GitHub
parent d010e50ad6
commit 3ed6367d3f
6 changed files with 8 additions and 23 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ dependencies = [
"setproctitle",
"sgl-deep-ep==0.1.2",
"sgl-deep-gemm==0.2.0",
"sglang-kernel==0.4.7",
"sglang-kernel==0.4.8",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
"tiktoken",
+1 -1
View File
@@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs):
if _is_cuda:
assert_pkg_version(
"sglang-kernel",
"0.4.7",
"0.4.8",
"Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`",
)
+1 -1
View File
@@ -1538,7 +1538,7 @@ class Envs:
SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False)
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
# fp4 compressed caches) or "auto" (v41 on SM100 when FlashMLA supports it).
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
SGLANG_DSV4_KV_LAYOUT = EnvStr("v4")
# Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded
# ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all.
@@ -394,6 +394,8 @@ class DeepseekSparseAttnBackend(
self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim
# FlashMLA cannot tell the 528 B/token zero-RoPE cache from V4.1 by shape.
self.flashmla_kv_format = "V32_NO_ROPE" if self.qk_rope_head_dim == 0 else "V32"
assert model_runner.req_to_token_pool is not None
self.req_to_token_pool = model_runner.req_to_token_pool
@@ -2984,6 +2986,7 @@ class DeepseekSparseAttnBackend(
(q_all.shape[0], 0), dtype=torch.int32, device=q_all.device
),
is_fp8_kvcache=True,
kv_format=self.flashmla_kv_format,
)
if target_q_heads != num_q_heads:
@@ -159,16 +159,6 @@ def resolve_compressed_kv_layout(
return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41
def flashmla_supports_v41_kv_layouts() -> bool:
"""Whether the installed FlashMLA decode kernel reads the V41 / V41_FP4
formats; its docstring lists the bytes-per-token it detects."""
try:
from sgl_kernel.flash_mla import flash_mla_with_kvcache
except Exception:
return False
return "528" in (flash_mla_with_kvcache.__doc__ or "")
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
"""The (main-cache layout, compressed-cache option) for a new DeepSeek-V4
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950
@@ -194,17 +184,9 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
)
return KVLayout.V41, option
supported = flashmla_supports_v41_kv_layouts()
if mode == "auto":
if is_sm100 and supported:
return KVLayout.V41, option
if mode == "auto" and not is_sm100:
return KVLayout.V4, None
assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU"
if not supported:
logger.warning(
"SGLANG_DSV4_KV_LAYOUT=v41 but the installed FlashMLA does not advertise "
"the V4.1 KV cache formats; the attention kernel will reject the cache."
)
return KVLayout.V41, option