[DSV4/DSA] Name the FlashMLA KV format and drop the V4.1 support probe (#41337)
This commit is contained in:
@@ -78,7 +78,7 @@ dependencies = [
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2",
|
||||
"sgl-deep-gemm==0.2.0",
|
||||
"sglang-kernel==0.4.7",
|
||||
"sglang-kernel==0.4.8",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
"tiktoken",
|
||||
|
||||
@@ -1856,7 +1856,7 @@ def _set_envs_and_config(server_args: ServerArgs):
|
||||
if _is_cuda:
|
||||
assert_pkg_version(
|
||||
"sglang-kernel",
|
||||
"0.4.7",
|
||||
"0.4.8",
|
||||
"Please reinstall the latest version with `pip install sglang-kernel --force-reinstall`",
|
||||
)
|
||||
|
||||
|
||||
@@ -1538,7 +1538,7 @@ class Envs:
|
||||
SGLANG_DSV4_USE_BF16_KV_QUANT_SOURCE = EnvBool(False)
|
||||
# Paged KV layout of the DeepSeek-V4 family pools: "v4" (584 B/token, every
|
||||
# GPU), "v41" (the SM100 FlashMLA V4.1 formats: 528 B fp8 SWA cache, fp8 or
|
||||
# fp4 compressed caches) or "auto" (v41 on SM100 when FlashMLA supports it).
|
||||
# fp4 compressed caches) or "auto" (v41 on SM100, v4 elsewhere).
|
||||
SGLANG_DSV4_KV_LAYOUT = EnvStr("v4")
|
||||
# Compressed-cache layout under "v41": "auto" (fp4 for the fp4-rounded
|
||||
# ratio-1 / ratio-2 latents, fp8 for ratios 4 / 128), "fp8" or "fp4" for all.
|
||||
|
||||
@@ -394,6 +394,8 @@ class DeepseekSparseAttnBackend(
|
||||
self.qk_nope_head_dim = model_runner.model_config.qk_nope_head_dim
|
||||
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
|
||||
self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim
|
||||
# FlashMLA cannot tell the 528 B/token zero-RoPE cache from V4.1 by shape.
|
||||
self.flashmla_kv_format = "V32_NO_ROPE" if self.qk_rope_head_dim == 0 else "V32"
|
||||
|
||||
assert model_runner.req_to_token_pool is not None
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
@@ -2984,6 +2986,7 @@ class DeepseekSparseAttnBackend(
|
||||
(q_all.shape[0], 0), dtype=torch.int32, device=q_all.device
|
||||
),
|
||||
is_fp8_kvcache=True,
|
||||
kv_format=self.flashmla_kv_format,
|
||||
)
|
||||
|
||||
if target_q_heads != num_q_heads:
|
||||
|
||||
@@ -159,16 +159,6 @@ def resolve_compressed_kv_layout(
|
||||
return KVLayout.V41_FP4 if compress_ratio in (1, 2) else KVLayout.V41
|
||||
|
||||
|
||||
def flashmla_supports_v41_kv_layouts() -> bool:
|
||||
"""Whether the installed FlashMLA decode kernel reads the V41 / V41_FP4
|
||||
formats; its docstring lists the bytes-per-token it detects."""
|
||||
try:
|
||||
from sgl_kernel.flash_mla import flash_mla_with_kvcache
|
||||
except Exception:
|
||||
return False
|
||||
return "528" in (flash_mla_with_kvcache.__doc__ or "")
|
||||
|
||||
|
||||
def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
"""The (main-cache layout, compressed-cache option) for a new DeepSeek-V4
|
||||
family pool; the V4.1 layouts exist only in SM100 / SM103 FlashMLA or gfx950
|
||||
@@ -194,17 +184,9 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
"V4.1 KV layouts on HIP require gfx950 with aiter_sparse attention"
|
||||
)
|
||||
return KVLayout.V41, option
|
||||
supported = flashmla_supports_v41_kv_layouts()
|
||||
if mode == "auto":
|
||||
if is_sm100 and supported:
|
||||
return KVLayout.V41, option
|
||||
if mode == "auto" and not is_sm100:
|
||||
return KVLayout.V4, None
|
||||
assert is_sm100, "the V4.1 KV cache layouts need an SM100 / SM103 GPU"
|
||||
if not supported:
|
||||
logger.warning(
|
||||
"SGLANG_DSV4_KV_LAYOUT=v41 but the installed FlashMLA does not advertise "
|
||||
"the V4.1 KV cache formats; the attention kernel will reject the cache."
|
||||
)
|
||||
return KVLayout.V41, option
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user