[Perf] deepseek-v4.1: Integrate FlashMLA MegaAttention with NVFP4 KV (#43922)

This commit is contained in:
Khoa Pham
2026-10-15 17:21:43 -07:00
committed by GitHub
parent d14fa5db9b
commit a93f16f0cf
12 changed files with 570 additions and 29 deletions
+1 -1
View File
@@ -78,7 +78,7 @@ dependencies = [
"setproctitle",
"sgl-deep-ep==0.1.2.post1",
"sgl-deep-gemm==0.2.2",
"sglang-kernel==0.4.9",
"sglang-kernel==0.4.10",
"smg-grpc-servicer>=0.9.0",
"soundfile==0.13.1",
"tiktoken",
@@ -68,6 +68,22 @@ def precompute_freqs_cis(
return freqs_cis
@lru_cache(2)
def precompute_cos_sin_table(
dim,
seqlen,
original_seq_len,
base,
factor,
beta_fast,
beta_slow,
) -> torch.Tensor:
freqs_cis = precompute_freqs_cis(
dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow
)
return torch.view_as_real(freqs_cis).transpose(1, 2).reshape(seqlen, -1)
@triton.jit
def apply_rotary_emb_triton_kernel(
x_ptr,
+2
View File
@@ -166,6 +166,8 @@ def mega_moe_pre_dispatch(
shared_x_sf: Optional[torch.Tensor] = None,
shared_block_m: int = 0,
) -> None:
if x.shape[0] == 0 and x.stride(0) != x.shape[1]:
x = x.new_empty(x.shape)
# shared_block_m must match the consuming MoE's paged scale layout
module = _jit_mega_moe_pre_dispatch_module(
quant_group_size, shared_x_sf is not None
@@ -26,6 +26,23 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _default_dsv4_attention_backend(server_args: ServerArgs) -> str:
from sglang.srt.utils.common import is_sm100_supported
cfg = resolving_view(server_args)
layout = envs.SGLANG_DSV4_KV_LAYOUT
if (
model_config_of(server_args).hf_config.model_type != "deepseek_v41"
or cfg.device != "cuda"
or not is_sm100_supported()
or cfg.kv_cache_dtype != "fp8_e4m3"
or cfg.disaggregation_mode != "null"
or (layout.is_set() and layout.get().lower() not in ("auto", "v41"))
):
return "flashmla"
return "flashmla_mega"
def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None:
"""Apply DeepSeek V4 environment defaults, request limits, and validation."""
cfg = resolving_view(server_args)
@@ -48,6 +65,22 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None
run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype)
if cfg.dsv4_attn_backend == "auto":
declare_resolution(
server_args,
"apply_deepseek_v4_defaults",
dsv4_attn_backend=_default_dsv4_attention_backend(server_args),
)
if cfg.dsv4_attn_backend == "flashmla_mega":
from sglang.srt.utils.common import is_sm100_supported
assert (
model_config_of(server_args).hf_config.model_type == "deepseek_v41"
and cfg.device == "cuda"
and is_sm100_supported()
), "--dsv4-attn-backend flashmla_mega requires DeepSeek-V4.1 on SM100/SM103."
if cfg.dsv4_attn_backend == "trtllm":
from sglang.srt.utils.common import is_sm100_supported
+7 -4
View File
@@ -316,12 +316,15 @@ class ExecKernel(msgspec.Struct):
str,
Arg(
help="DeepSeek V4 attention backend. 'auto' (default) resolves to "
"'flashmla'. 'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) "
"'flashmla_mega' where it is supported and 'flashmla' otherwise. "
"'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) "
"switches the SWA/compressed KV pools to a "
"uniform 512-dim FP8 layout and runs decode and sparse prefill "
"through the flashinfer trtllm-gen sparse MLA kernel. The backend "
"choice is shared by prefill and decode.",
choices=["auto", "flashmla", "trtllm"],
"through the flashinfer trtllm-gen sparse MLA kernel. "
"'flashmla_mega' (DeepSeek-V4.1 on SM100/SM103) runs the fused "
"Q-RoPE + sparse attention + inverse-RoPE + MXFP8 FlashMLA kernel. "
"The backend choice is shared by prefill and decode.",
choices=["auto", "flashmla", "flashmla_mega", "trtllm"],
resolvable=True,
),
] = "auto"
@@ -993,9 +993,6 @@ class DSV4Metadata:
# Captured with the late layers on a tail block (decoder SWA bounded replay
# inside the prefill CUDA graph); replay rebuilds the tail metadata iff set.
prefill_graph_tail: bool = False
# Captured with the late layers on a tail block (decoder SWA bounded replay
# inside the prefill CUDA graph); replay rebuilds the tail metadata iff set.
prefill_graph_tail: bool = False
@property
def core_metadata(self) -> DSV4AttnMetadata:
@@ -1113,6 +1110,7 @@ class DeepseekV4AttnBackend(
folds_encoder_swa_replay: bool = True
needs_cpu_seq_lens: bool = False
trtllm_attn: bool = False
mega_attn: bool = False
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
# Breakable-graph verify rereads shared state across segments.
@@ -3581,6 +3579,7 @@ class DeepseekV4AttnBackend(
# to True. Paged attention under active CP needs the page indices.
return (
self.trtllm_attn
or self.mega_attn
or self.tail_forward_metadata is not None
or self.forward_metadata.late_layer_tail is not None
or self.token_to_kv_pool.request_window is not None
@@ -3657,8 +3656,9 @@ class DeepseekV4AttnBackend(
compress_ratio: Literal[0, 1, 2, 4, 128],
save_kv_cache: bool = True,
attn_sink: Optional[torch.Tensor] = None,
cos_sin_cache: Optional[torch.Tensor] = None,
**_,
) -> torch.Tensor:
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
return q.new_empty(
(q.shape[0], q.shape[1], layer.v_head_dim),
@@ -3757,10 +3757,19 @@ class DeepseekV4AttnBackend(
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
)
use_mega_paged = cos_sin_cache is not None and self._mega_paged_prefill(
forward_batch,
num_qo_tokens=q.shape[0],
layer_id=layer_id,
compress_ratio=compress_ratio,
num_heads=q.shape[-2],
)
# The tail stays dense: its window floor lives in swa_page_indices,
# which the sparse chunk cache ignores.
if (
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
and not use_mega_paged
and self.forward_metadata.late_layer_tail is None
and token_to_kv_pool.request_window is None
):
@@ -3782,6 +3791,23 @@ class DeepseekV4AttnBackend(
token_to_kv_pool=token_to_kv_pool,
core_attn_metadata=core_attn_metadata,
attn_sink=attn_sink,
cos_sin_cache=cos_sin_cache,
)
if cos_sin_cache is not None:
return self._forward_mega_decode(
q=q.squeeze(1),
swa_k_cache=swa_k_cache,
swa_page_indices=swa_page_indices.squeeze(1),
positions=match_num_queries(core_attn_metadata.positions, value=0),
cos_sin_cache=cos_sin_cache,
attn_sink=attn_sink,
swa_topk_lengths=swa_topk_lengths,
extra_k_cache=extra_k_cache,
extra_indices=None
if extra_indices is None
else extra_indices.squeeze(1),
extra_topk_lengths=extra_topk_lengths,
)
if (
@@ -3889,7 +3915,8 @@ class DeepseekV4AttnBackend(
token_to_kv_pool: DeepSeekV4TokenToKVPool,
core_attn_metadata: DSV4AttnMetadata,
attn_sink: torch.Tensor,
) -> torch.Tensor:
cos_sin_cache: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""Gathers each request's SWA window and c4/c128 cache into a flat bf16
workspace that flash_mla_sparse_fwd reads through per-query rebased indices."""
if _is_xpu:
@@ -3948,6 +3975,17 @@ class DeepseekV4AttnBackend(
)
kv = workspace
if cos_sin_cache is not None:
return self._forward_mega_prefill(
q=q_flat,
kv=kv,
indices=combined_indices.unsqueeze(1),
positions=core_attn_metadata.positions[: q_flat.shape[0]],
cos_sin_cache=cos_sin_cache,
attn_sink=attn_sink,
topk_length=combined_lens,
)
o, _, _ = flash_mla_sparse_fwd(
q=q_flat,
kv=kv,
@@ -0,0 +1,121 @@
from __future__ import annotations
from typing import Optional, Tuple
import torch
from sglang.kernels.ops.quantization.fp8_kernel import (
sglang_per_token_group_quant_fp8,
)
from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
HEAD_DIM = 512
def interleave_query_rows(x: torch.Tensor, num_heads: int) -> torch.Tensor:
return x.view(num_heads, 32, 16, -1).transpose(0, 1).reshape(x.shape)
def pack_query(q: torch.Tensor, padded_heads: int) -> torch.Tensor:
tokens, heads, _ = q.shape
out = q.new_zeros(tokens, 32, padded_heads, 16)
out[:, :, :heads] = q.view(tokens, heads, 32, 16).transpose(1, 2)
return out.view(tokens, padded_heads, HEAD_DIM)
def interleave_output_columns(w: torch.Tensor) -> torch.Tensor:
return w.view(*w.shape[:-1], 8, 16, 32).transpose(-3, -2).reshape(w.shape)
def quantize_wo_a(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
groups, rank, _ = weight.shape
return sglang_per_token_group_quant_fp8(
weight.contiguous(),
group_size=32,
column_major_scales=True,
scale_tma_aligned=True,
scale_ue8m0=True,
masked_m=torch.full((groups,), rank, dtype=torch.int32, device=weight.device),
)
class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
mega_attn: bool = True
def _mega_paged_prefill(
self,
forward_batch: ForwardBatch,
*,
num_qo_tokens: int,
layer_id: int,
compress_ratio: int,
num_heads: int,
) -> bool:
if (
num_heads != 64
or compress_ratio not in (1, 2)
or num_qo_tokens > 1024
or not forward_batch.forward_mode.is_extend_without_speculative()
or forward_batch.seq_lens_cpu is None
or self.token_to_kv_pool.get_extra_key_bytes_per_token(layer_id) != 288
):
return False
gather_tokens = int(forward_batch.seq_lens_cpu.sum()) // compress_ratio
return gather_tokens >= max(32768, 192 * num_qo_tokens)
def _forward_mega_decode(
self,
*,
q: torch.Tensor,
swa_k_cache: torch.Tensor,
swa_page_indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
attn_sink: torch.Tensor,
swa_topk_lengths: torch.Tensor,
extra_k_cache: Optional[torch.Tensor],
extra_indices: Optional[torch.Tensor],
extra_topk_lengths: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
from sgl_kernel.flash_mla_mega import flash_mla_mega_decode
data, scale, _ = flash_mla_mega_decode(
q,
swa_k_cache,
swa_page_indices,
positions,
cos_sin_cache,
self.softmax_scale,
attn_sink=attn_sink,
topk_length=swa_topk_lengths,
extra_kv=extra_k_cache,
extra_indices=extra_indices,
extra_topk_length=extra_topk_lengths,
)
return data, scale
def _forward_mega_prefill(
self,
*,
q: torch.Tensor,
kv: torch.Tensor,
indices: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
attn_sink: torch.Tensor,
topk_length: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
data, scale, _, _ = flash_mla_mega_prefill(
q,
kv,
indices,
positions,
cos_sin_cache,
self.softmax_scale,
attn_sink=attn_sink,
topk_length=topk_length,
)
return data, scale
@@ -18,6 +18,9 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
DeepseekV4AttnBackend,
DeepseekV4MultiStepBackend,
)
from sglang.srt.layers.attention.deepseek_v4_mega_backend import (
DeepseekV4MegaAttnBackend,
)
from sglang.srt.runtime_context import (
get_buffer,
get_exec,
@@ -535,15 +538,20 @@ def is_dsv4_trtllm_attn_enabled() -> bool:
return get_exec().kernel.dsv4_attn_backend == "trtllm"
def is_dsv4_mega_attn_enabled() -> bool:
return get_exec().kernel.dsv4_attn_backend == "flashmla_mega"
def create_deepseek_v4_attn_backend(
model_runner: ModelRunner, **kwargs
) -> DeepseekV4AttnBackend:
"""Construct the DSV4 backend matching --dsv4-attn-backend."""
cls = (
DeepseekV4TrtllmAttnBackend
if is_dsv4_trtllm_attn_enabled()
else DeepseekV4AttnBackend
)
if is_dsv4_trtllm_attn_enabled():
cls = DeepseekV4TrtllmAttnBackend
elif is_dsv4_mega_attn_enabled():
cls = DeepseekV4MegaAttnBackend
else:
cls = DeepseekV4AttnBackend
return cls(model_runner, **kwargs)
@@ -168,6 +168,8 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
AITER attention."""
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega":
mode = "v41"
if get_exec().kernel.dsv4_attn_backend == "trtllm":
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
"trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts"
+145 -14
View File
@@ -638,6 +638,25 @@ def _freqs_cis_to_cos_sin(
return cos, sin
def _fp8_wo_a_einsum(
o_fp8: torch.Tensor,
o_s: torch.Tensor,
weight: torch.Tensor,
weight_scale: torch.Tensor,
recipe: Tuple[int, int, int],
) -> torch.Tensor:
import deep_gemm
groups, rank = weight.shape[:2]
output = torch.empty(
o_fp8.shape[0], groups, rank, device=o_fp8.device, dtype=torch.bfloat16
)
deep_gemm.fp8_einsum(
"bhr,hdr->bhd", (o_fp8, o_s), (weight, weight_scale), output, recipe=recipe
)
return output
def _apply_gguf_grouped_wo_a(
o: torch.Tensor,
qweight: torch.Tensor,
@@ -755,6 +774,23 @@ bcg_deepseek_v4_attention_with_output = eager_on_graph(True)(
)
def deepseek_v4_mega_attention_with_output(layer, query, output, attn_sink) -> None:
forward_batch = get_tc_piecewise_forward_context().forward_batch
real_num_tokens = forward_batch.global_num_token_non_padded_cpu
if real_num_tokens:
output[:real_num_tokens].copy_(
layer._mega_core(
query[:real_num_tokens], attn_sink, forward_batch, get_attn_backend()
)
)
output[real_num_tokens:].zero_()
bcg_deepseek_v4_mega_attention_with_output = eager_on_graph(True)(
deepseek_v4_mega_attention_with_output
)
def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
# The compressor and prefill indexer sync with the host, like the attention.
forward_batch = get_tc_piecewise_forward_context().forward_batch
@@ -994,7 +1030,7 @@ class MqaAttentionBase(nn.Module):
else 0
)
)
freqs_cis = precompute_freqs_cis(
self.rope_kwargs = dict(
dim=self.qk_rope_head_dim,
seqlen=config.max_position_embeddings,
original_seq_len=original_seq_len,
@@ -1003,6 +1039,7 @@ class MqaAttentionBase(nn.Module):
beta_fast=scaling.get("beta_fast", 32),
beta_slow=scaling.get("beta_slow", 1),
)
freqs_cis = precompute_freqs_cis(**self.rope_kwargs)
self.register_buffer("freqs_cis", freqs_cis, persistent=False)
self.freqs_cis: torch.Tensor
@@ -1101,6 +1138,7 @@ class MqaAttentionBase(nn.Module):
class MQALayer(MqaAttentionBase):
is_dsv41: bool = False
use_mega_attention: bool = False
def __init__(
self,
@@ -1237,6 +1275,21 @@ class MQALayer(MqaAttentionBase):
prefix=add_prefix("attn_mqa", prefix),
)
self.use_mega_attention = (
self.is_dsv41
and _is_cuda
and get_exec().kernel.dsv4_attn_backend == "flashmla_mega"
)
self.mega_cos_sin_cache = None
if self.use_mega_attention:
from sglang.kernels.ops.attention.deepseek_v4_rope import (
precompute_cos_sin_table,
)
self.mega_cos_sin_cache = precompute_cos_sin_table(**self.rope_kwargs)
self.mega_wo_a = None
self.mega_wq_b_permuted = False
self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self)
# Static eligibility; token count and wo_b's output format are checked
# in forward, after weights have loaded.
@@ -1264,6 +1317,77 @@ class MQALayer(MqaAttentionBase):
self.indexer.compressor.fp4_sin = self.sin_cache[:, 0, 0, :]
return result
def setup_mega_attention(self) -> None:
if not self.use_mega_attention:
return
from sglang.srt.layers.attention.deepseek_v4_mega_backend import (
interleave_output_columns,
interleave_query_rows,
quantize_wo_a,
)
self.mega_wq_b_permuted = (
self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8()
)
if self.mega_wq_b_permuted:
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
param.data.copy_(interleave_query_rows(param.data, self.n_local_heads))
self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b)
wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1)
self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a))
def _wq_b_uses_flashinfer_mxfp8(self) -> bool:
method = self.wq_b.quant_method
backend = getattr(method, "mxfp8_dense_backend", None)
return bool(
getattr(method, "use_mxfp8", False)
and backend is not None
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
)
def _forward_mega(
self,
q: torch.Tensor,
kernel_num_heads: int,
attn_sink: torch.Tensor,
forward_batch: ForwardBatch,
attn_backend,
) -> torch.Tensor:
from sglang.srt.layers.attention.deepseek_v4_mega_backend import pack_query
if forward_batch.forward_mode.is_idle():
return q.new_zeros(q.shape[0], self.n_local_groups, self.o_lora_rank)
if not self.mega_wq_b_permuted:
q = pack_query(q, kernel_num_heads)
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
o = q.new_empty((q.shape[0], self.n_local_groups, self.o_lora_rank))
bcg_deepseek_v4_mega_attention_with_output(self, q, o, attn_sink)
return o
return self._mega_core(q, attn_sink, forward_batch, attn_backend)
def _mega_core(
self,
q: torch.Tensor,
attn_sink: torch.Tensor,
forward_batch: ForwardBatch,
attn_backend,
) -> torch.Tensor:
data, scale = attn_backend.forward(
q=q,
k=q,
v=q,
layer=self.attn_mqa,
forward_batch=forward_batch,
compress_ratio=self.compress_ratio,
attn_sink=attn_sink,
save_kv_cache=False,
cos_sin_cache=self.mega_cos_sin_cache,
)
groups = self.n_local_groups
return _fp8_wo_a_einsum(
data[:, :groups], scale[:, :groups], *self.mega_wo_a, recipe=(1, 1, 32)
)
def _get_npu_rope_position_cache(
self,
forward_batch: ForwardBatch,
@@ -1374,6 +1498,8 @@ class MQALayer(MqaAttentionBase):
) -> torch.Tensor:
q, _ = self.wq_b(q)
q = q.view(-1, self.n_local_heads, self.head_dim)
if self.use_mega_attention:
return q
if not self.q_head_norm:
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
@@ -2278,7 +2404,9 @@ class MQALayer(MqaAttentionBase):
tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None
k_nope, k_rope = None, None
if (
if self.use_mega_attention:
kernel_num_heads = 64 if self.n_local_heads <= 64 else 128
elif (
self.is_dsv41
and not self.q_head_norm
and get_token_to_kv_pool().uniform_fp8
@@ -2403,6 +2531,12 @@ class MQALayer(MqaAttentionBase):
kv, k_rope, forward_batch, torch.cuda.current_stream()
)
if self.use_mega_attention:
o = self._forward_mega(
q, kernel_num_heads, attn_sink, forward_batch, attn_backend
)
return self._project_wo_b(o, defer_all_reduce)
# save_kv_cache = kv is not None selects who writes the ring. When kv is
# None the store was already fused into _forward_prepare* (decode) or
# done inline, so the backend skips its own store_cache; pass `q` as a
@@ -2567,8 +2701,6 @@ class MQALayer(MqaAttentionBase):
self.wo_a.weight_scale_inv.data,
)
elif self.wo_a_fp8:
import deep_gemm
T, G, D = o.shape
R = self.o_lora_rank
if _FP8_WO_A_UE8M0:
@@ -2586,18 +2718,13 @@ class MQALayer(MqaAttentionBase):
o_fp8 = o_fp8.view(T, G, D)
o_s = o_s.view(T, G, -1)
recipe = (1, 128, 128)
output = torch.empty(T, G, R, device=o.device, dtype=torch.bfloat16)
deep_gemm.fp8_einsum(
"bhr,hdr->bhd",
(o_fp8, o_s),
(
self.wo_a.weight.view(G, R, D),
self.wo_a.weight_scale_inv.data,
),
output,
o = _fp8_wo_a_einsum(
o_fp8,
o_s,
self.wo_a.weight.view(G, R, D),
self.wo_a.weight_scale_inv.data,
recipe=recipe,
)
o = output
else:
wo_a_weight = getattr(self.wo_a, "weight", None)
if wo_a_weight is not None:
@@ -4477,6 +4604,10 @@ class DeepseekV4ForCausalLM(nn.Module):
if self.wo_a_fp8:
self._setup_fp8_wo_a_scales(is_nextn)
for module in self.modules():
if isinstance(module, MQALayer):
module.setup_mega_attention()
if is_nextn:
return
for layer_id in range(self.model.start_layer, self.model.end_layer):