diff --git a/python/pyproject.toml b/python/pyproject.toml index c1cff46ed9..899c600d24 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -78,7 +78,7 @@ dependencies = [ "setproctitle", "sgl-deep-ep==0.1.2.post1", "sgl-deep-gemm==0.2.2", - "sglang-kernel==0.4.9", + "sglang-kernel==0.4.10", "smg-grpc-servicer>=0.9.0", "soundfile==0.13.1", "tiktoken", diff --git a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py b/python/sglang/kernels/ops/attention/deepseek_v4_rope.py index fb0ee2c79d..da61ee69f2 100644 --- a/python/sglang/kernels/ops/attention/deepseek_v4_rope.py +++ b/python/sglang/kernels/ops/attention/deepseek_v4_rope.py @@ -68,6 +68,22 @@ def precompute_freqs_cis( return freqs_cis +@lru_cache(2) +def precompute_cos_sin_table( + dim, + seqlen, + original_seq_len, + base, + factor, + beta_fast, + beta_slow, +) -> torch.Tensor: + freqs_cis = precompute_freqs_cis( + dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow + ) + return torch.view_as_real(freqs_cis).transpose(1, 2).reshape(seqlen, -1) + + @triton.jit def apply_rotary_emb_triton_kernel( x_ptr, diff --git a/python/sglang/kernels/ops/moe/dsv4.py b/python/sglang/kernels/ops/moe/dsv4.py index a9cf7a34f7..7c9fa71f77 100644 --- a/python/sglang/kernels/ops/moe/dsv4.py +++ b/python/sglang/kernels/ops/moe/dsv4.py @@ -166,6 +166,8 @@ def mega_moe_pre_dispatch( shared_x_sf: Optional[torch.Tensor] = None, shared_block_m: int = 0, ) -> None: + if x.shape[0] == 0 and x.stride(0) != x.shape[1]: + x = x.new_empty(x.shape) # shared_block_m must match the consuming MoE's paged scale layout module = _jit_mega_moe_pre_dispatch_module( quant_group_size, shared_x_sf is not None diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 37a22e5ef1..4c0b31094b 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -26,6 +26,23 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _default_dsv4_attention_backend(server_args: ServerArgs) -> str: + from sglang.srt.utils.common import is_sm100_supported + + cfg = resolving_view(server_args) + layout = envs.SGLANG_DSV4_KV_LAYOUT + if ( + model_config_of(server_args).hf_config.model_type != "deepseek_v41" + or cfg.device != "cuda" + or not is_sm100_supported() + or cfg.kv_cache_dtype != "fp8_e4m3" + or cfg.disaggregation_mode != "null" + or (layout.is_set() and layout.get().lower() not in ("auto", "v41")) + ): + return "flashmla" + return "flashmla_mega" + + def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None: """Apply DeepSeek V4 environment defaults, request limits, and validation.""" cfg = resolving_view(server_args) @@ -48,6 +65,22 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype) + if cfg.dsv4_attn_backend == "auto": + declare_resolution( + server_args, + "apply_deepseek_v4_defaults", + dsv4_attn_backend=_default_dsv4_attention_backend(server_args), + ) + + if cfg.dsv4_attn_backend == "flashmla_mega": + from sglang.srt.utils.common import is_sm100_supported + + assert ( + model_config_of(server_args).hf_config.model_type == "deepseek_v41" + and cfg.device == "cuda" + and is_sm100_supported() + ), "--dsv4-attn-backend flashmla_mega requires DeepSeek-V4.1 on SM100/SM103." + if cfg.dsv4_attn_backend == "trtllm": from sglang.srt.utils.common import is_sm100_supported diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index ac0e6db5b9..994d9ccd9f 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -316,12 +316,15 @@ class ExecKernel(msgspec.Struct): str, Arg( help="DeepSeek V4 attention backend. 'auto' (default) resolves to " - "'flashmla'. 'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) " + "'flashmla_mega' where it is supported and 'flashmla' otherwise. " + "'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) " "switches the SWA/compressed KV pools to a " "uniform 512-dim FP8 layout and runs decode and sparse prefill " - "through the flashinfer trtllm-gen sparse MLA kernel. The backend " - "choice is shared by prefill and decode.", - choices=["auto", "flashmla", "trtllm"], + "through the flashinfer trtllm-gen sparse MLA kernel. " + "'flashmla_mega' (DeepSeek-V4.1 on SM100/SM103) runs the fused " + "Q-RoPE + sparse attention + inverse-RoPE + MXFP8 FlashMLA kernel. " + "The backend choice is shared by prefill and decode.", + choices=["auto", "flashmla", "flashmla_mega", "trtllm"], resolvable=True, ), ] = "auto" diff --git a/python/sglang/srt/layers/attention/deepseek_v4_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_backend.py index a608ea3935..0517587277 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_backend.py @@ -993,9 +993,6 @@ class DSV4Metadata: # Captured with the late layers on a tail block (decoder SWA bounded replay # inside the prefill CUDA graph); replay rebuilds the tail metadata iff set. prefill_graph_tail: bool = False - # Captured with the late layers on a tail block (decoder SWA bounded replay - # inside the prefill CUDA graph); replay rebuilds the tail metadata iff set. - prefill_graph_tail: bool = False @property def core_metadata(self) -> DSV4AttnMetadata: @@ -1113,6 +1110,7 @@ class DeepseekV4AttnBackend( folds_encoder_swa_replay: bool = True needs_cpu_seq_lens: bool = False trtllm_attn: bool = False + mega_attn: bool = False def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: # Breakable-graph verify rereads shared state across segments. @@ -3581,6 +3579,7 @@ class DeepseekV4AttnBackend( # to True. Paged attention under active CP needs the page indices. return ( self.trtllm_attn + or self.mega_attn or self.tail_forward_metadata is not None or self.forward_metadata.late_layer_tail is not None or self.token_to_kv_pool.request_window is not None @@ -3657,8 +3656,9 @@ class DeepseekV4AttnBackend( compress_ratio: Literal[0, 1, 2, 4, 128], save_kv_cache: bool = True, attn_sink: Optional[torch.Tensor] = None, + cos_sin_cache: Optional[torch.Tensor] = None, **_, - ) -> torch.Tensor: + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: if self.mtp_enabled and forward_batch.forward_mode.is_idle(): return q.new_empty( (q.shape[0], q.shape[1], layer.v_head_dim), @@ -3757,10 +3757,19 @@ class DeepseekV4AttnBackend( f"{extra_indices.shape=}'s last dimension is not aligned to 64" ) + use_mega_paged = cos_sin_cache is not None and self._mega_paged_prefill( + forward_batch, + num_qo_tokens=q.shape[0], + layer_id=layer_id, + compress_ratio=compress_ratio, + num_heads=q.shape[-2], + ) + # The tail stays dense: its window floor lives in swa_page_indices, # which the sparse chunk cache ignores. if ( self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0]) + and not use_mega_paged and self.forward_metadata.late_layer_tail is None and token_to_kv_pool.request_window is None ): @@ -3782,6 +3791,23 @@ class DeepseekV4AttnBackend( token_to_kv_pool=token_to_kv_pool, core_attn_metadata=core_attn_metadata, attn_sink=attn_sink, + cos_sin_cache=cos_sin_cache, + ) + + if cos_sin_cache is not None: + return self._forward_mega_decode( + q=q.squeeze(1), + swa_k_cache=swa_k_cache, + swa_page_indices=swa_page_indices.squeeze(1), + positions=match_num_queries(core_attn_metadata.positions, value=0), + cos_sin_cache=cos_sin_cache, + attn_sink=attn_sink, + swa_topk_lengths=swa_topk_lengths, + extra_k_cache=extra_k_cache, + extra_indices=None + if extra_indices is None + else extra_indices.squeeze(1), + extra_topk_lengths=extra_topk_lengths, ) if ( @@ -3889,7 +3915,8 @@ class DeepseekV4AttnBackend( token_to_kv_pool: DeepSeekV4TokenToKVPool, core_attn_metadata: DSV4AttnMetadata, attn_sink: torch.Tensor, - ) -> torch.Tensor: + cos_sin_cache: Optional[torch.Tensor] = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: """Gathers each request's SWA window and c4/c128 cache into a flat bf16 workspace that flash_mla_sparse_fwd reads through per-query rebased indices.""" if _is_xpu: @@ -3948,6 +3975,17 @@ class DeepseekV4AttnBackend( ) kv = workspace + if cos_sin_cache is not None: + return self._forward_mega_prefill( + q=q_flat, + kv=kv, + indices=combined_indices.unsqueeze(1), + positions=core_attn_metadata.positions[: q_flat.shape[0]], + cos_sin_cache=cos_sin_cache, + attn_sink=attn_sink, + topk_length=combined_lens, + ) + o, _, _ = flash_mla_sparse_fwd( q=q_flat, kv=kv, diff --git a/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py new file mode 100644 index 0000000000..25f1bd227b --- /dev/null +++ b/python/sglang/srt/layers/attention/deepseek_v4_mega_backend.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from typing import Optional, Tuple + +import torch + +from sglang.kernels.ops.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8, +) +from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend +from sglang.srt.model_executor.forward_batch_info import ForwardBatch + +HEAD_DIM = 512 + + +def interleave_query_rows(x: torch.Tensor, num_heads: int) -> torch.Tensor: + return x.view(num_heads, 32, 16, -1).transpose(0, 1).reshape(x.shape) + + +def pack_query(q: torch.Tensor, padded_heads: int) -> torch.Tensor: + tokens, heads, _ = q.shape + out = q.new_zeros(tokens, 32, padded_heads, 16) + out[:, :, :heads] = q.view(tokens, heads, 32, 16).transpose(1, 2) + return out.view(tokens, padded_heads, HEAD_DIM) + + +def interleave_output_columns(w: torch.Tensor) -> torch.Tensor: + return w.view(*w.shape[:-1], 8, 16, 32).transpose(-3, -2).reshape(w.shape) + + +def quantize_wo_a(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + groups, rank, _ = weight.shape + return sglang_per_token_group_quant_fp8( + weight.contiguous(), + group_size=32, + column_major_scales=True, + scale_tma_aligned=True, + scale_ue8m0=True, + masked_m=torch.full((groups,), rank, dtype=torch.int32, device=weight.device), + ) + + +class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend): + mega_attn: bool = True + + def _mega_paged_prefill( + self, + forward_batch: ForwardBatch, + *, + num_qo_tokens: int, + layer_id: int, + compress_ratio: int, + num_heads: int, + ) -> bool: + if ( + num_heads != 64 + or compress_ratio not in (1, 2) + or num_qo_tokens > 1024 + or not forward_batch.forward_mode.is_extend_without_speculative() + or forward_batch.seq_lens_cpu is None + or self.token_to_kv_pool.get_extra_key_bytes_per_token(layer_id) != 288 + ): + return False + gather_tokens = int(forward_batch.seq_lens_cpu.sum()) // compress_ratio + return gather_tokens >= max(32768, 192 * num_qo_tokens) + + def _forward_mega_decode( + self, + *, + q: torch.Tensor, + swa_k_cache: torch.Tensor, + swa_page_indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + attn_sink: torch.Tensor, + swa_topk_lengths: torch.Tensor, + extra_k_cache: Optional[torch.Tensor], + extra_indices: Optional[torch.Tensor], + extra_topk_lengths: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + from sgl_kernel.flash_mla_mega import flash_mla_mega_decode + + data, scale, _ = flash_mla_mega_decode( + q, + swa_k_cache, + swa_page_indices, + positions, + cos_sin_cache, + self.softmax_scale, + attn_sink=attn_sink, + topk_length=swa_topk_lengths, + extra_kv=extra_k_cache, + extra_indices=extra_indices, + extra_topk_length=extra_topk_lengths, + ) + return data, scale + + def _forward_mega_prefill( + self, + *, + q: torch.Tensor, + kv: torch.Tensor, + indices: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + attn_sink: torch.Tensor, + topk_length: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill + + data, scale, _, _ = flash_mla_mega_prefill( + q, + kv, + indices, + positions, + cos_sin_cache, + self.softmax_scale, + attn_sink=attn_sink, + topk_length=topk_length, + ) + return data, scale diff --git a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py index 37919fb479..f056ba9e49 100644 --- a/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py +++ b/python/sglang/srt/layers/attention/deepseek_v4_trtllm_backend.py @@ -18,6 +18,9 @@ from sglang.srt.layers.attention.deepseek_v4_backend import ( DeepseekV4AttnBackend, DeepseekV4MultiStepBackend, ) +from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( + DeepseekV4MegaAttnBackend, +) from sglang.srt.runtime_context import ( get_buffer, get_exec, @@ -535,15 +538,20 @@ def is_dsv4_trtllm_attn_enabled() -> bool: return get_exec().kernel.dsv4_attn_backend == "trtllm" +def is_dsv4_mega_attn_enabled() -> bool: + return get_exec().kernel.dsv4_attn_backend == "flashmla_mega" + + def create_deepseek_v4_attn_backend( model_runner: ModelRunner, **kwargs ) -> DeepseekV4AttnBackend: """Construct the DSV4 backend matching --dsv4-attn-backend.""" - cls = ( - DeepseekV4TrtllmAttnBackend - if is_dsv4_trtllm_attn_enabled() - else DeepseekV4AttnBackend - ) + if is_dsv4_trtllm_attn_enabled(): + cls = DeepseekV4TrtllmAttnBackend + elif is_dsv4_mega_attn_enabled(): + cls = DeepseekV4MegaAttnBackend + else: + cls = DeepseekV4AttnBackend return cls(model_runner, **kwargs) diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index c26c7a1eb1..054c2f4988 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -168,6 +168,8 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]: AITER attention.""" mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower() option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower() + if get_exec().kernel.dsv4_attn_backend == "flashmla_mega": + mode = "v41" if get_exec().kernel.dsv4_attn_backend == "trtllm": assert mode in ("auto", "v4") and option in ("auto", "fp8"), ( "trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts" diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index ef6df00ba2..eab5c7daee 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -638,6 +638,25 @@ def _freqs_cis_to_cos_sin( return cos, sin +def _fp8_wo_a_einsum( + o_fp8: torch.Tensor, + o_s: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + recipe: Tuple[int, int, int], +) -> torch.Tensor: + import deep_gemm + + groups, rank = weight.shape[:2] + output = torch.empty( + o_fp8.shape[0], groups, rank, device=o_fp8.device, dtype=torch.bfloat16 + ) + deep_gemm.fp8_einsum( + "bhr,hdr->bhd", (o_fp8, o_s), (weight, weight_scale), output, recipe=recipe + ) + return output + + def _apply_gguf_grouped_wo_a( o: torch.Tensor, qweight: torch.Tensor, @@ -755,6 +774,23 @@ bcg_deepseek_v4_attention_with_output = eager_on_graph(True)( ) +def deepseek_v4_mega_attention_with_output(layer, query, output, attn_sink) -> None: + forward_batch = get_tc_piecewise_forward_context().forward_batch + real_num_tokens = forward_batch.global_num_token_non_padded_cpu + if real_num_tokens: + output[:real_num_tokens].copy_( + layer._mega_core( + query[:real_num_tokens], attn_sink, forward_batch, get_attn_backend() + ) + ) + output[real_num_tokens:].zero_() + + +bcg_deepseek_v4_mega_attention_with_output = eager_on_graph(True)( + deepseek_v4_mega_attention_with_output +) + + def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None: # The compressor and prefill indexer sync with the host, like the attention. forward_batch = get_tc_piecewise_forward_context().forward_batch @@ -994,7 +1030,7 @@ class MqaAttentionBase(nn.Module): else 0 ) ) - freqs_cis = precompute_freqs_cis( + self.rope_kwargs = dict( dim=self.qk_rope_head_dim, seqlen=config.max_position_embeddings, original_seq_len=original_seq_len, @@ -1003,6 +1039,7 @@ class MqaAttentionBase(nn.Module): beta_fast=scaling.get("beta_fast", 32), beta_slow=scaling.get("beta_slow", 1), ) + freqs_cis = precompute_freqs_cis(**self.rope_kwargs) self.register_buffer("freqs_cis", freqs_cis, persistent=False) self.freqs_cis: torch.Tensor @@ -1101,6 +1138,7 @@ class MqaAttentionBase(nn.Module): class MQALayer(MqaAttentionBase): is_dsv41: bool = False + use_mega_attention: bool = False def __init__( self, @@ -1237,6 +1275,21 @@ class MQALayer(MqaAttentionBase): prefix=add_prefix("attn_mqa", prefix), ) + self.use_mega_attention = ( + self.is_dsv41 + and _is_cuda + and get_exec().kernel.dsv4_attn_backend == "flashmla_mega" + ) + self.mega_cos_sin_cache = None + if self.use_mega_attention: + from sglang.kernels.ops.attention.deepseek_v4_rope import ( + precompute_cos_sin_table, + ) + + self.mega_cos_sin_cache = precompute_cos_sin_table(**self.rope_kwargs) + self.mega_wo_a = None + self.mega_wq_b_permuted = False + self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self) # Static eligibility; token count and wo_b's output format are checked # in forward, after weights have loaded. @@ -1264,6 +1317,77 @@ class MQALayer(MqaAttentionBase): self.indexer.compressor.fp4_sin = self.sin_cache[:, 0, 0, :] return result + def setup_mega_attention(self) -> None: + if not self.use_mega_attention: + return + from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( + interleave_output_columns, + interleave_query_rows, + quantize_wo_a, + ) + + self.mega_wq_b_permuted = ( + self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8() + ) + if self.mega_wq_b_permuted: + for param in (self.wq_b.weight, self.wq_b.weight_scale_inv): + param.data.copy_(interleave_query_rows(param.data, self.n_local_heads)) + self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b) + wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1) + self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a)) + + def _wq_b_uses_flashinfer_mxfp8(self) -> bool: + method = self.wq_b.quant_method + backend = getattr(method, "mxfp8_dense_backend", None) + return bool( + getattr(method, "use_mxfp8", False) + and backend is not None + and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl()) + ) + + def _forward_mega( + self, + q: torch.Tensor, + kernel_num_heads: int, + attn_sink: torch.Tensor, + forward_batch: ForwardBatch, + attn_backend, + ) -> torch.Tensor: + from sglang.srt.layers.attention.deepseek_v4_mega_backend import pack_query + + if forward_batch.forward_mode.is_idle(): + return q.new_zeros(q.shape[0], self.n_local_groups, self.o_lora_rank) + if not self.mega_wq_b_permuted: + q = pack_query(q, kernel_num_heads) + if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph(): + o = q.new_empty((q.shape[0], self.n_local_groups, self.o_lora_rank)) + bcg_deepseek_v4_mega_attention_with_output(self, q, o, attn_sink) + return o + return self._mega_core(q, attn_sink, forward_batch, attn_backend) + + def _mega_core( + self, + q: torch.Tensor, + attn_sink: torch.Tensor, + forward_batch: ForwardBatch, + attn_backend, + ) -> torch.Tensor: + data, scale = attn_backend.forward( + q=q, + k=q, + v=q, + layer=self.attn_mqa, + forward_batch=forward_batch, + compress_ratio=self.compress_ratio, + attn_sink=attn_sink, + save_kv_cache=False, + cos_sin_cache=self.mega_cos_sin_cache, + ) + groups = self.n_local_groups + return _fp8_wo_a_einsum( + data[:, :groups], scale[:, :groups], *self.mega_wo_a, recipe=(1, 1, 32) + ) + def _get_npu_rope_position_cache( self, forward_batch: ForwardBatch, @@ -1374,6 +1498,8 @@ class MQALayer(MqaAttentionBase): ) -> torch.Tensor: q, _ = self.wq_b(q) q = q.view(-1, self.n_local_heads, self.head_dim) + if self.use_mega_attention: + return q if not self.q_head_norm: if q_out is not None and q_out.dtype == torch.float8_e4m3fn: fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions) @@ -2278,7 +2404,9 @@ class MQALayer(MqaAttentionBase): tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None k_nope, k_rope = None, None - if ( + if self.use_mega_attention: + kernel_num_heads = 64 if self.n_local_heads <= 64 else 128 + elif ( self.is_dsv41 and not self.q_head_norm and get_token_to_kv_pool().uniform_fp8 @@ -2403,6 +2531,12 @@ class MQALayer(MqaAttentionBase): kv, k_rope, forward_batch, torch.cuda.current_stream() ) + if self.use_mega_attention: + o = self._forward_mega( + q, kernel_num_heads, attn_sink, forward_batch, attn_backend + ) + return self._project_wo_b(o, defer_all_reduce) + # save_kv_cache = kv is not None selects who writes the ring. When kv is # None the store was already fused into _forward_prepare* (decode) or # done inline, so the backend skips its own store_cache; pass `q` as a @@ -2567,8 +2701,6 @@ class MQALayer(MqaAttentionBase): self.wo_a.weight_scale_inv.data, ) elif self.wo_a_fp8: - import deep_gemm - T, G, D = o.shape R = self.o_lora_rank if _FP8_WO_A_UE8M0: @@ -2586,18 +2718,13 @@ class MQALayer(MqaAttentionBase): o_fp8 = o_fp8.view(T, G, D) o_s = o_s.view(T, G, -1) recipe = (1, 128, 128) - output = torch.empty(T, G, R, device=o.device, dtype=torch.bfloat16) - deep_gemm.fp8_einsum( - "bhr,hdr->bhd", - (o_fp8, o_s), - ( - self.wo_a.weight.view(G, R, D), - self.wo_a.weight_scale_inv.data, - ), - output, + o = _fp8_wo_a_einsum( + o_fp8, + o_s, + self.wo_a.weight.view(G, R, D), + self.wo_a.weight_scale_inv.data, recipe=recipe, ) - o = output else: wo_a_weight = getattr(self.wo_a, "weight", None) if wo_a_weight is not None: @@ -4477,6 +4604,10 @@ class DeepseekV4ForCausalLM(nn.Module): if self.wo_a_fp8: self._setup_fp8_wo_a_scales(is_nextn) + for module in self.modules(): + if isinstance(module, MQALayer): + module.setup_mega_attention() + if is_nextn: return for layer_id in range(self.model.start_layer, self.model.end_layer): diff --git a/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py new file mode 100644 index 0000000000..340e95a22e --- /dev/null +++ b/test/registered/kernels/ops/attention/dsv4/test_mega_attention.py @@ -0,0 +1,162 @@ +import copy +import unittest +from unittest.mock import patch + +import torch +from sgl_kernel.flash_mla_mega import flash_mla_mega_decode + +from sglang.kernels.ops.attention.dsv4.torch_quant import quantize_k_cache_v41 +from sglang.srt.layers.attention.deepseek_v4_mega_backend import ( + interleave_output_columns, + interleave_query_rows, + pack_query, + quantize_wo_a, +) +from sglang.srt.models.deepseek_v4 import _fp8_wo_a_einsum +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kernels.deepseek_v4.dsv41_kv_quant_reference import ( + quantize_k_cache_v41_fp4, +) +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200") + + +def unpack_output(data, scale): + tokens, groups, hidden = data.shape + sf = scale.contiguous().view(torch.uint8).float() + values = data.float().view(tokens, groups, hidden // 32, 32) + values *= torch.exp2(sf - 127).unsqueeze(-1) + return ( + values.view(tokens, groups, 16, 8, 32) + .transpose(2, 3) + .reshape(tokens, groups, hidden) + ) + + +def paged_view(raw, page_size, bytes_per_token): + return raw[:, : page_size * bytes_per_token].view( + raw.shape[0], page_size, 1, bytes_per_token + ) + + +@unittest.skipUnless( + torch.cuda.is_available() and torch.cuda.get_device_capability()[0] == 10, + "MegaAttention requires SM100/SM103", +) +class TestMegaAttention(CustomTestCase): + def test_wq_b_permutation_matches_gemm_then_pack(self): + from sglang.srt.layers.quantization import fp8_utils + from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8LinearMethod + from sglang.srt.layers.quantization.mxfp8_input import Mxfp8SwizzledInput + + for backend in ( + fp8_utils.Fp8GemmRunnerBackend.FLASHINFER_CUTLASS, + fp8_utils.Fp8GemmRunnerBackend.FLASHINFER_CUTEDSL, + ): + with patch.object(fp8_utils, "FP8_GEMM_RUNNER_BACKEND", backend): + method = Fp8LinearMethod( + Fp8Config( + is_checkpoint_fp8_serialized=True, + use_mxfp8=True, + weight_block_size=[1, 32], + scale_fmt="ue8m0", + ) + ) + with self.subTest(backend=backend): + linear = torch.nn.Module() + linear.quant_method = method + linear.weight = torch.randn(64 * 512, 1280, device="cuda").to( + torch.float8_e4m3fn + ) + linear.weight_scale_inv = torch.randint( + 119, 129, (64 * 512, 40), device="cuda", dtype=torch.uint8 + ) + method._process_mxfp8_linear_weight_scale(linear) + reference = copy.deepcopy(linear) + for param in (linear.weight, linear.weight_scale_inv): + param.copy_(interleave_query_rows(param, 64)) + method._process_mxfp8_linear_weight_scale(linear) + x = torch.randn(1536, 1280, device="cuda", dtype=torch.bfloat16) + quantized = Mxfp8SwizzledInput( + *fp8_utils.flashinfer_mxfp8_quantize( + x[:3], is_sf_swizzled_layout=True, alignment=32 + ) + ) + for inputs in (x, quantized): + torch.testing.assert_close( + method.apply(linear, inputs), + pack_query( + method.apply(reference, inputs).view(-1, 64, 512), 64 + ).view(-1, 64 * 512), + rtol=0, + atol=0, + ) + + def test_wo_a_projection_matches_reference(self): + torch.manual_seed(456) + positions = torch.tensor([0, 7, 127], device="cuda", dtype=torch.int32) + angle = ( + torch.arange(256, device="cuda")[:, None] + / torch.arange(1, 33, device="cuda")[None] + ) + table = torch.cat((angle.cos(), angle.sin()), dim=-1).float() + main = paged_view( + quantize_k_cache_v41( + torch.randn(1, 128, 512, device="cuda", dtype=torch.bfloat16) + ), + 128, + 528, + ) + extra = paged_view( + quantize_k_cache_v41_fp4( + torch.randn(1, 64, 512, device="cuda", dtype=torch.bfloat16) + ), + 64, + 288, + ) + indices = torch.arange(128, device="cuda", dtype=torch.int32).repeat(3, 1) + extra_indices = torch.arange(64, device="cuda", dtype=torch.int32).repeat(3, 1) + for heads in (16, 64): + with self.subTest(heads=heads): + q = torch.randn(3, heads, 512, device="cuda", dtype=torch.bfloat16) + data, scale, _ = flash_mla_mega_decode( + pack_query(q, 64), + main, + indices, + positions, + table, + 512**-0.5, + attn_sink=torch.randn(64, device="cuda", dtype=torch.float32), + topk_length=torch.tensor( + [128, 96, 1], device="cuda", dtype=torch.int32 + ), + extra_kv=extra, + extra_indices=extra_indices, + extra_topk_length=torch.tensor( + [64, 32, 0], device="cuda", dtype=torch.int32 + ), + ) + groups = heads // 8 + weight = torch.randn( + groups, 64, 4096, device="cuda", dtype=torch.bfloat16 + ) + actual = _fp8_wo_a_einsum( + data[:, :groups], + scale[:, :groups], + *quantize_wo_a(interleave_output_columns(weight)), + recipe=(1, 1, 32), + ) + expected = torch.einsum( + "tgd,grd->tgr", + unpack_output(data, scale)[:, :groups], + weight.float(), + ) + error = (actual.float() - expected).square().mean().sqrt() + self.assertLess( + error.item() / expected.square().mean().sqrt().item(), 0.04 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py b/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py index 708d4a5e82..7af9bf56c7 100644 --- a/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py +++ b/test/registered/kernels/ops/moe/test_mega_moe_pre_dispatch.py @@ -66,5 +66,30 @@ def test_mxfp8_scale_output_uses_padded_row_stride() -> None: assert torch.all(buf_topk_weights[num_tokens:] == 0) +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] < 10, + reason="Requires Blackwell GPU (sm_100+)", +) +def test_empty_mhc_view_clears_idle_dispatch() -> None: + hidden, padded_max, top_k = 2304, 8, 8 + x = torch.empty(1, 4, hidden, device="cuda", dtype=torch.bfloat16)[:0, 0] + assert x.stride(0) == 4 * hidden + buf_x = torch.zeros(padded_max, hidden, device="cuda", dtype=torch.float8_e4m3fn) + buf_x_sf = torch.zeros(padded_max, 20, device="cuda", dtype=torch.int32)[:, :18] + buf_idx = torch.full((padded_max, top_k), 42, device="cuda", dtype=torch.int64) + buf_weights = torch.ones(padded_max, top_k, device="cuda", dtype=torch.float32) + mega_moe_pre_dispatch( + x, + torch.empty(0, top_k, device="cuda", dtype=torch.int32), + torch.empty(0, top_k, device="cuda", dtype=torch.float32), + buf_x, + buf_x_sf, + buf_idx, + buf_weights, + ) + torch.testing.assert_close(buf_idx, torch.full_like(buf_idx, -1)) + torch.testing.assert_close(buf_weights, torch.zeros_like(buf_weights)) + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"]))