[Perf] deepseek-v4.1: Integrate FlashMLA MegaAttention with NVFP4 KV (#43922)
This commit is contained in:
@@ -78,7 +78,7 @@ dependencies = [
|
||||
"setproctitle",
|
||||
"sgl-deep-ep==0.1.2.post1",
|
||||
"sgl-deep-gemm==0.2.2",
|
||||
"sglang-kernel==0.4.9",
|
||||
"sglang-kernel==0.4.10",
|
||||
"smg-grpc-servicer>=0.9.0",
|
||||
"soundfile==0.13.1",
|
||||
"tiktoken",
|
||||
|
||||
@@ -68,6 +68,22 @@ def precompute_freqs_cis(
|
||||
return freqs_cis
|
||||
|
||||
|
||||
@lru_cache(2)
|
||||
def precompute_cos_sin_table(
|
||||
dim,
|
||||
seqlen,
|
||||
original_seq_len,
|
||||
base,
|
||||
factor,
|
||||
beta_fast,
|
||||
beta_slow,
|
||||
) -> torch.Tensor:
|
||||
freqs_cis = precompute_freqs_cis(
|
||||
dim, seqlen, original_seq_len, base, factor, beta_fast, beta_slow
|
||||
)
|
||||
return torch.view_as_real(freqs_cis).transpose(1, 2).reshape(seqlen, -1)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def apply_rotary_emb_triton_kernel(
|
||||
x_ptr,
|
||||
|
||||
@@ -166,6 +166,8 @@ def mega_moe_pre_dispatch(
|
||||
shared_x_sf: Optional[torch.Tensor] = None,
|
||||
shared_block_m: int = 0,
|
||||
) -> None:
|
||||
if x.shape[0] == 0 and x.stride(0) != x.shape[1]:
|
||||
x = x.new_empty(x.shape)
|
||||
# shared_block_m must match the consuming MoE's paged scale layout
|
||||
module = _jit_mega_moe_pre_dispatch_module(
|
||||
quant_group_size, shared_x_sf is not None
|
||||
|
||||
@@ -26,6 +26,23 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_dsv4_attention_backend(server_args: ServerArgs) -> str:
|
||||
from sglang.srt.utils.common import is_sm100_supported
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
layout = envs.SGLANG_DSV4_KV_LAYOUT
|
||||
if (
|
||||
model_config_of(server_args).hf_config.model_type != "deepseek_v41"
|
||||
or cfg.device != "cuda"
|
||||
or not is_sm100_supported()
|
||||
or cfg.kv_cache_dtype != "fp8_e4m3"
|
||||
or cfg.disaggregation_mode != "null"
|
||||
or (layout.is_set() and layout.get().lower() not in ("auto", "v41"))
|
||||
):
|
||||
return "flashmla"
|
||||
return "flashmla_mega"
|
||||
|
||||
|
||||
def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None:
|
||||
"""Apply DeepSeek V4 environment defaults, request limits, and validation."""
|
||||
cfg = resolving_view(server_args)
|
||||
@@ -48,6 +65,22 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None
|
||||
|
||||
run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype)
|
||||
|
||||
if cfg.dsv4_attn_backend == "auto":
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"apply_deepseek_v4_defaults",
|
||||
dsv4_attn_backend=_default_dsv4_attention_backend(server_args),
|
||||
)
|
||||
|
||||
if cfg.dsv4_attn_backend == "flashmla_mega":
|
||||
from sglang.srt.utils.common import is_sm100_supported
|
||||
|
||||
assert (
|
||||
model_config_of(server_args).hf_config.model_type == "deepseek_v41"
|
||||
and cfg.device == "cuda"
|
||||
and is_sm100_supported()
|
||||
), "--dsv4-attn-backend flashmla_mega requires DeepSeek-V4.1 on SM100/SM103."
|
||||
|
||||
if cfg.dsv4_attn_backend == "trtllm":
|
||||
from sglang.srt.utils.common import is_sm100_supported
|
||||
|
||||
|
||||
@@ -316,12 +316,15 @@ class ExecKernel(msgspec.Struct):
|
||||
str,
|
||||
Arg(
|
||||
help="DeepSeek V4 attention backend. 'auto' (default) resolves to "
|
||||
"'flashmla'. 'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) "
|
||||
"'flashmla_mega' where it is supported and 'flashmla' otherwise. "
|
||||
"'trtllm' (opt-in, SM100/SM103 with FP8 KV cache) "
|
||||
"switches the SWA/compressed KV pools to a "
|
||||
"uniform 512-dim FP8 layout and runs decode and sparse prefill "
|
||||
"through the flashinfer trtllm-gen sparse MLA kernel. The backend "
|
||||
"choice is shared by prefill and decode.",
|
||||
choices=["auto", "flashmla", "trtllm"],
|
||||
"through the flashinfer trtllm-gen sparse MLA kernel. "
|
||||
"'flashmla_mega' (DeepSeek-V4.1 on SM100/SM103) runs the fused "
|
||||
"Q-RoPE + sparse attention + inverse-RoPE + MXFP8 FlashMLA kernel. "
|
||||
"The backend choice is shared by prefill and decode.",
|
||||
choices=["auto", "flashmla", "flashmla_mega", "trtllm"],
|
||||
resolvable=True,
|
||||
),
|
||||
] = "auto"
|
||||
|
||||
@@ -993,9 +993,6 @@ class DSV4Metadata:
|
||||
# Captured with the late layers on a tail block (decoder SWA bounded replay
|
||||
# inside the prefill CUDA graph); replay rebuilds the tail metadata iff set.
|
||||
prefill_graph_tail: bool = False
|
||||
# Captured with the late layers on a tail block (decoder SWA bounded replay
|
||||
# inside the prefill CUDA graph); replay rebuilds the tail metadata iff set.
|
||||
prefill_graph_tail: bool = False
|
||||
|
||||
@property
|
||||
def core_metadata(self) -> DSV4AttnMetadata:
|
||||
@@ -1113,6 +1110,7 @@ class DeepseekV4AttnBackend(
|
||||
folds_encoder_swa_replay: bool = True
|
||||
needs_cpu_seq_lens: bool = False
|
||||
trtllm_attn: bool = False
|
||||
mega_attn: bool = False
|
||||
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
# Breakable-graph verify rereads shared state across segments.
|
||||
@@ -3581,6 +3579,7 @@ class DeepseekV4AttnBackend(
|
||||
# to True. Paged attention under active CP needs the page indices.
|
||||
return (
|
||||
self.trtllm_attn
|
||||
or self.mega_attn
|
||||
or self.tail_forward_metadata is not None
|
||||
or self.forward_metadata.late_layer_tail is not None
|
||||
or self.token_to_kv_pool.request_window is not None
|
||||
@@ -3657,8 +3656,9 @@ class DeepseekV4AttnBackend(
|
||||
compress_ratio: Literal[0, 1, 2, 4, 128],
|
||||
save_kv_cache: bool = True,
|
||||
attn_sink: Optional[torch.Tensor] = None,
|
||||
cos_sin_cache: Optional[torch.Tensor] = None,
|
||||
**_,
|
||||
) -> torch.Tensor:
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
if self.mtp_enabled and forward_batch.forward_mode.is_idle():
|
||||
return q.new_empty(
|
||||
(q.shape[0], q.shape[1], layer.v_head_dim),
|
||||
@@ -3757,10 +3757,19 @@ class DeepseekV4AttnBackend(
|
||||
f"{extra_indices.shape=}'s last dimension is not aligned to 64"
|
||||
)
|
||||
|
||||
use_mega_paged = cos_sin_cache is not None and self._mega_paged_prefill(
|
||||
forward_batch,
|
||||
num_qo_tokens=q.shape[0],
|
||||
layer_id=layer_id,
|
||||
compress_ratio=compress_ratio,
|
||||
num_heads=q.shape[-2],
|
||||
)
|
||||
|
||||
# The tail stays dense: its window floor lives in swa_page_indices,
|
||||
# which the sparse chunk cache ignores.
|
||||
if (
|
||||
self._use_sparse_prefill(forward_batch, num_qo_tokens=q.shape[0])
|
||||
and not use_mega_paged
|
||||
and self.forward_metadata.late_layer_tail is None
|
||||
and token_to_kv_pool.request_window is None
|
||||
):
|
||||
@@ -3782,6 +3791,23 @@ class DeepseekV4AttnBackend(
|
||||
token_to_kv_pool=token_to_kv_pool,
|
||||
core_attn_metadata=core_attn_metadata,
|
||||
attn_sink=attn_sink,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
)
|
||||
|
||||
if cos_sin_cache is not None:
|
||||
return self._forward_mega_decode(
|
||||
q=q.squeeze(1),
|
||||
swa_k_cache=swa_k_cache,
|
||||
swa_page_indices=swa_page_indices.squeeze(1),
|
||||
positions=match_num_queries(core_attn_metadata.positions, value=0),
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
attn_sink=attn_sink,
|
||||
swa_topk_lengths=swa_topk_lengths,
|
||||
extra_k_cache=extra_k_cache,
|
||||
extra_indices=None
|
||||
if extra_indices is None
|
||||
else extra_indices.squeeze(1),
|
||||
extra_topk_lengths=extra_topk_lengths,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -3889,7 +3915,8 @@ class DeepseekV4AttnBackend(
|
||||
token_to_kv_pool: DeepSeekV4TokenToKVPool,
|
||||
core_attn_metadata: DSV4AttnMetadata,
|
||||
attn_sink: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
cos_sin_cache: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Gathers each request's SWA window and c4/c128 cache into a flat bf16
|
||||
workspace that flash_mla_sparse_fwd reads through per-query rebased indices."""
|
||||
if _is_xpu:
|
||||
@@ -3948,6 +3975,17 @@ class DeepseekV4AttnBackend(
|
||||
)
|
||||
kv = workspace
|
||||
|
||||
if cos_sin_cache is not None:
|
||||
return self._forward_mega_prefill(
|
||||
q=q_flat,
|
||||
kv=kv,
|
||||
indices=combined_indices.unsqueeze(1),
|
||||
positions=core_attn_metadata.positions[: q_flat.shape[0]],
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=combined_lens,
|
||||
)
|
||||
|
||||
o, _, _ = flash_mla_sparse_fwd(
|
||||
q=q_flat,
|
||||
kv=kv,
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import (
|
||||
sglang_per_token_group_quant_fp8,
|
||||
)
|
||||
from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def interleave_query_rows(x: torch.Tensor, num_heads: int) -> torch.Tensor:
|
||||
return x.view(num_heads, 32, 16, -1).transpose(0, 1).reshape(x.shape)
|
||||
|
||||
|
||||
def pack_query(q: torch.Tensor, padded_heads: int) -> torch.Tensor:
|
||||
tokens, heads, _ = q.shape
|
||||
out = q.new_zeros(tokens, 32, padded_heads, 16)
|
||||
out[:, :, :heads] = q.view(tokens, heads, 32, 16).transpose(1, 2)
|
||||
return out.view(tokens, padded_heads, HEAD_DIM)
|
||||
|
||||
|
||||
def interleave_output_columns(w: torch.Tensor) -> torch.Tensor:
|
||||
return w.view(*w.shape[:-1], 8, 16, 32).transpose(-3, -2).reshape(w.shape)
|
||||
|
||||
|
||||
def quantize_wo_a(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
groups, rank, _ = weight.shape
|
||||
return sglang_per_token_group_quant_fp8(
|
||||
weight.contiguous(),
|
||||
group_size=32,
|
||||
column_major_scales=True,
|
||||
scale_tma_aligned=True,
|
||||
scale_ue8m0=True,
|
||||
masked_m=torch.full((groups,), rank, dtype=torch.int32, device=weight.device),
|
||||
)
|
||||
|
||||
|
||||
class DeepseekV4MegaAttnBackend(DeepseekV4AttnBackend):
|
||||
mega_attn: bool = True
|
||||
|
||||
def _mega_paged_prefill(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
*,
|
||||
num_qo_tokens: int,
|
||||
layer_id: int,
|
||||
compress_ratio: int,
|
||||
num_heads: int,
|
||||
) -> bool:
|
||||
if (
|
||||
num_heads != 64
|
||||
or compress_ratio not in (1, 2)
|
||||
or num_qo_tokens > 1024
|
||||
or not forward_batch.forward_mode.is_extend_without_speculative()
|
||||
or forward_batch.seq_lens_cpu is None
|
||||
or self.token_to_kv_pool.get_extra_key_bytes_per_token(layer_id) != 288
|
||||
):
|
||||
return False
|
||||
gather_tokens = int(forward_batch.seq_lens_cpu.sum()) // compress_ratio
|
||||
return gather_tokens >= max(32768, 192 * num_qo_tokens)
|
||||
|
||||
def _forward_mega_decode(
|
||||
self,
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
swa_k_cache: torch.Tensor,
|
||||
swa_page_indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
swa_topk_lengths: torch.Tensor,
|
||||
extra_k_cache: Optional[torch.Tensor],
|
||||
extra_indices: Optional[torch.Tensor],
|
||||
extra_topk_lengths: Optional[torch.Tensor],
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
from sgl_kernel.flash_mla_mega import flash_mla_mega_decode
|
||||
|
||||
data, scale, _ = flash_mla_mega_decode(
|
||||
q,
|
||||
swa_k_cache,
|
||||
swa_page_indices,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=swa_topk_lengths,
|
||||
extra_kv=extra_k_cache,
|
||||
extra_indices=extra_indices,
|
||||
extra_topk_length=extra_topk_lengths,
|
||||
)
|
||||
return data, scale
|
||||
|
||||
def _forward_mega_prefill(
|
||||
self,
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
kv: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
topk_length: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
from sgl_kernel.flash_mla_mega import flash_mla_mega_prefill
|
||||
|
||||
data, scale, _, _ = flash_mla_mega_prefill(
|
||||
q,
|
||||
kv,
|
||||
indices,
|
||||
positions,
|
||||
cos_sin_cache,
|
||||
self.softmax_scale,
|
||||
attn_sink=attn_sink,
|
||||
topk_length=topk_length,
|
||||
)
|
||||
return data, scale
|
||||
@@ -18,6 +18,9 @@ from sglang.srt.layers.attention.deepseek_v4_backend import (
|
||||
DeepseekV4AttnBackend,
|
||||
DeepseekV4MultiStepBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.deepseek_v4_mega_backend import (
|
||||
DeepseekV4MegaAttnBackend,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
get_buffer,
|
||||
get_exec,
|
||||
@@ -535,15 +538,20 @@ def is_dsv4_trtllm_attn_enabled() -> bool:
|
||||
return get_exec().kernel.dsv4_attn_backend == "trtllm"
|
||||
|
||||
|
||||
def is_dsv4_mega_attn_enabled() -> bool:
|
||||
return get_exec().kernel.dsv4_attn_backend == "flashmla_mega"
|
||||
|
||||
|
||||
def create_deepseek_v4_attn_backend(
|
||||
model_runner: ModelRunner, **kwargs
|
||||
) -> DeepseekV4AttnBackend:
|
||||
"""Construct the DSV4 backend matching --dsv4-attn-backend."""
|
||||
cls = (
|
||||
DeepseekV4TrtllmAttnBackend
|
||||
if is_dsv4_trtllm_attn_enabled()
|
||||
else DeepseekV4AttnBackend
|
||||
)
|
||||
if is_dsv4_trtllm_attn_enabled():
|
||||
cls = DeepseekV4TrtllmAttnBackend
|
||||
elif is_dsv4_mega_attn_enabled():
|
||||
cls = DeepseekV4MegaAttnBackend
|
||||
else:
|
||||
cls = DeepseekV4AttnBackend
|
||||
return cls(model_runner, **kwargs)
|
||||
|
||||
|
||||
|
||||
@@ -168,6 +168,8 @@ def select_dsv4_kv_layout() -> Tuple[KVLayout, Optional[str]]:
|
||||
AITER attention."""
|
||||
mode = envs.SGLANG_DSV4_KV_LAYOUT.get().lower()
|
||||
option = envs.SGLANG_DSV4_COMPRESSED_KV_LAYOUT.get().lower()
|
||||
if get_exec().kernel.dsv4_attn_backend == "flashmla_mega":
|
||||
mode = "v41"
|
||||
if get_exec().kernel.dsv4_attn_backend == "trtllm":
|
||||
assert mode in ("auto", "v4") and option in ("auto", "fp8"), (
|
||||
"trtllm uses uniform FP8 pools, not the packed V4.1 KV layouts"
|
||||
|
||||
@@ -638,6 +638,25 @@ def _freqs_cis_to_cos_sin(
|
||||
return cos, sin
|
||||
|
||||
|
||||
def _fp8_wo_a_einsum(
|
||||
o_fp8: torch.Tensor,
|
||||
o_s: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
weight_scale: torch.Tensor,
|
||||
recipe: Tuple[int, int, int],
|
||||
) -> torch.Tensor:
|
||||
import deep_gemm
|
||||
|
||||
groups, rank = weight.shape[:2]
|
||||
output = torch.empty(
|
||||
o_fp8.shape[0], groups, rank, device=o_fp8.device, dtype=torch.bfloat16
|
||||
)
|
||||
deep_gemm.fp8_einsum(
|
||||
"bhr,hdr->bhd", (o_fp8, o_s), (weight, weight_scale), output, recipe=recipe
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def _apply_gguf_grouped_wo_a(
|
||||
o: torch.Tensor,
|
||||
qweight: torch.Tensor,
|
||||
@@ -755,6 +774,23 @@ bcg_deepseek_v4_attention_with_output = eager_on_graph(True)(
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v4_mega_attention_with_output(layer, query, output, attn_sink) -> None:
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
real_num_tokens = forward_batch.global_num_token_non_padded_cpu
|
||||
if real_num_tokens:
|
||||
output[:real_num_tokens].copy_(
|
||||
layer._mega_core(
|
||||
query[:real_num_tokens], attn_sink, forward_batch, get_attn_backend()
|
||||
)
|
||||
)
|
||||
output[real_num_tokens:].zero_()
|
||||
|
||||
|
||||
bcg_deepseek_v4_mega_attention_with_output = eager_on_graph(True)(
|
||||
deepseek_v4_mega_attention_with_output
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v4_low_ratio_sources(layer, x, q_lora, positions) -> None:
|
||||
# The compressor and prefill indexer sync with the host, like the attention.
|
||||
forward_batch = get_tc_piecewise_forward_context().forward_batch
|
||||
@@ -994,7 +1030,7 @@ class MqaAttentionBase(nn.Module):
|
||||
else 0
|
||||
)
|
||||
)
|
||||
freqs_cis = precompute_freqs_cis(
|
||||
self.rope_kwargs = dict(
|
||||
dim=self.qk_rope_head_dim,
|
||||
seqlen=config.max_position_embeddings,
|
||||
original_seq_len=original_seq_len,
|
||||
@@ -1003,6 +1039,7 @@ class MqaAttentionBase(nn.Module):
|
||||
beta_fast=scaling.get("beta_fast", 32),
|
||||
beta_slow=scaling.get("beta_slow", 1),
|
||||
)
|
||||
freqs_cis = precompute_freqs_cis(**self.rope_kwargs)
|
||||
self.register_buffer("freqs_cis", freqs_cis, persistent=False)
|
||||
self.freqs_cis: torch.Tensor
|
||||
|
||||
@@ -1101,6 +1138,7 @@ class MqaAttentionBase(nn.Module):
|
||||
|
||||
class MQALayer(MqaAttentionBase):
|
||||
is_dsv41: bool = False
|
||||
use_mega_attention: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1237,6 +1275,21 @@ class MQALayer(MqaAttentionBase):
|
||||
prefix=add_prefix("attn_mqa", prefix),
|
||||
)
|
||||
|
||||
self.use_mega_attention = (
|
||||
self.is_dsv41
|
||||
and _is_cuda
|
||||
and get_exec().kernel.dsv4_attn_backend == "flashmla_mega"
|
||||
)
|
||||
self.mega_cos_sin_cache = None
|
||||
if self.use_mega_attention:
|
||||
from sglang.kernels.ops.attention.deepseek_v4_rope import (
|
||||
precompute_cos_sin_table,
|
||||
)
|
||||
|
||||
self.mega_cos_sin_cache = precompute_cos_sin_table(**self.rope_kwargs)
|
||||
self.mega_wo_a = None
|
||||
self.mega_wq_b_permuted = False
|
||||
|
||||
self.use_fused_qk_norm_rope = _is_hip and _hip.use_fused_qk_norm_rope(self)
|
||||
# Static eligibility; token count and wo_b's output format are checked
|
||||
# in forward, after weights have loaded.
|
||||
@@ -1264,6 +1317,77 @@ class MQALayer(MqaAttentionBase):
|
||||
self.indexer.compressor.fp4_sin = self.sin_cache[:, 0, 0, :]
|
||||
return result
|
||||
|
||||
def setup_mega_attention(self) -> None:
|
||||
if not self.use_mega_attention:
|
||||
return
|
||||
from sglang.srt.layers.attention.deepseek_v4_mega_backend import (
|
||||
interleave_output_columns,
|
||||
interleave_query_rows,
|
||||
quantize_wo_a,
|
||||
)
|
||||
|
||||
self.mega_wq_b_permuted = (
|
||||
self.n_local_heads in (64, 128) and self._wq_b_uses_flashinfer_mxfp8()
|
||||
)
|
||||
if self.mega_wq_b_permuted:
|
||||
for param in (self.wq_b.weight, self.wq_b.weight_scale_inv):
|
||||
param.data.copy_(interleave_query_rows(param.data, self.n_local_heads))
|
||||
self.wq_b.quant_method._process_mxfp8_linear_weight_scale(self.wq_b)
|
||||
wo_a = self.wo_a.weight.view(self.n_local_groups, self.o_lora_rank, -1)
|
||||
self.mega_wo_a = quantize_wo_a(interleave_output_columns(wo_a))
|
||||
|
||||
def _wq_b_uses_flashinfer_mxfp8(self) -> bool:
|
||||
method = self.wq_b.quant_method
|
||||
backend = getattr(method, "mxfp8_dense_backend", None)
|
||||
return bool(
|
||||
getattr(method, "use_mxfp8", False)
|
||||
and backend is not None
|
||||
and (backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl())
|
||||
)
|
||||
|
||||
def _forward_mega(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kernel_num_heads: int,
|
||||
attn_sink: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
attn_backend,
|
||||
) -> torch.Tensor:
|
||||
from sglang.srt.layers.attention.deepseek_v4_mega_backend import pack_query
|
||||
|
||||
if forward_batch.forward_mode.is_idle():
|
||||
return q.new_zeros(q.shape[0], self.n_local_groups, self.o_lora_rank)
|
||||
if not self.mega_wq_b_permuted:
|
||||
q = pack_query(q, kernel_num_heads)
|
||||
if forward_batch.forward_mode.is_extend() and is_in_breakable_cuda_graph():
|
||||
o = q.new_empty((q.shape[0], self.n_local_groups, self.o_lora_rank))
|
||||
bcg_deepseek_v4_mega_attention_with_output(self, q, o, attn_sink)
|
||||
return o
|
||||
return self._mega_core(q, attn_sink, forward_batch, attn_backend)
|
||||
|
||||
def _mega_core(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
attn_sink: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
attn_backend,
|
||||
) -> torch.Tensor:
|
||||
data, scale = attn_backend.forward(
|
||||
q=q,
|
||||
k=q,
|
||||
v=q,
|
||||
layer=self.attn_mqa,
|
||||
forward_batch=forward_batch,
|
||||
compress_ratio=self.compress_ratio,
|
||||
attn_sink=attn_sink,
|
||||
save_kv_cache=False,
|
||||
cos_sin_cache=self.mega_cos_sin_cache,
|
||||
)
|
||||
groups = self.n_local_groups
|
||||
return _fp8_wo_a_einsum(
|
||||
data[:, :groups], scale[:, :groups], *self.mega_wo_a, recipe=(1, 1, 32)
|
||||
)
|
||||
|
||||
def _get_npu_rope_position_cache(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
@@ -1374,6 +1498,8 @@ class MQALayer(MqaAttentionBase):
|
||||
) -> torch.Tensor:
|
||||
q, _ = self.wq_b(q)
|
||||
q = q.view(-1, self.n_local_heads, self.head_dim)
|
||||
if self.use_mega_attention:
|
||||
return q
|
||||
if not self.q_head_norm:
|
||||
if q_out is not None and q_out.dtype == torch.float8_e4m3fn:
|
||||
fused_q_norm_rope(q, q_out, None, self.freqs_cis, positions)
|
||||
@@ -2278,7 +2404,9 @@ class MQALayer(MqaAttentionBase):
|
||||
|
||||
tp_slice, q_padded, q_out, q_rope = slice(None), None, None, None
|
||||
k_nope, k_rope = None, None
|
||||
if (
|
||||
if self.use_mega_attention:
|
||||
kernel_num_heads = 64 if self.n_local_heads <= 64 else 128
|
||||
elif (
|
||||
self.is_dsv41
|
||||
and not self.q_head_norm
|
||||
and get_token_to_kv_pool().uniform_fp8
|
||||
@@ -2403,6 +2531,12 @@ class MQALayer(MqaAttentionBase):
|
||||
kv, k_rope, forward_batch, torch.cuda.current_stream()
|
||||
)
|
||||
|
||||
if self.use_mega_attention:
|
||||
o = self._forward_mega(
|
||||
q, kernel_num_heads, attn_sink, forward_batch, attn_backend
|
||||
)
|
||||
return self._project_wo_b(o, defer_all_reduce)
|
||||
|
||||
# save_kv_cache = kv is not None selects who writes the ring. When kv is
|
||||
# None the store was already fused into _forward_prepare* (decode) or
|
||||
# done inline, so the backend skips its own store_cache; pass `q` as a
|
||||
@@ -2567,8 +2701,6 @@ class MQALayer(MqaAttentionBase):
|
||||
self.wo_a.weight_scale_inv.data,
|
||||
)
|
||||
elif self.wo_a_fp8:
|
||||
import deep_gemm
|
||||
|
||||
T, G, D = o.shape
|
||||
R = self.o_lora_rank
|
||||
if _FP8_WO_A_UE8M0:
|
||||
@@ -2586,18 +2718,13 @@ class MQALayer(MqaAttentionBase):
|
||||
o_fp8 = o_fp8.view(T, G, D)
|
||||
o_s = o_s.view(T, G, -1)
|
||||
recipe = (1, 128, 128)
|
||||
output = torch.empty(T, G, R, device=o.device, dtype=torch.bfloat16)
|
||||
deep_gemm.fp8_einsum(
|
||||
"bhr,hdr->bhd",
|
||||
(o_fp8, o_s),
|
||||
(
|
||||
self.wo_a.weight.view(G, R, D),
|
||||
self.wo_a.weight_scale_inv.data,
|
||||
),
|
||||
output,
|
||||
o = _fp8_wo_a_einsum(
|
||||
o_fp8,
|
||||
o_s,
|
||||
self.wo_a.weight.view(G, R, D),
|
||||
self.wo_a.weight_scale_inv.data,
|
||||
recipe=recipe,
|
||||
)
|
||||
o = output
|
||||
else:
|
||||
wo_a_weight = getattr(self.wo_a, "weight", None)
|
||||
if wo_a_weight is not None:
|
||||
@@ -4477,6 +4604,10 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
if self.wo_a_fp8:
|
||||
self._setup_fp8_wo_a_scales(is_nextn)
|
||||
|
||||
for module in self.modules():
|
||||
if isinstance(module, MQALayer):
|
||||
module.setup_mega_attention()
|
||||
|
||||
if is_nextn:
|
||||
return
|
||||
for layer_id in range(self.model.start_layer, self.model.end_layer):
|
||||
|
||||
Reference in New Issue
Block a user