[Chore] attention: Remove the tokenspeed_mla backend and dependency (#44803)

This commit is contained in:
Chunan Zeng
2026-10-20 18:50:05 -07:00
committed by GitHub
parent c0d18c0254
commit 93088901fb
40 changed files with 269 additions and 1014 deletions
-1
View File
@@ -83,7 +83,6 @@ dependencies = [
"tilelang==0.1.12",
"timm==1.0.16",
"tokenizers==0.23.2",
"tokenspeed_mla==0.1.8",
"torch==2.14.1",
"torch_memory_saver>=0.0.9.post1",
"torchaudio==2.11.0",
@@ -182,7 +182,7 @@ def handle_attention_backend_compatibility(server_args: Any):
# fallback stay below.
run_post_process_pass(server_args, _mla_backend_page_constraints)
# The TRT-LLM / tokenspeed MLA kv-dtype validations moved to the
# The TRT-LLM MLA kv-dtype validations moved to the
# resolution pipeline (arg_groups/overrides.py:
# _mla_kv_cache_dtype_checks), invoked here at their legacy slot.
+1 -2
View File
@@ -84,7 +84,7 @@ ATTENTION_BACKEND_CHOICES = [
"flashmla",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"tokenspeed_mla", # Deprecated alias for "cutedsl_mla"
"trtllm_mha",
"hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), Hopper (SM90) only, requires --page-size 64
"minicpm_flashattn",
@@ -234,7 +234,6 @@ CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
"flashmla",
"cutedsl_mla",
"trtllm_mla",
"tokenspeed_mla",
]
add_chunked_prefix_cache_attention_backend = (
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append
@@ -481,7 +481,6 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
@@ -749,7 +748,7 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
# The trtllm_mla family builds its DCP block table through the pool's v2p
# gather (create_mla_kv_page_table_for_dcp), so it speaks the same
# two-stage contract as flashinfer.
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= dcp_allowed, (
@@ -806,7 +805,6 @@ def handle_page_major_kv_layout(server_args: Any):
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
}
elif model_config.has_asymmetric_kv:
@@ -351,14 +351,10 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
return "triton"
def _dspark_verify_on_decode_backend(
backend: Optional[str], q_len: int, kv_cache_dtype: Optional[str]
) -> bool:
def _dspark_verify_on_decode_backend(backend: Optional[str], q_len: int) -> bool:
"""Whether the MLA decode backend can serve a q_len-wide target verify."""
if backend == "trtllm_mla":
return True
if backend == "tokenspeed_mla":
return kv_cache_dtype == "fp8_e4m3" and q_len <= 8
if backend == "cutedsl_mla":
# cute-dsl monolithic MLA decode folds the verify tokens into the head
# dim (fold_sq), so it serves any DSPARK verify width. Needs flashinfer
@@ -92,21 +92,6 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
prefill_attention_backend="trtllm_mla",
decode_attention_backend="cutedsl_mla",
)
elif decode_backend == "tokenspeed_mla":
logger.info(
"Kimi-K3 DCP overrides attention backends: "
f"prefill={prefill_backend!r}, decode={decode_backend!r} -> "
"'tokenspeed_mla'."
)
logger.info(
"Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: "
f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'."
)
overrides.update(
prefill_attention_backend="tokenspeed_mla",
decode_attention_backend="tokenspeed_mla",
kv_cache_dtype="fp8_e4m3",
)
elif decode_backend == "aiter":
_require_kimi_k3_aiter_gluon_dcp_support()
# Override prefill backend to aiter by default
@@ -123,7 +108,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
)
else:
raise AssertionError(
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla', 'tokenspeed_mla' or 'aiter', got {decode_backend!r}."
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'aiter', got {decode_backend!r}."
)
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in (
@@ -167,7 +152,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# that still needs declaring -- else verify stays on the prefill backend,
# whose host-side plan (flashinfer by default) forces a per-step D2H.
_, backend = attention_backends_of(cfg)
if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype):
if _dspark_verify_on_decode_backend(backend, q_len):
overrides["speculative_attention_mode"] = "decode"
logger.info(
"Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend "
+1 -23
View File
@@ -1143,15 +1143,6 @@ def _mla_backend_page_constraints(view: Any) -> dict:
f"TensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
)
page_size = 64
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if page_size not in [32, 64]:
logger.warning(
f"tokenspeed_mla only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
)
page_size = 64
if (
view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla"
@@ -1193,7 +1184,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
"""Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
"""Validate resolved KV-cache dtype for the TRTLLM MLA backend."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
@@ -1206,19 +1197,6 @@ def _mla_kv_cache_dtype_checks(view: Any) -> dict:
raise ValueError(
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, bf16, or auto."
)
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if not get_platform().is_blackwell:
raise ValueError(
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
)
if view.kv_cache_dtype not in ["fp8_e4m3"]:
raise ValueError(
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
f"got {view.kv_cache_dtype}."
)
return {}
+23 -4
View File
@@ -293,7 +293,8 @@ def handle_deprecated_args(server_args: Any):
"_handle_deprecated_args",
enable_flashinfer_allreduce_fusion=False,
)
# Deprecated attention-backend alias: "compressed" -> "dsv4".
# Deprecated attention-backend aliases.
aliases = {"compressed": "dsv4", "tokenspeed_mla": "cutedsl_mla"}
renamed = {}
for attr in (
"attention_backend",
@@ -301,12 +302,30 @@ def handle_deprecated_args(server_args: Any):
"prefill_attention_backend",
"speculative_draft_attention_backend",
):
if getattr(server_args, attr, None) == "compressed":
old = getattr(server_args, attr, None)
if old in aliases:
logger.warning(
"--%s=compressed is deprecated; use 'dsv4' instead.",
"--%s=%s is deprecated; use '%s' instead.",
attr.replace("_", "-"),
old,
aliases[old],
)
renamed[attr] = "dsv4"
renamed[attr] = aliases[old]
if renamed.get("prefill_attention_backend") == "cutedsl_mla":
# cutedsl_mla is decode-only.
renamed["prefill_attention_backend"] = "trtllm_mla"
if "tokenspeed_mla" in (cfg.attention_backend, cfg.decode_attention_backend):
# Keep what tokenspeed_mla implied: FP8 KV, verify on the decode kernel.
if cfg.kv_cache_dtype != "fp8_e4m3":
logger.warning(
"tokenspeed_mla required FP8 KV; setting --kv-cache-dtype=fp8_e4m3."
)
renamed["kv_cache_dtype"] = "fp8_e4m3"
if (
cfg.speculative_algorithm is not None
and cfg.speculative_attention_mode == "prefill"
):
renamed["speculative_attention_mode"] = "decode"
if renamed:
declare_resolution(server_args, "_handle_deprecated_args", **renamed)
@@ -2,6 +2,8 @@ import logging
import warnings
from typing import TYPE_CHECKING
import torch
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
@@ -52,8 +54,6 @@ def register_attention_backend(name):
@register_attention_backend("flashinfer")
def create_flashinfer_backend(runner):
import torch
if not runner.use_mla_backend:
from sglang.srt.layers.attention.flashinfer_backend import (
FlashInferAttnBackend,
@@ -95,31 +95,26 @@ def create_llada2_cfg_flashinfer_backend(runner):
def create_trtllm_mla_backend(runner):
if not runner.use_mla_backend:
raise ValueError("trtllm_mla backend can only be used with MLA models.")
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if get_parallel().dcp_enabled and get_spec().speculative_algorithm is not None:
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if decode_backend == "trtllm_mla":
raise ValueError(
"trtllm_mla cannot serve decode context parallelism with speculative "
"decoding: it does not forward the cyclic DCP metadata to its decode "
"kernel and returns no rank-local LSE for the cross-rank merge. "
"Select cutedsl_mla or tokenspeed_mla."
"Select cutedsl_mla."
)
if decode_backend == "cutedsl_mla" and runner.kv_cache_dtype == torch.float8_e4m3fn:
from sglang.srt.layers.attention.cutedsl_mla_backend import (
CuteDslMLAPrefillBackend,
)
return CuteDslMLAPrefillBackend(runner)
from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend
return TRTLLMMLABackend(runner)
@register_attention_backend("tokenspeed_mla")
def create_tokenspeed_mla_backend(runner):
if not runner.use_mla_backend:
raise ValueError("tokenspeed_mla backend can only be used with MLA models.")
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLABackend,
)
return TokenspeedMLABackend(runner)
@register_attention_backend("cutedsl_mla")
def create_cutedsl_mla_backend(runner):
if not runner.use_mla_backend:
@@ -21,12 +21,17 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
mla_kv_pack_quantize_fp8,
)
from sglang.kernels.ops.attention.utils import (
concat_mla_absorb_q_general,
mla_quantize_and_rope_for_fp8,
mla_quantize_without_rope_for_fp8,
)
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
from sglang.srt.environ import envs
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
@@ -40,11 +45,13 @@ from sglang.srt.utils import is_flashinfer_available
if is_flashinfer_available():
import flashinfer
import flashinfer.rope as _flashinfer_rope
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
logger = logging.getLogger(__name__)
@@ -296,3 +303,139 @@ class CuteDslMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
kv_indptr_buf=self.kv_indptr[i],
q_indptr_decode_buf=self.q_indptr_decode,
)
class CuteDslMLAPrefillBackend(TRTLLMMLABackend):
"""trtllm_mla prefill for cutedsl_mla deployments with FP8 KV.
The model-side MHA hooks below build FP8 Q/K/V with fused RoPE + quantize
and write the FP8 KV cache directly, so the ragged prefill kernel consumes
them without the per-tensor casts of the BF16 path.
"""
@staticmethod
def _fused_rope_fp8_quantize(
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
cos_sin_cache: Optional[torch.Tensor],
positions: torch.Tensor,
is_neox: bool,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
FP8 quantize and the packed layout, only the rotation is dropped.
"""
num_heads = q_nope.shape[1]
seq_len = q_nope.shape[0]
q_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=q_nope.device,
)
k_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=k_nope.device,
)
if seq_len == 0:
return q_fp8, k_fp8
# Broadcast the shared latent k_pe across heads — RoPE is position-only
# so per-head outputs are identical, and the cache write below reuses
# head 0.
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
else:
k_pe_expanded = k_pe
if cos_sin_cache is None:
# NoPE layer: quantize straight into the packed buffers.
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
return q_fp8, k_fp8
_flashinfer_rope.mla_rope_quantize_fp8(
q_rope=q_pe,
k_rope=k_pe_expanded,
q_nope=q_nope,
k_nope=k_nope,
cos_sin_cache=cos_sin_cache,
pos_ids=positions,
is_neox=is_neox,
quantize_dtype=torch.float8_e4m3fn,
q_rope_out=q_fp8[..., qk_nope_head_dim:],
k_rope_out=k_fp8[..., qk_nope_head_dim:],
q_nope_out=q_fp8[..., :qk_nope_head_dim],
k_nope_out=k_fp8[..., :qk_nope_head_dim],
quant_scale_q=1.0,
quant_scale_kv=1.0,
enable_pdl=is_arch_support_pdl(),
)
return q_fp8, k_fp8
def prepare_prefill_qkv(
self,
*,
q: torch.Tensor,
q_pe: torch.Tensor,
kv_a: torch.Tensor,
k_pe: torch.Tensor,
positions: torch.Tensor,
layer: DeepseekV2AttentionMLA,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
kv = layer.kv_b_proj(kv_a)[0]
kv = kv.view(
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
)
k_nope = kv[..., : layer.qk_nope_head_dim]
v_bf16 = kv[..., layer.qk_nope_head_dim :]
q_nope = q[..., : layer.qk_nope_head_dim]
rotary_emb = layer.rotary_emb
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
q_nope=q_nope,
q_pe=q_pe,
k_nope=k_nope,
k_pe=k_pe,
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
positions=positions,
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
qk_nope_head_dim=layer.qk_nope_head_dim,
qk_rope_head_dim=layer.qk_rope_head_dim,
)
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
# k_pe is shared across heads (RoPE is position-only), so head 0
# reproduces the original [tokens, 1, qk_rope] latent layout.
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
self.token_to_kv_pool.set_mla_kv_buffer(
layer.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a_fp8.unsqueeze(1),
k_pe_fp8,
)
return q_fp8, k_fp8, v_fp8
def pack_prefix_chunk_kv(
self,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
v: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
strided ``v`` into contig FP8 V in a single kernel.
"""
return mla_kv_pack_quantize_fp8(
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
)
@@ -1,516 +0,0 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import annotations
"""Attention backend for the tokenspeed-mla CuTe DSL kernels on Blackwell.
Subclasses :class:`TRTLLMMLABackend` to share its MLA data preparation, prefill
plumbing, and DCP metadata (rank-local KV lengths and page table). The decode
forward lives here because the TokenSpeed decode kernel natively accepts CP
rank/world metadata and returns the partial log-sum-exp needed by the cross-rank
merge.
"""
import logging
from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
mla_kv_pack_quantize_fp8,
)
from sglang.kernels.ops.attention.utils import (
mla_quantize_and_rope_for_fp8,
mla_quantize_without_rope_for_fp8,
)
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
TRTLLMMLAMultiStepDraftBackend,
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.runtime_context import (
get_parallel,
get_resources,
max_speculative_num_draft_tokens,
)
from sglang.srt.utils import is_flashinfer_available, is_tokenspeed_mla_available
if is_flashinfer_available():
import flashinfer.rope as _flashinfer_rope
if is_tokenspeed_mla_available():
import tokenspeed_mla
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
logger = logging.getLogger(__name__)
# Workspace upper bound for tokenspeed_mla_decode:
# num_sms * num_heads * max_q_len * (kv_lora_rank + 1) * sizeof(float32)
# MAX_Q_LEN=8 covers EAGLE3 num_draft_tokens=4 plus headroom. Larger
# speculative widths grow the bound at backend initialization.
_TOKENSPEED_MAX_Q_LEN = 8
def _get_tokenspeed_workspace(
device: torch.device,
num_heads: int,
kv_lora_rank: int,
max_q_len: int = _TOKENSPEED_MAX_Q_LEN,
) -> torch.Tensor:
# DCP target verification gathers Q to the full head count before launching
# TokenSpeed; size for that launch shape, not the rank-local head count.
num_heads *= get_parallel().attn_dcp_size
max_q_len = max(max_q_len, _TOKENSPEED_MAX_Q_LEN)
needed = (
tokenspeed_mla.get_num_sm(device)
* num_heads
* max_q_len
* (kv_lora_rank + 1)
* 4
)
buffers = get_resources().buffers
key = f"tokenspeed_mla_workspace:{device}"
existing = buffers.get(key)
if existing is None or existing.numel() < needed:
buffers[key] = torch.empty(needed, dtype=torch.int8, device=device)
return buffers[key]
# TODO(Qiaolin-Yu): Merge this attention backend into trtllm_mla_backend.py
# once the same CuteDSL kernels in flashinfer_trtllm are stable
# and there is no performance gap compared to this backend.
class TokenspeedMLABackend(TRTLLMMLABackend):
"""tokenspeed-mla CuTe DSL attention backend (Blackwell SM100, FP8 KV)."""
def __init__(
self,
model_runner: ModelRunner,
skip_prefill: bool = False,
kv_indptr_buf: Optional[torch.Tensor] = None,
q_indptr_decode_buf: Optional[torch.Tensor] = None,
):
super().__init__(
model_runner,
skip_prefill,
kv_indptr_buf,
q_indptr_decode_buf,
)
if self.data_type != torch.float8_e4m3fn:
raise ValueError(
"tokenspeed_mla backend requires --kv-cache-dtype fp8_e4m3, "
f"got data_type={self.data_type}."
)
if self.page_size not in (32, 64):
raise ValueError(
"tokenspeed_mla backend requires page_size in {32, 64}, "
f"got page_size={self.page_size}."
)
self._tokenspeed_workspace: Optional[torch.Tensor] = None
if is_tokenspeed_mla_available():
self._tokenspeed_workspace = _get_tokenspeed_workspace(
self.device,
self.num_q_heads,
self.kv_lora_rank,
max_q_len=(max_speculative_num_draft_tokens() or 1),
)
# Pre-JIT the prefill kernel variants. Each cute.compile takes 1-2
# min; without warm-up the first request trips the 300 s scheduler
# watchdog.
_compile_prefill_kernel = tokenspeed_mla.mla_prefill._compile_prefill_kernel
_compiled_kernels = tokenspeed_mla.mla_prefill._compiled_kernels
head_dim_qk = self.qk_nope_head_dim + self.qk_rope_head_dim
enable_ex2_emulation = tokenspeed_mla.mla_prefill._enable_ex2_emulation()
use_pdl = is_arch_support_pdl()
for is_causal in (True, False):
for return_lse in (True, False):
# Non-causal is only entered from the chunked-prefix
# branch, which always asks for the LSE.
if is_causal is False and return_lse is False:
continue
# Runtime feeds fp8_e4m3fn q/k/v
config = (
torch.float8_e4m3fn,
head_dim_qk,
self.v_head_dim,
is_causal,
return_lse,
use_pdl,
enable_ex2_emulation,
)
if config in _compiled_kernels:
continue
_compiled_kernels[config] = _compile_prefill_kernel(
torch.float8_e4m3fn,
head_dim_qk,
self.v_head_dim,
is_causal,
return_lse,
use_pdl=use_pdl,
enable_ex2_emulation=enable_ex2_emulation,
)
@staticmethod
def _fused_rope_fp8_quantize(
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
cos_sin_cache: Optional[torch.Tensor],
positions: torch.Tensor,
is_neox: bool,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
FP8 quantize and the packed layout, only the rotation is dropped.
"""
num_heads = q_nope.shape[1]
seq_len = q_nope.shape[0]
q_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=q_nope.device,
)
k_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=k_nope.device,
)
if seq_len == 0:
return q_fp8, k_fp8
# Broadcast the shared latent k_pe across heads — RoPE is position-only
# so per-head outputs are identical, and the cache write below reuses
# head 0.
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
else:
k_pe_expanded = k_pe
if cos_sin_cache is None:
# NoPE layer: quantize straight into the packed buffers.
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
return q_fp8, k_fp8
_flashinfer_rope.mla_rope_quantize_fp8(
q_rope=q_pe,
k_rope=k_pe_expanded,
q_nope=q_nope,
k_nope=k_nope,
cos_sin_cache=cos_sin_cache,
pos_ids=positions,
is_neox=is_neox,
quantize_dtype=torch.float8_e4m3fn,
q_rope_out=q_fp8[..., qk_nope_head_dim:],
k_rope_out=k_fp8[..., qk_nope_head_dim:],
q_nope_out=q_fp8[..., :qk_nope_head_dim],
k_nope_out=k_fp8[..., :qk_nope_head_dim],
quant_scale_q=1.0,
quant_scale_kv=1.0,
enable_pdl=is_arch_support_pdl(),
)
return q_fp8, k_fp8
def prepare_prefill_qkv(
self,
*,
q: torch.Tensor,
q_pe: torch.Tensor,
kv_a: torch.Tensor,
k_pe: torch.Tensor,
positions: torch.Tensor,
layer: DeepseekV2AttentionMLA,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
kv = layer.kv_b_proj(kv_a)[0]
kv = kv.view(
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
)
k_nope = kv[..., : layer.qk_nope_head_dim]
v_bf16 = kv[..., layer.qk_nope_head_dim :]
q_nope = q[..., : layer.qk_nope_head_dim]
rotary_emb = layer.rotary_emb
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
q_nope=q_nope,
q_pe=q_pe,
k_nope=k_nope,
k_pe=k_pe,
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
positions=positions,
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
qk_nope_head_dim=layer.qk_nope_head_dim,
qk_rope_head_dim=layer.qk_rope_head_dim,
)
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
# k_pe is shared across heads (RoPE is position-only), so head 0
# reproduces the original [tokens, 1, qk_rope] latent layout.
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
self.token_to_kv_pool.set_mla_kv_buffer(
layer.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a_fp8.unsqueeze(1),
k_pe_fp8,
)
return q_fp8, k_fp8, v_fp8
def pack_prefix_chunk_kv(
self,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
v: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
strided ``v`` into contig FP8 V in a single kernel.
"""
return mla_kv_pack_quantize_fp8(
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
)
def _run_decode_kernel(
self,
query: torch.Tensor,
kv_cache: torch.Tensor,
block_tables: torch.Tensor,
seq_lens: torch.Tensor,
max_seq_len: int,
layer: RadixAttention,
*,
causal_seqs: Optional[torch.Tensor] = None,
cp_world: int = 1,
cp_rank: int = 0,
return_lse: bool = False,
):
k_scale = getattr(layer, "k_scale_float", None)
if k_scale is None:
k_scale = 1.0
softmax_scale = float(layer.scaling) * float(k_scale)
output_scale = float(k_scale)
seq_lens_i32 = (
seq_lens if seq_lens.dtype == torch.int32 else seq_lens.to(torch.int32)
)
return tokenspeed_mla.tokenspeed_mla_decode(
query=query,
kv_cache=kv_cache,
workspace_buffer=self._tokenspeed_workspace,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
block_tables=block_tables,
seq_lens=seq_lens_i32,
max_seq_len=int(max_seq_len),
softmax_scale=softmax_scale,
output_scale=output_scale,
enable_pdl=is_arch_support_pdl(),
return_lse=return_lse,
causal_seqs=causal_seqs,
cp_world=cp_world,
cp_rank=cp_rank,
)
def forward_decode(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
layer: RadixAttention,
forward_batch: ForwardBatch,
save_kv_cache: bool = True,
q_rope: Optional[torch.Tensor] = None,
k_rope: Optional[torch.Tensor] = None,
cos_sin_cache: Optional[torch.Tensor] = None,
is_neox: Optional[bool] = False,
llama_4_scaling: Optional[torch.Tensor] = None,
):
parallel = get_parallel()
if parallel.dcp_enabled and get_in_autotune_dummy_run():
return self._dummy_dcp_decode_for_autotune(q, layer)
if not parallel.dcp_enabled:
return super().forward_decode(
q,
k,
v,
layer,
forward_batch,
save_kv_cache,
q_rope,
k_rope,
cos_sin_cache,
is_neox,
llama_4_scaling,
)
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
q, k, k_rope = mla_quantize_without_rope_for_fp8(
q, q_rope, k.squeeze(1), k_rope.squeeze(1)
)
else:
q, k, k_rope = mla_quantize_and_rope_for_fp8(
q,
q_rope,
k.squeeze(1),
k_rope.squeeze(1),
forward_batch.positions,
cos_sin_cache,
is_neox,
self.kv_lora_rank,
self.qk_rope_head_dim,
)
if save_kv_cache:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
query = q.view(-1, layer.tp_q_head_num, layer.head_dim)
if llama_4_scaling is not None:
query = (query.to(self.q_data_type) * llama_4_scaling).to(self.data_type)
if query.dim() == 3:
query = query.unsqueeze(1)
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
metadata = (
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
or self.forward_decode_metadata
)
metadata_batch_size = getattr(metadata, "batch_size", None)
if (
metadata_batch_size is not None
and metadata_batch_size < forward_batch.batch_size
):
self.init_forward_metadata(forward_batch)
metadata = forward_batch.decode_trtllm_mla_metadata
global_seq_lens = forward_batch.seq_lens[: forward_batch.batch_size]
local_seq_lens = self._get_dcp_local_seq_lens(global_seq_lens)
raw_out, lse = self._run_decode_kernel(
query=query,
kv_cache=kv_cache,
block_tables=metadata.block_kv_indices,
seq_lens=local_seq_lens,
max_seq_len=metadata.max_seq_len_k,
layer=layer,
causal_seqs=global_seq_lens,
cp_world=parallel.dcp_size,
cp_rank=parallel.dcp_rank,
return_lse=True,
)
output = raw_out.view(-1, layer.tp_q_head_num, layer.v_head_dim)
lse = lse.view(-1, layer.tp_q_head_num)
fixup_zero_kv_rows(
output,
lse,
local_seq_lens,
self.q_indptr_decode[: forward_batch.batch_size + 1],
1,
)
return output.flatten(1), lse
def _run_prefill_kernel(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
layer: RadixAttention,
batch_size: int,
cum_seq_lens_q: torch.Tensor,
max_q_len: int,
seq_lens_kv: torch.Tensor,
cum_seq_lens_kv: torch.Tensor,
max_kv_len: int,
is_causal: bool,
return_lse: bool,
out_buffer: torch.Tensor,
o_sf_scale: float = 1.0,
): # Q/K/V arrive already in FP8 via the model-side fused path
# (prepare_prefill_qkv / pack_prefix_chunk_kv); no quantize here.
# Hybrid MLA models resolve the model-side hook through the outer
# HybridLinearAttnBackend, so their fallback MHA path can pass V as a
# last-dimension slice of kv_b_proj (stride(-2) > size(-1)). The
# TokenSpeed prefill kernel requires dense Q/K/V layouts even though
# the public wrapper accepts arbitrary torch tensors.
q = q.contiguous()
k = k.contiguous()
v = v.contiguous()
return tokenspeed_mla.tokenspeed_mla_prefill(
query=q,
key=k,
value=v,
seq_lens=seq_lens_kv,
cum_seq_lens=cum_seq_lens_kv,
max_seq_len=int(max_kv_len),
batch_size=int(batch_size),
softmax_scale=float(layer.scaling),
is_causal=is_causal,
return_lse=return_lse,
cum_seq_lens_q=cum_seq_lens_q,
max_seq_len_q=int(max_q_len),
enable_pdl=is_arch_support_pdl(),
)
class TokenspeedMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
"""Multi-step draft backend for tokenspeed_mla used by EAGLE."""
def __init__(
self, model_runner: ModelRunner, topk: int, speculative_num_steps: int
):
super().__init__(model_runner, topk, speculative_num_steps)
# Parent populates self.attn_backends with TRT-LLM instances; replace
# them with tokenspeed instances sharing the parent's index buffers.
for i in range(self.speculative_num_steps - 1):
self.attn_backends[i] = TokenspeedMLABackend(
model_runner,
skip_prefill=True,
kv_indptr_buf=self.kv_indptr[i],
q_indptr_decode_buf=self.q_indptr_decode,
)
@@ -1017,8 +1017,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
raise NotImplementedError(
"trtllm_mla cannot forward a global causal bound to its decode "
"kernel, which is required for DCP with q_len > 1 (speculative "
"target-verify / draft-extend); select cutedsl_mla or "
"tokenspeed_mla for a DCP speculative run"
"target-verify / draft-extend); select cutedsl_mla "
"for a DCP speculative run"
)
# Scale computation for TRTLLM MLA kernel BMM1 operation:
@@ -1073,10 +1073,11 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
o_sf_scale: float = 1.0,
):
"""Hook for subclasses to swap the ragged prefill kernel. Q/K/V arrive
in model-native dtype; subclasses do any kernel-specific quantization.
in model-native dtype, or already FP8 from a model-side prefill hook;
subclasses do any kernel-specific quantization.
Returns the output tensor or (output, lse) if return_lse."""
q_scale = k_scale = v_scale = 1.0
if self.data_type == torch.float8_e4m3fn:
if self.data_type == torch.float8_e4m3fn and q.dtype != torch.float8_e4m3fn:
q, k, v, k_scale, v_scale = _quantize_fp8_qkv(q, k, v, layer)
return flashinfer.prefill.trtllm_ragged_attention_deepseek(
query=q,
@@ -181,12 +181,6 @@ def handle_attention_trtllm_mla(attn, forward_batch):
return _dispatch_mla_subtype(attn, forward_batch)
def handle_attention_tokenspeed_mla(attn, forward_batch):
# tokenspeed_mla shares the trtllm_mla dispatch pattern: pure prefill goes
# via MHA chunked KV (TRT-LLM ragged), spec decode / decode goes via MLA.
return handle_attention_trtllm_mla(attn, forward_batch)
def handle_attention_aiter(attn, forward_batch):
# During PCG/BCG capture on ROCm, aiter fp8 MLA prefill has no capture
# kernels; route through the MHA path (radix_attention swaps attn_mqa for
@@ -268,7 +262,6 @@ AttentionBackendRegistry.register("fa3", handle_attention_fa3)
AttentionBackendRegistry.register("flashmla", handle_attention_flashmla)
AttentionBackendRegistry.register("fa4", handle_attention_fa4)
AttentionBackendRegistry.register("trtllm_mla", handle_attention_trtllm_mla)
AttentionBackendRegistry.register("tokenspeed_mla", handle_attention_tokenspeed_mla)
AttentionBackendRegistry.register("aiter", handle_attention_aiter)
AttentionBackendRegistry.register("dsa", handle_attention_dsa)
AttentionBackendRegistry.register(
@@ -942,8 +942,7 @@ class DeepseekMLAForwardMixin:
return (
self.rotary_emb is not None
and self.current_attention_backend
in ("trtllm_mla", "tokenspeed_mla", "cutedsl_mla")
and self.current_attention_backend in ("trtllm_mla", "cutedsl_mla")
and (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
@@ -70,7 +70,6 @@ FORWARD_ABSORB_CORE_ATTENTION_BACKENDS = [
"flashinfer",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"ascend",
"intel_xpu",
]
@@ -105,7 +105,6 @@ class DraftBackendFactory:
"trtllm_mha": self._create_trtllm_mha_decode_backend,
"trtllm_mla": self._create_trtllm_mla_decode_backend,
"cutedsl_mla": self._create_cutedsl_mla_decode_backend,
"tokenspeed_mla": self._create_tokenspeed_mla_decode_backend,
"dsa": self._create_dsa_decode_backend,
"nsa": self._create_dsa_decode_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_decode_backend,
@@ -137,7 +136,6 @@ class DraftBackendFactory:
"trtllm_mla": self._create_trtllm_mla_prefill_backend,
# cute-dsl MLA only supports decode; draft-extend falls back to trtllm-gen.
"cutedsl_mla": self._create_trtllm_mla_prefill_backend,
"tokenspeed_mla": self._create_tokenspeed_mla_prefill_backend,
"dsa": self._create_dsa_prefill_backend,
"nsa": self._create_dsa_prefill_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_prefill_backend,
@@ -394,23 +392,6 @@ class DraftBackendFactory:
),
)
def _create_tokenspeed_mla_decode_backend(self):
if not self.draft_model_runner.use_mla_backend:
raise ValueError(
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLAMultiStepDraftBackend,
)
return (
"tokenspeed_mla",
TokenspeedMLAMultiStepDraftBackend(
self.draft_model_runner, self.topk, self.speculative_num_steps
),
)
def _create_ascend_decode_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnMultiStepDraftBackend,
@@ -541,21 +522,6 @@ class DraftBackendFactory:
TRTLLMMLABackend(self.draft_model_runner, skip_prefill=False),
)
def _create_tokenspeed_mla_prefill_backend(self):
if not self.draft_model_runner.use_mla_backend:
raise ValueError(
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLABackend,
)
return (
"tokenspeed_mla",
TokenspeedMLABackend(self.draft_model_runner, skip_prefill=False),
)
def _create_ascend_prefill_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnBackend,
@@ -27,7 +27,6 @@ from sglang.srt.layers.attention.qwen_sparse_attn_backend import (
QwenSparseAttnBackend,
QwenSparseMultiStepDraftBackend,
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend
from sglang.srt.layers.attention.trtllm_mla_backend import (
@@ -536,7 +535,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
TritonAttnBackend,
TRTLLMMLABackend,
TRTLLMHAAttnBackend,
TokenspeedMLABackend,
FlashInferAttnBackend,
QwenSparseAttnBackend,
]
-12
View File
@@ -394,18 +394,6 @@ def is_flashinfer_available():
return importlib.util.find_spec("flashinfer") is not None and is_cuda()
@lru_cache(maxsize=1)
def is_tokenspeed_mla_available():
"""
Check whether the tokenspeed_mla CuTe DSL kernels are available.
Only available on NVIDIA Blackwell (SM100) at the moment.
"""
return (
importlib.util.find_spec("tokenspeed_mla") is not None
and is_blackwell_supported()
)
def is_nvidia_cublas_version_ge_12_9():
"""
temporary fix for issue #11272 (cublas 12.9+)
@@ -228,8 +228,8 @@ class MockMLAModelRunner(ModelRunner):
self.device = device
self.dtype = dtype
# `kv_cache_dtype` is the dtype the *storage* uses. For FP8 KV
# cache (the production deployment dtype for tokenspeed_mla and
# some trtllm_mla configs), the pool stores quantized bytes
# cache (the production deployment dtype for some trtllm_mla
# configs), the pool stores quantized bytes
# while the model still projects K/V in bf16; `set_mla_kv_buffer`
# does the BF16->FP8 cast on the way in.
self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype