[Chore] attention: Remove the tokenspeed_mla backend and dependency (#44803)

This commit is contained in:
Chunan Zeng
2026-10-20 18:50:05 -07:00
committed by GitHub
parent c0d18c0254
commit 93088901fb
40 changed files with 269 additions and 1014 deletions
@@ -509,8 +509,8 @@ Named slots + two keyed-lazy registries:
side stream is `"lora_side"`. Two call sites may share a name only if their work belongs
on one stream — sharing across roles serializes intended overlap.
- `get_buffer(name, factory)` — get-or-create a named persistent buffer. Grow-only or
per-device semantics manage their `resources.buffers` entries directly (see tokenspeed /
SM120 split / Marlin workspace). Buffer names are per-backend today; do not silently
per-device semantics manage their `resources.buffers` entries directly (see SM120
split / Marlin workspace). Buffer names are per-backend today; do not silently
share.
- Singletons with manager semantics (EP dispatcher buffers, EPLB recorder/metadata, graph
memory pool) keep their owning accessors/classes as facades; only the *state* lives in a
@@ -207,7 +207,6 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request
- DCP is the only axis that shards the TP-replicated MLA KV; Low-Latency skips it.
- Leave `--dcp-comm-backend` unset (fabric-resolved: `fi_a2a` on GB200/GB300, `a2a` on B200/B300).
- No `--enable-symm-mem` under DCP (force-disabled for decode-graph correctness).
- Explicit `tokenspeed_mla` force-rewrites `--kv-cache-dtype` to fp8; the default `cutedsl_mla` serves either dtype.
- Calculator ratios run well above 1 here (`r > 1` is legal): `bfloat16` state buys admission, `fp8` KV buys context.
- Don't use EP with an a2a backend: a2a buffers reclaim the KV that DCP buys. Compose only to measure. a2a backend is set when `--moe-a2a-backend` is set.
@@ -254,15 +254,6 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>✅</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>**TokenSpeed MLA (Blackwell)**</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>32 or 64</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>✅ (required)</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>✅</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>✅</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>**FA3 (FlashAttention 3)**</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>n/a</td>
@@ -330,7 +321,6 @@ MLA page-size constraints:
- Cutlass MLA: page_size = 128.
- TRTLLM MLA: page_size ∈ &#123;32, 64&#125;.
- CuteDSL MLA: page_size ∈ &#123;32, 64&#125; (decode-only; prefill falls back to `trtllm_mla` when unset).
- TokenSpeed MLA: page_size ∈ &#123;32, 64&#125; (Blackwell SM100/SM12x only; requires `--kv-cache-dtype fp8_e4m3`).
### GDN Attention Backends
@@ -196,15 +196,18 @@ export const KimiK25Deployment = () => {
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla';
}
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
// Blackwell (B300/GB300): tokenspeed MLA attention backend
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
// Blackwell (B300/GB300): CuteDSL MLA attention backend
if (usesCutedslMla) {
cmd += ' \\\n --attention-backend cutedsl_mla';
if (speculative === 'enabled') {
cmd += ' \\\n --speculative-attention-mode decode';
}
}
// FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility
if (isAMD || usesTokenspeedMla) {
// FP8 KV cache for AMD and Blackwell memory efficiency
if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
@@ -194,13 +194,16 @@ export const KimiK26Deployment = () => {
cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla';
}
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
if (usesCutedslMla) {
cmd += ' \\\n --attention-backend cutedsl_mla';
if (speculative === 'enabled') {
cmd += ' \\\n --speculative-attention-mode decode';
}
}
if (isAMD || usesTokenspeedMla) {
if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
@@ -189,13 +189,13 @@ export const KimiK27CodeDeployment = () => {
cmd += ' \\\n --tool-call-parser kimi_k2';
}
const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
if (usesTokenspeedMla) {
cmd += ' \\\n --attention-backend tokenspeed_mla';
if (usesCutedslMla) {
cmd += ' \\\n --attention-backend cutedsl_mla';
}
if (isAMD || usesTokenspeedMla) {
if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
-1
View File
@@ -83,7 +83,6 @@ dependencies = [
"tilelang==0.1.12",
"timm==1.0.16",
"tokenizers==0.23.2",
"tokenspeed_mla==0.1.8",
"torch==2.14.1",
"torch_memory_saver>=0.0.9.post1",
"torchaudio==2.11.0",
@@ -182,7 +182,7 @@ def handle_attention_backend_compatibility(server_args: Any):
# fallback stay below.
run_post_process_pass(server_args, _mla_backend_page_constraints)
# The TRT-LLM / tokenspeed MLA kv-dtype validations moved to the
# The TRT-LLM MLA kv-dtype validations moved to the
# resolution pipeline (arg_groups/overrides.py:
# _mla_kv_cache_dtype_checks), invoked here at their legacy slot.
+1 -2
View File
@@ -84,7 +84,7 @@ ATTENTION_BACKEND_CHOICES = [
"flashmla",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"tokenspeed_mla", # Deprecated alias for "cutedsl_mla"
"trtllm_mha",
"hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), Hopper (SM90) only, requires --page-size 64
"minicpm_flashattn",
@@ -234,7 +234,6 @@ CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
"flashmla",
"cutedsl_mla",
"trtllm_mla",
"tokenspeed_mla",
]
add_chunked_prefix_cache_attention_backend = (
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append
@@ -481,7 +481,6 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
@@ -749,7 +748,7 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
# The trtllm_mla family builds its DCP block table through the pool's v2p
# gather (create_mla_kv_page_table_for_dcp), so it speaks the same
# two-stage contract as flashinfer.
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= dcp_allowed, (
@@ -806,7 +805,6 @@ def handle_page_major_kv_layout(server_args: Any):
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
}
elif model_config.has_asymmetric_kv:
@@ -351,14 +351,10 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
return "triton"
def _dspark_verify_on_decode_backend(
backend: Optional[str], q_len: int, kv_cache_dtype: Optional[str]
) -> bool:
def _dspark_verify_on_decode_backend(backend: Optional[str], q_len: int) -> bool:
"""Whether the MLA decode backend can serve a q_len-wide target verify."""
if backend == "trtllm_mla":
return True
if backend == "tokenspeed_mla":
return kv_cache_dtype == "fp8_e4m3" and q_len <= 8
if backend == "cutedsl_mla":
# cute-dsl monolithic MLA decode folds the verify tokens into the head
# dim (fold_sq), so it serves any DSPARK verify width. Needs flashinfer
@@ -92,21 +92,6 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
prefill_attention_backend="trtllm_mla",
decode_attention_backend="cutedsl_mla",
)
elif decode_backend == "tokenspeed_mla":
logger.info(
"Kimi-K3 DCP overrides attention backends: "
f"prefill={prefill_backend!r}, decode={decode_backend!r} -> "
"'tokenspeed_mla'."
)
logger.info(
"Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: "
f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'."
)
overrides.update(
prefill_attention_backend="tokenspeed_mla",
decode_attention_backend="tokenspeed_mla",
kv_cache_dtype="fp8_e4m3",
)
elif decode_backend == "aiter":
_require_kimi_k3_aiter_gluon_dcp_support()
# Override prefill backend to aiter by default
@@ -123,7 +108,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
)
else:
raise AssertionError(
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla', 'tokenspeed_mla' or 'aiter', got {decode_backend!r}."
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'aiter', got {decode_backend!r}."
)
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in (
@@ -167,7 +152,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# that still needs declaring -- else verify stays on the prefill backend,
# whose host-side plan (flashinfer by default) forces a per-step D2H.
_, backend = attention_backends_of(cfg)
if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype):
if _dspark_verify_on_decode_backend(backend, q_len):
overrides["speculative_attention_mode"] = "decode"
logger.info(
"Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend "
+1 -23
View File
@@ -1143,15 +1143,6 @@ def _mla_backend_page_constraints(view: Any) -> dict:
f"TensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
)
page_size = 64
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if page_size not in [32, 64]:
logger.warning(
f"tokenspeed_mla only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
)
page_size = 64
if (
view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla"
@@ -1193,7 +1184,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
"""Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
"""Validate resolved KV-cache dtype for the TRTLLM MLA backend."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
@@ -1206,19 +1197,6 @@ def _mla_kv_cache_dtype_checks(view: Any) -> dict:
raise ValueError(
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, bf16, or auto."
)
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if not get_platform().is_blackwell:
raise ValueError(
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
)
if view.kv_cache_dtype not in ["fp8_e4m3"]:
raise ValueError(
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
f"got {view.kv_cache_dtype}."
)
return {}
+23 -4
View File
@@ -293,7 +293,8 @@ def handle_deprecated_args(server_args: Any):
"_handle_deprecated_args",
enable_flashinfer_allreduce_fusion=False,
)
# Deprecated attention-backend alias: "compressed" -> "dsv4".
# Deprecated attention-backend aliases.
aliases = {"compressed": "dsv4", "tokenspeed_mla": "cutedsl_mla"}
renamed = {}
for attr in (
"attention_backend",
@@ -301,12 +302,30 @@ def handle_deprecated_args(server_args: Any):
"prefill_attention_backend",
"speculative_draft_attention_backend",
):
if getattr(server_args, attr, None) == "compressed":
old = getattr(server_args, attr, None)
if old in aliases:
logger.warning(
"--%s=compressed is deprecated; use 'dsv4' instead.",
"--%s=%s is deprecated; use '%s' instead.",
attr.replace("_", "-"),
old,
aliases[old],
)
renamed[attr] = "dsv4"
renamed[attr] = aliases[old]
if renamed.get("prefill_attention_backend") == "cutedsl_mla":
# cutedsl_mla is decode-only.
renamed["prefill_attention_backend"] = "trtllm_mla"
if "tokenspeed_mla" in (cfg.attention_backend, cfg.decode_attention_backend):
# Keep what tokenspeed_mla implied: FP8 KV, verify on the decode kernel.
if cfg.kv_cache_dtype != "fp8_e4m3":
logger.warning(
"tokenspeed_mla required FP8 KV; setting --kv-cache-dtype=fp8_e4m3."
)
renamed["kv_cache_dtype"] = "fp8_e4m3"
if (
cfg.speculative_algorithm is not None
and cfg.speculative_attention_mode == "prefill"
):
renamed["speculative_attention_mode"] = "decode"
if renamed:
declare_resolution(server_args, "_handle_deprecated_args", **renamed)
@@ -2,6 +2,8 @@ import logging
import warnings
from typing import TYPE_CHECKING
import torch
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
@@ -52,8 +54,6 @@ def register_attention_backend(name):
@register_attention_backend("flashinfer")
def create_flashinfer_backend(runner):
import torch
if not runner.use_mla_backend:
from sglang.srt.layers.attention.flashinfer_backend import (
FlashInferAttnBackend,
@@ -95,31 +95,26 @@ def create_llada2_cfg_flashinfer_backend(runner):
def create_trtllm_mla_backend(runner):
if not runner.use_mla_backend:
raise ValueError("trtllm_mla backend can only be used with MLA models.")
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if get_parallel().dcp_enabled and get_spec().speculative_algorithm is not None:
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if decode_backend == "trtllm_mla":
raise ValueError(
"trtllm_mla cannot serve decode context parallelism with speculative "
"decoding: it does not forward the cyclic DCP metadata to its decode "
"kernel and returns no rank-local LSE for the cross-rank merge. "
"Select cutedsl_mla or tokenspeed_mla."
"Select cutedsl_mla."
)
if decode_backend == "cutedsl_mla" and runner.kv_cache_dtype == torch.float8_e4m3fn:
from sglang.srt.layers.attention.cutedsl_mla_backend import (
CuteDslMLAPrefillBackend,
)
return CuteDslMLAPrefillBackend(runner)
from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend
return TRTLLMMLABackend(runner)
@register_attention_backend("tokenspeed_mla")
def create_tokenspeed_mla_backend(runner):
if not runner.use_mla_backend:
raise ValueError("tokenspeed_mla backend can only be used with MLA models.")
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLABackend,
)
return TokenspeedMLABackend(runner)
@register_attention_backend("cutedsl_mla")
def create_cutedsl_mla_backend(runner):
if not runner.use_mla_backend:
@@ -21,12 +21,17 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
mla_kv_pack_quantize_fp8,
)
from sglang.kernels.ops.attention.utils import (
concat_mla_absorb_q_general,
mla_quantize_and_rope_for_fp8,
mla_quantize_without_rope_for_fp8,
)
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
from sglang.srt.environ import envs
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
@@ -40,11 +45,13 @@ from sglang.srt.utils import is_flashinfer_available
if is_flashinfer_available():
import flashinfer
import flashinfer.rope as _flashinfer_rope
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
logger = logging.getLogger(__name__)
@@ -296,3 +303,139 @@ class CuteDslMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
kv_indptr_buf=self.kv_indptr[i],
q_indptr_decode_buf=self.q_indptr_decode,
)
class CuteDslMLAPrefillBackend(TRTLLMMLABackend):
"""trtllm_mla prefill for cutedsl_mla deployments with FP8 KV.
The model-side MHA hooks below build FP8 Q/K/V with fused RoPE + quantize
and write the FP8 KV cache directly, so the ragged prefill kernel consumes
them without the per-tensor casts of the BF16 path.
"""
@staticmethod
def _fused_rope_fp8_quantize(
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
cos_sin_cache: Optional[torch.Tensor],
positions: torch.Tensor,
is_neox: bool,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
FP8 quantize and the packed layout, only the rotation is dropped.
"""
num_heads = q_nope.shape[1]
seq_len = q_nope.shape[0]
q_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=q_nope.device,
)
k_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=k_nope.device,
)
if seq_len == 0:
return q_fp8, k_fp8
# Broadcast the shared latent k_pe across heads — RoPE is position-only
# so per-head outputs are identical, and the cache write below reuses
# head 0.
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
else:
k_pe_expanded = k_pe
if cos_sin_cache is None:
# NoPE layer: quantize straight into the packed buffers.
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
return q_fp8, k_fp8
_flashinfer_rope.mla_rope_quantize_fp8(
q_rope=q_pe,
k_rope=k_pe_expanded,
q_nope=q_nope,
k_nope=k_nope,
cos_sin_cache=cos_sin_cache,
pos_ids=positions,
is_neox=is_neox,
quantize_dtype=torch.float8_e4m3fn,
q_rope_out=q_fp8[..., qk_nope_head_dim:],
k_rope_out=k_fp8[..., qk_nope_head_dim:],
q_nope_out=q_fp8[..., :qk_nope_head_dim],
k_nope_out=k_fp8[..., :qk_nope_head_dim],
quant_scale_q=1.0,
quant_scale_kv=1.0,
enable_pdl=is_arch_support_pdl(),
)
return q_fp8, k_fp8
def prepare_prefill_qkv(
self,
*,
q: torch.Tensor,
q_pe: torch.Tensor,
kv_a: torch.Tensor,
k_pe: torch.Tensor,
positions: torch.Tensor,
layer: DeepseekV2AttentionMLA,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
kv = layer.kv_b_proj(kv_a)[0]
kv = kv.view(
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
)
k_nope = kv[..., : layer.qk_nope_head_dim]
v_bf16 = kv[..., layer.qk_nope_head_dim :]
q_nope = q[..., : layer.qk_nope_head_dim]
rotary_emb = layer.rotary_emb
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
q_nope=q_nope,
q_pe=q_pe,
k_nope=k_nope,
k_pe=k_pe,
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
positions=positions,
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
qk_nope_head_dim=layer.qk_nope_head_dim,
qk_rope_head_dim=layer.qk_rope_head_dim,
)
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
# k_pe is shared across heads (RoPE is position-only), so head 0
# reproduces the original [tokens, 1, qk_rope] latent layout.
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
self.token_to_kv_pool.set_mla_kv_buffer(
layer.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a_fp8.unsqueeze(1),
k_pe_fp8,
)
return q_fp8, k_fp8, v_fp8
def pack_prefix_chunk_kv(
self,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
v: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
strided ``v`` into contig FP8 V in a single kernel.
"""
return mla_kv_pack_quantize_fp8(
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
)
@@ -1,516 +0,0 @@
# Copyright (c) 2026 LightSeek Foundation
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from __future__ import annotations
"""Attention backend for the tokenspeed-mla CuTe DSL kernels on Blackwell.
Subclasses :class:`TRTLLMMLABackend` to share its MLA data preparation, prefill
plumbing, and DCP metadata (rank-local KV lengths and page table). The decode
forward lives here because the TokenSpeed decode kernel natively accepts CP
rank/world metadata and returns the partial log-sum-exp needed by the cross-rank
merge.
"""
import logging
from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
mla_kv_pack_quantize_fp8,
)
from sglang.kernels.ops.attention.utils import (
mla_quantize_and_rope_for_fp8,
mla_quantize_without_rope_for_fp8,
)
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
TRTLLMMLAMultiStepDraftBackend,
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.runtime_context import (
get_parallel,
get_resources,
max_speculative_num_draft_tokens,
)
from sglang.srt.utils import is_flashinfer_available, is_tokenspeed_mla_available
if is_flashinfer_available():
import flashinfer.rope as _flashinfer_rope
if is_tokenspeed_mla_available():
import tokenspeed_mla
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
logger = logging.getLogger(__name__)
# Workspace upper bound for tokenspeed_mla_decode:
# num_sms * num_heads * max_q_len * (kv_lora_rank + 1) * sizeof(float32)
# MAX_Q_LEN=8 covers EAGLE3 num_draft_tokens=4 plus headroom. Larger
# speculative widths grow the bound at backend initialization.
_TOKENSPEED_MAX_Q_LEN = 8
def _get_tokenspeed_workspace(
device: torch.device,
num_heads: int,
kv_lora_rank: int,
max_q_len: int = _TOKENSPEED_MAX_Q_LEN,
) -> torch.Tensor:
# DCP target verification gathers Q to the full head count before launching
# TokenSpeed; size for that launch shape, not the rank-local head count.
num_heads *= get_parallel().attn_dcp_size
max_q_len = max(max_q_len, _TOKENSPEED_MAX_Q_LEN)
needed = (
tokenspeed_mla.get_num_sm(device)
* num_heads
* max_q_len
* (kv_lora_rank + 1)
* 4
)
buffers = get_resources().buffers
key = f"tokenspeed_mla_workspace:{device}"
existing = buffers.get(key)
if existing is None or existing.numel() < needed:
buffers[key] = torch.empty(needed, dtype=torch.int8, device=device)
return buffers[key]
# TODO(Qiaolin-Yu): Merge this attention backend into trtllm_mla_backend.py
# once the same CuteDSL kernels in flashinfer_trtllm are stable
# and there is no performance gap compared to this backend.
class TokenspeedMLABackend(TRTLLMMLABackend):
"""tokenspeed-mla CuTe DSL attention backend (Blackwell SM100, FP8 KV)."""
def __init__(
self,
model_runner: ModelRunner,
skip_prefill: bool = False,
kv_indptr_buf: Optional[torch.Tensor] = None,
q_indptr_decode_buf: Optional[torch.Tensor] = None,
):
super().__init__(
model_runner,
skip_prefill,
kv_indptr_buf,
q_indptr_decode_buf,
)
if self.data_type != torch.float8_e4m3fn:
raise ValueError(
"tokenspeed_mla backend requires --kv-cache-dtype fp8_e4m3, "
f"got data_type={self.data_type}."
)
if self.page_size not in (32, 64):
raise ValueError(
"tokenspeed_mla backend requires page_size in {32, 64}, "
f"got page_size={self.page_size}."
)
self._tokenspeed_workspace: Optional[torch.Tensor] = None
if is_tokenspeed_mla_available():
self._tokenspeed_workspace = _get_tokenspeed_workspace(
self.device,
self.num_q_heads,
self.kv_lora_rank,
max_q_len=(max_speculative_num_draft_tokens() or 1),
)
# Pre-JIT the prefill kernel variants. Each cute.compile takes 1-2
# min; without warm-up the first request trips the 300 s scheduler
# watchdog.
_compile_prefill_kernel = tokenspeed_mla.mla_prefill._compile_prefill_kernel
_compiled_kernels = tokenspeed_mla.mla_prefill._compiled_kernels
head_dim_qk = self.qk_nope_head_dim + self.qk_rope_head_dim
enable_ex2_emulation = tokenspeed_mla.mla_prefill._enable_ex2_emulation()
use_pdl = is_arch_support_pdl()
for is_causal in (True, False):
for return_lse in (True, False):
# Non-causal is only entered from the chunked-prefix
# branch, which always asks for the LSE.
if is_causal is False and return_lse is False:
continue
# Runtime feeds fp8_e4m3fn q/k/v
config = (
torch.float8_e4m3fn,
head_dim_qk,
self.v_head_dim,
is_causal,
return_lse,
use_pdl,
enable_ex2_emulation,
)
if config in _compiled_kernels:
continue
_compiled_kernels[config] = _compile_prefill_kernel(
torch.float8_e4m3fn,
head_dim_qk,
self.v_head_dim,
is_causal,
return_lse,
use_pdl=use_pdl,
enable_ex2_emulation=enable_ex2_emulation,
)
@staticmethod
def _fused_rope_fp8_quantize(
q_nope: torch.Tensor,
q_pe: torch.Tensor,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
cos_sin_cache: Optional[torch.Tensor],
positions: torch.Tensor,
is_neox: bool,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
FP8 quantize and the packed layout, only the rotation is dropped.
"""
num_heads = q_nope.shape[1]
seq_len = q_nope.shape[0]
q_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=q_nope.device,
)
k_fp8 = torch.empty(
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
dtype=torch.float8_e4m3fn,
device=k_nope.device,
)
if seq_len == 0:
return q_fp8, k_fp8
# Broadcast the shared latent k_pe across heads — RoPE is position-only
# so per-head outputs are identical, and the cache write below reuses
# head 0.
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
else:
k_pe_expanded = k_pe
if cos_sin_cache is None:
# NoPE layer: quantize straight into the packed buffers.
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
return q_fp8, k_fp8
_flashinfer_rope.mla_rope_quantize_fp8(
q_rope=q_pe,
k_rope=k_pe_expanded,
q_nope=q_nope,
k_nope=k_nope,
cos_sin_cache=cos_sin_cache,
pos_ids=positions,
is_neox=is_neox,
quantize_dtype=torch.float8_e4m3fn,
q_rope_out=q_fp8[..., qk_nope_head_dim:],
k_rope_out=k_fp8[..., qk_nope_head_dim:],
q_nope_out=q_fp8[..., :qk_nope_head_dim],
k_nope_out=k_fp8[..., :qk_nope_head_dim],
quant_scale_q=1.0,
quant_scale_kv=1.0,
enable_pdl=is_arch_support_pdl(),
)
return q_fp8, k_fp8
def prepare_prefill_qkv(
self,
*,
q: torch.Tensor,
q_pe: torch.Tensor,
kv_a: torch.Tensor,
k_pe: torch.Tensor,
positions: torch.Tensor,
layer: DeepseekV2AttentionMLA,
forward_batch: ForwardBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
kv = layer.kv_b_proj(kv_a)[0]
kv = kv.view(
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
)
k_nope = kv[..., : layer.qk_nope_head_dim]
v_bf16 = kv[..., layer.qk_nope_head_dim :]
q_nope = q[..., : layer.qk_nope_head_dim]
rotary_emb = layer.rotary_emb
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
q_nope=q_nope,
q_pe=q_pe,
k_nope=k_nope,
k_pe=k_pe,
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
positions=positions,
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
qk_nope_head_dim=layer.qk_nope_head_dim,
qk_rope_head_dim=layer.qk_rope_head_dim,
)
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
# k_pe is shared across heads (RoPE is position-only), so head 0
# reproduces the original [tokens, 1, qk_rope] latent layout.
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
self.token_to_kv_pool.set_mla_kv_buffer(
layer.attn_mha,
KVWriteLoc.for_batch(forward_batch),
kv_a_fp8.unsqueeze(1),
k_pe_fp8,
)
return q_fp8, k_fp8, v_fp8
def pack_prefix_chunk_kv(
self,
k_nope: torch.Tensor,
k_pe: torch.Tensor,
v: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
strided ``v`` into contig FP8 V in a single kernel.
"""
return mla_kv_pack_quantize_fp8(
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
)
def _run_decode_kernel(
self,
query: torch.Tensor,
kv_cache: torch.Tensor,
block_tables: torch.Tensor,
seq_lens: torch.Tensor,
max_seq_len: int,
layer: RadixAttention,
*,
causal_seqs: Optional[torch.Tensor] = None,
cp_world: int = 1,
cp_rank: int = 0,
return_lse: bool = False,
):
k_scale = getattr(layer, "k_scale_float", None)
if k_scale is None:
k_scale = 1.0
softmax_scale = float(layer.scaling) * float(k_scale)
output_scale = float(k_scale)
seq_lens_i32 = (
seq_lens if seq_lens.dtype == torch.int32 else seq_lens.to(torch.int32)
)
return tokenspeed_mla.tokenspeed_mla_decode(
query=query,
kv_cache=kv_cache,
workspace_buffer=self._tokenspeed_workspace,
kv_lora_rank=self.kv_lora_rank,
qk_rope_head_dim=self.qk_rope_head_dim,
block_tables=block_tables,
seq_lens=seq_lens_i32,
max_seq_len=int(max_seq_len),
softmax_scale=softmax_scale,
output_scale=output_scale,
enable_pdl=is_arch_support_pdl(),
return_lse=return_lse,
causal_seqs=causal_seqs,
cp_world=cp_world,
cp_rank=cp_rank,
)
def forward_decode(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
layer: RadixAttention,
forward_batch: ForwardBatch,
save_kv_cache: bool = True,
q_rope: Optional[torch.Tensor] = None,
k_rope: Optional[torch.Tensor] = None,
cos_sin_cache: Optional[torch.Tensor] = None,
is_neox: Optional[bool] = False,
llama_4_scaling: Optional[torch.Tensor] = None,
):
parallel = get_parallel()
if parallel.dcp_enabled and get_in_autotune_dummy_run():
return self._dummy_dcp_decode_for_autotune(q, layer)
if not parallel.dcp_enabled:
return super().forward_decode(
q,
k,
v,
layer,
forward_batch,
save_kv_cache,
q_rope,
k_rope,
cos_sin_cache,
is_neox,
llama_4_scaling,
)
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
q, k, k_rope = mla_quantize_without_rope_for_fp8(
q, q_rope, k.squeeze(1), k_rope.squeeze(1)
)
else:
q, k, k_rope = mla_quantize_and_rope_for_fp8(
q,
q_rope,
k.squeeze(1),
k_rope.squeeze(1),
forward_batch.positions,
cos_sin_cache,
is_neox,
self.kv_lora_rank,
self.qk_rope_head_dim,
)
if save_kv_cache:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
query = q.view(-1, layer.tp_q_head_num, layer.head_dim)
if llama_4_scaling is not None:
query = (query.to(self.q_data_type) * llama_4_scaling).to(self.data_type)
if query.dim() == 3:
query = query.unsqueeze(1)
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
metadata = (
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
or self.forward_decode_metadata
)
metadata_batch_size = getattr(metadata, "batch_size", None)
if (
metadata_batch_size is not None
and metadata_batch_size < forward_batch.batch_size
):
self.init_forward_metadata(forward_batch)
metadata = forward_batch.decode_trtllm_mla_metadata
global_seq_lens = forward_batch.seq_lens[: forward_batch.batch_size]
local_seq_lens = self._get_dcp_local_seq_lens(global_seq_lens)
raw_out, lse = self._run_decode_kernel(
query=query,
kv_cache=kv_cache,
block_tables=metadata.block_kv_indices,
seq_lens=local_seq_lens,
max_seq_len=metadata.max_seq_len_k,
layer=layer,
causal_seqs=global_seq_lens,
cp_world=parallel.dcp_size,
cp_rank=parallel.dcp_rank,
return_lse=True,
)
output = raw_out.view(-1, layer.tp_q_head_num, layer.v_head_dim)
lse = lse.view(-1, layer.tp_q_head_num)
fixup_zero_kv_rows(
output,
lse,
local_seq_lens,
self.q_indptr_decode[: forward_batch.batch_size + 1],
1,
)
return output.flatten(1), lse
def _run_prefill_kernel(
self,
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
layer: RadixAttention,
batch_size: int,
cum_seq_lens_q: torch.Tensor,
max_q_len: int,
seq_lens_kv: torch.Tensor,
cum_seq_lens_kv: torch.Tensor,
max_kv_len: int,
is_causal: bool,
return_lse: bool,
out_buffer: torch.Tensor,
o_sf_scale: float = 1.0,
): # Q/K/V arrive already in FP8 via the model-side fused path
# (prepare_prefill_qkv / pack_prefix_chunk_kv); no quantize here.
# Hybrid MLA models resolve the model-side hook through the outer
# HybridLinearAttnBackend, so their fallback MHA path can pass V as a
# last-dimension slice of kv_b_proj (stride(-2) > size(-1)). The
# TokenSpeed prefill kernel requires dense Q/K/V layouts even though
# the public wrapper accepts arbitrary torch tensors.
q = q.contiguous()
k = k.contiguous()
v = v.contiguous()
return tokenspeed_mla.tokenspeed_mla_prefill(
query=q,
key=k,
value=v,
seq_lens=seq_lens_kv,
cum_seq_lens=cum_seq_lens_kv,
max_seq_len=int(max_kv_len),
batch_size=int(batch_size),
softmax_scale=float(layer.scaling),
is_causal=is_causal,
return_lse=return_lse,
cum_seq_lens_q=cum_seq_lens_q,
max_seq_len_q=int(max_q_len),
enable_pdl=is_arch_support_pdl(),
)
class TokenspeedMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
"""Multi-step draft backend for tokenspeed_mla used by EAGLE."""
def __init__(
self, model_runner: ModelRunner, topk: int, speculative_num_steps: int
):
super().__init__(model_runner, topk, speculative_num_steps)
# Parent populates self.attn_backends with TRT-LLM instances; replace
# them with tokenspeed instances sharing the parent's index buffers.
for i in range(self.speculative_num_steps - 1):
self.attn_backends[i] = TokenspeedMLABackend(
model_runner,
skip_prefill=True,
kv_indptr_buf=self.kv_indptr[i],
q_indptr_decode_buf=self.q_indptr_decode,
)
@@ -1017,8 +1017,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
raise NotImplementedError(
"trtllm_mla cannot forward a global causal bound to its decode "
"kernel, which is required for DCP with q_len > 1 (speculative "
"target-verify / draft-extend); select cutedsl_mla or "
"tokenspeed_mla for a DCP speculative run"
"target-verify / draft-extend); select cutedsl_mla "
"for a DCP speculative run"
)
# Scale computation for TRTLLM MLA kernel BMM1 operation:
@@ -1073,10 +1073,11 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
o_sf_scale: float = 1.0,
):
"""Hook for subclasses to swap the ragged prefill kernel. Q/K/V arrive
in model-native dtype; subclasses do any kernel-specific quantization.
in model-native dtype, or already FP8 from a model-side prefill hook;
subclasses do any kernel-specific quantization.
Returns the output tensor or (output, lse) if return_lse."""
q_scale = k_scale = v_scale = 1.0
if self.data_type == torch.float8_e4m3fn:
if self.data_type == torch.float8_e4m3fn and q.dtype != torch.float8_e4m3fn:
q, k, v, k_scale, v_scale = _quantize_fp8_qkv(q, k, v, layer)
return flashinfer.prefill.trtllm_ragged_attention_deepseek(
query=q,
@@ -181,12 +181,6 @@ def handle_attention_trtllm_mla(attn, forward_batch):
return _dispatch_mla_subtype(attn, forward_batch)
def handle_attention_tokenspeed_mla(attn, forward_batch):
# tokenspeed_mla shares the trtllm_mla dispatch pattern: pure prefill goes
# via MHA chunked KV (TRT-LLM ragged), spec decode / decode goes via MLA.
return handle_attention_trtllm_mla(attn, forward_batch)
def handle_attention_aiter(attn, forward_batch):
# During PCG/BCG capture on ROCm, aiter fp8 MLA prefill has no capture
# kernels; route through the MHA path (radix_attention swaps attn_mqa for
@@ -268,7 +262,6 @@ AttentionBackendRegistry.register("fa3", handle_attention_fa3)
AttentionBackendRegistry.register("flashmla", handle_attention_flashmla)
AttentionBackendRegistry.register("fa4", handle_attention_fa4)
AttentionBackendRegistry.register("trtllm_mla", handle_attention_trtllm_mla)
AttentionBackendRegistry.register("tokenspeed_mla", handle_attention_tokenspeed_mla)
AttentionBackendRegistry.register("aiter", handle_attention_aiter)
AttentionBackendRegistry.register("dsa", handle_attention_dsa)
AttentionBackendRegistry.register(
@@ -942,8 +942,7 @@ class DeepseekMLAForwardMixin:
return (
self.rotary_emb is not None
and self.current_attention_backend
in ("trtllm_mla", "tokenspeed_mla", "cutedsl_mla")
and self.current_attention_backend in ("trtllm_mla", "cutedsl_mla")
and (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
@@ -70,7 +70,6 @@ FORWARD_ABSORB_CORE_ATTENTION_BACKENDS = [
"flashinfer",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"ascend",
"intel_xpu",
]
@@ -105,7 +105,6 @@ class DraftBackendFactory:
"trtllm_mha": self._create_trtllm_mha_decode_backend,
"trtllm_mla": self._create_trtllm_mla_decode_backend,
"cutedsl_mla": self._create_cutedsl_mla_decode_backend,
"tokenspeed_mla": self._create_tokenspeed_mla_decode_backend,
"dsa": self._create_dsa_decode_backend,
"nsa": self._create_dsa_decode_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_decode_backend,
@@ -137,7 +136,6 @@ class DraftBackendFactory:
"trtllm_mla": self._create_trtllm_mla_prefill_backend,
# cute-dsl MLA only supports decode; draft-extend falls back to trtllm-gen.
"cutedsl_mla": self._create_trtllm_mla_prefill_backend,
"tokenspeed_mla": self._create_tokenspeed_mla_prefill_backend,
"dsa": self._create_dsa_prefill_backend,
"nsa": self._create_dsa_prefill_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_prefill_backend,
@@ -394,23 +392,6 @@ class DraftBackendFactory:
),
)
def _create_tokenspeed_mla_decode_backend(self):
if not self.draft_model_runner.use_mla_backend:
raise ValueError(
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLAMultiStepDraftBackend,
)
return (
"tokenspeed_mla",
TokenspeedMLAMultiStepDraftBackend(
self.draft_model_runner, self.topk, self.speculative_num_steps
),
)
def _create_ascend_decode_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnMultiStepDraftBackend,
@@ -541,21 +522,6 @@ class DraftBackendFactory:
TRTLLMMLABackend(self.draft_model_runner, skip_prefill=False),
)
def _create_tokenspeed_mla_prefill_backend(self):
if not self.draft_model_runner.use_mla_backend:
raise ValueError(
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
TokenspeedMLABackend,
)
return (
"tokenspeed_mla",
TokenspeedMLABackend(self.draft_model_runner, skip_prefill=False),
)
def _create_ascend_prefill_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnBackend,
@@ -27,7 +27,6 @@ from sglang.srt.layers.attention.qwen_sparse_attn_backend import (
QwenSparseAttnBackend,
QwenSparseMultiStepDraftBackend,
)
from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend
from sglang.srt.layers.attention.trtllm_mla_backend import (
@@ -536,7 +535,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
TritonAttnBackend,
TRTLLMMLABackend,
TRTLLMHAAttnBackend,
TokenspeedMLABackend,
FlashInferAttnBackend,
QwenSparseAttnBackend,
]
-12
View File
@@ -394,18 +394,6 @@ def is_flashinfer_available():
return importlib.util.find_spec("flashinfer") is not None and is_cuda()
@lru_cache(maxsize=1)
def is_tokenspeed_mla_available():
"""
Check whether the tokenspeed_mla CuTe DSL kernels are available.
Only available on NVIDIA Blackwell (SM100) at the moment.
"""
return (
importlib.util.find_spec("tokenspeed_mla") is not None
and is_blackwell_supported()
)
def is_nvidia_cublas_version_ge_12_9():
"""
temporary fix for issue #11272 (cublas 12.9+)
@@ -228,8 +228,8 @@ class MockMLAModelRunner(ModelRunner):
self.device = device
self.dtype = dtype
# `kv_cache_dtype` is the dtype the *storage* uses. For FP8 KV
# cache (the production deployment dtype for tokenspeed_mla and
# some trtllm_mla configs), the pool stores quantized bytes
# cache (the production deployment dtype for some trtllm_mla
# configs), the pool stores quantized bytes
# while the model still projects K/V in bf16; `set_mla_kv_buffer`
# does the BF16->FP8 cast on the way in.
self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype
@@ -1,6 +1,6 @@
"""The paged MLA decode kernels must not depend on the KV cache's slot stride.
trtllm_mla, cutedsl_mla and tokenspeed_mla hand their decode kernels
trtllm_mla and cutedsl_mla hand their decode kernels
`paged_row_view(k_cache, page_size).unsqueeze(1)`: `[pages, 1, page_size,
kv_cache_dim]` whose in-page token stride is the pool's SLOT stride. Under the
unified pool's token-major layout that stride spans every layer's latent row,
@@ -19,8 +19,7 @@ import unittest
import torch
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_sm100_supported, is_tokenspeed_mla_available
from sglang.srt.utils import is_sm100_supported
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=15, stage="base-b", runner_config="4-gpu-b200")
@@ -119,38 +118,6 @@ class TestPagedMLADecodeStrideParity(unittest.TestCase):
self._assert_stride_independent(run, kv_dtype)
def test_tokenspeed_mla_decode_is_identical_across_slot_strides(self):
if not is_tokenspeed_mla_available():
self.skipTest("tokenspeed_mla is not installed")
import tokenspeed_mla
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
_get_tokenspeed_workspace,
)
# The workspace is sized for the DCP-gathered head count.
with get_parallel().override(attn_dcp_size=1):
workspace = _get_tokenspeed_workspace(
torch.device("cuda"), _Q_HEADS, _KV_LORA_RANK
)
kv_dtype = torch.float8_e4m3fn
q = torch.randn(_BS, 1, _Q_HEADS, _KV_CACHE_DIM, device="cuda").to(kv_dtype)
def run(kv_cache):
return tokenspeed_mla.tokenspeed_mla_decode(
query=q,
kv_cache=kv_cache,
workspace_buffer=workspace,
kv_lora_rank=_KV_LORA_RANK,
qk_rope_head_dim=_QK_ROPE_HEAD_DIM,
block_tables=self.block_tables,
seq_lens=self.seq_lens,
max_seq_len=_PAGES * _PAGE_SIZE,
softmax_scale=_KV_CACHE_DIM**-0.5,
)
self._assert_stride_independent(run, kv_dtype)
if __name__ == "__main__":
unittest.main()
@@ -68,7 +68,6 @@ so that "skipped: ..." results have a quick lookup.
|---|---|---|---|
| `flashmla` decode/verify | SM 9.0 (Hopper) only | `mla/test_flashmla.py:_DECODE_REQUIRES_SM90A` | `Dense decode MLA is only supported on SM90a architecture` |
| `trtllm_mla` | SM 12.0a / 12.1a | `mla/test_trtllm_mla.py::_supported` | FlashInfer XQA MLA dispatch reject |
| `tokenspeed_mla` | SM ≥ 10.0 + FP8 KV + pkg | `mla/test_tokenspeed_mla.py::_supported` | `tokenspeed_mla` import or kernel dispatch |
| `trtllm_mha` prefill | SM ≥ 10.0 | `dense/test_trtllm_mha.py` decode-only matrix | FlashInfer TRT-LLM Gen FMHA reject (`Unsupported architecture`) |
| `dsa` `fa3` impl | SM 9.x only | `dsa_impl_capability("fa3")` | `flash_attn at sgl-kernel is only supported on sm90 and above` |
| `dsa` `trtllm` impl | exactly SM 10.0 | `dsa_impl_capability("trtllm")` | `Missing TRTLLM-GEN kernel` (compiled for SM10.0) |
@@ -176,7 +175,6 @@ fail at backend init.
|---|---|---|
| FlashMLA | `64` only | `server_args.py:2767-2770` |
| TRT-LLM MLA | `{32, 64}` | `server_args.py:2790-2794` |
| Tokenspeed MLA | `{32, 64}` | `server_args.py:2809-2813`, `tokenspeed_mla_backend.py:111-113` |
| TRT-LLM MHA | `{16, 32, 64}` | `server_args.py:2849-2853` |
| FA4 (non-MLA) | `128` when default-selected | `server_args.py:2862-2870` |
| DSV4 | `256` only | `deepseek_v4_backend.py:355`, `dsv4/metadata.py:134` |
@@ -191,7 +189,6 @@ fail at backend init.
| `flashinfer_mla` | `1` only | `flashinfer_mla_backend.py:910-913` |
| `flashmla` | `1` only | `flashmla_backend.py:555-558` |
| `trtllm_mla` | `1` only | `trtllm_mla_backend.py:1223-1229` (inherits) |
| `tokenspeed_mla` | `1` only | `tokenspeed_mla_backend.py:341-347` (inherits) |
| `dsv4` | `0` or `1` | `deepseek_v4_backend.py:369`, `:363` (HIP) |
| `trtllm_mha` (graph replay) | `1` only | `trtllm_mha_backend.py:459,492`; `server_args.py:2391-2392` |
@@ -199,7 +196,6 @@ fail at backend init.
| Backend | Allowed dtype | Citation |
|---|---|---|
| `tokenspeed_mla` | `fp8_e4m3` only | `server_args.py:2814-2818` |
| `trtllm_mla` | `{fp8_e4m3, fp4_e2m1, bf16, auto}` | `server_args.py:2796-2799` |
| `fa3` | not `fp8_e5m2` (silently falls back to `triton`) | `server_args.py:2855-2860` |
| `dsv4` | Packed FP8/BF16 layout enforced by `DeepSeekV4TokenToKVPool` | `deepseek_v4_backend.py:363` |
@@ -21,7 +21,6 @@ Columns are runner modes; rows are attention backends. Cells use:
| `flashinfer` | ✓ 10 input layouts with DeepSeek-like `kv_lora_rank=512`, `qk_rope_head_dim=64` | ✓ MLA decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | ✓ EAGLE ragged-accept | blocked: `is_draft_extend()` default `include_v2=False` (`flashinfer_mla_backend.py:432,501,454-455,512`) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashinfer_mla_backend.py:910-913`) | ✓ EAGLE ragged-accept (V1) | — (no FKVMTP wiring for MLA) |
| `flashmla` | ✓ FlashMLA-compatible page-size-64 cases (zero-prefix exact page, input page edges 63/64/65, prefix exact page, total exact page, cross page, ragged, decode page-boundary, decode bsz=1 nonzero prefix) | ✓ page-size-64 decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | deferred: parent FlashInfer-MLA capture path expects 1D `cuda_graph_kv_indices`, FlashMLA allocates 2D `[max_bs, (max_context+PAGE_SIZE)//PAGE_SIZE]` (`flashmla_backend.py:347-348` + parent `init_forward_metadata_capture_cuda_graph`) | — (FlashMLA does not implement V2) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashmla_backend.py:555-558`) | — (DE CG deferred above) | — |
| `trtllm_mla` | skip:hw — needs SM 12.0a / 12.1a (`is_sm120_supported`) | — | — | — | blocked: `topk=1` only (`trtllm_mla_backend.py:1223-1229` inherits from FlashInfer MLA) | — | — | — | — | — | — | — |
| `tokenspeed_mla` | skip:hw — needs `find_spec("tokenspeed_mla")`, SM 10.0+, and `kv_cache_dtype=fp8_e4m3` (`server_args.py:2814-2818`); current MLA fixture does not emit FP8 KV cache | — | — | — | blocked: `topk=1` only (`tokenspeed_mla_backend.py:341-347` inherits from TRT-LLM MLA) | — | — | — | — | — | — | — |
## Input And Config Coverage
@@ -56,15 +55,12 @@ multi-step draft backends and cannot ever appear at runtime.
- **TRT-LLM MLA tree verify / draft-extend with `topk > 1`** —
`TRTLLMMLAMultiStepDraftBackend` inherits from
`FlashInferMLAMultiStepDraftBackend` (`trtllm_mla_backend.py:1223-1229`).
- **Tokenspeed MLA tree verify / draft-extend with `topk > 1`** —
`TokenspeedMLAMultiStepDraftBackend` inherits from
`TRTLLMMLAMultiStepDraftBackend` (`tokenspeed_mla_backend.py:341-347`).
- **FlashInfer-MLA `DRAFT_EXTEND_V2` graph capture/replay** —
`flashinfer_mla_backend.py:432,501` only route through `is_draft_extend()`
(default `include_v2=False`); `else: raise ValueError("Invalid mode")` at
`flashinfer_mla_backend.py:454-455,512`.
- **All MLA backends fixed page size** — FlashMLA forces `page_size=64`,
TRT-LLM MLA and Tokenspeed MLA force `page_size in {32, 64}`.
TRT-LLM MLA forces `page_size in {32, 64}`.
## Backend Container Gate (SM10.x)
@@ -85,6 +81,6 @@ See `KNOWN_FAILURES.md` §3 for the full root cause + fix.
override capture/replay in `FlashMLABackend` to use its 2D layout, or
allocate both parent-style 1D and FlashMLA-style 2D buffers and route
`DRAFT_EXTEND` to the parent path).
- Add hardware-gated tests for `trtllm_mla` and `tokenspeed_mla`
- Add hardware-gated tests for `trtllm_mla`
decode (chain spec only) when the appropriate hardware/KV dtype fixtures are
available.
@@ -1,265 +0,0 @@
import importlib.util
import unittest
import torch
from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.test.kits.attention_unittest.attention_methods.mla_attention import (
MLAAttentionCase,
run_mla_attention_case,
)
from sglang.test.test_utils import CustomTestCase
# tokenspeed_mla is a CuTe DSL backend for Blackwell (SM100). It additionally
# enforces:
# - kv_cache_dtype == torch.float8_e4m3fn (kv_cache_dtype=fp8_e4m3)
# - page_size in {32, 64}
# See python/sglang/srt/layers/attention/tokenspeed_mla_backend.py and
# is_tokenspeed_mla_available() in python/sglang/srt/utils/common.py.
#
# The shared MLAAttentionCase fixture now supports `fp8_kv_cache=True`:
# `MockMLAModelRunner` decouples `kv_cache_dtype` from the model `dtype`
# and routes K writes through the FP8 quantize path. The reference still
# computes against BF16 K (independent of the cache bytes) and tolerates
# FP8 quant noise via a looser tolerance.
_MIN_SM = 100
def _supported() -> tuple[bool, str]:
if not torch.cuda.is_available():
return False, "CUDA is required"
if importlib.util.find_spec("tokenspeed_mla") is None:
return False, "tokenspeed_mla python package is not installed"
major, minor = torch.cuda.get_device_capability()
sm = major * 10 + minor
if sm < _MIN_SM:
return (
False,
f"tokenspeed_mla requires SM {_MIN_SM // 10}.{_MIN_SM % 10}+ (Blackwell), "
f"got SM {major}.{minor}",
)
return True, ""
_SUPPORTED, _SKIP_REASON = _supported()
MLA_SHAPE_KWARGS = dict(
kv_lora_rank=512,
qk_rope_head_dim=64,
hidden_size=1024,
max_context_len=256,
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200")
register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-large")
@unittest.skipIf(not _SUPPORTED, _SKIP_REASON)
class TestTokenspeedMLAAttentionBackendCorrectness(CustomTestCase):
# tokenspeed_mla allows page_size in {32, 64} (server_args.py:2809-2813)
# and requires kv_cache_dtype==fp8_e4m3 (server_args.py:2814-2818).
# Cover both page sizes, with extend + decode + ragged + page-boundary.
CASES = (
# ----- page_size=64 -----
MLAAttentionCase(
name="mla_extend_tokenspeed_zero_prefix_exact_page_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(0,),
extend_lens=(64,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_zero_prefix_below_page_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(0,),
extend_lens=(63,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_zero_prefix_above_page_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(0,),
extend_lens=(65,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_prefix_exact_page_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(64,),
extend_lens=(4,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_cross_page_boundary_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(63,),
extend_lens=(2,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_ragged_page_boundary_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=64,
prefix_lens=(0, 32, 64),
extend_lens=(63, 32, 1),
),
MLAAttentionCase(
name="mla_decode_tokenspeed_page_boundary_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.DECODE,
num_heads=4,
page_size=64,
prefix_lens=(62, 63, 64),
),
MLAAttentionCase(
name="mla_decode_tokenspeed_bsz1_nonzero_prefix_64",
backend="tokenspeed_mla",
forward_mode=ForwardMode.DECODE,
num_heads=4,
page_size=64,
prefix_lens=(31,),
),
# ----- page_size=32 -----
MLAAttentionCase(
name="mla_extend_tokenspeed_zero_prefix_exact_page_32",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=32,
prefix_lens=(0,),
extend_lens=(32,),
),
MLAAttentionCase(
name="mla_extend_tokenspeed_cross_page_boundary_32",
backend="tokenspeed_mla",
forward_mode=ForwardMode.EXTEND,
num_heads=4,
page_size=32,
prefix_lens=(31,),
extend_lens=(2,),
),
MLAAttentionCase(
name="mla_decode_tokenspeed_page_boundary_32",
backend="tokenspeed_mla",
forward_mode=ForwardMode.DECODE,
num_heads=4,
page_size=32,
prefix_lens=(30, 31, 32),
),
)
def test_projected_mla_attention_cases(self):
for case in self.CASES:
with self.subTest(case=case.name, backend=case.backend):
# Looser tolerance to absorb FP8 quant noise (the reference
# reads BF16 K independent of the FP8 cache, so per-element
# drift from the BF16->FP8 cast accumulates through the
# attention reduction).
run_mla_attention_case(
self,
case,
fp8_kv_cache=True,
atol=2e-1,
rtol=2e-1,
**MLA_SHAPE_KWARGS,
)
class TestTokenspeedMLANoPEPrefillQuantize(CustomTestCase):
"""NoPE MLA layers must still get packed FP8 prefill Q/K.
Layers built with ``skip_rope`` carry no rotary embedding, so the prefill
quantize runs with ``cos_sin_cache=None``. It must return the same packed
``[nope | pe]`` FP8 layout as the RoPE path, and the head-0 pe slice it
writes to the KV cache must equal the plain FP8 cast of the unroped
``k_pe`` that the decode path reads back.
"""
T = 7
NUM_HEADS = 4
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
KV_LORA_RANK = 512
def test_nope_prefill_quantize_packs_without_rope(self):
device = "cuda" if torch.cuda.is_available() else "cpu"
head_dim = self.QK_NOPE_HEAD_DIM + self.QK_ROPE_HEAD_DIM
q_nope = torch.randn(
self.T,
self.NUM_HEADS,
self.QK_NOPE_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
q_pe = torch.randn(
self.T,
self.NUM_HEADS,
self.QK_ROPE_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
k_nope = torch.randn(
self.T,
self.NUM_HEADS,
self.QK_NOPE_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
# k_pe reaches the backend as the strided tail of the latent cache.
latent_cache = torch.randn(
self.T,
1,
self.KV_LORA_RANK + self.QK_ROPE_HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
k_pe = latent_cache[:, :, self.KV_LORA_RANK :]
q_fp8, k_fp8 = TokenspeedMLABackend._fused_rope_fp8_quantize(
q_nope=q_nope,
q_pe=q_pe,
k_nope=k_nope,
k_pe=k_pe,
cos_sin_cache=None,
positions=torch.arange(self.T, device=device),
is_neox=True,
qk_nope_head_dim=self.QK_NOPE_HEAD_DIM,
qk_rope_head_dim=self.QK_ROPE_HEAD_DIM,
)
for name, out in (("q", q_fp8), ("k", k_fp8)):
with self.subTest(tensor=name):
self.assertEqual(out.shape, (self.T, self.NUM_HEADS, head_dim))
self.assertEqual(out.dtype, torch.float8_e4m3fn)
self.assertTrue(out.is_contiguous())
fp8 = torch.float8_e4m3fn
nope = self.QK_NOPE_HEAD_DIM
self.assertTrue(torch.equal(q_fp8[..., :nope], q_nope.to(fp8)))
self.assertTrue(torch.equal(q_fp8[..., nope:], q_pe.to(fp8)))
self.assertTrue(torch.equal(k_fp8[..., :nope], k_nope.to(fp8)))
# The slice prepare_prefill_qkv writes into the KV cache; the decode
# NoPE path reads it back as a plain cast of the unroped k_pe.
self.assertTrue(torch.equal(k_fp8[:, 0:1, nope:], k_pe.to(fp8)))
if __name__ == "__main__":
unittest.main()
+3 -3
View File
@@ -1,4 +1,4 @@
"""Four-Blackwell acceptance coverage for Kimi Linear TokenSpeed MLA DCP.
"""Four-Blackwell acceptance coverage for Kimi Linear CuteDSL MLA DCP.
The captured-shape and eager-shape requests deliberately straddle
``--cuda-graph-max-bs-decode``. This guards both the regular CUDA graph
@@ -37,7 +37,7 @@ def _has_four_blackwell_gpus() -> bool:
@unittest.skipUnless(
_has_four_blackwell_gpus(),
"TokenSpeed MLA DCP acceptance requires four Blackwell GPUs",
"CuteDSL MLA DCP acceptance requires four Blackwell GPUs",
)
class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase):
model = KIMI_LINEAR_MODEL
@@ -62,7 +62,7 @@ class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase):
"--dcp-size",
"4",
"--attention-backend",
"tokenspeed_mla",
"cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dcp-comm-backend",
@@ -1,4 +1,4 @@
"""Four-Blackwell Kimi Linear TokenSpeed DCP + DSpark static acceptance test."""
"""Four-Blackwell Kimi Linear CuteDSL MLA DCP + DSpark static acceptance test."""
import json
import socket
@@ -96,7 +96,7 @@ def _wait_for_port_release(base_url: str, timeout: float = 30.0) -> None:
@unittest.skipUnless(
_has_four_blackwell_gpus(),
"Kimi Linear TokenSpeed DCP + DSpark requires four Blackwell GPUs",
"Kimi Linear CuteDSL MLA DCP + DSpark requires four Blackwell GPUs",
)
class TestKimiLinearDCPDSpark4(CustomTestCase):
base_url = DEFAULT_URL_FOR_TEST
@@ -132,7 +132,7 @@ class TestKimiLinearDCPDSpark4(CustomTestCase):
"--max-running-requests",
str(MAX_RUNNING_REQUESTS),
"--attention-backend",
"tokenspeed_mla",
"cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dcp-comm-backend",
@@ -14,7 +14,6 @@ import torch
from sglang.srt.layers.attention import trtllm_mla_backend as backend_module
from sglang.srt.layers.attention.cutedsl_mla_backend import CuteDslMLABackend
from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
TRTLLMMLADecodeMetadata,
@@ -168,10 +167,6 @@ class TestTRTLLMMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
backend_cls = TRTLLMMLABackend
class TestTokenspeedMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
backend_cls = TokenspeedMLABackend
class TestCuteDslMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
backend_cls = CuteDslMLABackend
@@ -30,7 +30,7 @@ NIAH_KEY = "739391"
COMMON_ARGS = [
"--attention-backend",
"tokenspeed_mla",
"cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dtype",
@@ -14,9 +14,9 @@ prefix-cache replay below are here to catch.
`cutedsl_mla` stands in for the family: it is the DCP-native MLA decode kernel
on Blackwell (the only backend flashinfer accepts `enable_dcp=True` for) and
what the B300 hosts run, and prefill resolves to `trtllm_mla`, so one server
covers both halves of the page-table contract. `trtllm_mla` and
`tokenspeed_mla` share that builder and were checked by hand against the
static pool (see below) rather than given a CI server each.
covers both halves of the page-table contract. `trtllm_mla` shares that
builder and was checked by hand against the static pool (see below) rather
than given a CI server.
Blackwell-only, hence its own module: the Hopper suite next door cannot host
it. In extra-b rather than nightly because what it guards is a per-PR argument
@@ -29,13 +29,11 @@ pools agree:
unified / static
TP2 DCP2 cutedsl_mla 0.905 / 0.905
TP2 DCP2 trtllm_mla 0.895 / 0.895
TP2 DCP2 tokenspeed_mla 0.890 / 0.895, repeat 0.890 / 0.885
TP4 DCP4 cutedsl_mla 0.910 / 0.900
The cutedsl_mla and trtllm_mla pairs reproduced exactly. tokenspeed_mla moved
on a repeat of the same questions on BOTH pools, so its spread is the backend,
not the pool. The TP4 DCP4 row is there because it widens the virtual page to
4 x 64 = 256, past what this file exercises.
The cutedsl_mla and trtllm_mla pairs reproduced exactly. The TP4 DCP4 row is
there because it widens the virtual page to 4 x 64 = 256, past what this file
exercises.
python -m pytest test/registered/models_e2e/test_kimi_linear_unified_memory_dcp_blackwell.py -v
"""
@@ -256,7 +256,6 @@ class TestLSEBaseByBackend(CustomTestCase):
self.assertTrue(is_mla_dcp_lse_base_on_e("flashmla"))
self.assertTrue(is_mla_dcp_lse_base_on_e("cutedsl_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e("flashinfer_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e("tokenspeed_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e("trtllm_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e(None))
@@ -243,7 +243,6 @@ _UNIFIED_BACKEND_FILES = {
"trtllm_mha": ("layers/attention/trtllm_mha_backend.py",),
"trtllm_mla": ("layers/attention/trtllm_mla_backend.py",),
"cutedsl_mla": ("layers/attention/cutedsl_mla_backend.py",),
"tokenspeed_mla": ("layers/attention/tokenspeed_mla_backend.py",),
"flashmla": ("layers/attention/flashmla_backend.py",),
}
# Model code and the rest of the forward path also write KV directly.
@@ -24,7 +24,7 @@ page ids:
ONE builder computes that formula for every family -- the iteration plan's
read table (`KVLocPlan.read_table`) -- and the backends only differ in how
they consume it:
- trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows
- trtllm_mla / cutedsl_mla / flashmla: the plan's rows
copied into their padded block tables (`KVIndexTranslator.copy_page_table`);
- the flashinfer updaters: token ids reconstructed from the table by
`create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`;
@@ -98,13 +98,12 @@ class TestPageMajorBackendAllowlist(unittest.TestCase):
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
)
# Wired for the per-layer MHA/SWA views.
PER_LAYER_VIEW_MHA_BACKENDS = ("fa3", "fa4", "flashinfer", "trtllm_mha")
# MLA-family kernels that must never leak into the MHA arm.
MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "tokenspeed_mla", "flashmla")
MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "flashmla")
# No virtual-to-physical id wiring anywhere: must stay rejected until they get one.
UNWIRED_BACKENDS = ("aiter",)
@@ -710,6 +710,43 @@ class TestImageProcessorBackend(CustomTestCase):
handle_deprecated_args(server_args)
class TestTokenspeedMlaAlias(CustomTestCase):
def test_maps_to_cutedsl_with_fp8_kv(self):
server_args = ServerArgs(model_path="dummy", attention_backend="tokenspeed_mla")
with self.assertLogs(serving_hook.logger, level="WARNING"):
handle_deprecated_args(server_args)
self.assertEqual(
resolution_result(server_args, "attention_backend"), "cutedsl_mla"
)
self.assertEqual(resolution_result(server_args, "kv_cache_dtype"), "fp8_e4m3")
self.assertEqual(
resolution_result(server_args, "speculative_attention_mode"), "prefill"
)
def test_speculative_verify_moves_to_decode(self):
server_args = ServerArgs(
model_path="dummy",
decode_attention_backend="tokenspeed_mla",
prefill_attention_backend="tokenspeed_mla",
kv_cache_dtype="fp8_e4m3",
speculative_algorithm="EAGLE3",
)
handle_deprecated_args(server_args)
self.assertEqual(
resolution_result(server_args, "decode_attention_backend"), "cutedsl_mla"
)
self.assertEqual(
resolution_result(server_args, "prefill_attention_backend"), "trtllm_mla"
)
self.assertEqual(
resolution_result(server_args, "speculative_attention_mode"), "decode"
)
class TestMultimodalFeatureTransport(CustomTestCase):
@staticmethod
def _set_model_type(server_args, *, is_multimodal):
@@ -93,7 +93,6 @@ _VERIFY_BACKENDS = (
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",