diff --git a/.agents/skills/sglang-runtime-context/SKILL.md b/.agents/skills/sglang-runtime-context/SKILL.md
index 99628611e0..b1790a9187 100644
--- a/.agents/skills/sglang-runtime-context/SKILL.md
+++ b/.agents/skills/sglang-runtime-context/SKILL.md
@@ -509,8 +509,8 @@ Named slots + two keyed-lazy registries:
side stream is `"lora_side"`. Two call sites may share a name only if their work belongs
on one stream — sharing across roles serializes intended overlap.
- `get_buffer(name, factory)` — get-or-create a named persistent buffer. Grow-only or
- per-device semantics manage their `resources.buffers` entries directly (see tokenspeed /
- SM120 split / Marlin workspace). Buffer names are per-backend today; do not silently
+ per-device semantics manage their `resources.buffers` entries directly (see SM120
+ split / Marlin workspace). Buffer names are per-backend today; do not silently
share.
- Singletons with manager semantics (EP dispatcher buffers, EPLB recorder/metadata, graph
memory pool) keep their owning accessors/classes as facades; only the *state* lives in a
diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx
index 916a4b037b..1d457d5c99 100644
--- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx
+++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx
@@ -207,7 +207,6 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request
- DCP is the only axis that shards the TP-replicated MLA KV; Low-Latency skips it.
- Leave `--dcp-comm-backend` unset (fabric-resolved: `fi_a2a` on GB200/GB300, `a2a` on B200/B300).
- No `--enable-symm-mem` under DCP (force-disabled for decode-graph correctness).
-- Explicit `tokenspeed_mla` force-rewrites `--kv-cache-dtype` to fp8; the default `cutedsl_mla` serves either dtype.
- Calculator ratios run well above 1 here (`r > 1` is legal): `bfloat16` state buys admission, `fp8` KV buys context.
- Don't use EP with an a2a backend: a2a buffers reclaim the KV that DCP buys. Compose only to measure. a2a backend is set when `--moe-a2a-backend` is set.
diff --git a/docs/docs/advanced_features/attention_backend.mdx b/docs/docs/advanced_features/attention_backend.mdx
index a842451236..7d4dad9ec3 100644
--- a/docs/docs/advanced_features/attention_backend.mdx
+++ b/docs/docs/advanced_features/attention_backend.mdx
@@ -254,15 +254,6 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu
✅ |
❌ |
-
- | **TokenSpeed MLA (Blackwell)** |
- 32 or 64 |
- ✅ (required) |
- ❌ |
- ✅ |
- ✅ |
- ❌ |
-
| **FA3 (FlashAttention 3)** |
n/a |
@@ -330,7 +321,6 @@ MLA page-size constraints:
- Cutlass MLA: page_size = 128.
- TRTLLM MLA: page_size ∈ {32, 64}.
- CuteDSL MLA: page_size ∈ {32, 64} (decode-only; prefill falls back to `trtllm_mla` when unset).
-- TokenSpeed MLA: page_size ∈ {32, 64} (Blackwell SM100/SM12x only; requires `--kv-cache-dtype fp8_e4m3`).
### GDN Attention Backends
diff --git a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx
index ad21e8df6a..a630daa27d 100644
--- a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx
+++ b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx
@@ -196,15 +196,18 @@ export const KimiK25Deployment = () => {
cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla';
}
- const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
+ const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
- // Blackwell (B300/GB300): tokenspeed MLA attention backend
- if (usesTokenspeedMla) {
- cmd += ' \\\n --attention-backend tokenspeed_mla';
+ // Blackwell (B300/GB300): CuteDSL MLA attention backend
+ if (usesCutedslMla) {
+ cmd += ' \\\n --attention-backend cutedsl_mla';
+ if (speculative === 'enabled') {
+ cmd += ' \\\n --speculative-attention-mode decode';
+ }
}
- // FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility
- if (isAMD || usesTokenspeedMla) {
+ // FP8 KV cache for AMD and Blackwell memory efficiency
+ if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
diff --git a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx
index 251155c307..1e222c1781 100644
--- a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx
+++ b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx
@@ -194,13 +194,16 @@ export const KimiK26Deployment = () => {
cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla';
}
- const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
+ const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
- if (usesTokenspeedMla) {
- cmd += ' \\\n --attention-backend tokenspeed_mla';
+ if (usesCutedslMla) {
+ cmd += ' \\\n --attention-backend cutedsl_mla';
+ if (speculative === 'enabled') {
+ cmd += ' \\\n --speculative-attention-mode decode';
+ }
}
- if (isAMD || usesTokenspeedMla) {
+ if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
index ec7b5b88aa..af53b7eee8 100644
--- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
+++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx
@@ -189,13 +189,13 @@ export const KimiK27CodeDeployment = () => {
cmd += ' \\\n --tool-call-parser kimi_k2';
}
- const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300';
+ const usesCutedslMla = hardware === 'b300' || hardware === 'gb300';
- if (usesTokenspeedMla) {
- cmd += ' \\\n --attention-backend tokenspeed_mla';
+ if (usesCutedslMla) {
+ cmd += ' \\\n --attention-backend cutedsl_mla';
}
- if (isAMD || usesTokenspeedMla) {
+ if (isAMD || usesCutedslMla) {
cmd += ' \\\n --kv-cache-dtype fp8_e4m3';
}
diff --git a/python/pyproject.toml b/python/pyproject.toml
index c6d53beebc..60bb7b7cc9 100755
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -83,7 +83,6 @@ dependencies = [
"tilelang==0.1.12",
"timm==1.0.16",
"tokenizers==0.23.2",
- "tokenspeed_mla==0.1.8",
"torch==2.14.1",
"torch_memory_saver>=0.0.9.post1",
"torchaudio==2.11.0",
diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py
index 92a23018a0..4bf3b28178 100644
--- a/python/sglang/srt/arg_groups/attention_hook.py
+++ b/python/sglang/srt/arg_groups/attention_hook.py
@@ -182,7 +182,7 @@ def handle_attention_backend_compatibility(server_args: Any):
# fallback stay below.
run_post_process_pass(server_args, _mla_backend_page_constraints)
- # The TRT-LLM / tokenspeed MLA kv-dtype validations moved to the
+ # The TRT-LLM MLA kv-dtype validations moved to the
# resolution pipeline (arg_groups/overrides.py:
# _mla_kv_cache_dtype_checks), invoked here at their legacy slot.
diff --git a/python/sglang/srt/arg_groups/choices.py b/python/sglang/srt/arg_groups/choices.py
index 20a6253b9a..e96ed1e0d6 100644
--- a/python/sglang/srt/arg_groups/choices.py
+++ b/python/sglang/srt/arg_groups/choices.py
@@ -84,7 +84,7 @@ ATTENTION_BACKEND_CHOICES = [
"flashmla",
"trtllm_mla",
"cutedsl_mla",
- "tokenspeed_mla",
+ "tokenspeed_mla", # Deprecated alias for "cutedsl_mla"
"trtllm_mha",
"hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), Hopper (SM90) only, requires --page-size 64
"minicpm_flashattn",
@@ -234,7 +234,6 @@ CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
"flashmla",
"cutedsl_mla",
"trtllm_mla",
- "tokenspeed_mla",
]
add_chunked_prefix_cache_attention_backend = (
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append
diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py
index 2d4ab9a5ee..2fe7977c2c 100644
--- a/python/sglang/srt/arg_groups/kv_cache_hook.py
+++ b/python/sglang/srt/arg_groups/kv_cache_hook.py
@@ -481,7 +481,6 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
"triton",
"trtllm_mla",
"cutedsl_mla",
- "tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
@@ -749,7 +748,7 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
# The trtllm_mla family builds its DCP block table through the pool's v2p
# gather (create_mla_kv_page_table_for_dcp), so it speaks the same
# two-stage contract as flashinfer.
- dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
+ dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= dcp_allowed, (
@@ -806,7 +805,6 @@ def handle_page_major_kv_layout(server_args: Any):
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
- "tokenspeed_mla",
"flashmla",
}
elif model_config.has_asymmetric_kv:
diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py
index 3b965ddbac..68aa916c84 100644
--- a/python/sglang/srt/arg_groups/model_override_base.py
+++ b/python/sglang/srt/arg_groups/model_override_base.py
@@ -351,14 +351,10 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
return "triton"
-def _dspark_verify_on_decode_backend(
- backend: Optional[str], q_len: int, kv_cache_dtype: Optional[str]
-) -> bool:
+def _dspark_verify_on_decode_backend(backend: Optional[str], q_len: int) -> bool:
"""Whether the MLA decode backend can serve a q_len-wide target verify."""
if backend == "trtllm_mla":
return True
- if backend == "tokenspeed_mla":
- return kv_cache_dtype == "fp8_e4m3" and q_len <= 8
if backend == "cutedsl_mla":
# cute-dsl monolithic MLA decode folds the verify tokens into the head
# dim (fold_sq), so it serves any DSPARK verify width. Needs flashinfer
diff --git a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py
index 6a0157314e..fe615e3f6c 100644
--- a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py
+++ b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py
@@ -92,21 +92,6 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
prefill_attention_backend="trtllm_mla",
decode_attention_backend="cutedsl_mla",
)
- elif decode_backend == "tokenspeed_mla":
- logger.info(
- "Kimi-K3 DCP overrides attention backends: "
- f"prefill={prefill_backend!r}, decode={decode_backend!r} -> "
- "'tokenspeed_mla'."
- )
- logger.info(
- "Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: "
- f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'."
- )
- overrides.update(
- prefill_attention_backend="tokenspeed_mla",
- decode_attention_backend="tokenspeed_mla",
- kv_cache_dtype="fp8_e4m3",
- )
elif decode_backend == "aiter":
_require_kimi_k3_aiter_gluon_dcp_support()
# Override prefill backend to aiter by default
@@ -123,7 +108,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
)
else:
raise AssertionError(
- f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla', 'tokenspeed_mla' or 'aiter', got {decode_backend!r}."
+ f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'aiter', got {decode_backend!r}."
)
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in (
@@ -167,7 +152,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
# that still needs declaring -- else verify stays on the prefill backend,
# whose host-side plan (flashinfer by default) forces a per-step D2H.
_, backend = attention_backends_of(cfg)
- if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype):
+ if _dspark_verify_on_decode_backend(backend, q_len):
overrides["speculative_attention_mode"] = "decode"
logger.info(
"Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend "
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index 101817dc90..9a672420d6 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -1143,15 +1143,6 @@ def _mla_backend_page_constraints(view: Any) -> dict:
f"TensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
)
page_size = 64
- if (
- view.attention_backend == "tokenspeed_mla"
- or view.decode_attention_backend == "tokenspeed_mla"
- ):
- if page_size not in [32, 64]:
- logger.warning(
- f"tokenspeed_mla only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
- )
- page_size = 64
if (
view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla"
@@ -1193,7 +1184,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
- """Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
+ """Validate resolved KV-cache dtype for the TRTLLM MLA backend."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
@@ -1206,19 +1197,6 @@ def _mla_kv_cache_dtype_checks(view: Any) -> dict:
raise ValueError(
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, bf16, or auto."
)
- if (
- view.attention_backend == "tokenspeed_mla"
- or view.decode_attention_backend == "tokenspeed_mla"
- ):
- if not get_platform().is_blackwell:
- raise ValueError(
- "tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
- )
- if view.kv_cache_dtype not in ["fp8_e4m3"]:
- raise ValueError(
- "tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
- f"got {view.kv_cache_dtype}."
- )
return {}
diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py
index c04056ebb0..f7583dd010 100644
--- a/python/sglang/srt/arg_groups/serving_hook.py
+++ b/python/sglang/srt/arg_groups/serving_hook.py
@@ -293,7 +293,8 @@ def handle_deprecated_args(server_args: Any):
"_handle_deprecated_args",
enable_flashinfer_allreduce_fusion=False,
)
- # Deprecated attention-backend alias: "compressed" -> "dsv4".
+ # Deprecated attention-backend aliases.
+ aliases = {"compressed": "dsv4", "tokenspeed_mla": "cutedsl_mla"}
renamed = {}
for attr in (
"attention_backend",
@@ -301,12 +302,30 @@ def handle_deprecated_args(server_args: Any):
"prefill_attention_backend",
"speculative_draft_attention_backend",
):
- if getattr(server_args, attr, None) == "compressed":
+ old = getattr(server_args, attr, None)
+ if old in aliases:
logger.warning(
- "--%s=compressed is deprecated; use 'dsv4' instead.",
+ "--%s=%s is deprecated; use '%s' instead.",
attr.replace("_", "-"),
+ old,
+ aliases[old],
)
- renamed[attr] = "dsv4"
+ renamed[attr] = aliases[old]
+ if renamed.get("prefill_attention_backend") == "cutedsl_mla":
+ # cutedsl_mla is decode-only.
+ renamed["prefill_attention_backend"] = "trtllm_mla"
+ if "tokenspeed_mla" in (cfg.attention_backend, cfg.decode_attention_backend):
+ # Keep what tokenspeed_mla implied: FP8 KV, verify on the decode kernel.
+ if cfg.kv_cache_dtype != "fp8_e4m3":
+ logger.warning(
+ "tokenspeed_mla required FP8 KV; setting --kv-cache-dtype=fp8_e4m3."
+ )
+ renamed["kv_cache_dtype"] = "fp8_e4m3"
+ if (
+ cfg.speculative_algorithm is not None
+ and cfg.speculative_attention_mode == "prefill"
+ ):
+ renamed["speculative_attention_mode"] = "decode"
if renamed:
declare_resolution(server_args, "_handle_deprecated_args", **renamed)
diff --git a/python/sglang/srt/layers/attention/attention_registry.py b/python/sglang/srt/layers/attention/attention_registry.py
index e79d5b014e..92aae6e8ca 100644
--- a/python/sglang/srt/layers/attention/attention_registry.py
+++ b/python/sglang/srt/layers/attention/attention_registry.py
@@ -2,6 +2,8 @@ import logging
import warnings
from typing import TYPE_CHECKING
+import torch
+
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
@@ -52,8 +54,6 @@ def register_attention_backend(name):
@register_attention_backend("flashinfer")
def create_flashinfer_backend(runner):
- import torch
-
if not runner.use_mla_backend:
from sglang.srt.layers.attention.flashinfer_backend import (
FlashInferAttnBackend,
@@ -95,31 +95,26 @@ def create_llada2_cfg_flashinfer_backend(runner):
def create_trtllm_mla_backend(runner):
if not runner.use_mla_backend:
raise ValueError("trtllm_mla backend can only be used with MLA models.")
+ _, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if get_parallel().dcp_enabled and get_spec().speculative_algorithm is not None:
- _, decode_backend = attention_backends_of(resolved_view(runner.server_args))
if decode_backend == "trtllm_mla":
raise ValueError(
"trtllm_mla cannot serve decode context parallelism with speculative "
"decoding: it does not forward the cyclic DCP metadata to its decode "
"kernel and returns no rank-local LSE for the cross-rank merge. "
- "Select cutedsl_mla or tokenspeed_mla."
+ "Select cutedsl_mla."
)
+ if decode_backend == "cutedsl_mla" and runner.kv_cache_dtype == torch.float8_e4m3fn:
+ from sglang.srt.layers.attention.cutedsl_mla_backend import (
+ CuteDslMLAPrefillBackend,
+ )
+
+ return CuteDslMLAPrefillBackend(runner)
from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend
return TRTLLMMLABackend(runner)
-@register_attention_backend("tokenspeed_mla")
-def create_tokenspeed_mla_backend(runner):
- if not runner.use_mla_backend:
- raise ValueError("tokenspeed_mla backend can only be used with MLA models.")
- from sglang.srt.layers.attention.tokenspeed_mla_backend import (
- TokenspeedMLABackend,
- )
-
- return TokenspeedMLABackend(runner)
-
-
@register_attention_backend("cutedsl_mla")
def create_cutedsl_mla_backend(runner):
if not runner.use_mla_backend:
diff --git a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py
index ea867c7c05..fa2ce98044 100644
--- a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py
+++ b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py
@@ -21,12 +21,17 @@ from typing import TYPE_CHECKING, Optional
import torch
+from sglang.kernels.jit.utils import is_arch_support_pdl
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
+from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
+ mla_kv_pack_quantize_fp8,
+)
from sglang.kernels.ops.attention.utils import (
concat_mla_absorb_q_general,
mla_quantize_and_rope_for_fp8,
mla_quantize_without_rope_for_fp8,
)
+from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
from sglang.srt.environ import envs
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
@@ -40,11 +45,13 @@ from sglang.srt.utils import is_flashinfer_available
if is_flashinfer_available():
import flashinfer
+ import flashinfer.rope as _flashinfer_rope
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.model_runner import ModelRunner
+ from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
logger = logging.getLogger(__name__)
@@ -296,3 +303,139 @@ class CuteDslMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
kv_indptr_buf=self.kv_indptr[i],
q_indptr_decode_buf=self.q_indptr_decode,
)
+
+
+class CuteDslMLAPrefillBackend(TRTLLMMLABackend):
+ """trtllm_mla prefill for cutedsl_mla deployments with FP8 KV.
+
+ The model-side MHA hooks below build FP8 Q/K/V with fused RoPE + quantize
+ and write the FP8 KV cache directly, so the ragged prefill kernel consumes
+ them without the per-tensor casts of the BF16 path.
+ """
+
+ @staticmethod
+ def _fused_rope_fp8_quantize(
+ q_nope: torch.Tensor,
+ q_pe: torch.Tensor,
+ k_nope: torch.Tensor,
+ k_pe: torch.Tensor,
+ cos_sin_cache: Optional[torch.Tensor],
+ positions: torch.Tensor,
+ is_neox: bool,
+ qk_nope_head_dim: int,
+ qk_rope_head_dim: int,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """Fused RoPE + FP8 quantize that also packs nope+pe along the last
+ dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
+
+ ``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
+ FP8 quantize and the packed layout, only the rotation is dropped.
+ """
+ num_heads = q_nope.shape[1]
+ seq_len = q_nope.shape[0]
+ q_fp8 = torch.empty(
+ (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
+ dtype=torch.float8_e4m3fn,
+ device=q_nope.device,
+ )
+ k_fp8 = torch.empty(
+ (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
+ dtype=torch.float8_e4m3fn,
+ device=k_nope.device,
+ )
+ if seq_len == 0:
+ return q_fp8, k_fp8
+
+ # Broadcast the shared latent k_pe across heads — RoPE is position-only
+ # so per-head outputs are identical, and the cache write below reuses
+ # head 0.
+ if k_pe.dim() == 3 and k_pe.shape[1] == 1:
+ k_pe_expanded = k_pe.expand(-1, num_heads, -1)
+ else:
+ k_pe_expanded = k_pe
+
+ if cos_sin_cache is None:
+ # NoPE layer: quantize straight into the packed buffers.
+ q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
+ q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
+ k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
+ k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
+ return q_fp8, k_fp8
+
+ _flashinfer_rope.mla_rope_quantize_fp8(
+ q_rope=q_pe,
+ k_rope=k_pe_expanded,
+ q_nope=q_nope,
+ k_nope=k_nope,
+ cos_sin_cache=cos_sin_cache,
+ pos_ids=positions,
+ is_neox=is_neox,
+ quantize_dtype=torch.float8_e4m3fn,
+ q_rope_out=q_fp8[..., qk_nope_head_dim:],
+ k_rope_out=k_fp8[..., qk_nope_head_dim:],
+ q_nope_out=q_fp8[..., :qk_nope_head_dim],
+ k_nope_out=k_fp8[..., :qk_nope_head_dim],
+ quant_scale_q=1.0,
+ quant_scale_kv=1.0,
+ enable_pdl=is_arch_support_pdl(),
+ )
+ return q_fp8, k_fp8
+
+ def prepare_prefill_qkv(
+ self,
+ *,
+ q: torch.Tensor,
+ q_pe: torch.Tensor,
+ kv_a: torch.Tensor,
+ k_pe: torch.Tensor,
+ positions: torch.Tensor,
+ layer: DeepseekV2AttentionMLA,
+ forward_batch: ForwardBatch,
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
+ kv = layer.kv_b_proj(kv_a)[0]
+ kv = kv.view(
+ -1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
+ )
+ k_nope = kv[..., : layer.qk_nope_head_dim]
+ v_bf16 = kv[..., layer.qk_nope_head_dim :]
+ q_nope = q[..., : layer.qk_nope_head_dim]
+
+ rotary_emb = layer.rotary_emb
+ q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
+ q_nope=q_nope,
+ q_pe=q_pe,
+ k_nope=k_nope,
+ k_pe=k_pe,
+ cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
+ positions=positions,
+ is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
+ qk_nope_head_dim=layer.qk_nope_head_dim,
+ qk_rope_head_dim=layer.qk_rope_head_dim,
+ )
+ v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
+
+ # k_pe is shared across heads (RoPE is position-only), so head 0
+ # reproduces the original [tokens, 1, qk_rope] latent layout.
+ kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
+ k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
+ self.token_to_kv_pool.set_mla_kv_buffer(
+ layer.attn_mha,
+ KVWriteLoc.for_batch(forward_batch),
+ kv_a_fp8.unsqueeze(1),
+ k_pe_fp8,
+ )
+ return q_fp8, k_fp8, v_fp8
+
+ def pack_prefix_chunk_kv(
+ self,
+ k_nope: torch.Tensor,
+ k_pe: torch.Tensor,
+ v: torch.Tensor,
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
+ strided ``v`` into contig FP8 V in a single kernel.
+ """
+ return mla_kv_pack_quantize_fp8(
+ k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
+ )
diff --git a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py
deleted file mode 100644
index 4b36a9506b..0000000000
--- a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py
+++ /dev/null
@@ -1,516 +0,0 @@
-# Copyright (c) 2026 LightSeek Foundation
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-from __future__ import annotations
-
-"""Attention backend for the tokenspeed-mla CuTe DSL kernels on Blackwell.
-
-Subclasses :class:`TRTLLMMLABackend` to share its MLA data preparation, prefill
-plumbing, and DCP metadata (rank-local KV lengths and page table). The decode
-forward lives here because the TokenSpeed decode kernel natively accepts CP
-rank/world metadata and returns the partial log-sum-exp needed by the cross-rank
-merge.
-"""
-
-import logging
-from typing import TYPE_CHECKING, Optional
-
-import torch
-
-from sglang.kernels.jit.utils import is_arch_support_pdl
-from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
-from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
- mla_kv_pack_quantize_fp8,
-)
-from sglang.kernels.ops.attention.utils import (
- mla_quantize_and_rope_for_fp8,
- mla_quantize_without_rope_for_fp8,
-)
-from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
-from sglang.srt.layers.attention.trtllm_mla_backend import (
- TRTLLMMLABackend,
- TRTLLMMLAMultiStepDraftBackend,
-)
-from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
-from sglang.srt.mem_cache.layout.paged_view import paged_row_view
-from sglang.srt.mem_cache.memory_pool import KVWriteLoc
-from sglang.srt.runtime_context import (
- get_parallel,
- get_resources,
- max_speculative_num_draft_tokens,
-)
-from sglang.srt.utils import is_flashinfer_available, is_tokenspeed_mla_available
-
-if is_flashinfer_available():
- import flashinfer.rope as _flashinfer_rope
-
-if is_tokenspeed_mla_available():
- import tokenspeed_mla
-
-if TYPE_CHECKING:
- from sglang.srt.layers.radix_attention import RadixAttention
- from sglang.srt.model_executor.forward_batch_info import ForwardBatch
- from sglang.srt.model_executor.model_runner import ModelRunner
- from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
-
-logger = logging.getLogger(__name__)
-
-
-# Workspace upper bound for tokenspeed_mla_decode:
-# num_sms * num_heads * max_q_len * (kv_lora_rank + 1) * sizeof(float32)
-# MAX_Q_LEN=8 covers EAGLE3 num_draft_tokens=4 plus headroom. Larger
-# speculative widths grow the bound at backend initialization.
-_TOKENSPEED_MAX_Q_LEN = 8
-
-
-def _get_tokenspeed_workspace(
- device: torch.device,
- num_heads: int,
- kv_lora_rank: int,
- max_q_len: int = _TOKENSPEED_MAX_Q_LEN,
-) -> torch.Tensor:
-
- # DCP target verification gathers Q to the full head count before launching
- # TokenSpeed; size for that launch shape, not the rank-local head count.
- num_heads *= get_parallel().attn_dcp_size
- max_q_len = max(max_q_len, _TOKENSPEED_MAX_Q_LEN)
- needed = (
- tokenspeed_mla.get_num_sm(device)
- * num_heads
- * max_q_len
- * (kv_lora_rank + 1)
- * 4
- )
- buffers = get_resources().buffers
- key = f"tokenspeed_mla_workspace:{device}"
- existing = buffers.get(key)
- if existing is None or existing.numel() < needed:
- buffers[key] = torch.empty(needed, dtype=torch.int8, device=device)
- return buffers[key]
-
-
-# TODO(Qiaolin-Yu): Merge this attention backend into trtllm_mla_backend.py
-# once the same CuteDSL kernels in flashinfer_trtllm are stable
-# and there is no performance gap compared to this backend.
-class TokenspeedMLABackend(TRTLLMMLABackend):
- """tokenspeed-mla CuTe DSL attention backend (Blackwell SM100, FP8 KV)."""
-
- def __init__(
- self,
- model_runner: ModelRunner,
- skip_prefill: bool = False,
- kv_indptr_buf: Optional[torch.Tensor] = None,
- q_indptr_decode_buf: Optional[torch.Tensor] = None,
- ):
- super().__init__(
- model_runner,
- skip_prefill,
- kv_indptr_buf,
- q_indptr_decode_buf,
- )
-
- if self.data_type != torch.float8_e4m3fn:
- raise ValueError(
- "tokenspeed_mla backend requires --kv-cache-dtype fp8_e4m3, "
- f"got data_type={self.data_type}."
- )
- if self.page_size not in (32, 64):
- raise ValueError(
- "tokenspeed_mla backend requires page_size in {32, 64}, "
- f"got page_size={self.page_size}."
- )
-
- self._tokenspeed_workspace: Optional[torch.Tensor] = None
- if is_tokenspeed_mla_available():
- self._tokenspeed_workspace = _get_tokenspeed_workspace(
- self.device,
- self.num_q_heads,
- self.kv_lora_rank,
- max_q_len=(max_speculative_num_draft_tokens() or 1),
- )
-
- # Pre-JIT the prefill kernel variants. Each cute.compile takes 1-2
- # min; without warm-up the first request trips the 300 s scheduler
- # watchdog.
- _compile_prefill_kernel = tokenspeed_mla.mla_prefill._compile_prefill_kernel
- _compiled_kernels = tokenspeed_mla.mla_prefill._compiled_kernels
- head_dim_qk = self.qk_nope_head_dim + self.qk_rope_head_dim
- enable_ex2_emulation = tokenspeed_mla.mla_prefill._enable_ex2_emulation()
- use_pdl = is_arch_support_pdl()
- for is_causal in (True, False):
- for return_lse in (True, False):
- # Non-causal is only entered from the chunked-prefix
- # branch, which always asks for the LSE.
- if is_causal is False and return_lse is False:
- continue
- # Runtime feeds fp8_e4m3fn q/k/v
- config = (
- torch.float8_e4m3fn,
- head_dim_qk,
- self.v_head_dim,
- is_causal,
- return_lse,
- use_pdl,
- enable_ex2_emulation,
- )
- if config in _compiled_kernels:
- continue
- _compiled_kernels[config] = _compile_prefill_kernel(
- torch.float8_e4m3fn,
- head_dim_qk,
- self.v_head_dim,
- is_causal,
- return_lse,
- use_pdl=use_pdl,
- enable_ex2_emulation=enable_ex2_emulation,
- )
-
- @staticmethod
- def _fused_rope_fp8_quantize(
- q_nope: torch.Tensor,
- q_pe: torch.Tensor,
- k_nope: torch.Tensor,
- k_pe: torch.Tensor,
- cos_sin_cache: Optional[torch.Tensor],
- positions: torch.Tensor,
- is_neox: bool,
- qk_nope_head_dim: int,
- qk_rope_head_dim: int,
- ) -> tuple[torch.Tensor, torch.Tensor]:
- """Fused RoPE + FP8 quantize that also packs nope+pe along the last
- dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
-
- ``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
- FP8 quantize and the packed layout, only the rotation is dropped.
- """
- num_heads = q_nope.shape[1]
- seq_len = q_nope.shape[0]
- q_fp8 = torch.empty(
- (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
- dtype=torch.float8_e4m3fn,
- device=q_nope.device,
- )
- k_fp8 = torch.empty(
- (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
- dtype=torch.float8_e4m3fn,
- device=k_nope.device,
- )
- if seq_len == 0:
- return q_fp8, k_fp8
-
- # Broadcast the shared latent k_pe across heads — RoPE is position-only
- # so per-head outputs are identical, and the cache write below reuses
- # head 0.
- if k_pe.dim() == 3 and k_pe.shape[1] == 1:
- k_pe_expanded = k_pe.expand(-1, num_heads, -1)
- else:
- k_pe_expanded = k_pe
-
- if cos_sin_cache is None:
- # NoPE layer: quantize straight into the packed buffers.
- q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
- q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
- k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
- k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
- return q_fp8, k_fp8
-
- _flashinfer_rope.mla_rope_quantize_fp8(
- q_rope=q_pe,
- k_rope=k_pe_expanded,
- q_nope=q_nope,
- k_nope=k_nope,
- cos_sin_cache=cos_sin_cache,
- pos_ids=positions,
- is_neox=is_neox,
- quantize_dtype=torch.float8_e4m3fn,
- q_rope_out=q_fp8[..., qk_nope_head_dim:],
- k_rope_out=k_fp8[..., qk_nope_head_dim:],
- q_nope_out=q_fp8[..., :qk_nope_head_dim],
- k_nope_out=k_fp8[..., :qk_nope_head_dim],
- quant_scale_q=1.0,
- quant_scale_kv=1.0,
- enable_pdl=is_arch_support_pdl(),
- )
- return q_fp8, k_fp8
-
- def prepare_prefill_qkv(
- self,
- *,
- q: torch.Tensor,
- q_pe: torch.Tensor,
- kv_a: torch.Tensor,
- k_pe: torch.Tensor,
- positions: torch.Tensor,
- layer: DeepseekV2AttentionMLA,
- forward_batch: ForwardBatch,
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
- """Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
- kv = layer.kv_b_proj(kv_a)[0]
- kv = kv.view(
- -1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
- )
- k_nope = kv[..., : layer.qk_nope_head_dim]
- v_bf16 = kv[..., layer.qk_nope_head_dim :]
- q_nope = q[..., : layer.qk_nope_head_dim]
-
- rotary_emb = layer.rotary_emb
- q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
- q_nope=q_nope,
- q_pe=q_pe,
- k_nope=k_nope,
- k_pe=k_pe,
- cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
- positions=positions,
- is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
- qk_nope_head_dim=layer.qk_nope_head_dim,
- qk_rope_head_dim=layer.qk_rope_head_dim,
- )
- v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
-
- # k_pe is shared across heads (RoPE is position-only), so head 0
- # reproduces the original [tokens, 1, qk_rope] latent layout.
- kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
- k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
- self.token_to_kv_pool.set_mla_kv_buffer(
- layer.attn_mha,
- KVWriteLoc.for_batch(forward_batch),
- kv_a_fp8.unsqueeze(1),
- k_pe_fp8,
- )
- return q_fp8, k_fp8, v_fp8
-
- def pack_prefix_chunk_kv(
- self,
- k_nope: torch.Tensor,
- k_pe: torch.Tensor,
- v: torch.Tensor,
- ) -> tuple[torch.Tensor, torch.Tensor]:
- """Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
- strided ``v`` into contig FP8 V in a single kernel.
- """
- return mla_kv_pack_quantize_fp8(
- k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
- )
-
- def _run_decode_kernel(
- self,
- query: torch.Tensor,
- kv_cache: torch.Tensor,
- block_tables: torch.Tensor,
- seq_lens: torch.Tensor,
- max_seq_len: int,
- layer: RadixAttention,
- *,
- causal_seqs: Optional[torch.Tensor] = None,
- cp_world: int = 1,
- cp_rank: int = 0,
- return_lse: bool = False,
- ):
- k_scale = getattr(layer, "k_scale_float", None)
- if k_scale is None:
- k_scale = 1.0
- softmax_scale = float(layer.scaling) * float(k_scale)
- output_scale = float(k_scale)
-
- seq_lens_i32 = (
- seq_lens if seq_lens.dtype == torch.int32 else seq_lens.to(torch.int32)
- )
- return tokenspeed_mla.tokenspeed_mla_decode(
- query=query,
- kv_cache=kv_cache,
- workspace_buffer=self._tokenspeed_workspace,
- kv_lora_rank=self.kv_lora_rank,
- qk_rope_head_dim=self.qk_rope_head_dim,
- block_tables=block_tables,
- seq_lens=seq_lens_i32,
- max_seq_len=int(max_seq_len),
- softmax_scale=softmax_scale,
- output_scale=output_scale,
- enable_pdl=is_arch_support_pdl(),
- return_lse=return_lse,
- causal_seqs=causal_seqs,
- cp_world=cp_world,
- cp_rank=cp_rank,
- )
-
- def forward_decode(
- self,
- q: torch.Tensor,
- k: torch.Tensor,
- v: torch.Tensor,
- layer: RadixAttention,
- forward_batch: ForwardBatch,
- save_kv_cache: bool = True,
- q_rope: Optional[torch.Tensor] = None,
- k_rope: Optional[torch.Tensor] = None,
- cos_sin_cache: Optional[torch.Tensor] = None,
- is_neox: Optional[bool] = False,
- llama_4_scaling: Optional[torch.Tensor] = None,
- ):
- parallel = get_parallel()
- if parallel.dcp_enabled and get_in_autotune_dummy_run():
- return self._dummy_dcp_decode_for_autotune(q, layer)
-
- if not parallel.dcp_enabled:
- return super().forward_decode(
- q,
- k,
- v,
- layer,
- forward_batch,
- save_kv_cache,
- q_rope,
- k_rope,
- cos_sin_cache,
- is_neox,
- llama_4_scaling,
- )
-
- assert q_rope is not None and k_rope is not None
- if cos_sin_cache is None:
- q, k, k_rope = mla_quantize_without_rope_for_fp8(
- q, q_rope, k.squeeze(1), k_rope.squeeze(1)
- )
- else:
- q, k, k_rope = mla_quantize_and_rope_for_fp8(
- q,
- q_rope,
- k.squeeze(1),
- k_rope.squeeze(1),
- forward_batch.positions,
- cos_sin_cache,
- is_neox,
- self.kv_lora_rank,
- self.qk_rope_head_dim,
- )
-
- if save_kv_cache:
- self.token_to_kv_pool.set_mla_kv_buffer(
- layer,
- KVWriteLoc.for_batch(forward_batch),
- k,
- k_rope,
- )
-
- query = q.view(-1, layer.tp_q_head_num, layer.head_dim)
- if llama_4_scaling is not None:
- query = (query.to(self.q_data_type) * llama_4_scaling).to(self.data_type)
- if query.dim() == 3:
- query = query.unsqueeze(1)
-
- k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
- kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
- metadata = (
- getattr(forward_batch, "decode_trtllm_mla_metadata", None)
- or self.forward_decode_metadata
- )
- metadata_batch_size = getattr(metadata, "batch_size", None)
- if (
- metadata_batch_size is not None
- and metadata_batch_size < forward_batch.batch_size
- ):
- self.init_forward_metadata(forward_batch)
- metadata = forward_batch.decode_trtllm_mla_metadata
-
- global_seq_lens = forward_batch.seq_lens[: forward_batch.batch_size]
- local_seq_lens = self._get_dcp_local_seq_lens(global_seq_lens)
- raw_out, lse = self._run_decode_kernel(
- query=query,
- kv_cache=kv_cache,
- block_tables=metadata.block_kv_indices,
- seq_lens=local_seq_lens,
- max_seq_len=metadata.max_seq_len_k,
- layer=layer,
- causal_seqs=global_seq_lens,
- cp_world=parallel.dcp_size,
- cp_rank=parallel.dcp_rank,
- return_lse=True,
- )
-
- output = raw_out.view(-1, layer.tp_q_head_num, layer.v_head_dim)
- lse = lse.view(-1, layer.tp_q_head_num)
- fixup_zero_kv_rows(
- output,
- lse,
- local_seq_lens,
- self.q_indptr_decode[: forward_batch.batch_size + 1],
- 1,
- )
- return output.flatten(1), lse
-
- def _run_prefill_kernel(
- self,
- q: torch.Tensor,
- k: torch.Tensor,
- v: torch.Tensor,
- layer: RadixAttention,
- batch_size: int,
- cum_seq_lens_q: torch.Tensor,
- max_q_len: int,
- seq_lens_kv: torch.Tensor,
- cum_seq_lens_kv: torch.Tensor,
- max_kv_len: int,
- is_causal: bool,
- return_lse: bool,
- out_buffer: torch.Tensor,
- o_sf_scale: float = 1.0,
- ): # Q/K/V arrive already in FP8 via the model-side fused path
- # (prepare_prefill_qkv / pack_prefix_chunk_kv); no quantize here.
- # Hybrid MLA models resolve the model-side hook through the outer
- # HybridLinearAttnBackend, so their fallback MHA path can pass V as a
- # last-dimension slice of kv_b_proj (stride(-2) > size(-1)). The
- # TokenSpeed prefill kernel requires dense Q/K/V layouts even though
- # the public wrapper accepts arbitrary torch tensors.
- q = q.contiguous()
- k = k.contiguous()
- v = v.contiguous()
- return tokenspeed_mla.tokenspeed_mla_prefill(
- query=q,
- key=k,
- value=v,
- seq_lens=seq_lens_kv,
- cum_seq_lens=cum_seq_lens_kv,
- max_seq_len=int(max_kv_len),
- batch_size=int(batch_size),
- softmax_scale=float(layer.scaling),
- is_causal=is_causal,
- return_lse=return_lse,
- cum_seq_lens_q=cum_seq_lens_q,
- max_seq_len_q=int(max_q_len),
- enable_pdl=is_arch_support_pdl(),
- )
-
-
-class TokenspeedMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
- """Multi-step draft backend for tokenspeed_mla used by EAGLE."""
-
- def __init__(
- self, model_runner: ModelRunner, topk: int, speculative_num_steps: int
- ):
- super().__init__(model_runner, topk, speculative_num_steps)
- # Parent populates self.attn_backends with TRT-LLM instances; replace
- # them with tokenspeed instances sharing the parent's index buffers.
- for i in range(self.speculative_num_steps - 1):
- self.attn_backends[i] = TokenspeedMLABackend(
- model_runner,
- skip_prefill=True,
- kv_indptr_buf=self.kv_indptr[i],
- q_indptr_decode_buf=self.q_indptr_decode,
- )
diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py
index 44989bc726..643613a0a9 100755
--- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py
+++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py
@@ -1017,8 +1017,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
raise NotImplementedError(
"trtllm_mla cannot forward a global causal bound to its decode "
"kernel, which is required for DCP with q_len > 1 (speculative "
- "target-verify / draft-extend); select cutedsl_mla or "
- "tokenspeed_mla for a DCP speculative run"
+ "target-verify / draft-extend); select cutedsl_mla "
+ "for a DCP speculative run"
)
# Scale computation for TRTLLM MLA kernel BMM1 operation:
@@ -1073,10 +1073,11 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
o_sf_scale: float = 1.0,
):
"""Hook for subclasses to swap the ragged prefill kernel. Q/K/V arrive
- in model-native dtype; subclasses do any kernel-specific quantization.
+ in model-native dtype, or already FP8 from a model-side prefill hook;
+ subclasses do any kernel-specific quantization.
Returns the output tensor or (output, lse) if return_lse."""
q_scale = k_scale = v_scale = 1.0
- if self.data_type == torch.float8_e4m3fn:
+ if self.data_type == torch.float8_e4m3fn and q.dtype != torch.float8_e4m3fn:
q, k, v, k_scale, v_scale = _quantize_fp8_qkv(q, k, v, layer)
return flashinfer.prefill.trtllm_ragged_attention_deepseek(
query=q,
diff --git a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py
index 3ceab0d877..9a4c145419 100644
--- a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py
+++ b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py
@@ -181,12 +181,6 @@ def handle_attention_trtllm_mla(attn, forward_batch):
return _dispatch_mla_subtype(attn, forward_batch)
-def handle_attention_tokenspeed_mla(attn, forward_batch):
- # tokenspeed_mla shares the trtllm_mla dispatch pattern: pure prefill goes
- # via MHA chunked KV (TRT-LLM ragged), spec decode / decode goes via MLA.
- return handle_attention_trtllm_mla(attn, forward_batch)
-
-
def handle_attention_aiter(attn, forward_batch):
# During PCG/BCG capture on ROCm, aiter fp8 MLA prefill has no capture
# kernels; route through the MHA path (radix_attention swaps attn_mqa for
@@ -268,7 +262,6 @@ AttentionBackendRegistry.register("fa3", handle_attention_fa3)
AttentionBackendRegistry.register("flashmla", handle_attention_flashmla)
AttentionBackendRegistry.register("fa4", handle_attention_fa4)
AttentionBackendRegistry.register("trtllm_mla", handle_attention_trtllm_mla)
-AttentionBackendRegistry.register("tokenspeed_mla", handle_attention_tokenspeed_mla)
AttentionBackendRegistry.register("aiter", handle_attention_aiter)
AttentionBackendRegistry.register("dsa", handle_attention_dsa)
AttentionBackendRegistry.register(
diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py
index 0fae375ec4..e5fa8b7d6b 100644
--- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py
+++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py
@@ -942,8 +942,7 @@ class DeepseekMLAForwardMixin:
return (
self.rotary_emb is not None
- and self.current_attention_backend
- in ("trtllm_mla", "tokenspeed_mla", "cutedsl_mla")
+ and self.current_attention_backend in ("trtllm_mla", "cutedsl_mla")
and (
forward_batch.forward_mode.is_decode_or_idle()
or forward_batch.forward_mode.is_target_verify()
diff --git a/python/sglang/srt/models/deepseek_common/utils.py b/python/sglang/srt/models/deepseek_common/utils.py
index ceefce6d80..71dd1bad13 100644
--- a/python/sglang/srt/models/deepseek_common/utils.py
+++ b/python/sglang/srt/models/deepseek_common/utils.py
@@ -70,7 +70,6 @@ FORWARD_ABSORB_CORE_ATTENTION_BACKENDS = [
"flashinfer",
"trtllm_mla",
"cutedsl_mla",
- "tokenspeed_mla",
"ascend",
"intel_xpu",
]
diff --git a/python/sglang/srt/speculative/draft_utils.py b/python/sglang/srt/speculative/draft_utils.py
index 5c13066884..085ff19976 100644
--- a/python/sglang/srt/speculative/draft_utils.py
+++ b/python/sglang/srt/speculative/draft_utils.py
@@ -105,7 +105,6 @@ class DraftBackendFactory:
"trtllm_mha": self._create_trtllm_mha_decode_backend,
"trtllm_mla": self._create_trtllm_mla_decode_backend,
"cutedsl_mla": self._create_cutedsl_mla_decode_backend,
- "tokenspeed_mla": self._create_tokenspeed_mla_decode_backend,
"dsa": self._create_dsa_decode_backend,
"nsa": self._create_dsa_decode_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_decode_backend,
@@ -137,7 +136,6 @@ class DraftBackendFactory:
"trtllm_mla": self._create_trtllm_mla_prefill_backend,
# cute-dsl MLA only supports decode; draft-extend falls back to trtllm-gen.
"cutedsl_mla": self._create_trtllm_mla_prefill_backend,
- "tokenspeed_mla": self._create_tokenspeed_mla_prefill_backend,
"dsa": self._create_dsa_prefill_backend,
"nsa": self._create_dsa_prefill_backend, # Deprecated alias for "dsa"
"ascend": self._create_ascend_prefill_backend,
@@ -394,23 +392,6 @@ class DraftBackendFactory:
),
)
- def _create_tokenspeed_mla_decode_backend(self):
- if not self.draft_model_runner.use_mla_backend:
- raise ValueError(
- "tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
- )
-
- from sglang.srt.layers.attention.tokenspeed_mla_backend import (
- TokenspeedMLAMultiStepDraftBackend,
- )
-
- return (
- "tokenspeed_mla",
- TokenspeedMLAMultiStepDraftBackend(
- self.draft_model_runner, self.topk, self.speculative_num_steps
- ),
- )
-
def _create_ascend_decode_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnMultiStepDraftBackend,
@@ -541,21 +522,6 @@ class DraftBackendFactory:
TRTLLMMLABackend(self.draft_model_runner, skip_prefill=False),
)
- def _create_tokenspeed_mla_prefill_backend(self):
- if not self.draft_model_runner.use_mla_backend:
- raise ValueError(
- "tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
- )
-
- from sglang.srt.layers.attention.tokenspeed_mla_backend import (
- TokenspeedMLABackend,
- )
-
- return (
- "tokenspeed_mla",
- TokenspeedMLABackend(self.draft_model_runner, skip_prefill=False),
- )
-
def _create_ascend_prefill_backend(self):
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnBackend,
diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py
index bf29c06429..36c1410bf7 100644
--- a/python/sglang/srt/speculative/eagle_worker_v2.py
+++ b/python/sglang/srt/speculative/eagle_worker_v2.py
@@ -27,7 +27,6 @@ from sglang.srt.layers.attention.qwen_sparse_attn_backend import (
QwenSparseAttnBackend,
QwenSparseMultiStepDraftBackend,
)
-from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend
from sglang.srt.layers.attention.trtllm_mla_backend import (
@@ -536,7 +535,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
TritonAttnBackend,
TRTLLMMLABackend,
TRTLLMHAAttnBackend,
- TokenspeedMLABackend,
FlashInferAttnBackend,
QwenSparseAttnBackend,
]
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index d980bc8c1f..c118396bb0 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -394,18 +394,6 @@ def is_flashinfer_available():
return importlib.util.find_spec("flashinfer") is not None and is_cuda()
-@lru_cache(maxsize=1)
-def is_tokenspeed_mla_available():
- """
- Check whether the tokenspeed_mla CuTe DSL kernels are available.
- Only available on NVIDIA Blackwell (SM100) at the moment.
- """
- return (
- importlib.util.find_spec("tokenspeed_mla") is not None
- and is_blackwell_supported()
- )
-
-
def is_nvidia_cublas_version_ge_12_9():
"""
temporary fix for issue #11272 (cublas 12.9+)
diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py
index 132c7f1bbc..e8018cfd22 100644
--- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py
+++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py
@@ -228,8 +228,8 @@ class MockMLAModelRunner(ModelRunner):
self.device = device
self.dtype = dtype
# `kv_cache_dtype` is the dtype the *storage* uses. For FP8 KV
- # cache (the production deployment dtype for tokenspeed_mla and
- # some trtllm_mla configs), the pool stores quantized bytes
+ # cache (the production deployment dtype for some trtllm_mla
+ # configs), the pool stores quantized bytes
# while the model still projects K/V in bf16; `set_mla_kv_buffer`
# does the BF16->FP8 cast on the way in.
self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype
diff --git a/test/registered/attention/test_trtllm_mla_stride_parity.py b/test/registered/attention/test_trtllm_mla_stride_parity.py
index 3d83bb6375..431e6bbdbc 100644
--- a/test/registered/attention/test_trtllm_mla_stride_parity.py
+++ b/test/registered/attention/test_trtllm_mla_stride_parity.py
@@ -1,6 +1,6 @@
"""The paged MLA decode kernels must not depend on the KV cache's slot stride.
-trtllm_mla, cutedsl_mla and tokenspeed_mla hand their decode kernels
+trtllm_mla and cutedsl_mla hand their decode kernels
`paged_row_view(k_cache, page_size).unsqueeze(1)`: `[pages, 1, page_size,
kv_cache_dim]` whose in-page token stride is the pool's SLOT stride. Under the
unified pool's token-major layout that stride spans every layer's latent row,
@@ -19,8 +19,7 @@ import unittest
import torch
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
-from sglang.srt.runtime_context import get_parallel
-from sglang.srt.utils import is_sm100_supported, is_tokenspeed_mla_available
+from sglang.srt.utils import is_sm100_supported
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=15, stage="base-b", runner_config="4-gpu-b200")
@@ -119,38 +118,6 @@ class TestPagedMLADecodeStrideParity(unittest.TestCase):
self._assert_stride_independent(run, kv_dtype)
- def test_tokenspeed_mla_decode_is_identical_across_slot_strides(self):
- if not is_tokenspeed_mla_available():
- self.skipTest("tokenspeed_mla is not installed")
- import tokenspeed_mla
-
- from sglang.srt.layers.attention.tokenspeed_mla_backend import (
- _get_tokenspeed_workspace,
- )
-
- # The workspace is sized for the DCP-gathered head count.
- with get_parallel().override(attn_dcp_size=1):
- workspace = _get_tokenspeed_workspace(
- torch.device("cuda"), _Q_HEADS, _KV_LORA_RANK
- )
- kv_dtype = torch.float8_e4m3fn
- q = torch.randn(_BS, 1, _Q_HEADS, _KV_CACHE_DIM, device="cuda").to(kv_dtype)
-
- def run(kv_cache):
- return tokenspeed_mla.tokenspeed_mla_decode(
- query=q,
- kv_cache=kv_cache,
- workspace_buffer=workspace,
- kv_lora_rank=_KV_LORA_RANK,
- qk_rope_head_dim=_QK_ROPE_HEAD_DIM,
- block_tables=self.block_tables,
- seq_lens=self.seq_lens,
- max_seq_len=_PAGES * _PAGE_SIZE,
- softmax_scale=_KV_CACHE_DIM**-0.5,
- )
-
- self._assert_stride_independent(run, kv_dtype)
-
if __name__ == "__main__":
unittest.main()
diff --git a/test/registered/attention/unittests/KNOWN_FAILURES.md b/test/registered/attention/unittests/KNOWN_FAILURES.md
index 67a7b7742f..fac847ec74 100644
--- a/test/registered/attention/unittests/KNOWN_FAILURES.md
+++ b/test/registered/attention/unittests/KNOWN_FAILURES.md
@@ -68,7 +68,6 @@ so that "skipped: ..." results have a quick lookup.
|---|---|---|---|
| `flashmla` decode/verify | SM 9.0 (Hopper) only | `mla/test_flashmla.py:_DECODE_REQUIRES_SM90A` | `Dense decode MLA is only supported on SM90a architecture` |
| `trtllm_mla` | SM 12.0a / 12.1a | `mla/test_trtllm_mla.py::_supported` | FlashInfer XQA MLA dispatch reject |
-| `tokenspeed_mla` | SM ≥ 10.0 + FP8 KV + pkg | `mla/test_tokenspeed_mla.py::_supported` | `tokenspeed_mla` import or kernel dispatch |
| `trtllm_mha` prefill | SM ≥ 10.0 | `dense/test_trtllm_mha.py` decode-only matrix | FlashInfer TRT-LLM Gen FMHA reject (`Unsupported architecture`) |
| `dsa` `fa3` impl | SM 9.x only | `dsa_impl_capability("fa3")` | `flash_attn at sgl-kernel is only supported on sm90 and above` |
| `dsa` `trtllm` impl | exactly SM 10.0 | `dsa_impl_capability("trtllm")` | `Missing TRTLLM-GEN kernel` (compiled for SM10.0) |
@@ -176,7 +175,6 @@ fail at backend init.
|---|---|---|
| FlashMLA | `64` only | `server_args.py:2767-2770` |
| TRT-LLM MLA | `{32, 64}` | `server_args.py:2790-2794` |
-| Tokenspeed MLA | `{32, 64}` | `server_args.py:2809-2813`, `tokenspeed_mla_backend.py:111-113` |
| TRT-LLM MHA | `{16, 32, 64}` | `server_args.py:2849-2853` |
| FA4 (non-MLA) | `128` when default-selected | `server_args.py:2862-2870` |
| DSV4 | `256` only | `deepseek_v4_backend.py:355`, `dsv4/metadata.py:134` |
@@ -191,7 +189,6 @@ fail at backend init.
| `flashinfer_mla` | `1` only | `flashinfer_mla_backend.py:910-913` |
| `flashmla` | `1` only | `flashmla_backend.py:555-558` |
| `trtllm_mla` | `1` only | `trtllm_mla_backend.py:1223-1229` (inherits) |
-| `tokenspeed_mla` | `1` only | `tokenspeed_mla_backend.py:341-347` (inherits) |
| `dsv4` | `0` or `1` | `deepseek_v4_backend.py:369`, `:363` (HIP) |
| `trtllm_mha` (graph replay) | `1` only | `trtllm_mha_backend.py:459,492`; `server_args.py:2391-2392` |
@@ -199,7 +196,6 @@ fail at backend init.
| Backend | Allowed dtype | Citation |
|---|---|---|
-| `tokenspeed_mla` | `fp8_e4m3` only | `server_args.py:2814-2818` |
| `trtllm_mla` | `{fp8_e4m3, fp4_e2m1, bf16, auto}` | `server_args.py:2796-2799` |
| `fa3` | not `fp8_e5m2` (silently falls back to `triton`) | `server_args.py:2855-2860` |
| `dsv4` | Packed FP8/BF16 layout enforced by `DeepSeekV4TokenToKVPool` | `deepseek_v4_backend.py:363` |
diff --git a/test/registered/attention/unittests/mla/README.md b/test/registered/attention/unittests/mla/README.md
index e4a08cc227..52774f0794 100644
--- a/test/registered/attention/unittests/mla/README.md
+++ b/test/registered/attention/unittests/mla/README.md
@@ -21,7 +21,6 @@ Columns are runner modes; rows are attention backends. Cells use:
| `flashinfer` | ✓ 10 input layouts with DeepSeek-like `kv_lora_rank=512`, `qk_rope_head_dim=64` | ✓ MLA decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | ✓ EAGLE ragged-accept | blocked: `is_draft_extend()` default `include_v2=False` (`flashinfer_mla_backend.py:432,501,454-455,512`) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashinfer_mla_backend.py:910-913`) | ✓ EAGLE ragged-accept (V1) | — (no FKVMTP wiring for MLA) |
| `flashmla` | ✓ FlashMLA-compatible page-size-64 cases (zero-prefix exact page, input page edges 63/64/65, prefix exact page, total exact page, cross page, ragged, decode page-boundary, decode bsz=1 nonzero prefix) | ✓ page-size-64 decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | deferred: parent FlashInfer-MLA capture path expects 1D `cuda_graph_kv_indices`, FlashMLA allocates 2D `[max_bs, (max_context+PAGE_SIZE)//PAGE_SIZE]` (`flashmla_backend.py:347-348` + parent `init_forward_metadata_capture_cuda_graph`) | — (FlashMLA does not implement V2) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashmla_backend.py:555-558`) | — (DE CG deferred above) | — |
| `trtllm_mla` | skip:hw — needs SM 12.0a / 12.1a (`is_sm120_supported`) | — | — | — | blocked: `topk=1` only (`trtllm_mla_backend.py:1223-1229` inherits from FlashInfer MLA) | — | — | — | — | — | — | — |
-| `tokenspeed_mla` | skip:hw — needs `find_spec("tokenspeed_mla")`, SM 10.0+, and `kv_cache_dtype=fp8_e4m3` (`server_args.py:2814-2818`); current MLA fixture does not emit FP8 KV cache | — | — | — | blocked: `topk=1` only (`tokenspeed_mla_backend.py:341-347` inherits from TRT-LLM MLA) | — | — | — | — | — | — | — |
## Input And Config Coverage
@@ -56,15 +55,12 @@ multi-step draft backends and cannot ever appear at runtime.
- **TRT-LLM MLA tree verify / draft-extend with `topk > 1`** —
`TRTLLMMLAMultiStepDraftBackend` inherits from
`FlashInferMLAMultiStepDraftBackend` (`trtllm_mla_backend.py:1223-1229`).
-- **Tokenspeed MLA tree verify / draft-extend with `topk > 1`** —
- `TokenspeedMLAMultiStepDraftBackend` inherits from
- `TRTLLMMLAMultiStepDraftBackend` (`tokenspeed_mla_backend.py:341-347`).
- **FlashInfer-MLA `DRAFT_EXTEND_V2` graph capture/replay** —
`flashinfer_mla_backend.py:432,501` only route through `is_draft_extend()`
(default `include_v2=False`); `else: raise ValueError("Invalid mode")` at
`flashinfer_mla_backend.py:454-455,512`.
- **All MLA backends fixed page size** — FlashMLA forces `page_size=64`,
- TRT-LLM MLA and Tokenspeed MLA force `page_size in {32, 64}`.
+ TRT-LLM MLA forces `page_size in {32, 64}`.
## Backend Container Gate (SM10.x)
@@ -85,6 +81,6 @@ See `KNOWN_FAILURES.md` §3 for the full root cause + fix.
override capture/replay in `FlashMLABackend` to use its 2D layout, or
allocate both parent-style 1D and FlashMLA-style 2D buffers and route
`DRAFT_EXTEND` to the parent path).
-- Add hardware-gated tests for `trtllm_mla` and `tokenspeed_mla`
+- Add hardware-gated tests for `trtllm_mla`
decode (chain spec only) when the appropriate hardware/KV dtype fixtures are
available.
diff --git a/test/registered/attention/unittests/mla/test_tokenspeed_mla.py b/test/registered/attention/unittests/mla/test_tokenspeed_mla.py
deleted file mode 100644
index 5ba21ab877..0000000000
--- a/test/registered/attention/unittests/mla/test_tokenspeed_mla.py
+++ /dev/null
@@ -1,265 +0,0 @@
-import importlib.util
-import unittest
-
-import torch
-
-from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
-from sglang.srt.model_executor.forward_batch_info import ForwardMode
-from sglang.test.kits.attention_unittest.attention_methods.mla_attention import (
- MLAAttentionCase,
- run_mla_attention_case,
-)
-from sglang.test.test_utils import CustomTestCase
-
-# tokenspeed_mla is a CuTe DSL backend for Blackwell (SM100). It additionally
-# enforces:
-# - kv_cache_dtype == torch.float8_e4m3fn (kv_cache_dtype=fp8_e4m3)
-# - page_size in {32, 64}
-# See python/sglang/srt/layers/attention/tokenspeed_mla_backend.py and
-# is_tokenspeed_mla_available() in python/sglang/srt/utils/common.py.
-#
-# The shared MLAAttentionCase fixture now supports `fp8_kv_cache=True`:
-# `MockMLAModelRunner` decouples `kv_cache_dtype` from the model `dtype`
-# and routes K writes through the FP8 quantize path. The reference still
-# computes against BF16 K (independent of the cache bytes) and tolerates
-# FP8 quant noise via a looser tolerance.
-_MIN_SM = 100
-
-
-def _supported() -> tuple[bool, str]:
- if not torch.cuda.is_available():
- return False, "CUDA is required"
- if importlib.util.find_spec("tokenspeed_mla") is None:
- return False, "tokenspeed_mla python package is not installed"
- major, minor = torch.cuda.get_device_capability()
- sm = major * 10 + minor
- if sm < _MIN_SM:
- return (
- False,
- f"tokenspeed_mla requires SM {_MIN_SM // 10}.{_MIN_SM % 10}+ (Blackwell), "
- f"got SM {major}.{minor}",
- )
- return True, ""
-
-
-_SUPPORTED, _SKIP_REASON = _supported()
-
-
-MLA_SHAPE_KWARGS = dict(
- kv_lora_rank=512,
- qk_rope_head_dim=64,
- hidden_size=1024,
- max_context_len=256,
-)
-
-
-from sglang.test.ci.ci_register import register_cuda_ci
-
-register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200")
-register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-large")
-
-
-@unittest.skipIf(not _SUPPORTED, _SKIP_REASON)
-class TestTokenspeedMLAAttentionBackendCorrectness(CustomTestCase):
- # tokenspeed_mla allows page_size in {32, 64} (server_args.py:2809-2813)
- # and requires kv_cache_dtype==fp8_e4m3 (server_args.py:2814-2818).
- # Cover both page sizes, with extend + decode + ragged + page-boundary.
- CASES = (
- # ----- page_size=64 -----
- MLAAttentionCase(
- name="mla_extend_tokenspeed_zero_prefix_exact_page_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(0,),
- extend_lens=(64,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_zero_prefix_below_page_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(0,),
- extend_lens=(63,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_zero_prefix_above_page_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(0,),
- extend_lens=(65,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_prefix_exact_page_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(64,),
- extend_lens=(4,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_cross_page_boundary_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(63,),
- extend_lens=(2,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_ragged_page_boundary_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=64,
- prefix_lens=(0, 32, 64),
- extend_lens=(63, 32, 1),
- ),
- MLAAttentionCase(
- name="mla_decode_tokenspeed_page_boundary_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.DECODE,
- num_heads=4,
- page_size=64,
- prefix_lens=(62, 63, 64),
- ),
- MLAAttentionCase(
- name="mla_decode_tokenspeed_bsz1_nonzero_prefix_64",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.DECODE,
- num_heads=4,
- page_size=64,
- prefix_lens=(31,),
- ),
- # ----- page_size=32 -----
- MLAAttentionCase(
- name="mla_extend_tokenspeed_zero_prefix_exact_page_32",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=32,
- prefix_lens=(0,),
- extend_lens=(32,),
- ),
- MLAAttentionCase(
- name="mla_extend_tokenspeed_cross_page_boundary_32",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.EXTEND,
- num_heads=4,
- page_size=32,
- prefix_lens=(31,),
- extend_lens=(2,),
- ),
- MLAAttentionCase(
- name="mla_decode_tokenspeed_page_boundary_32",
- backend="tokenspeed_mla",
- forward_mode=ForwardMode.DECODE,
- num_heads=4,
- page_size=32,
- prefix_lens=(30, 31, 32),
- ),
- )
-
- def test_projected_mla_attention_cases(self):
- for case in self.CASES:
- with self.subTest(case=case.name, backend=case.backend):
- # Looser tolerance to absorb FP8 quant noise (the reference
- # reads BF16 K independent of the FP8 cache, so per-element
- # drift from the BF16->FP8 cast accumulates through the
- # attention reduction).
- run_mla_attention_case(
- self,
- case,
- fp8_kv_cache=True,
- atol=2e-1,
- rtol=2e-1,
- **MLA_SHAPE_KWARGS,
- )
-
-
-class TestTokenspeedMLANoPEPrefillQuantize(CustomTestCase):
- """NoPE MLA layers must still get packed FP8 prefill Q/K.
-
- Layers built with ``skip_rope`` carry no rotary embedding, so the prefill
- quantize runs with ``cos_sin_cache=None``. It must return the same packed
- ``[nope | pe]`` FP8 layout as the RoPE path, and the head-0 pe slice it
- writes to the KV cache must equal the plain FP8 cast of the unroped
- ``k_pe`` that the decode path reads back.
- """
-
- T = 7
- NUM_HEADS = 4
- QK_NOPE_HEAD_DIM = 128
- QK_ROPE_HEAD_DIM = 64
- KV_LORA_RANK = 512
-
- def test_nope_prefill_quantize_packs_without_rope(self):
- device = "cuda" if torch.cuda.is_available() else "cpu"
- head_dim = self.QK_NOPE_HEAD_DIM + self.QK_ROPE_HEAD_DIM
-
- q_nope = torch.randn(
- self.T,
- self.NUM_HEADS,
- self.QK_NOPE_HEAD_DIM,
- dtype=torch.bfloat16,
- device=device,
- )
- q_pe = torch.randn(
- self.T,
- self.NUM_HEADS,
- self.QK_ROPE_HEAD_DIM,
- dtype=torch.bfloat16,
- device=device,
- )
- k_nope = torch.randn(
- self.T,
- self.NUM_HEADS,
- self.QK_NOPE_HEAD_DIM,
- dtype=torch.bfloat16,
- device=device,
- )
- # k_pe reaches the backend as the strided tail of the latent cache.
- latent_cache = torch.randn(
- self.T,
- 1,
- self.KV_LORA_RANK + self.QK_ROPE_HEAD_DIM,
- dtype=torch.bfloat16,
- device=device,
- )
- k_pe = latent_cache[:, :, self.KV_LORA_RANK :]
-
- q_fp8, k_fp8 = TokenspeedMLABackend._fused_rope_fp8_quantize(
- q_nope=q_nope,
- q_pe=q_pe,
- k_nope=k_nope,
- k_pe=k_pe,
- cos_sin_cache=None,
- positions=torch.arange(self.T, device=device),
- is_neox=True,
- qk_nope_head_dim=self.QK_NOPE_HEAD_DIM,
- qk_rope_head_dim=self.QK_ROPE_HEAD_DIM,
- )
-
- for name, out in (("q", q_fp8), ("k", k_fp8)):
- with self.subTest(tensor=name):
- self.assertEqual(out.shape, (self.T, self.NUM_HEADS, head_dim))
- self.assertEqual(out.dtype, torch.float8_e4m3fn)
- self.assertTrue(out.is_contiguous())
-
- fp8 = torch.float8_e4m3fn
- nope = self.QK_NOPE_HEAD_DIM
- self.assertTrue(torch.equal(q_fp8[..., :nope], q_nope.to(fp8)))
- self.assertTrue(torch.equal(q_fp8[..., nope:], q_pe.to(fp8)))
- self.assertTrue(torch.equal(k_fp8[..., :nope], k_nope.to(fp8)))
- # The slice prepare_prefill_qkv writes into the KV cache; the decode
- # NoPE path reads it back as a plain cast of the unroped k_pe.
- self.assertTrue(torch.equal(k_fp8[:, 0:1, nope:], k_pe.to(fp8)))
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/test/registered/dcp/test_kimi_linear_dcp4.py b/test/registered/dcp/test_kimi_linear_dcp4.py
index a565553b26..6819ba3183 100644
--- a/test/registered/dcp/test_kimi_linear_dcp4.py
+++ b/test/registered/dcp/test_kimi_linear_dcp4.py
@@ -1,4 +1,4 @@
-"""Four-Blackwell acceptance coverage for Kimi Linear TokenSpeed MLA DCP.
+"""Four-Blackwell acceptance coverage for Kimi Linear CuteDSL MLA DCP.
The captured-shape and eager-shape requests deliberately straddle
``--cuda-graph-max-bs-decode``. This guards both the regular CUDA graph
@@ -37,7 +37,7 @@ def _has_four_blackwell_gpus() -> bool:
@unittest.skipUnless(
_has_four_blackwell_gpus(),
- "TokenSpeed MLA DCP acceptance requires four Blackwell GPUs",
+ "CuteDSL MLA DCP acceptance requires four Blackwell GPUs",
)
class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase):
model = KIMI_LINEAR_MODEL
@@ -62,7 +62,7 @@ class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase):
"--dcp-size",
"4",
"--attention-backend",
- "tokenspeed_mla",
+ "cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dcp-comm-backend",
diff --git a/test/registered/dcp/test_kimi_linear_dcp_dspark4.py b/test/registered/dcp/test_kimi_linear_dcp_dspark4.py
index 49e9f448b6..19f4e17861 100644
--- a/test/registered/dcp/test_kimi_linear_dcp_dspark4.py
+++ b/test/registered/dcp/test_kimi_linear_dcp_dspark4.py
@@ -1,4 +1,4 @@
-"""Four-Blackwell Kimi Linear TokenSpeed DCP + DSpark static acceptance test."""
+"""Four-Blackwell Kimi Linear CuteDSL MLA DCP + DSpark static acceptance test."""
import json
import socket
@@ -96,7 +96,7 @@ def _wait_for_port_release(base_url: str, timeout: float = 30.0) -> None:
@unittest.skipUnless(
_has_four_blackwell_gpus(),
- "Kimi Linear TokenSpeed DCP + DSpark requires four Blackwell GPUs",
+ "Kimi Linear CuteDSL MLA DCP + DSpark requires four Blackwell GPUs",
)
class TestKimiLinearDCPDSpark4(CustomTestCase):
base_url = DEFAULT_URL_FOR_TEST
@@ -132,7 +132,7 @@ class TestKimiLinearDCPDSpark4(CustomTestCase):
"--max-running-requests",
str(MAX_RUNNING_REQUESTS),
"--attention-backend",
- "tokenspeed_mla",
+ "cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dcp-comm-backend",
diff --git a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py
index f8296a0efc..594ff04cb3 100644
--- a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py
+++ b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py
@@ -14,7 +14,6 @@ import torch
from sglang.srt.layers.attention import trtllm_mla_backend as backend_module
from sglang.srt.layers.attention.cutedsl_mla_backend import CuteDslMLABackend
-from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLABackend,
TRTLLMMLADecodeMetadata,
@@ -168,10 +167,6 @@ class TestTRTLLMMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
backend_cls = TRTLLMMLABackend
-class TestTokenspeedMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
- backend_cls = TokenspeedMLABackend
-
-
class TestCuteDslMLADCPMetadata(_DCPMetadataTests, CustomTestCase):
backend_cls = CuteDslMLABackend
diff --git a/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py b/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py
index 175c2dd850..3fbb52d1ac 100644
--- a/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py
+++ b/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py
@@ -30,7 +30,7 @@ NIAH_KEY = "739391"
COMMON_ARGS = [
"--attention-backend",
- "tokenspeed_mla",
+ "cutedsl_mla",
"--kv-cache-dtype",
"fp8_e4m3",
"--dtype",
diff --git a/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py b/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py
index e9f954cf8f..c484f47657 100644
--- a/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py
+++ b/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py
@@ -14,9 +14,9 @@ prefix-cache replay below are here to catch.
`cutedsl_mla` stands in for the family: it is the DCP-native MLA decode kernel
on Blackwell (the only backend flashinfer accepts `enable_dcp=True` for) and
what the B300 hosts run, and prefill resolves to `trtllm_mla`, so one server
-covers both halves of the page-table contract. `trtllm_mla` and
-`tokenspeed_mla` share that builder and were checked by hand against the
-static pool (see below) rather than given a CI server each.
+covers both halves of the page-table contract. `trtllm_mla` shares that
+builder and was checked by hand against the static pool (see below) rather
+than given a CI server.
Blackwell-only, hence its own module: the Hopper suite next door cannot host
it. In extra-b rather than nightly because what it guards is a per-PR argument
@@ -29,13 +29,11 @@ pools agree:
unified / static
TP2 DCP2 cutedsl_mla 0.905 / 0.905
TP2 DCP2 trtllm_mla 0.895 / 0.895
- TP2 DCP2 tokenspeed_mla 0.890 / 0.895, repeat 0.890 / 0.885
TP4 DCP4 cutedsl_mla 0.910 / 0.900
-The cutedsl_mla and trtllm_mla pairs reproduced exactly. tokenspeed_mla moved
-on a repeat of the same questions on BOTH pools, so its spread is the backend,
-not the pool. The TP4 DCP4 row is there because it widens the virtual page to
-4 x 64 = 256, past what this file exercises.
+The cutedsl_mla and trtllm_mla pairs reproduced exactly. The TP4 DCP4 row is
+there because it widens the virtual page to 4 x 64 = 256, past what this file
+exercises.
python -m pytest test/registered/models_e2e/test_kimi_linear_unified_memory_dcp_blackwell.py -v
"""
diff --git a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py
index 23bc9a6de1..76561d4edf 100644
--- a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py
+++ b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py
@@ -256,7 +256,6 @@ class TestLSEBaseByBackend(CustomTestCase):
self.assertTrue(is_mla_dcp_lse_base_on_e("flashmla"))
self.assertTrue(is_mla_dcp_lse_base_on_e("cutedsl_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e("flashinfer_mla"))
- self.assertFalse(is_mla_dcp_lse_base_on_e("tokenspeed_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e("trtllm_mla"))
self.assertFalse(is_mla_dcp_lse_base_on_e(None))
diff --git a/test/registered/unit/mem_cache/test_physical_write_loc.py b/test/registered/unit/mem_cache/test_physical_write_loc.py
index 78f18700ab..c85c5dee74 100644
--- a/test/registered/unit/mem_cache/test_physical_write_loc.py
+++ b/test/registered/unit/mem_cache/test_physical_write_loc.py
@@ -243,7 +243,6 @@ _UNIFIED_BACKEND_FILES = {
"trtllm_mha": ("layers/attention/trtllm_mha_backend.py",),
"trtllm_mla": ("layers/attention/trtllm_mla_backend.py",),
"cutedsl_mla": ("layers/attention/cutedsl_mla_backend.py",),
- "tokenspeed_mla": ("layers/attention/tokenspeed_mla_backend.py",),
"flashmla": ("layers/attention/flashmla_backend.py",),
}
# Model code and the rest of the forward path also write KV directly.
diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py
index d8ed36e6c3..bc781268fb 100644
--- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py
+++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py
@@ -24,7 +24,7 @@ page ids:
ONE builder computes that formula for every family -- the iteration plan's
read table (`KVLocPlan.read_table`) -- and the backends only differ in how
they consume it:
- - trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows
+ - trtllm_mla / cutedsl_mla / flashmla: the plan's rows
copied into their padded block tables (`KVIndexTranslator.copy_page_table`);
- the flashinfer updaters: token ids reconstructed from the table by
`create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`;
diff --git a/test/registered/unit/server_args/test_page_major_backend_allowlist.py b/test/registered/unit/server_args/test_page_major_backend_allowlist.py
index e35d1c3835..9969b82b71 100644
--- a/test/registered/unit/server_args/test_page_major_backend_allowlist.py
+++ b/test/registered/unit/server_args/test_page_major_backend_allowlist.py
@@ -98,13 +98,12 @@ class TestPageMajorBackendAllowlist(unittest.TestCase):
"trtllm_mla",
"flashinfer",
"cutedsl_mla",
- "tokenspeed_mla",
"flashmla",
)
# Wired for the per-layer MHA/SWA views.
PER_LAYER_VIEW_MHA_BACKENDS = ("fa3", "fa4", "flashinfer", "trtllm_mha")
# MLA-family kernels that must never leak into the MHA arm.
- MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "tokenspeed_mla", "flashmla")
+ MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "flashmla")
# No virtual-to-physical id wiring anywhere: must stay rejected until they get one.
UNWIRED_BACKENDS = ("aiter",)
diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
index 6e02dc78ec..023fb5b8b5 100644
--- a/test/registered/unit/server_args/test_server_args.py
+++ b/test/registered/unit/server_args/test_server_args.py
@@ -710,6 +710,43 @@ class TestImageProcessorBackend(CustomTestCase):
handle_deprecated_args(server_args)
+class TestTokenspeedMlaAlias(CustomTestCase):
+ def test_maps_to_cutedsl_with_fp8_kv(self):
+ server_args = ServerArgs(model_path="dummy", attention_backend="tokenspeed_mla")
+
+ with self.assertLogs(serving_hook.logger, level="WARNING"):
+ handle_deprecated_args(server_args)
+
+ self.assertEqual(
+ resolution_result(server_args, "attention_backend"), "cutedsl_mla"
+ )
+ self.assertEqual(resolution_result(server_args, "kv_cache_dtype"), "fp8_e4m3")
+ self.assertEqual(
+ resolution_result(server_args, "speculative_attention_mode"), "prefill"
+ )
+
+ def test_speculative_verify_moves_to_decode(self):
+ server_args = ServerArgs(
+ model_path="dummy",
+ decode_attention_backend="tokenspeed_mla",
+ prefill_attention_backend="tokenspeed_mla",
+ kv_cache_dtype="fp8_e4m3",
+ speculative_algorithm="EAGLE3",
+ )
+
+ handle_deprecated_args(server_args)
+
+ self.assertEqual(
+ resolution_result(server_args, "decode_attention_backend"), "cutedsl_mla"
+ )
+ self.assertEqual(
+ resolution_result(server_args, "prefill_attention_backend"), "trtllm_mla"
+ )
+ self.assertEqual(
+ resolution_result(server_args, "speculative_attention_mode"), "decode"
+ )
+
+
class TestMultimodalFeatureTransport(CustomTestCase):
@staticmethod
def _set_model_type(server_args, *, is_multimodal):
diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py
index d4195df061..dd1659de36 100644
--- a/test/registered/unit/server_args/test_unified_memory_spec_gate.py
+++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py
@@ -93,7 +93,6 @@ _VERIFY_BACKENDS = (
"triton",
"trtllm_mla",
"cutedsl_mla",
- "tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",