From 93088901fb413e5a81eeb81b675fc21b4adc5b1f Mon Sep 17 00:00:00 2001 From: Chunan Zeng Date: Tue, 20 Oct 2026 18:50:05 -0700 Subject: [PATCH] [Chore] attention: Remove the tokenspeed_mla backend and dependency (#44803) --- .../skills/sglang-runtime-context/SKILL.md | 4 +- .../autoregressive/Moonshotai/Kimi-K3.mdx | 1 - .../advanced_features/attention_backend.mdx | 10 - .../autoregressive/kimi-k25-deployment.jsx | 15 +- .../autoregressive/kimi-k26-deployment.jsx | 11 +- .../kimi-k27-code-deployment.jsx | 8 +- python/pyproject.toml | 1 - .../sglang/srt/arg_groups/attention_hook.py | 2 +- python/sglang/srt/arg_groups/choices.py | 3 +- python/sglang/srt/arg_groups/kv_cache_hook.py | 4 +- .../srt/arg_groups/model_override_base.py | 6 +- .../srt/arg_groups/model_overrides/kimi_k3.py | 19 +- python/sglang/srt/arg_groups/overrides.py | 24 +- python/sglang/srt/arg_groups/serving_hook.py | 27 +- .../layers/attention/attention_registry.py | 25 +- .../layers/attention/cutedsl_mla_backend.py | 143 +++++ .../attention/tokenspeed_mla_backend.py | 516 ------------------ .../layers/attention/trtllm_mla_backend.py | 9 +- .../attention_backend_handler.py | 7 - .../attention_forward_methods/forward_mla.py | 3 +- .../srt/models/deepseek_common/utils.py | 1 - python/sglang/srt/speculative/draft_utils.py | 34 -- .../sglang/srt/speculative/eagle_worker_v2.py | 2 - python/sglang/srt/utils/common.py | 12 - .../attention_methods/mla_attention.py | 4 +- .../test_trtllm_mla_stride_parity.py | 37 +- .../attention/unittests/KNOWN_FAILURES.md | 4 - .../attention/unittests/mla/README.md | 8 +- .../unittests/mla/test_tokenspeed_mla.py | 265 --------- test/registered/dcp/test_kimi_linear_dcp4.py | 6 +- .../dcp/test_kimi_linear_dcp_dspark4.py | 6 +- .../test_trtllm_mla_family_dcp_metadata.py | 5 - .../test_kimi_linear_pd_dcp4.py | 2 +- ...imi_linear_unified_memory_dcp_blackwell.py | 14 +- .../ops/attention/test_dcp_lse_combine.py | 1 - .../unit/mem_cache/test_physical_write_loc.py | 1 - .../mem_cache/test_unified_mla_block_table.py | 2 +- .../test_page_major_backend_allowlist.py | 3 +- .../unit/server_args/test_server_args.py | 37 ++ .../test_unified_memory_spec_gate.py | 1 - 40 files changed, 269 insertions(+), 1014 deletions(-) delete mode 100644 python/sglang/srt/layers/attention/tokenspeed_mla_backend.py delete mode 100644 test/registered/attention/unittests/mla/test_tokenspeed_mla.py diff --git a/.agents/skills/sglang-runtime-context/SKILL.md b/.agents/skills/sglang-runtime-context/SKILL.md index 99628611e0..b1790a9187 100644 --- a/.agents/skills/sglang-runtime-context/SKILL.md +++ b/.agents/skills/sglang-runtime-context/SKILL.md @@ -509,8 +509,8 @@ Named slots + two keyed-lazy registries: side stream is `"lora_side"`. Two call sites may share a name only if their work belongs on one stream — sharing across roles serializes intended overlap. - `get_buffer(name, factory)` — get-or-create a named persistent buffer. Grow-only or - per-device semantics manage their `resources.buffers` entries directly (see tokenspeed / - SM120 split / Marlin workspace). Buffer names are per-backend today; do not silently + per-device semantics manage their `resources.buffers` entries directly (see SM120 + split / Marlin workspace). Buffer names are per-backend today; do not silently share. - Singletons with manager semantics (EP dispatcher buffers, EPLB recorder/metadata, graph memory pool) keep their owning accessors/classes as facades; only the *state* lives in a diff --git a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx index 916a4b037b..1d457d5c99 100644 --- a/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx +++ b/docs/cookbook/autoregressive/Moonshotai/Kimi-K3.mdx @@ -207,7 +207,6 @@ Speculation: DSPARK holds block size + 1 (= 8) intermediate states per request - DCP is the only axis that shards the TP-replicated MLA KV; Low-Latency skips it. - Leave `--dcp-comm-backend` unset (fabric-resolved: `fi_a2a` on GB200/GB300, `a2a` on B200/B300). - No `--enable-symm-mem` under DCP (force-disabled for decode-graph correctness). -- Explicit `tokenspeed_mla` force-rewrites `--kv-cache-dtype` to fp8; the default `cutedsl_mla` serves either dtype. - Calculator ratios run well above 1 here (`r > 1` is legal): `bfloat16` state buys admission, `fp8` KV buys context. - Don't use EP with an a2a backend: a2a buffers reclaim the KV that DCP buys. Compose only to measure. a2a backend is set when `--moe-a2a-backend` is set. diff --git a/docs/docs/advanced_features/attention_backend.mdx b/docs/docs/advanced_features/attention_backend.mdx index a842451236..7d4dad9ec3 100644 --- a/docs/docs/advanced_features/attention_backend.mdx +++ b/docs/docs/advanced_features/attention_backend.mdx @@ -254,15 +254,6 @@ The support matrix is split into two parts: MHA (standard attention) and MLA (mu ✅ ❌ - - **TokenSpeed MLA (Blackwell)** - 32 or 64 - ✅ (required) - ❌ - ✅ - ✅ - ❌ - **FA3 (FlashAttention 3)** n/a @@ -330,7 +321,6 @@ MLA page-size constraints: - Cutlass MLA: page_size = 128. - TRTLLM MLA: page_size ∈ {32, 64}. - CuteDSL MLA: page_size ∈ {32, 64} (decode-only; prefill falls back to `trtllm_mla` when unset). -- TokenSpeed MLA: page_size ∈ {32, 64} (Blackwell SM100/SM12x only; requires `--kv-cache-dtype fp8_e4m3`). ### GDN Attention Backends diff --git a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx index ad21e8df6a..a630daa27d 100644 --- a/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k25-deployment.jsx @@ -196,15 +196,18 @@ export const KimiK25Deployment = () => { cmd += ' \\\n --speculative-algorithm EAGLE3 \\\n --speculative-num-steps 3 \\\n --speculative-eagle-topk 1 \\\n --speculative-num-draft-tokens 4 \\\n --speculative-draft-model-path lightseekorg/kimi-k2.5-eagle3-mla'; } - const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300'; + const usesCutedslMla = hardware === 'b300' || hardware === 'gb300'; - // Blackwell (B300/GB300): tokenspeed MLA attention backend - if (usesTokenspeedMla) { - cmd += ' \\\n --attention-backend tokenspeed_mla'; + // Blackwell (B300/GB300): CuteDSL MLA attention backend + if (usesCutedslMla) { + cmd += ' \\\n --attention-backend cutedsl_mla'; + if (speculative === 'enabled') { + cmd += ' \\\n --speculative-attention-mode decode'; + } } - // FP8 KV cache for AMD memory efficiency and tokenspeed MLA compatibility - if (isAMD || usesTokenspeedMla) { + // FP8 KV cache for AMD and Blackwell memory efficiency + if (isAMD || usesCutedslMla) { cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; } diff --git a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx index 251155c307..1e222c1781 100644 --- a/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k26-deployment.jsx @@ -194,13 +194,16 @@ export const KimiK26Deployment = () => { cmd += ' \\\n --speculative-draft-model-path lightseekorg/kimi-k2.6-eagle3.1-mla'; } - const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300'; + const usesCutedslMla = hardware === 'b300' || hardware === 'gb300'; - if (usesTokenspeedMla) { - cmd += ' \\\n --attention-backend tokenspeed_mla'; + if (usesCutedslMla) { + cmd += ' \\\n --attention-backend cutedsl_mla'; + if (speculative === 'enabled') { + cmd += ' \\\n --speculative-attention-mode decode'; + } } - if (isAMD || usesTokenspeedMla) { + if (isAMD || usesCutedslMla) { cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; } diff --git a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx index ec7b5b88aa..af53b7eee8 100644 --- a/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx +++ b/docs/src/snippets/autoregressive/kimi-k27-code-deployment.jsx @@ -189,13 +189,13 @@ export const KimiK27CodeDeployment = () => { cmd += ' \\\n --tool-call-parser kimi_k2'; } - const usesTokenspeedMla = hardware === 'b300' || hardware === 'gb300'; + const usesCutedslMla = hardware === 'b300' || hardware === 'gb300'; - if (usesTokenspeedMla) { - cmd += ' \\\n --attention-backend tokenspeed_mla'; + if (usesCutedslMla) { + cmd += ' \\\n --attention-backend cutedsl_mla'; } - if (isAMD || usesTokenspeedMla) { + if (isAMD || usesCutedslMla) { cmd += ' \\\n --kv-cache-dtype fp8_e4m3'; } diff --git a/python/pyproject.toml b/python/pyproject.toml index c6d53beebc..60bb7b7cc9 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -83,7 +83,6 @@ dependencies = [ "tilelang==0.1.12", "timm==1.0.16", "tokenizers==0.23.2", - "tokenspeed_mla==0.1.8", "torch==2.14.1", "torch_memory_saver>=0.0.9.post1", "torchaudio==2.11.0", diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index 92a23018a0..4bf3b28178 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -182,7 +182,7 @@ def handle_attention_backend_compatibility(server_args: Any): # fallback stay below. run_post_process_pass(server_args, _mla_backend_page_constraints) - # The TRT-LLM / tokenspeed MLA kv-dtype validations moved to the + # The TRT-LLM MLA kv-dtype validations moved to the # resolution pipeline (arg_groups/overrides.py: # _mla_kv_cache_dtype_checks), invoked here at their legacy slot. diff --git a/python/sglang/srt/arg_groups/choices.py b/python/sglang/srt/arg_groups/choices.py index 20a6253b9a..e96ed1e0d6 100644 --- a/python/sglang/srt/arg_groups/choices.py +++ b/python/sglang/srt/arg_groups/choices.py @@ -84,7 +84,7 @@ ATTENTION_BACKEND_CHOICES = [ "flashmla", "trtllm_mla", "cutedsl_mla", - "tokenspeed_mla", + "tokenspeed_mla", # Deprecated alias for "cutedsl_mla" "trtllm_mha", "hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), Hopper (SM90) only, requires --page-size 64 "minicpm_flashattn", @@ -234,7 +234,6 @@ CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [ "flashmla", "cutedsl_mla", "trtllm_mla", - "tokenspeed_mla", ] add_chunked_prefix_cache_attention_backend = ( CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 2d4ab9a5ee..2fe7977c2c 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -481,7 +481,6 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset( "triton", "trtllm_mla", "cutedsl_mla", - "tokenspeed_mla", "flashmla", "flashinfer", "fa3", @@ -749,7 +748,7 @@ def _validate_unified_memory_dcp(server_args: Any) -> None: # The trtllm_mla family builds its DCP block table through the pool's v2p # gather (create_mla_kv_page_table_for_dcp), so it speaks the same # two-stage contract as flashinfer. - dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"} + dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla"} backends = set(attention_backends_of(resolved_view(server_args))) backends.discard(None) assert backends <= dcp_allowed, ( @@ -806,7 +805,6 @@ def handle_page_major_kv_layout(server_args: Any): "trtllm_mla", "flashinfer", "cutedsl_mla", - "tokenspeed_mla", "flashmla", } elif model_config.has_asymmetric_kv: diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py index 3b965ddbac..68aa916c84 100644 --- a/python/sglang/srt/arg_groups/model_override_base.py +++ b/python/sglang/srt/arg_groups/model_override_base.py @@ -351,14 +351,10 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf return "triton" -def _dspark_verify_on_decode_backend( - backend: Optional[str], q_len: int, kv_cache_dtype: Optional[str] -) -> bool: +def _dspark_verify_on_decode_backend(backend: Optional[str], q_len: int) -> bool: """Whether the MLA decode backend can serve a q_len-wide target verify.""" if backend == "trtllm_mla": return True - if backend == "tokenspeed_mla": - return kv_cache_dtype == "fp8_e4m3" and q_len <= 8 if backend == "cutedsl_mla": # cute-dsl monolithic MLA decode folds the verify tokens into the head # dim (fold_sq), so it serves any DSPARK verify width. Needs flashinfer diff --git a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py index 6a0157314e..fe615e3f6c 100644 --- a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py +++ b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py @@ -92,21 +92,6 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: prefill_attention_backend="trtllm_mla", decode_attention_backend="cutedsl_mla", ) - elif decode_backend == "tokenspeed_mla": - logger.info( - "Kimi-K3 DCP overrides attention backends: " - f"prefill={prefill_backend!r}, decode={decode_backend!r} -> " - "'tokenspeed_mla'." - ) - logger.info( - "Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: " - f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'." - ) - overrides.update( - prefill_attention_backend="tokenspeed_mla", - decode_attention_backend="tokenspeed_mla", - kv_cache_dtype="fp8_e4m3", - ) elif decode_backend == "aiter": _require_kimi_k3_aiter_gluon_dcp_support() # Override prefill backend to aiter by default @@ -123,7 +108,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: ) else: raise AssertionError( - f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla', 'tokenspeed_mla' or 'aiter', got {decode_backend!r}." + f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'aiter', got {decode_backend!r}." ) if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in ( @@ -167,7 +152,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: # that still needs declaring -- else verify stays on the prefill backend, # whose host-side plan (flashinfer by default) forces a per-step D2H. _, backend = attention_backends_of(cfg) - if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype): + if _dspark_verify_on_decode_backend(backend, q_len): overrides["speculative_attention_mode"] = "decode" logger.info( "Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend " diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 101817dc90..9a672420d6 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1143,15 +1143,6 @@ def _mla_backend_page_constraints(view: Any) -> dict: f"TensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from {page_size} to 64." ) page_size = 64 - if ( - view.attention_backend == "tokenspeed_mla" - or view.decode_attention_backend == "tokenspeed_mla" - ): - if page_size not in [32, 64]: - logger.warning( - f"tokenspeed_mla only supports page_size of 32 or 64, changing page_size from {page_size} to 64." - ) - page_size = 64 if ( view.attention_backend == "cutedsl_mla" or view.decode_attention_backend == "cutedsl_mla" @@ -1193,7 +1184,7 @@ def _mla_backend_page_constraints(view: Any) -> dict: @register_post_process def _mla_kv_cache_dtype_checks(view: Any) -> dict: - """Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends.""" + """Validate resolved KV-cache dtype for the TRTLLM MLA backend.""" if ( view.attention_backend == "trtllm_mla" or view.decode_attention_backend == "trtllm_mla" @@ -1206,19 +1197,6 @@ def _mla_kv_cache_dtype_checks(view: Any) -> dict: raise ValueError( "TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, bf16, or auto." ) - if ( - view.attention_backend == "tokenspeed_mla" - or view.decode_attention_backend == "tokenspeed_mla" - ): - if not get_platform().is_blackwell: - raise ValueError( - "tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)." - ) - if view.kv_cache_dtype not in ["fp8_e4m3"]: - raise ValueError( - "tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, " - f"got {view.kv_cache_dtype}." - ) return {} diff --git a/python/sglang/srt/arg_groups/serving_hook.py b/python/sglang/srt/arg_groups/serving_hook.py index c04056ebb0..f7583dd010 100644 --- a/python/sglang/srt/arg_groups/serving_hook.py +++ b/python/sglang/srt/arg_groups/serving_hook.py @@ -293,7 +293,8 @@ def handle_deprecated_args(server_args: Any): "_handle_deprecated_args", enable_flashinfer_allreduce_fusion=False, ) - # Deprecated attention-backend alias: "compressed" -> "dsv4". + # Deprecated attention-backend aliases. + aliases = {"compressed": "dsv4", "tokenspeed_mla": "cutedsl_mla"} renamed = {} for attr in ( "attention_backend", @@ -301,12 +302,30 @@ def handle_deprecated_args(server_args: Any): "prefill_attention_backend", "speculative_draft_attention_backend", ): - if getattr(server_args, attr, None) == "compressed": + old = getattr(server_args, attr, None) + if old in aliases: logger.warning( - "--%s=compressed is deprecated; use 'dsv4' instead.", + "--%s=%s is deprecated; use '%s' instead.", attr.replace("_", "-"), + old, + aliases[old], ) - renamed[attr] = "dsv4" + renamed[attr] = aliases[old] + if renamed.get("prefill_attention_backend") == "cutedsl_mla": + # cutedsl_mla is decode-only. + renamed["prefill_attention_backend"] = "trtllm_mla" + if "tokenspeed_mla" in (cfg.attention_backend, cfg.decode_attention_backend): + # Keep what tokenspeed_mla implied: FP8 KV, verify on the decode kernel. + if cfg.kv_cache_dtype != "fp8_e4m3": + logger.warning( + "tokenspeed_mla required FP8 KV; setting --kv-cache-dtype=fp8_e4m3." + ) + renamed["kv_cache_dtype"] = "fp8_e4m3" + if ( + cfg.speculative_algorithm is not None + and cfg.speculative_attention_mode == "prefill" + ): + renamed["speculative_attention_mode"] = "decode" if renamed: declare_resolution(server_args, "_handle_deprecated_args", **renamed) diff --git a/python/sglang/srt/layers/attention/attention_registry.py b/python/sglang/srt/layers/attention/attention_registry.py index e79d5b014e..92aae6e8ca 100644 --- a/python/sglang/srt/layers/attention/attention_registry.py +++ b/python/sglang/srt/layers/attention/attention_registry.py @@ -2,6 +2,8 @@ import logging import warnings from typing import TYPE_CHECKING +import torch + from sglang.srt.arg_groups.overrides import ( attention_backends_of, resolved_view, @@ -52,8 +54,6 @@ def register_attention_backend(name): @register_attention_backend("flashinfer") def create_flashinfer_backend(runner): - import torch - if not runner.use_mla_backend: from sglang.srt.layers.attention.flashinfer_backend import ( FlashInferAttnBackend, @@ -95,31 +95,26 @@ def create_llada2_cfg_flashinfer_backend(runner): def create_trtllm_mla_backend(runner): if not runner.use_mla_backend: raise ValueError("trtllm_mla backend can only be used with MLA models.") + _, decode_backend = attention_backends_of(resolved_view(runner.server_args)) if get_parallel().dcp_enabled and get_spec().speculative_algorithm is not None: - _, decode_backend = attention_backends_of(resolved_view(runner.server_args)) if decode_backend == "trtllm_mla": raise ValueError( "trtllm_mla cannot serve decode context parallelism with speculative " "decoding: it does not forward the cyclic DCP metadata to its decode " "kernel and returns no rank-local LSE for the cross-rank merge. " - "Select cutedsl_mla or tokenspeed_mla." + "Select cutedsl_mla." ) + if decode_backend == "cutedsl_mla" and runner.kv_cache_dtype == torch.float8_e4m3fn: + from sglang.srt.layers.attention.cutedsl_mla_backend import ( + CuteDslMLAPrefillBackend, + ) + + return CuteDslMLAPrefillBackend(runner) from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend return TRTLLMMLABackend(runner) -@register_attention_backend("tokenspeed_mla") -def create_tokenspeed_mla_backend(runner): - if not runner.use_mla_backend: - raise ValueError("tokenspeed_mla backend can only be used with MLA models.") - from sglang.srt.layers.attention.tokenspeed_mla_backend import ( - TokenspeedMLABackend, - ) - - return TokenspeedMLABackend(runner) - - @register_attention_backend("cutedsl_mla") def create_cutedsl_mla_backend(runner): if not runner.use_mla_backend: diff --git a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py index ea867c7c05..fa2ce98044 100644 --- a/python/sglang/srt/layers/attention/cutedsl_mla_backend.py +++ b/python/sglang/srt/layers/attention/cutedsl_mla_backend.py @@ -21,12 +21,17 @@ from typing import TYPE_CHECKING, Optional import torch +from sglang.kernels.jit.utils import is_arch_support_pdl from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows +from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import ( + mla_kv_pack_quantize_fp8, +) from sglang.kernels.ops.attention.utils import ( concat_mla_absorb_q_general, mla_quantize_and_rope_for_fp8, mla_quantize_without_rope_for_fp8, ) +from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize from sglang.srt.environ import envs from sglang.srt.layers.attention.trtllm_mla_backend import ( TRTLLMMLABackend, @@ -40,11 +45,13 @@ from sglang.srt.utils import is_flashinfer_available if is_flashinfer_available(): import flashinfer + import flashinfer.rope as _flashinfer_rope if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.model_runner import ModelRunner + from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA logger = logging.getLogger(__name__) @@ -296,3 +303,139 @@ class CuteDslMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend): kv_indptr_buf=self.kv_indptr[i], q_indptr_decode_buf=self.q_indptr_decode, ) + + +class CuteDslMLAPrefillBackend(TRTLLMMLABackend): + """trtllm_mla prefill for cutedsl_mla deployments with FP8 KV. + + The model-side MHA hooks below build FP8 Q/K/V with fused RoPE + quantize + and write the FP8 KV cache directly, so the ragged prefill kernel consumes + them without the per-tensor casts of the BF16 path. + """ + + @staticmethod + def _fused_rope_fp8_quantize( + q_nope: torch.Tensor, + q_pe: torch.Tensor, + k_nope: torch.Tensor, + k_pe: torch.Tensor, + cos_sin_cache: Optional[torch.Tensor], + positions: torch.Tensor, + is_neox: bool, + qk_nope_head_dim: int, + qk_rope_head_dim: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Fused RoPE + FP8 quantize that also packs nope+pe along the last + dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast. + + ``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the + FP8 quantize and the packed layout, only the rotation is dropped. + """ + num_heads = q_nope.shape[1] + seq_len = q_nope.shape[0] + q_fp8 = torch.empty( + (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim), + dtype=torch.float8_e4m3fn, + device=q_nope.device, + ) + k_fp8 = torch.empty( + (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim), + dtype=torch.float8_e4m3fn, + device=k_nope.device, + ) + if seq_len == 0: + return q_fp8, k_fp8 + + # Broadcast the shared latent k_pe across heads — RoPE is position-only + # so per-head outputs are identical, and the cache write below reuses + # head 0. + if k_pe.dim() == 3 and k_pe.shape[1] == 1: + k_pe_expanded = k_pe.expand(-1, num_heads, -1) + else: + k_pe_expanded = k_pe + + if cos_sin_cache is None: + # NoPE layer: quantize straight into the packed buffers. + q_fp8[..., :qk_nope_head_dim].copy_(q_nope) + q_fp8[..., qk_nope_head_dim:].copy_(q_pe) + k_fp8[..., :qk_nope_head_dim].copy_(k_nope) + k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded) + return q_fp8, k_fp8 + + _flashinfer_rope.mla_rope_quantize_fp8( + q_rope=q_pe, + k_rope=k_pe_expanded, + q_nope=q_nope, + k_nope=k_nope, + cos_sin_cache=cos_sin_cache, + pos_ids=positions, + is_neox=is_neox, + quantize_dtype=torch.float8_e4m3fn, + q_rope_out=q_fp8[..., qk_nope_head_dim:], + k_rope_out=k_fp8[..., qk_nope_head_dim:], + q_nope_out=q_fp8[..., :qk_nope_head_dim], + k_nope_out=k_fp8[..., :qk_nope_head_dim], + quant_scale_q=1.0, + quant_scale_kv=1.0, + enable_pdl=is_arch_support_pdl(), + ) + return q_fp8, k_fp8 + + def prepare_prefill_qkv( + self, + *, + q: torch.Tensor, + q_pe: torch.Tensor, + kv_a: torch.Tensor, + k_pe: torch.Tensor, + positions: torch.Tensor, + layer: DeepseekV2AttentionMLA, + forward_batch: ForwardBatch, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache.""" + kv = layer.kv_b_proj(kv_a)[0] + kv = kv.view( + -1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim + ) + k_nope = kv[..., : layer.qk_nope_head_dim] + v_bf16 = kv[..., layer.qk_nope_head_dim :] + q_nope = q[..., : layer.qk_nope_head_dim] + + rotary_emb = layer.rotary_emb + q_fp8, k_fp8 = self._fused_rope_fp8_quantize( + q_nope=q_nope, + q_pe=q_pe, + k_nope=k_nope, + k_pe=k_pe, + cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache, + positions=positions, + is_neox=True if rotary_emb is None else rotary_emb.is_neox_style, + qk_nope_head_dim=layer.qk_nope_head_dim, + qk_rope_head_dim=layer.qk_rope_head_dim, + ) + v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl()) + + # k_pe is shared across heads (RoPE is position-only), so head 0 + # reproduces the original [tokens, 1, qk_rope] latent layout. + kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl()) + k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :] + self.token_to_kv_pool.set_mla_kv_buffer( + layer.attn_mha, + KVWriteLoc.for_batch(forward_batch), + kv_a_fp8.unsqueeze(1), + k_pe_fp8, + ) + return q_fp8, k_fp8, v_fp8 + + def pack_prefix_chunk_kv( + self, + k_nope: torch.Tensor, + k_pe: torch.Tensor, + v: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize + strided ``v`` into contig FP8 V in a single kernel. + """ + return mla_kv_pack_quantize_fp8( + k_nope, k_pe, v, enable_pdl=is_arch_support_pdl() + ) diff --git a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py b/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py deleted file mode 100644 index 4b36a9506b..0000000000 --- a/python/sglang/srt/layers/attention/tokenspeed_mla_backend.py +++ /dev/null @@ -1,516 +0,0 @@ -# Copyright (c) 2026 LightSeek Foundation -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from __future__ import annotations - -"""Attention backend for the tokenspeed-mla CuTe DSL kernels on Blackwell. - -Subclasses :class:`TRTLLMMLABackend` to share its MLA data preparation, prefill -plumbing, and DCP metadata (rank-local KV lengths and page table). The decode -forward lives here because the TokenSpeed decode kernel natively accepts CP -rank/world metadata and returns the partial log-sum-exp needed by the cross-rank -merge. -""" - -import logging -from typing import TYPE_CHECKING, Optional - -import torch - -from sglang.kernels.jit.utils import is_arch_support_pdl -from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows -from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import ( - mla_kv_pack_quantize_fp8, -) -from sglang.kernels.ops.attention.utils import ( - mla_quantize_and_rope_for_fp8, - mla_quantize_without_rope_for_fp8, -) -from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize -from sglang.srt.layers.attention.trtllm_mla_backend import ( - TRTLLMMLABackend, - TRTLLMMLAMultiStepDraftBackend, -) -from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run -from sglang.srt.mem_cache.layout.paged_view import paged_row_view -from sglang.srt.mem_cache.memory_pool import KVWriteLoc -from sglang.srt.runtime_context import ( - get_parallel, - get_resources, - max_speculative_num_draft_tokens, -) -from sglang.srt.utils import is_flashinfer_available, is_tokenspeed_mla_available - -if is_flashinfer_available(): - import flashinfer.rope as _flashinfer_rope - -if is_tokenspeed_mla_available(): - import tokenspeed_mla - -if TYPE_CHECKING: - from sglang.srt.layers.radix_attention import RadixAttention - from sglang.srt.model_executor.forward_batch_info import ForwardBatch - from sglang.srt.model_executor.model_runner import ModelRunner - from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA - -logger = logging.getLogger(__name__) - - -# Workspace upper bound for tokenspeed_mla_decode: -# num_sms * num_heads * max_q_len * (kv_lora_rank + 1) * sizeof(float32) -# MAX_Q_LEN=8 covers EAGLE3 num_draft_tokens=4 plus headroom. Larger -# speculative widths grow the bound at backend initialization. -_TOKENSPEED_MAX_Q_LEN = 8 - - -def _get_tokenspeed_workspace( - device: torch.device, - num_heads: int, - kv_lora_rank: int, - max_q_len: int = _TOKENSPEED_MAX_Q_LEN, -) -> torch.Tensor: - - # DCP target verification gathers Q to the full head count before launching - # TokenSpeed; size for that launch shape, not the rank-local head count. - num_heads *= get_parallel().attn_dcp_size - max_q_len = max(max_q_len, _TOKENSPEED_MAX_Q_LEN) - needed = ( - tokenspeed_mla.get_num_sm(device) - * num_heads - * max_q_len - * (kv_lora_rank + 1) - * 4 - ) - buffers = get_resources().buffers - key = f"tokenspeed_mla_workspace:{device}" - existing = buffers.get(key) - if existing is None or existing.numel() < needed: - buffers[key] = torch.empty(needed, dtype=torch.int8, device=device) - return buffers[key] - - -# TODO(Qiaolin-Yu): Merge this attention backend into trtllm_mla_backend.py -# once the same CuteDSL kernels in flashinfer_trtllm are stable -# and there is no performance gap compared to this backend. -class TokenspeedMLABackend(TRTLLMMLABackend): - """tokenspeed-mla CuTe DSL attention backend (Blackwell SM100, FP8 KV).""" - - def __init__( - self, - model_runner: ModelRunner, - skip_prefill: bool = False, - kv_indptr_buf: Optional[torch.Tensor] = None, - q_indptr_decode_buf: Optional[torch.Tensor] = None, - ): - super().__init__( - model_runner, - skip_prefill, - kv_indptr_buf, - q_indptr_decode_buf, - ) - - if self.data_type != torch.float8_e4m3fn: - raise ValueError( - "tokenspeed_mla backend requires --kv-cache-dtype fp8_e4m3, " - f"got data_type={self.data_type}." - ) - if self.page_size not in (32, 64): - raise ValueError( - "tokenspeed_mla backend requires page_size in {32, 64}, " - f"got page_size={self.page_size}." - ) - - self._tokenspeed_workspace: Optional[torch.Tensor] = None - if is_tokenspeed_mla_available(): - self._tokenspeed_workspace = _get_tokenspeed_workspace( - self.device, - self.num_q_heads, - self.kv_lora_rank, - max_q_len=(max_speculative_num_draft_tokens() or 1), - ) - - # Pre-JIT the prefill kernel variants. Each cute.compile takes 1-2 - # min; without warm-up the first request trips the 300 s scheduler - # watchdog. - _compile_prefill_kernel = tokenspeed_mla.mla_prefill._compile_prefill_kernel - _compiled_kernels = tokenspeed_mla.mla_prefill._compiled_kernels - head_dim_qk = self.qk_nope_head_dim + self.qk_rope_head_dim - enable_ex2_emulation = tokenspeed_mla.mla_prefill._enable_ex2_emulation() - use_pdl = is_arch_support_pdl() - for is_causal in (True, False): - for return_lse in (True, False): - # Non-causal is only entered from the chunked-prefix - # branch, which always asks for the LSE. - if is_causal is False and return_lse is False: - continue - # Runtime feeds fp8_e4m3fn q/k/v - config = ( - torch.float8_e4m3fn, - head_dim_qk, - self.v_head_dim, - is_causal, - return_lse, - use_pdl, - enable_ex2_emulation, - ) - if config in _compiled_kernels: - continue - _compiled_kernels[config] = _compile_prefill_kernel( - torch.float8_e4m3fn, - head_dim_qk, - self.v_head_dim, - is_causal, - return_lse, - use_pdl=use_pdl, - enable_ex2_emulation=enable_ex2_emulation, - ) - - @staticmethod - def _fused_rope_fp8_quantize( - q_nope: torch.Tensor, - q_pe: torch.Tensor, - k_nope: torch.Tensor, - k_pe: torch.Tensor, - cos_sin_cache: Optional[torch.Tensor], - positions: torch.Tensor, - is_neox: bool, - qk_nope_head_dim: int, - qk_rope_head_dim: int, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Fused RoPE + FP8 quantize that also packs nope+pe along the last - dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast. - - ``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the - FP8 quantize and the packed layout, only the rotation is dropped. - """ - num_heads = q_nope.shape[1] - seq_len = q_nope.shape[0] - q_fp8 = torch.empty( - (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim), - dtype=torch.float8_e4m3fn, - device=q_nope.device, - ) - k_fp8 = torch.empty( - (seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim), - dtype=torch.float8_e4m3fn, - device=k_nope.device, - ) - if seq_len == 0: - return q_fp8, k_fp8 - - # Broadcast the shared latent k_pe across heads — RoPE is position-only - # so per-head outputs are identical, and the cache write below reuses - # head 0. - if k_pe.dim() == 3 and k_pe.shape[1] == 1: - k_pe_expanded = k_pe.expand(-1, num_heads, -1) - else: - k_pe_expanded = k_pe - - if cos_sin_cache is None: - # NoPE layer: quantize straight into the packed buffers. - q_fp8[..., :qk_nope_head_dim].copy_(q_nope) - q_fp8[..., qk_nope_head_dim:].copy_(q_pe) - k_fp8[..., :qk_nope_head_dim].copy_(k_nope) - k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded) - return q_fp8, k_fp8 - - _flashinfer_rope.mla_rope_quantize_fp8( - q_rope=q_pe, - k_rope=k_pe_expanded, - q_nope=q_nope, - k_nope=k_nope, - cos_sin_cache=cos_sin_cache, - pos_ids=positions, - is_neox=is_neox, - quantize_dtype=torch.float8_e4m3fn, - q_rope_out=q_fp8[..., qk_nope_head_dim:], - k_rope_out=k_fp8[..., qk_nope_head_dim:], - q_nope_out=q_fp8[..., :qk_nope_head_dim], - k_nope_out=k_fp8[..., :qk_nope_head_dim], - quant_scale_q=1.0, - quant_scale_kv=1.0, - enable_pdl=is_arch_support_pdl(), - ) - return q_fp8, k_fp8 - - def prepare_prefill_qkv( - self, - *, - q: torch.Tensor, - q_pe: torch.Tensor, - kv_a: torch.Tensor, - k_pe: torch.Tensor, - positions: torch.Tensor, - layer: DeepseekV2AttentionMLA, - forward_batch: ForwardBatch, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache.""" - kv = layer.kv_b_proj(kv_a)[0] - kv = kv.view( - -1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim - ) - k_nope = kv[..., : layer.qk_nope_head_dim] - v_bf16 = kv[..., layer.qk_nope_head_dim :] - q_nope = q[..., : layer.qk_nope_head_dim] - - rotary_emb = layer.rotary_emb - q_fp8, k_fp8 = self._fused_rope_fp8_quantize( - q_nope=q_nope, - q_pe=q_pe, - k_nope=k_nope, - k_pe=k_pe, - cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache, - positions=positions, - is_neox=True if rotary_emb is None else rotary_emb.is_neox_style, - qk_nope_head_dim=layer.qk_nope_head_dim, - qk_rope_head_dim=layer.qk_rope_head_dim, - ) - v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl()) - - # k_pe is shared across heads (RoPE is position-only), so head 0 - # reproduces the original [tokens, 1, qk_rope] latent layout. - kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl()) - k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :] - self.token_to_kv_pool.set_mla_kv_buffer( - layer.attn_mha, - KVWriteLoc.for_batch(forward_batch), - kv_a_fp8.unsqueeze(1), - k_pe_fp8, - ) - return q_fp8, k_fp8, v_fp8 - - def pack_prefix_chunk_kv( - self, - k_nope: torch.Tensor, - k_pe: torch.Tensor, - v: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - """Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize - strided ``v`` into contig FP8 V in a single kernel. - """ - return mla_kv_pack_quantize_fp8( - k_nope, k_pe, v, enable_pdl=is_arch_support_pdl() - ) - - def _run_decode_kernel( - self, - query: torch.Tensor, - kv_cache: torch.Tensor, - block_tables: torch.Tensor, - seq_lens: torch.Tensor, - max_seq_len: int, - layer: RadixAttention, - *, - causal_seqs: Optional[torch.Tensor] = None, - cp_world: int = 1, - cp_rank: int = 0, - return_lse: bool = False, - ): - k_scale = getattr(layer, "k_scale_float", None) - if k_scale is None: - k_scale = 1.0 - softmax_scale = float(layer.scaling) * float(k_scale) - output_scale = float(k_scale) - - seq_lens_i32 = ( - seq_lens if seq_lens.dtype == torch.int32 else seq_lens.to(torch.int32) - ) - return tokenspeed_mla.tokenspeed_mla_decode( - query=query, - kv_cache=kv_cache, - workspace_buffer=self._tokenspeed_workspace, - kv_lora_rank=self.kv_lora_rank, - qk_rope_head_dim=self.qk_rope_head_dim, - block_tables=block_tables, - seq_lens=seq_lens_i32, - max_seq_len=int(max_seq_len), - softmax_scale=softmax_scale, - output_scale=output_scale, - enable_pdl=is_arch_support_pdl(), - return_lse=return_lse, - causal_seqs=causal_seqs, - cp_world=cp_world, - cp_rank=cp_rank, - ) - - def forward_decode( - self, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - layer: RadixAttention, - forward_batch: ForwardBatch, - save_kv_cache: bool = True, - q_rope: Optional[torch.Tensor] = None, - k_rope: Optional[torch.Tensor] = None, - cos_sin_cache: Optional[torch.Tensor] = None, - is_neox: Optional[bool] = False, - llama_4_scaling: Optional[torch.Tensor] = None, - ): - parallel = get_parallel() - if parallel.dcp_enabled and get_in_autotune_dummy_run(): - return self._dummy_dcp_decode_for_autotune(q, layer) - - if not parallel.dcp_enabled: - return super().forward_decode( - q, - k, - v, - layer, - forward_batch, - save_kv_cache, - q_rope, - k_rope, - cos_sin_cache, - is_neox, - llama_4_scaling, - ) - - assert q_rope is not None and k_rope is not None - if cos_sin_cache is None: - q, k, k_rope = mla_quantize_without_rope_for_fp8( - q, q_rope, k.squeeze(1), k_rope.squeeze(1) - ) - else: - q, k, k_rope = mla_quantize_and_rope_for_fp8( - q, - q_rope, - k.squeeze(1), - k_rope.squeeze(1), - forward_batch.positions, - cos_sin_cache, - is_neox, - self.kv_lora_rank, - self.qk_rope_head_dim, - ) - - if save_kv_cache: - self.token_to_kv_pool.set_mla_kv_buffer( - layer, - KVWriteLoc.for_batch(forward_batch), - k, - k_rope, - ) - - query = q.view(-1, layer.tp_q_head_num, layer.head_dim) - if llama_4_scaling is not None: - query = (query.to(self.q_data_type) * llama_4_scaling).to(self.data_type) - if query.dim() == 3: - query = query.unsqueeze(1) - - k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id) - kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1) - metadata = ( - getattr(forward_batch, "decode_trtllm_mla_metadata", None) - or self.forward_decode_metadata - ) - metadata_batch_size = getattr(metadata, "batch_size", None) - if ( - metadata_batch_size is not None - and metadata_batch_size < forward_batch.batch_size - ): - self.init_forward_metadata(forward_batch) - metadata = forward_batch.decode_trtllm_mla_metadata - - global_seq_lens = forward_batch.seq_lens[: forward_batch.batch_size] - local_seq_lens = self._get_dcp_local_seq_lens(global_seq_lens) - raw_out, lse = self._run_decode_kernel( - query=query, - kv_cache=kv_cache, - block_tables=metadata.block_kv_indices, - seq_lens=local_seq_lens, - max_seq_len=metadata.max_seq_len_k, - layer=layer, - causal_seqs=global_seq_lens, - cp_world=parallel.dcp_size, - cp_rank=parallel.dcp_rank, - return_lse=True, - ) - - output = raw_out.view(-1, layer.tp_q_head_num, layer.v_head_dim) - lse = lse.view(-1, layer.tp_q_head_num) - fixup_zero_kv_rows( - output, - lse, - local_seq_lens, - self.q_indptr_decode[: forward_batch.batch_size + 1], - 1, - ) - return output.flatten(1), lse - - def _run_prefill_kernel( - self, - q: torch.Tensor, - k: torch.Tensor, - v: torch.Tensor, - layer: RadixAttention, - batch_size: int, - cum_seq_lens_q: torch.Tensor, - max_q_len: int, - seq_lens_kv: torch.Tensor, - cum_seq_lens_kv: torch.Tensor, - max_kv_len: int, - is_causal: bool, - return_lse: bool, - out_buffer: torch.Tensor, - o_sf_scale: float = 1.0, - ): # Q/K/V arrive already in FP8 via the model-side fused path - # (prepare_prefill_qkv / pack_prefix_chunk_kv); no quantize here. - # Hybrid MLA models resolve the model-side hook through the outer - # HybridLinearAttnBackend, so their fallback MHA path can pass V as a - # last-dimension slice of kv_b_proj (stride(-2) > size(-1)). The - # TokenSpeed prefill kernel requires dense Q/K/V layouts even though - # the public wrapper accepts arbitrary torch tensors. - q = q.contiguous() - k = k.contiguous() - v = v.contiguous() - return tokenspeed_mla.tokenspeed_mla_prefill( - query=q, - key=k, - value=v, - seq_lens=seq_lens_kv, - cum_seq_lens=cum_seq_lens_kv, - max_seq_len=int(max_kv_len), - batch_size=int(batch_size), - softmax_scale=float(layer.scaling), - is_causal=is_causal, - return_lse=return_lse, - cum_seq_lens_q=cum_seq_lens_q, - max_seq_len_q=int(max_q_len), - enable_pdl=is_arch_support_pdl(), - ) - - -class TokenspeedMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend): - """Multi-step draft backend for tokenspeed_mla used by EAGLE.""" - - def __init__( - self, model_runner: ModelRunner, topk: int, speculative_num_steps: int - ): - super().__init__(model_runner, topk, speculative_num_steps) - # Parent populates self.attn_backends with TRT-LLM instances; replace - # them with tokenspeed instances sharing the parent's index buffers. - for i in range(self.speculative_num_steps - 1): - self.attn_backends[i] = TokenspeedMLABackend( - model_runner, - skip_prefill=True, - kv_indptr_buf=self.kv_indptr[i], - q_indptr_decode_buf=self.q_indptr_decode, - ) diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 44989bc726..643613a0a9 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -1017,8 +1017,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): raise NotImplementedError( "trtllm_mla cannot forward a global causal bound to its decode " "kernel, which is required for DCP with q_len > 1 (speculative " - "target-verify / draft-extend); select cutedsl_mla or " - "tokenspeed_mla for a DCP speculative run" + "target-verify / draft-extend); select cutedsl_mla " + "for a DCP speculative run" ) # Scale computation for TRTLLM MLA kernel BMM1 operation: @@ -1073,10 +1073,11 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): o_sf_scale: float = 1.0, ): """Hook for subclasses to swap the ragged prefill kernel. Q/K/V arrive - in model-native dtype; subclasses do any kernel-specific quantization. + in model-native dtype, or already FP8 from a model-side prefill hook; + subclasses do any kernel-specific quantization. Returns the output tensor or (output, lse) if return_lse.""" q_scale = k_scale = v_scale = 1.0 - if self.data_type == torch.float8_e4m3fn: + if self.data_type == torch.float8_e4m3fn and q.dtype != torch.float8_e4m3fn: q, k, v, k_scale, v_scale = _quantize_fp8_qkv(q, k, v, layer) return flashinfer.prefill.trtllm_ragged_attention_deepseek( query=q, diff --git a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py index 3ceab0d877..9a4c145419 100644 --- a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py +++ b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py @@ -181,12 +181,6 @@ def handle_attention_trtllm_mla(attn, forward_batch): return _dispatch_mla_subtype(attn, forward_batch) -def handle_attention_tokenspeed_mla(attn, forward_batch): - # tokenspeed_mla shares the trtllm_mla dispatch pattern: pure prefill goes - # via MHA chunked KV (TRT-LLM ragged), spec decode / decode goes via MLA. - return handle_attention_trtllm_mla(attn, forward_batch) - - def handle_attention_aiter(attn, forward_batch): # During PCG/BCG capture on ROCm, aiter fp8 MLA prefill has no capture # kernels; route through the MHA path (radix_attention swaps attn_mqa for @@ -268,7 +262,6 @@ AttentionBackendRegistry.register("fa3", handle_attention_fa3) AttentionBackendRegistry.register("flashmla", handle_attention_flashmla) AttentionBackendRegistry.register("fa4", handle_attention_fa4) AttentionBackendRegistry.register("trtllm_mla", handle_attention_trtllm_mla) -AttentionBackendRegistry.register("tokenspeed_mla", handle_attention_tokenspeed_mla) AttentionBackendRegistry.register("aiter", handle_attention_aiter) AttentionBackendRegistry.register("dsa", handle_attention_dsa) AttentionBackendRegistry.register( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index 0fae375ec4..e5fa8b7d6b 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -942,8 +942,7 @@ class DeepseekMLAForwardMixin: return ( self.rotary_emb is not None - and self.current_attention_backend - in ("trtllm_mla", "tokenspeed_mla", "cutedsl_mla") + and self.current_attention_backend in ("trtllm_mla", "cutedsl_mla") and ( forward_batch.forward_mode.is_decode_or_idle() or forward_batch.forward_mode.is_target_verify() diff --git a/python/sglang/srt/models/deepseek_common/utils.py b/python/sglang/srt/models/deepseek_common/utils.py index ceefce6d80..71dd1bad13 100644 --- a/python/sglang/srt/models/deepseek_common/utils.py +++ b/python/sglang/srt/models/deepseek_common/utils.py @@ -70,7 +70,6 @@ FORWARD_ABSORB_CORE_ATTENTION_BACKENDS = [ "flashinfer", "trtllm_mla", "cutedsl_mla", - "tokenspeed_mla", "ascend", "intel_xpu", ] diff --git a/python/sglang/srt/speculative/draft_utils.py b/python/sglang/srt/speculative/draft_utils.py index 5c13066884..085ff19976 100644 --- a/python/sglang/srt/speculative/draft_utils.py +++ b/python/sglang/srt/speculative/draft_utils.py @@ -105,7 +105,6 @@ class DraftBackendFactory: "trtllm_mha": self._create_trtllm_mha_decode_backend, "trtllm_mla": self._create_trtllm_mla_decode_backend, "cutedsl_mla": self._create_cutedsl_mla_decode_backend, - "tokenspeed_mla": self._create_tokenspeed_mla_decode_backend, "dsa": self._create_dsa_decode_backend, "nsa": self._create_dsa_decode_backend, # Deprecated alias for "dsa" "ascend": self._create_ascend_decode_backend, @@ -137,7 +136,6 @@ class DraftBackendFactory: "trtllm_mla": self._create_trtllm_mla_prefill_backend, # cute-dsl MLA only supports decode; draft-extend falls back to trtllm-gen. "cutedsl_mla": self._create_trtllm_mla_prefill_backend, - "tokenspeed_mla": self._create_tokenspeed_mla_prefill_backend, "dsa": self._create_dsa_prefill_backend, "nsa": self._create_dsa_prefill_backend, # Deprecated alias for "dsa" "ascend": self._create_ascend_prefill_backend, @@ -394,23 +392,6 @@ class DraftBackendFactory: ), ) - def _create_tokenspeed_mla_decode_backend(self): - if not self.draft_model_runner.use_mla_backend: - raise ValueError( - "tokenspeed_mla backend requires MLA model (use_mla_backend=True)." - ) - - from sglang.srt.layers.attention.tokenspeed_mla_backend import ( - TokenspeedMLAMultiStepDraftBackend, - ) - - return ( - "tokenspeed_mla", - TokenspeedMLAMultiStepDraftBackend( - self.draft_model_runner, self.topk, self.speculative_num_steps - ), - ) - def _create_ascend_decode_backend(self): from sglang.srt.hardware_backend.npu.attention.ascend_backend import ( AscendAttnMultiStepDraftBackend, @@ -541,21 +522,6 @@ class DraftBackendFactory: TRTLLMMLABackend(self.draft_model_runner, skip_prefill=False), ) - def _create_tokenspeed_mla_prefill_backend(self): - if not self.draft_model_runner.use_mla_backend: - raise ValueError( - "tokenspeed_mla backend requires MLA model (use_mla_backend=True)." - ) - - from sglang.srt.layers.attention.tokenspeed_mla_backend import ( - TokenspeedMLABackend, - ) - - return ( - "tokenspeed_mla", - TokenspeedMLABackend(self.draft_model_runner, skip_prefill=False), - ) - def _create_ascend_prefill_backend(self): from sglang.srt.hardware_backend.npu.attention.ascend_backend import ( AscendAttnBackend, diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index bf29c06429..36c1410bf7 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -27,7 +27,6 @@ from sglang.srt.layers.attention.qwen_sparse_attn_backend import ( QwenSparseAttnBackend, QwenSparseMultiStepDraftBackend, ) -from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend from sglang.srt.layers.attention.triton_backend import TritonAttnBackend from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend from sglang.srt.layers.attention.trtllm_mla_backend import ( @@ -536,7 +535,6 @@ class EagleDraftWorker(EagleDraftWorkerBase): TritonAttnBackend, TRTLLMMLABackend, TRTLLMHAAttnBackend, - TokenspeedMLABackend, FlashInferAttnBackend, QwenSparseAttnBackend, ] diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index d980bc8c1f..c118396bb0 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -394,18 +394,6 @@ def is_flashinfer_available(): return importlib.util.find_spec("flashinfer") is not None and is_cuda() -@lru_cache(maxsize=1) -def is_tokenspeed_mla_available(): - """ - Check whether the tokenspeed_mla CuTe DSL kernels are available. - Only available on NVIDIA Blackwell (SM100) at the moment. - """ - return ( - importlib.util.find_spec("tokenspeed_mla") is not None - and is_blackwell_supported() - ) - - def is_nvidia_cublas_version_ge_12_9(): """ temporary fix for issue #11272 (cublas 12.9+) diff --git a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py index 132c7f1bbc..e8018cfd22 100644 --- a/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py +++ b/python/sglang/test/kits/attention_unittest/attention_methods/mla_attention.py @@ -228,8 +228,8 @@ class MockMLAModelRunner(ModelRunner): self.device = device self.dtype = dtype # `kv_cache_dtype` is the dtype the *storage* uses. For FP8 KV - # cache (the production deployment dtype for tokenspeed_mla and - # some trtllm_mla configs), the pool stores quantized bytes + # cache (the production deployment dtype for some trtllm_mla + # configs), the pool stores quantized bytes # while the model still projects K/V in bf16; `set_mla_kv_buffer` # does the BF16->FP8 cast on the way in. self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype diff --git a/test/registered/attention/test_trtllm_mla_stride_parity.py b/test/registered/attention/test_trtllm_mla_stride_parity.py index 3d83bb6375..431e6bbdbc 100644 --- a/test/registered/attention/test_trtllm_mla_stride_parity.py +++ b/test/registered/attention/test_trtllm_mla_stride_parity.py @@ -1,6 +1,6 @@ """The paged MLA decode kernels must not depend on the KV cache's slot stride. -trtllm_mla, cutedsl_mla and tokenspeed_mla hand their decode kernels +trtllm_mla and cutedsl_mla hand their decode kernels `paged_row_view(k_cache, page_size).unsqueeze(1)`: `[pages, 1, page_size, kv_cache_dim]` whose in-page token stride is the pool's SLOT stride. Under the unified pool's token-major layout that stride spans every layer's latent row, @@ -19,8 +19,7 @@ import unittest import torch from sglang.srt.mem_cache.layout.paged_view import paged_row_view -from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import is_sm100_supported, is_tokenspeed_mla_available +from sglang.srt.utils import is_sm100_supported from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=15, stage="base-b", runner_config="4-gpu-b200") @@ -119,38 +118,6 @@ class TestPagedMLADecodeStrideParity(unittest.TestCase): self._assert_stride_independent(run, kv_dtype) - def test_tokenspeed_mla_decode_is_identical_across_slot_strides(self): - if not is_tokenspeed_mla_available(): - self.skipTest("tokenspeed_mla is not installed") - import tokenspeed_mla - - from sglang.srt.layers.attention.tokenspeed_mla_backend import ( - _get_tokenspeed_workspace, - ) - - # The workspace is sized for the DCP-gathered head count. - with get_parallel().override(attn_dcp_size=1): - workspace = _get_tokenspeed_workspace( - torch.device("cuda"), _Q_HEADS, _KV_LORA_RANK - ) - kv_dtype = torch.float8_e4m3fn - q = torch.randn(_BS, 1, _Q_HEADS, _KV_CACHE_DIM, device="cuda").to(kv_dtype) - - def run(kv_cache): - return tokenspeed_mla.tokenspeed_mla_decode( - query=q, - kv_cache=kv_cache, - workspace_buffer=workspace, - kv_lora_rank=_KV_LORA_RANK, - qk_rope_head_dim=_QK_ROPE_HEAD_DIM, - block_tables=self.block_tables, - seq_lens=self.seq_lens, - max_seq_len=_PAGES * _PAGE_SIZE, - softmax_scale=_KV_CACHE_DIM**-0.5, - ) - - self._assert_stride_independent(run, kv_dtype) - if __name__ == "__main__": unittest.main() diff --git a/test/registered/attention/unittests/KNOWN_FAILURES.md b/test/registered/attention/unittests/KNOWN_FAILURES.md index 67a7b7742f..fac847ec74 100644 --- a/test/registered/attention/unittests/KNOWN_FAILURES.md +++ b/test/registered/attention/unittests/KNOWN_FAILURES.md @@ -68,7 +68,6 @@ so that "skipped: ..." results have a quick lookup. |---|---|---|---| | `flashmla` decode/verify | SM 9.0 (Hopper) only | `mla/test_flashmla.py:_DECODE_REQUIRES_SM90A` | `Dense decode MLA is only supported on SM90a architecture` | | `trtllm_mla` | SM 12.0a / 12.1a | `mla/test_trtllm_mla.py::_supported` | FlashInfer XQA MLA dispatch reject | -| `tokenspeed_mla` | SM ≥ 10.0 + FP8 KV + pkg | `mla/test_tokenspeed_mla.py::_supported` | `tokenspeed_mla` import or kernel dispatch | | `trtllm_mha` prefill | SM ≥ 10.0 | `dense/test_trtllm_mha.py` decode-only matrix | FlashInfer TRT-LLM Gen FMHA reject (`Unsupported architecture`) | | `dsa` `fa3` impl | SM 9.x only | `dsa_impl_capability("fa3")` | `flash_attn at sgl-kernel is only supported on sm90 and above` | | `dsa` `trtllm` impl | exactly SM 10.0 | `dsa_impl_capability("trtllm")` | `Missing TRTLLM-GEN kernel` (compiled for SM10.0) | @@ -176,7 +175,6 @@ fail at backend init. |---|---|---| | FlashMLA | `64` only | `server_args.py:2767-2770` | | TRT-LLM MLA | `{32, 64}` | `server_args.py:2790-2794` | -| Tokenspeed MLA | `{32, 64}` | `server_args.py:2809-2813`, `tokenspeed_mla_backend.py:111-113` | | TRT-LLM MHA | `{16, 32, 64}` | `server_args.py:2849-2853` | | FA4 (non-MLA) | `128` when default-selected | `server_args.py:2862-2870` | | DSV4 | `256` only | `deepseek_v4_backend.py:355`, `dsv4/metadata.py:134` | @@ -191,7 +189,6 @@ fail at backend init. | `flashinfer_mla` | `1` only | `flashinfer_mla_backend.py:910-913` | | `flashmla` | `1` only | `flashmla_backend.py:555-558` | | `trtllm_mla` | `1` only | `trtllm_mla_backend.py:1223-1229` (inherits) | -| `tokenspeed_mla` | `1` only | `tokenspeed_mla_backend.py:341-347` (inherits) | | `dsv4` | `0` or `1` | `deepseek_v4_backend.py:369`, `:363` (HIP) | | `trtllm_mha` (graph replay) | `1` only | `trtllm_mha_backend.py:459,492`; `server_args.py:2391-2392` | @@ -199,7 +196,6 @@ fail at backend init. | Backend | Allowed dtype | Citation | |---|---|---| -| `tokenspeed_mla` | `fp8_e4m3` only | `server_args.py:2814-2818` | | `trtllm_mla` | `{fp8_e4m3, fp4_e2m1, bf16, auto}` | `server_args.py:2796-2799` | | `fa3` | not `fp8_e5m2` (silently falls back to `triton`) | `server_args.py:2855-2860` | | `dsv4` | Packed FP8/BF16 layout enforced by `DeepSeekV4TokenToKVPool` | `deepseek_v4_backend.py:363` | diff --git a/test/registered/attention/unittests/mla/README.md b/test/registered/attention/unittests/mla/README.md index e4a08cc227..52774f0794 100644 --- a/test/registered/attention/unittests/mla/README.md +++ b/test/registered/attention/unittests/mla/README.md @@ -21,7 +21,6 @@ Columns are runner modes; rows are attention backends. Cells use: | `flashinfer` | ✓ 10 input layouts with DeepSeek-like `kv_lora_rank=512`, `qk_rope_head_dim=64` | ✓ MLA decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | ✓ EAGLE ragged-accept | blocked: `is_draft_extend()` default `include_v2=False` (`flashinfer_mla_backend.py:432,501,454-455,512`) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashinfer_mla_backend.py:910-913`) | ✓ EAGLE ragged-accept (V1) | — (no FKVMTP wiring for MLA) | | `flashmla` | ✓ FlashMLA-compatible page-size-64 cases (zero-prefix exact page, input page edges 63/64/65, prefix exact page, total exact page, cross page, ragged, decode page-boundary, decode bsz=1 nonzero prefix) | ✓ page-size-64 decode page-boundary | ✓ ragged page-boundary extend | ✓ ragged page-boundary extend | ✓ EAGLE chain (topk=1) | ✓ EAGLE chain (topk=1) | ✓ EAGLE ragged-accept | deferred: parent FlashInfer-MLA capture path expects 1D `cuda_graph_kv_indices`, FlashMLA allocates 2D `[max_bs, (max_context+PAGE_SIZE)//PAGE_SIZE]` (`flashmla_backend.py:347-348` + parent `init_forward_metadata_capture_cuda_graph`) | — (FlashMLA does not implement V2) | ✓ chain (topk=1) only — tree blocked by `topk=1` reject (`flashmla_backend.py:555-558`) | — (DE CG deferred above) | — | | `trtllm_mla` | skip:hw — needs SM 12.0a / 12.1a (`is_sm120_supported`) | — | — | — | blocked: `topk=1` only (`trtllm_mla_backend.py:1223-1229` inherits from FlashInfer MLA) | — | — | — | — | — | — | — | -| `tokenspeed_mla` | skip:hw — needs `find_spec("tokenspeed_mla")`, SM 10.0+, and `kv_cache_dtype=fp8_e4m3` (`server_args.py:2814-2818`); current MLA fixture does not emit FP8 KV cache | — | — | — | blocked: `topk=1` only (`tokenspeed_mla_backend.py:341-347` inherits from TRT-LLM MLA) | — | — | — | — | — | — | — | ## Input And Config Coverage @@ -56,15 +55,12 @@ multi-step draft backends and cannot ever appear at runtime. - **TRT-LLM MLA tree verify / draft-extend with `topk > 1`** — `TRTLLMMLAMultiStepDraftBackend` inherits from `FlashInferMLAMultiStepDraftBackend` (`trtllm_mla_backend.py:1223-1229`). -- **Tokenspeed MLA tree verify / draft-extend with `topk > 1`** — - `TokenspeedMLAMultiStepDraftBackend` inherits from - `TRTLLMMLAMultiStepDraftBackend` (`tokenspeed_mla_backend.py:341-347`). - **FlashInfer-MLA `DRAFT_EXTEND_V2` graph capture/replay** — `flashinfer_mla_backend.py:432,501` only route through `is_draft_extend()` (default `include_v2=False`); `else: raise ValueError("Invalid mode")` at `flashinfer_mla_backend.py:454-455,512`. - **All MLA backends fixed page size** — FlashMLA forces `page_size=64`, - TRT-LLM MLA and Tokenspeed MLA force `page_size in {32, 64}`. + TRT-LLM MLA forces `page_size in {32, 64}`. ## Backend Container Gate (SM10.x) @@ -85,6 +81,6 @@ See `KNOWN_FAILURES.md` §3 for the full root cause + fix. override capture/replay in `FlashMLABackend` to use its 2D layout, or allocate both parent-style 1D and FlashMLA-style 2D buffers and route `DRAFT_EXTEND` to the parent path). -- Add hardware-gated tests for `trtllm_mla` and `tokenspeed_mla` +- Add hardware-gated tests for `trtllm_mla` decode (chain spec only) when the appropriate hardware/KV dtype fixtures are available. diff --git a/test/registered/attention/unittests/mla/test_tokenspeed_mla.py b/test/registered/attention/unittests/mla/test_tokenspeed_mla.py deleted file mode 100644 index 5ba21ab877..0000000000 --- a/test/registered/attention/unittests/mla/test_tokenspeed_mla.py +++ /dev/null @@ -1,265 +0,0 @@ -import importlib.util -import unittest - -import torch - -from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend -from sglang.srt.model_executor.forward_batch_info import ForwardMode -from sglang.test.kits.attention_unittest.attention_methods.mla_attention import ( - MLAAttentionCase, - run_mla_attention_case, -) -from sglang.test.test_utils import CustomTestCase - -# tokenspeed_mla is a CuTe DSL backend for Blackwell (SM100). It additionally -# enforces: -# - kv_cache_dtype == torch.float8_e4m3fn (kv_cache_dtype=fp8_e4m3) -# - page_size in {32, 64} -# See python/sglang/srt/layers/attention/tokenspeed_mla_backend.py and -# is_tokenspeed_mla_available() in python/sglang/srt/utils/common.py. -# -# The shared MLAAttentionCase fixture now supports `fp8_kv_cache=True`: -# `MockMLAModelRunner` decouples `kv_cache_dtype` from the model `dtype` -# and routes K writes through the FP8 quantize path. The reference still -# computes against BF16 K (independent of the cache bytes) and tolerates -# FP8 quant noise via a looser tolerance. -_MIN_SM = 100 - - -def _supported() -> tuple[bool, str]: - if not torch.cuda.is_available(): - return False, "CUDA is required" - if importlib.util.find_spec("tokenspeed_mla") is None: - return False, "tokenspeed_mla python package is not installed" - major, minor = torch.cuda.get_device_capability() - sm = major * 10 + minor - if sm < _MIN_SM: - return ( - False, - f"tokenspeed_mla requires SM {_MIN_SM // 10}.{_MIN_SM % 10}+ (Blackwell), " - f"got SM {major}.{minor}", - ) - return True, "" - - -_SUPPORTED, _SKIP_REASON = _supported() - - -MLA_SHAPE_KWARGS = dict( - kv_lora_rank=512, - qk_rope_head_dim=64, - hidden_size=1024, - max_context_len=256, -) - - -from sglang.test.ci.ci_register import register_cuda_ci - -register_cuda_ci(est_time=10, stage="base-b", runner_config="4-gpu-b200") -register_cuda_ci(est_time=9, stage="base-b", runner_config="1-gpu-large") - - -@unittest.skipIf(not _SUPPORTED, _SKIP_REASON) -class TestTokenspeedMLAAttentionBackendCorrectness(CustomTestCase): - # tokenspeed_mla allows page_size in {32, 64} (server_args.py:2809-2813) - # and requires kv_cache_dtype==fp8_e4m3 (server_args.py:2814-2818). - # Cover both page sizes, with extend + decode + ragged + page-boundary. - CASES = ( - # ----- page_size=64 ----- - MLAAttentionCase( - name="mla_extend_tokenspeed_zero_prefix_exact_page_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(0,), - extend_lens=(64,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_zero_prefix_below_page_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(0,), - extend_lens=(63,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_zero_prefix_above_page_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(0,), - extend_lens=(65,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_prefix_exact_page_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(64,), - extend_lens=(4,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_cross_page_boundary_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(63,), - extend_lens=(2,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_ragged_page_boundary_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=64, - prefix_lens=(0, 32, 64), - extend_lens=(63, 32, 1), - ), - MLAAttentionCase( - name="mla_decode_tokenspeed_page_boundary_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.DECODE, - num_heads=4, - page_size=64, - prefix_lens=(62, 63, 64), - ), - MLAAttentionCase( - name="mla_decode_tokenspeed_bsz1_nonzero_prefix_64", - backend="tokenspeed_mla", - forward_mode=ForwardMode.DECODE, - num_heads=4, - page_size=64, - prefix_lens=(31,), - ), - # ----- page_size=32 ----- - MLAAttentionCase( - name="mla_extend_tokenspeed_zero_prefix_exact_page_32", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=32, - prefix_lens=(0,), - extend_lens=(32,), - ), - MLAAttentionCase( - name="mla_extend_tokenspeed_cross_page_boundary_32", - backend="tokenspeed_mla", - forward_mode=ForwardMode.EXTEND, - num_heads=4, - page_size=32, - prefix_lens=(31,), - extend_lens=(2,), - ), - MLAAttentionCase( - name="mla_decode_tokenspeed_page_boundary_32", - backend="tokenspeed_mla", - forward_mode=ForwardMode.DECODE, - num_heads=4, - page_size=32, - prefix_lens=(30, 31, 32), - ), - ) - - def test_projected_mla_attention_cases(self): - for case in self.CASES: - with self.subTest(case=case.name, backend=case.backend): - # Looser tolerance to absorb FP8 quant noise (the reference - # reads BF16 K independent of the FP8 cache, so per-element - # drift from the BF16->FP8 cast accumulates through the - # attention reduction). - run_mla_attention_case( - self, - case, - fp8_kv_cache=True, - atol=2e-1, - rtol=2e-1, - **MLA_SHAPE_KWARGS, - ) - - -class TestTokenspeedMLANoPEPrefillQuantize(CustomTestCase): - """NoPE MLA layers must still get packed FP8 prefill Q/K. - - Layers built with ``skip_rope`` carry no rotary embedding, so the prefill - quantize runs with ``cos_sin_cache=None``. It must return the same packed - ``[nope | pe]`` FP8 layout as the RoPE path, and the head-0 pe slice it - writes to the KV cache must equal the plain FP8 cast of the unroped - ``k_pe`` that the decode path reads back. - """ - - T = 7 - NUM_HEADS = 4 - QK_NOPE_HEAD_DIM = 128 - QK_ROPE_HEAD_DIM = 64 - KV_LORA_RANK = 512 - - def test_nope_prefill_quantize_packs_without_rope(self): - device = "cuda" if torch.cuda.is_available() else "cpu" - head_dim = self.QK_NOPE_HEAD_DIM + self.QK_ROPE_HEAD_DIM - - q_nope = torch.randn( - self.T, - self.NUM_HEADS, - self.QK_NOPE_HEAD_DIM, - dtype=torch.bfloat16, - device=device, - ) - q_pe = torch.randn( - self.T, - self.NUM_HEADS, - self.QK_ROPE_HEAD_DIM, - dtype=torch.bfloat16, - device=device, - ) - k_nope = torch.randn( - self.T, - self.NUM_HEADS, - self.QK_NOPE_HEAD_DIM, - dtype=torch.bfloat16, - device=device, - ) - # k_pe reaches the backend as the strided tail of the latent cache. - latent_cache = torch.randn( - self.T, - 1, - self.KV_LORA_RANK + self.QK_ROPE_HEAD_DIM, - dtype=torch.bfloat16, - device=device, - ) - k_pe = latent_cache[:, :, self.KV_LORA_RANK :] - - q_fp8, k_fp8 = TokenspeedMLABackend._fused_rope_fp8_quantize( - q_nope=q_nope, - q_pe=q_pe, - k_nope=k_nope, - k_pe=k_pe, - cos_sin_cache=None, - positions=torch.arange(self.T, device=device), - is_neox=True, - qk_nope_head_dim=self.QK_NOPE_HEAD_DIM, - qk_rope_head_dim=self.QK_ROPE_HEAD_DIM, - ) - - for name, out in (("q", q_fp8), ("k", k_fp8)): - with self.subTest(tensor=name): - self.assertEqual(out.shape, (self.T, self.NUM_HEADS, head_dim)) - self.assertEqual(out.dtype, torch.float8_e4m3fn) - self.assertTrue(out.is_contiguous()) - - fp8 = torch.float8_e4m3fn - nope = self.QK_NOPE_HEAD_DIM - self.assertTrue(torch.equal(q_fp8[..., :nope], q_nope.to(fp8))) - self.assertTrue(torch.equal(q_fp8[..., nope:], q_pe.to(fp8))) - self.assertTrue(torch.equal(k_fp8[..., :nope], k_nope.to(fp8))) - # The slice prepare_prefill_qkv writes into the KV cache; the decode - # NoPE path reads it back as a plain cast of the unroped k_pe. - self.assertTrue(torch.equal(k_fp8[:, 0:1, nope:], k_pe.to(fp8))) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/dcp/test_kimi_linear_dcp4.py b/test/registered/dcp/test_kimi_linear_dcp4.py index a565553b26..6819ba3183 100644 --- a/test/registered/dcp/test_kimi_linear_dcp4.py +++ b/test/registered/dcp/test_kimi_linear_dcp4.py @@ -1,4 +1,4 @@ -"""Four-Blackwell acceptance coverage for Kimi Linear TokenSpeed MLA DCP. +"""Four-Blackwell acceptance coverage for Kimi Linear CuteDSL MLA DCP. The captured-shape and eager-shape requests deliberately straddle ``--cuda-graph-max-bs-decode``. This guards both the regular CUDA graph @@ -37,7 +37,7 @@ def _has_four_blackwell_gpus() -> bool: @unittest.skipUnless( _has_four_blackwell_gpus(), - "TokenSpeed MLA DCP acceptance requires four Blackwell GPUs", + "CuteDSL MLA DCP acceptance requires four Blackwell GPUs", ) class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase): model = KIMI_LINEAR_MODEL @@ -62,7 +62,7 @@ class TestKimiLinearDCP4(GSM8KMixin, CustomTestCase): "--dcp-size", "4", "--attention-backend", - "tokenspeed_mla", + "cutedsl_mla", "--kv-cache-dtype", "fp8_e4m3", "--dcp-comm-backend", diff --git a/test/registered/dcp/test_kimi_linear_dcp_dspark4.py b/test/registered/dcp/test_kimi_linear_dcp_dspark4.py index 49e9f448b6..19f4e17861 100644 --- a/test/registered/dcp/test_kimi_linear_dcp_dspark4.py +++ b/test/registered/dcp/test_kimi_linear_dcp_dspark4.py @@ -1,4 +1,4 @@ -"""Four-Blackwell Kimi Linear TokenSpeed DCP + DSpark static acceptance test.""" +"""Four-Blackwell Kimi Linear CuteDSL MLA DCP + DSpark static acceptance test.""" import json import socket @@ -96,7 +96,7 @@ def _wait_for_port_release(base_url: str, timeout: float = 30.0) -> None: @unittest.skipUnless( _has_four_blackwell_gpus(), - "Kimi Linear TokenSpeed DCP + DSpark requires four Blackwell GPUs", + "Kimi Linear CuteDSL MLA DCP + DSpark requires four Blackwell GPUs", ) class TestKimiLinearDCPDSpark4(CustomTestCase): base_url = DEFAULT_URL_FOR_TEST @@ -132,7 +132,7 @@ class TestKimiLinearDCPDSpark4(CustomTestCase): "--max-running-requests", str(MAX_RUNNING_REQUESTS), "--attention-backend", - "tokenspeed_mla", + "cutedsl_mla", "--kv-cache-dtype", "fp8_e4m3", "--dcp-comm-backend", diff --git a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py index f8296a0efc..594ff04cb3 100644 --- a/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py +++ b/test/registered/dcp/test_trtllm_mla_family_dcp_metadata.py @@ -14,7 +14,6 @@ import torch from sglang.srt.layers.attention import trtllm_mla_backend as backend_module from sglang.srt.layers.attention.cutedsl_mla_backend import CuteDslMLABackend -from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend from sglang.srt.layers.attention.trtllm_mla_backend import ( TRTLLMMLABackend, TRTLLMMLADecodeMetadata, @@ -168,10 +167,6 @@ class TestTRTLLMMLADCPMetadata(_DCPMetadataTests, CustomTestCase): backend_cls = TRTLLMMLABackend -class TestTokenspeedMLADCPMetadata(_DCPMetadataTests, CustomTestCase): - backend_cls = TokenspeedMLABackend - - class TestCuteDslMLADCPMetadata(_DCPMetadataTests, CustomTestCase): backend_cls = CuteDslMLABackend diff --git a/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py b/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py index 175c2dd850..3fbb52d1ac 100644 --- a/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py +++ b/test/registered/e2e/disaggregation/test_kimi_linear_pd_dcp4.py @@ -30,7 +30,7 @@ NIAH_KEY = "739391" COMMON_ARGS = [ "--attention-backend", - "tokenspeed_mla", + "cutedsl_mla", "--kv-cache-dtype", "fp8_e4m3", "--dtype", diff --git a/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py b/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py index e9f954cf8f..c484f47657 100644 --- a/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py +++ b/test/registered/e2e/models/test_kimi_linear_unified_memory_dcp_blackwell.py @@ -14,9 +14,9 @@ prefix-cache replay below are here to catch. `cutedsl_mla` stands in for the family: it is the DCP-native MLA decode kernel on Blackwell (the only backend flashinfer accepts `enable_dcp=True` for) and what the B300 hosts run, and prefill resolves to `trtllm_mla`, so one server -covers both halves of the page-table contract. `trtllm_mla` and -`tokenspeed_mla` share that builder and were checked by hand against the -static pool (see below) rather than given a CI server each. +covers both halves of the page-table contract. `trtllm_mla` shares that +builder and was checked by hand against the static pool (see below) rather +than given a CI server. Blackwell-only, hence its own module: the Hopper suite next door cannot host it. In extra-b rather than nightly because what it guards is a per-PR argument @@ -29,13 +29,11 @@ pools agree: unified / static TP2 DCP2 cutedsl_mla 0.905 / 0.905 TP2 DCP2 trtllm_mla 0.895 / 0.895 - TP2 DCP2 tokenspeed_mla 0.890 / 0.895, repeat 0.890 / 0.885 TP4 DCP4 cutedsl_mla 0.910 / 0.900 -The cutedsl_mla and trtllm_mla pairs reproduced exactly. tokenspeed_mla moved -on a repeat of the same questions on BOTH pools, so its spread is the backend, -not the pool. The TP4 DCP4 row is there because it widens the virtual page to -4 x 64 = 256, past what this file exercises. +The cutedsl_mla and trtllm_mla pairs reproduced exactly. The TP4 DCP4 row is +there because it widens the virtual page to 4 x 64 = 256, past what this file +exercises. python -m pytest test/registered/models_e2e/test_kimi_linear_unified_memory_dcp_blackwell.py -v """ diff --git a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py index 23bc9a6de1..76561d4edf 100644 --- a/test/registered/kernels/ops/attention/test_dcp_lse_combine.py +++ b/test/registered/kernels/ops/attention/test_dcp_lse_combine.py @@ -256,7 +256,6 @@ class TestLSEBaseByBackend(CustomTestCase): self.assertTrue(is_mla_dcp_lse_base_on_e("flashmla")) self.assertTrue(is_mla_dcp_lse_base_on_e("cutedsl_mla")) self.assertFalse(is_mla_dcp_lse_base_on_e("flashinfer_mla")) - self.assertFalse(is_mla_dcp_lse_base_on_e("tokenspeed_mla")) self.assertFalse(is_mla_dcp_lse_base_on_e("trtllm_mla")) self.assertFalse(is_mla_dcp_lse_base_on_e(None)) diff --git a/test/registered/unit/mem_cache/test_physical_write_loc.py b/test/registered/unit/mem_cache/test_physical_write_loc.py index 78f18700ab..c85c5dee74 100644 --- a/test/registered/unit/mem_cache/test_physical_write_loc.py +++ b/test/registered/unit/mem_cache/test_physical_write_loc.py @@ -243,7 +243,6 @@ _UNIFIED_BACKEND_FILES = { "trtllm_mha": ("layers/attention/trtllm_mha_backend.py",), "trtllm_mla": ("layers/attention/trtllm_mla_backend.py",), "cutedsl_mla": ("layers/attention/cutedsl_mla_backend.py",), - "tokenspeed_mla": ("layers/attention/tokenspeed_mla_backend.py",), "flashmla": ("layers/attention/flashmla_backend.py",), } # Model code and the rest of the forward path also write KV directly. diff --git a/test/registered/unit/mem_cache/test_unified_mla_block_table.py b/test/registered/unit/mem_cache/test_unified_mla_block_table.py index d8ed36e6c3..bc781268fb 100644 --- a/test/registered/unit/mem_cache/test_unified_mla_block_table.py +++ b/test/registered/unit/mem_cache/test_unified_mla_block_table.py @@ -24,7 +24,7 @@ page ids: ONE builder computes that formula for every family -- the iteration plan's read table (`KVLocPlan.read_table`) -- and the backends only differ in how they consume it: - - trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows + - trtllm_mla / cutedsl_mla / flashmla: the plan's rows copied into their padded block tables (`KVIndexTranslator.copy_page_table`); - the flashinfer updaters: token ids reconstructed from the table by `create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`; diff --git a/test/registered/unit/server_args/test_page_major_backend_allowlist.py b/test/registered/unit/server_args/test_page_major_backend_allowlist.py index e35d1c3835..9969b82b71 100644 --- a/test/registered/unit/server_args/test_page_major_backend_allowlist.py +++ b/test/registered/unit/server_args/test_page_major_backend_allowlist.py @@ -98,13 +98,12 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): "trtllm_mla", "flashinfer", "cutedsl_mla", - "tokenspeed_mla", "flashmla", ) # Wired for the per-layer MHA/SWA views. PER_LAYER_VIEW_MHA_BACKENDS = ("fa3", "fa4", "flashinfer", "trtllm_mha") # MLA-family kernels that must never leak into the MHA arm. - MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "tokenspeed_mla", "flashmla") + MLA_ONLY_BACKENDS = ("trtllm_mla", "cutedsl_mla", "flashmla") # No virtual-to-physical id wiring anywhere: must stay rejected until they get one. UNWIRED_BACKENDS = ("aiter",) diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 6e02dc78ec..023fb5b8b5 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -710,6 +710,43 @@ class TestImageProcessorBackend(CustomTestCase): handle_deprecated_args(server_args) +class TestTokenspeedMlaAlias(CustomTestCase): + def test_maps_to_cutedsl_with_fp8_kv(self): + server_args = ServerArgs(model_path="dummy", attention_backend="tokenspeed_mla") + + with self.assertLogs(serving_hook.logger, level="WARNING"): + handle_deprecated_args(server_args) + + self.assertEqual( + resolution_result(server_args, "attention_backend"), "cutedsl_mla" + ) + self.assertEqual(resolution_result(server_args, "kv_cache_dtype"), "fp8_e4m3") + self.assertEqual( + resolution_result(server_args, "speculative_attention_mode"), "prefill" + ) + + def test_speculative_verify_moves_to_decode(self): + server_args = ServerArgs( + model_path="dummy", + decode_attention_backend="tokenspeed_mla", + prefill_attention_backend="tokenspeed_mla", + kv_cache_dtype="fp8_e4m3", + speculative_algorithm="EAGLE3", + ) + + handle_deprecated_args(server_args) + + self.assertEqual( + resolution_result(server_args, "decode_attention_backend"), "cutedsl_mla" + ) + self.assertEqual( + resolution_result(server_args, "prefill_attention_backend"), "trtllm_mla" + ) + self.assertEqual( + resolution_result(server_args, "speculative_attention_mode"), "decode" + ) + + class TestMultimodalFeatureTransport(CustomTestCase): @staticmethod def _set_model_type(server_args, *, is_multimodal): diff --git a/test/registered/unit/server_args/test_unified_memory_spec_gate.py b/test/registered/unit/server_args/test_unified_memory_spec_gate.py index d4195df061..dd1659de36 100644 --- a/test/registered/unit/server_args/test_unified_memory_spec_gate.py +++ b/test/registered/unit/server_args/test_unified_memory_spec_gate.py @@ -93,7 +93,6 @@ _VERIFY_BACKENDS = ( "triton", "trtllm_mla", "cutedsl_mla", - "tokenspeed_mla", "flashmla", "flashinfer", "fa3",