[Chore] attention: Remove the tokenspeed_mla backend and dependency (#44803)
This commit is contained in:
@@ -83,7 +83,6 @@ dependencies = [
|
||||
"tilelang==0.1.12",
|
||||
"timm==1.0.16",
|
||||
"tokenizers==0.23.2",
|
||||
"tokenspeed_mla==0.1.8",
|
||||
"torch==2.14.1",
|
||||
"torch_memory_saver>=0.0.9.post1",
|
||||
"torchaudio==2.11.0",
|
||||
|
||||
@@ -182,7 +182,7 @@ def handle_attention_backend_compatibility(server_args: Any):
|
||||
# fallback stay below.
|
||||
run_post_process_pass(server_args, _mla_backend_page_constraints)
|
||||
|
||||
# The TRT-LLM / tokenspeed MLA kv-dtype validations moved to the
|
||||
# The TRT-LLM MLA kv-dtype validations moved to the
|
||||
# resolution pipeline (arg_groups/overrides.py:
|
||||
# _mla_kv_cache_dtype_checks), invoked here at their legacy slot.
|
||||
|
||||
|
||||
@@ -84,7 +84,7 @@ ATTENTION_BACKEND_CHOICES = [
|
||||
"flashmla",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"tokenspeed_mla", # Deprecated alias for "cutedsl_mla"
|
||||
"trtllm_mha",
|
||||
"hpc_ops", # HPC-Ops (https://github.com/Tencent/hpc-ops), Hopper (SM90) only, requires --page-size 64
|
||||
"minicpm_flashattn",
|
||||
@@ -234,7 +234,6 @@ CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
|
||||
"flashmla",
|
||||
"cutedsl_mla",
|
||||
"trtllm_mla",
|
||||
"tokenspeed_mla",
|
||||
]
|
||||
add_chunked_prefix_cache_attention_backend = (
|
||||
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append
|
||||
|
||||
@@ -481,7 +481,6 @@ _SPEC_VERIFY_AUDITED_BACKENDS = frozenset(
|
||||
"triton",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
"flashinfer",
|
||||
"fa3",
|
||||
@@ -749,7 +748,7 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
|
||||
# The trtllm_mla family builds its DCP block table through the pool's v2p
|
||||
# gather (create_mla_kv_page_table_for_dcp), so it speaks the same
|
||||
# two-stage contract as flashinfer.
|
||||
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla", "tokenspeed_mla"}
|
||||
dcp_allowed = {"flashinfer", "trtllm_mla", "cutedsl_mla"}
|
||||
backends = set(attention_backends_of(resolved_view(server_args)))
|
||||
backends.discard(None)
|
||||
assert backends <= dcp_allowed, (
|
||||
@@ -806,7 +805,6 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
"trtllm_mla",
|
||||
"flashinfer",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
}
|
||||
elif model_config.has_asymmetric_kv:
|
||||
|
||||
@@ -351,14 +351,10 @@ def get_default_attn_backend(server_args: Any, use_mla_backend: bool, model_conf
|
||||
return "triton"
|
||||
|
||||
|
||||
def _dspark_verify_on_decode_backend(
|
||||
backend: Optional[str], q_len: int, kv_cache_dtype: Optional[str]
|
||||
) -> bool:
|
||||
def _dspark_verify_on_decode_backend(backend: Optional[str], q_len: int) -> bool:
|
||||
"""Whether the MLA decode backend can serve a q_len-wide target verify."""
|
||||
if backend == "trtllm_mla":
|
||||
return True
|
||||
if backend == "tokenspeed_mla":
|
||||
return kv_cache_dtype == "fp8_e4m3" and q_len <= 8
|
||||
if backend == "cutedsl_mla":
|
||||
# cute-dsl monolithic MLA decode folds the verify tokens into the head
|
||||
# dim (fold_sq), so it serves any DSPARK verify width. Needs flashinfer
|
||||
|
||||
@@ -92,21 +92,6 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
prefill_attention_backend="trtllm_mla",
|
||||
decode_attention_backend="cutedsl_mla",
|
||||
)
|
||||
elif decode_backend == "tokenspeed_mla":
|
||||
logger.info(
|
||||
"Kimi-K3 DCP overrides attention backends: "
|
||||
f"prefill={prefill_backend!r}, decode={decode_backend!r} -> "
|
||||
"'tokenspeed_mla'."
|
||||
)
|
||||
logger.info(
|
||||
"Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: "
|
||||
f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'."
|
||||
)
|
||||
overrides.update(
|
||||
prefill_attention_backend="tokenspeed_mla",
|
||||
decode_attention_backend="tokenspeed_mla",
|
||||
kv_cache_dtype="fp8_e4m3",
|
||||
)
|
||||
elif decode_backend == "aiter":
|
||||
_require_kimi_k3_aiter_gluon_dcp_support()
|
||||
# Override prefill backend to aiter by default
|
||||
@@ -123,7 +108,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
)
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla', 'tokenspeed_mla' or 'aiter', got {decode_backend!r}."
|
||||
f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'aiter', got {decode_backend!r}."
|
||||
)
|
||||
|
||||
if cfg.dcp_replicate_q_proj is None and cfg.dcp_comm_backend in (
|
||||
@@ -167,7 +152,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
# that still needs declaring -- else verify stays on the prefill backend,
|
||||
# whose host-side plan (flashinfer by default) forces a per-step D2H.
|
||||
_, backend = attention_backends_of(cfg)
|
||||
if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype):
|
||||
if _dspark_verify_on_decode_backend(backend, q_len):
|
||||
overrides["speculative_attention_mode"] = "decode"
|
||||
logger.info(
|
||||
"Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend "
|
||||
|
||||
@@ -1143,15 +1143,6 @@ def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
f"TensorRT-LLM MLA only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
|
||||
)
|
||||
page_size = 64
|
||||
if (
|
||||
view.attention_backend == "tokenspeed_mla"
|
||||
or view.decode_attention_backend == "tokenspeed_mla"
|
||||
):
|
||||
if page_size not in [32, 64]:
|
||||
logger.warning(
|
||||
f"tokenspeed_mla only supports page_size of 32 or 64, changing page_size from {page_size} to 64."
|
||||
)
|
||||
page_size = 64
|
||||
if (
|
||||
view.attention_backend == "cutedsl_mla"
|
||||
or view.decode_attention_backend == "cutedsl_mla"
|
||||
@@ -1193,7 +1184,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
|
||||
"""Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
|
||||
"""Validate resolved KV-cache dtype for the TRTLLM MLA backend."""
|
||||
if (
|
||||
view.attention_backend == "trtllm_mla"
|
||||
or view.decode_attention_backend == "trtllm_mla"
|
||||
@@ -1206,19 +1197,6 @@ def _mla_kv_cache_dtype_checks(view: Any) -> dict:
|
||||
raise ValueError(
|
||||
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, bf16, or auto."
|
||||
)
|
||||
if (
|
||||
view.attention_backend == "tokenspeed_mla"
|
||||
or view.decode_attention_backend == "tokenspeed_mla"
|
||||
):
|
||||
if not get_platform().is_blackwell:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
|
||||
)
|
||||
if view.kv_cache_dtype not in ["fp8_e4m3"]:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
|
||||
f"got {view.kv_cache_dtype}."
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
@@ -293,7 +293,8 @@ def handle_deprecated_args(server_args: Any):
|
||||
"_handle_deprecated_args",
|
||||
enable_flashinfer_allreduce_fusion=False,
|
||||
)
|
||||
# Deprecated attention-backend alias: "compressed" -> "dsv4".
|
||||
# Deprecated attention-backend aliases.
|
||||
aliases = {"compressed": "dsv4", "tokenspeed_mla": "cutedsl_mla"}
|
||||
renamed = {}
|
||||
for attr in (
|
||||
"attention_backend",
|
||||
@@ -301,12 +302,30 @@ def handle_deprecated_args(server_args: Any):
|
||||
"prefill_attention_backend",
|
||||
"speculative_draft_attention_backend",
|
||||
):
|
||||
if getattr(server_args, attr, None) == "compressed":
|
||||
old = getattr(server_args, attr, None)
|
||||
if old in aliases:
|
||||
logger.warning(
|
||||
"--%s=compressed is deprecated; use 'dsv4' instead.",
|
||||
"--%s=%s is deprecated; use '%s' instead.",
|
||||
attr.replace("_", "-"),
|
||||
old,
|
||||
aliases[old],
|
||||
)
|
||||
renamed[attr] = "dsv4"
|
||||
renamed[attr] = aliases[old]
|
||||
if renamed.get("prefill_attention_backend") == "cutedsl_mla":
|
||||
# cutedsl_mla is decode-only.
|
||||
renamed["prefill_attention_backend"] = "trtllm_mla"
|
||||
if "tokenspeed_mla" in (cfg.attention_backend, cfg.decode_attention_backend):
|
||||
# Keep what tokenspeed_mla implied: FP8 KV, verify on the decode kernel.
|
||||
if cfg.kv_cache_dtype != "fp8_e4m3":
|
||||
logger.warning(
|
||||
"tokenspeed_mla required FP8 KV; setting --kv-cache-dtype=fp8_e4m3."
|
||||
)
|
||||
renamed["kv_cache_dtype"] = "fp8_e4m3"
|
||||
if (
|
||||
cfg.speculative_algorithm is not None
|
||||
and cfg.speculative_attention_mode == "prefill"
|
||||
):
|
||||
renamed["speculative_attention_mode"] = "decode"
|
||||
if renamed:
|
||||
declare_resolution(server_args, "_handle_deprecated_args", **renamed)
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ import logging
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
@@ -52,8 +54,6 @@ def register_attention_backend(name):
|
||||
|
||||
@register_attention_backend("flashinfer")
|
||||
def create_flashinfer_backend(runner):
|
||||
import torch
|
||||
|
||||
if not runner.use_mla_backend:
|
||||
from sglang.srt.layers.attention.flashinfer_backend import (
|
||||
FlashInferAttnBackend,
|
||||
@@ -95,31 +95,26 @@ def create_llada2_cfg_flashinfer_backend(runner):
|
||||
def create_trtllm_mla_backend(runner):
|
||||
if not runner.use_mla_backend:
|
||||
raise ValueError("trtllm_mla backend can only be used with MLA models.")
|
||||
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
|
||||
if get_parallel().dcp_enabled and get_spec().speculative_algorithm is not None:
|
||||
_, decode_backend = attention_backends_of(resolved_view(runner.server_args))
|
||||
if decode_backend == "trtllm_mla":
|
||||
raise ValueError(
|
||||
"trtllm_mla cannot serve decode context parallelism with speculative "
|
||||
"decoding: it does not forward the cyclic DCP metadata to its decode "
|
||||
"kernel and returns no rank-local LSE for the cross-rank merge. "
|
||||
"Select cutedsl_mla or tokenspeed_mla."
|
||||
"Select cutedsl_mla."
|
||||
)
|
||||
if decode_backend == "cutedsl_mla" and runner.kv_cache_dtype == torch.float8_e4m3fn:
|
||||
from sglang.srt.layers.attention.cutedsl_mla_backend import (
|
||||
CuteDslMLAPrefillBackend,
|
||||
)
|
||||
|
||||
return CuteDslMLAPrefillBackend(runner)
|
||||
from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend
|
||||
|
||||
return TRTLLMMLABackend(runner)
|
||||
|
||||
|
||||
@register_attention_backend("tokenspeed_mla")
|
||||
def create_tokenspeed_mla_backend(runner):
|
||||
if not runner.use_mla_backend:
|
||||
raise ValueError("tokenspeed_mla backend can only be used with MLA models.")
|
||||
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
|
||||
TokenspeedMLABackend,
|
||||
)
|
||||
|
||||
return TokenspeedMLABackend(runner)
|
||||
|
||||
|
||||
@register_attention_backend("cutedsl_mla")
|
||||
def create_cutedsl_mla_backend(runner):
|
||||
if not runner.use_mla_backend:
|
||||
|
||||
@@ -21,12 +21,17 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
|
||||
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
|
||||
mla_kv_pack_quantize_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.attention.utils import (
|
||||
concat_mla_absorb_q_general,
|
||||
mla_quantize_and_rope_for_fp8,
|
||||
mla_quantize_without_rope_for_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
TRTLLMMLABackend,
|
||||
@@ -40,11 +45,13 @@ from sglang.srt.utils import is_flashinfer_available
|
||||
|
||||
if is_flashinfer_available():
|
||||
import flashinfer
|
||||
import flashinfer.rope as _flashinfer_rope
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -296,3 +303,139 @@ class CuteDslMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
|
||||
kv_indptr_buf=self.kv_indptr[i],
|
||||
q_indptr_decode_buf=self.q_indptr_decode,
|
||||
)
|
||||
|
||||
|
||||
class CuteDslMLAPrefillBackend(TRTLLMMLABackend):
|
||||
"""trtllm_mla prefill for cutedsl_mla deployments with FP8 KV.
|
||||
|
||||
The model-side MHA hooks below build FP8 Q/K/V with fused RoPE + quantize
|
||||
and write the FP8 KV cache directly, so the ragged prefill kernel consumes
|
||||
them without the per-tensor casts of the BF16 path.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fused_rope_fp8_quantize(
|
||||
q_nope: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_nope: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
cos_sin_cache: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
is_neox: bool,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
|
||||
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
|
||||
|
||||
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
|
||||
FP8 quantize and the packed layout, only the rotation is dropped.
|
||||
"""
|
||||
num_heads = q_nope.shape[1]
|
||||
seq_len = q_nope.shape[0]
|
||||
q_fp8 = torch.empty(
|
||||
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=q_nope.device,
|
||||
)
|
||||
k_fp8 = torch.empty(
|
||||
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=k_nope.device,
|
||||
)
|
||||
if seq_len == 0:
|
||||
return q_fp8, k_fp8
|
||||
|
||||
# Broadcast the shared latent k_pe across heads — RoPE is position-only
|
||||
# so per-head outputs are identical, and the cache write below reuses
|
||||
# head 0.
|
||||
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
|
||||
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
|
||||
else:
|
||||
k_pe_expanded = k_pe
|
||||
|
||||
if cos_sin_cache is None:
|
||||
# NoPE layer: quantize straight into the packed buffers.
|
||||
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
|
||||
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
|
||||
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
|
||||
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
|
||||
return q_fp8, k_fp8
|
||||
|
||||
_flashinfer_rope.mla_rope_quantize_fp8(
|
||||
q_rope=q_pe,
|
||||
k_rope=k_pe_expanded,
|
||||
q_nope=q_nope,
|
||||
k_nope=k_nope,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
pos_ids=positions,
|
||||
is_neox=is_neox,
|
||||
quantize_dtype=torch.float8_e4m3fn,
|
||||
q_rope_out=q_fp8[..., qk_nope_head_dim:],
|
||||
k_rope_out=k_fp8[..., qk_nope_head_dim:],
|
||||
q_nope_out=q_fp8[..., :qk_nope_head_dim],
|
||||
k_nope_out=k_fp8[..., :qk_nope_head_dim],
|
||||
quant_scale_q=1.0,
|
||||
quant_scale_kv=1.0,
|
||||
enable_pdl=is_arch_support_pdl(),
|
||||
)
|
||||
return q_fp8, k_fp8
|
||||
|
||||
def prepare_prefill_qkv(
|
||||
self,
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
kv_a: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer: DeepseekV2AttentionMLA,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
|
||||
kv = layer.kv_b_proj(kv_a)[0]
|
||||
kv = kv.view(
|
||||
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
|
||||
)
|
||||
k_nope = kv[..., : layer.qk_nope_head_dim]
|
||||
v_bf16 = kv[..., layer.qk_nope_head_dim :]
|
||||
q_nope = q[..., : layer.qk_nope_head_dim]
|
||||
|
||||
rotary_emb = layer.rotary_emb
|
||||
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
|
||||
q_nope=q_nope,
|
||||
q_pe=q_pe,
|
||||
k_nope=k_nope,
|
||||
k_pe=k_pe,
|
||||
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
|
||||
positions=positions,
|
||||
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
|
||||
qk_nope_head_dim=layer.qk_nope_head_dim,
|
||||
qk_rope_head_dim=layer.qk_rope_head_dim,
|
||||
)
|
||||
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
|
||||
|
||||
# k_pe is shared across heads (RoPE is position-only), so head 0
|
||||
# reproduces the original [tokens, 1, qk_rope] latent layout.
|
||||
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
|
||||
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer.attn_mha,
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
kv_a_fp8.unsqueeze(1),
|
||||
k_pe_fp8,
|
||||
)
|
||||
return q_fp8, k_fp8, v_fp8
|
||||
|
||||
def pack_prefix_chunk_kv(
|
||||
self,
|
||||
k_nope: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
|
||||
strided ``v`` into contig FP8 V in a single kernel.
|
||||
"""
|
||||
return mla_kv_pack_quantize_fp8(
|
||||
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
|
||||
)
|
||||
|
||||
@@ -1,516 +0,0 @@
|
||||
# Copyright (c) 2026 LightSeek Foundation
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
"""Attention backend for the tokenspeed-mla CuTe DSL kernels on Blackwell.
|
||||
|
||||
Subclasses :class:`TRTLLMMLABackend` to share its MLA data preparation, prefill
|
||||
plumbing, and DCP metadata (rank-local KV lengths and page table). The decode
|
||||
forward lives here because the TokenSpeed decode kernel natively accepts CP
|
||||
rank/world metadata and returns the partial log-sum-exp needed by the cross-rank
|
||||
merge.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import is_arch_support_pdl
|
||||
from sglang.kernels.ops.attention.fixup_zero_kv import fixup_zero_kv_rows
|
||||
from sglang.kernels.ops.attention.mla_kv_pack_quantize_fp8 import (
|
||||
mla_kv_pack_quantize_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.attention.utils import (
|
||||
mla_quantize_and_rope_for_fp8,
|
||||
mla_quantize_without_rope_for_fp8,
|
||||
)
|
||||
from sglang.kernels.ops.quantization.fp8_quantize import fp8_quantize
|
||||
from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
TRTLLMMLABackend,
|
||||
TRTLLMMLAMultiStepDraftBackend,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_resources,
|
||||
max_speculative_num_draft_tokens,
|
||||
)
|
||||
from sglang.srt.utils import is_flashinfer_available, is_tokenspeed_mla_available
|
||||
|
||||
if is_flashinfer_available():
|
||||
import flashinfer.rope as _flashinfer_rope
|
||||
|
||||
if is_tokenspeed_mla_available():
|
||||
import tokenspeed_mla
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.model_runner import ModelRunner
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Workspace upper bound for tokenspeed_mla_decode:
|
||||
# num_sms * num_heads * max_q_len * (kv_lora_rank + 1) * sizeof(float32)
|
||||
# MAX_Q_LEN=8 covers EAGLE3 num_draft_tokens=4 plus headroom. Larger
|
||||
# speculative widths grow the bound at backend initialization.
|
||||
_TOKENSPEED_MAX_Q_LEN = 8
|
||||
|
||||
|
||||
def _get_tokenspeed_workspace(
|
||||
device: torch.device,
|
||||
num_heads: int,
|
||||
kv_lora_rank: int,
|
||||
max_q_len: int = _TOKENSPEED_MAX_Q_LEN,
|
||||
) -> torch.Tensor:
|
||||
|
||||
# DCP target verification gathers Q to the full head count before launching
|
||||
# TokenSpeed; size for that launch shape, not the rank-local head count.
|
||||
num_heads *= get_parallel().attn_dcp_size
|
||||
max_q_len = max(max_q_len, _TOKENSPEED_MAX_Q_LEN)
|
||||
needed = (
|
||||
tokenspeed_mla.get_num_sm(device)
|
||||
* num_heads
|
||||
* max_q_len
|
||||
* (kv_lora_rank + 1)
|
||||
* 4
|
||||
)
|
||||
buffers = get_resources().buffers
|
||||
key = f"tokenspeed_mla_workspace:{device}"
|
||||
existing = buffers.get(key)
|
||||
if existing is None or existing.numel() < needed:
|
||||
buffers[key] = torch.empty(needed, dtype=torch.int8, device=device)
|
||||
return buffers[key]
|
||||
|
||||
|
||||
# TODO(Qiaolin-Yu): Merge this attention backend into trtllm_mla_backend.py
|
||||
# once the same CuteDSL kernels in flashinfer_trtllm are stable
|
||||
# and there is no performance gap compared to this backend.
|
||||
class TokenspeedMLABackend(TRTLLMMLABackend):
|
||||
"""tokenspeed-mla CuTe DSL attention backend (Blackwell SM100, FP8 KV)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_runner: ModelRunner,
|
||||
skip_prefill: bool = False,
|
||||
kv_indptr_buf: Optional[torch.Tensor] = None,
|
||||
q_indptr_decode_buf: Optional[torch.Tensor] = None,
|
||||
):
|
||||
super().__init__(
|
||||
model_runner,
|
||||
skip_prefill,
|
||||
kv_indptr_buf,
|
||||
q_indptr_decode_buf,
|
||||
)
|
||||
|
||||
if self.data_type != torch.float8_e4m3fn:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires --kv-cache-dtype fp8_e4m3, "
|
||||
f"got data_type={self.data_type}."
|
||||
)
|
||||
if self.page_size not in (32, 64):
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires page_size in {32, 64}, "
|
||||
f"got page_size={self.page_size}."
|
||||
)
|
||||
|
||||
self._tokenspeed_workspace: Optional[torch.Tensor] = None
|
||||
if is_tokenspeed_mla_available():
|
||||
self._tokenspeed_workspace = _get_tokenspeed_workspace(
|
||||
self.device,
|
||||
self.num_q_heads,
|
||||
self.kv_lora_rank,
|
||||
max_q_len=(max_speculative_num_draft_tokens() or 1),
|
||||
)
|
||||
|
||||
# Pre-JIT the prefill kernel variants. Each cute.compile takes 1-2
|
||||
# min; without warm-up the first request trips the 300 s scheduler
|
||||
# watchdog.
|
||||
_compile_prefill_kernel = tokenspeed_mla.mla_prefill._compile_prefill_kernel
|
||||
_compiled_kernels = tokenspeed_mla.mla_prefill._compiled_kernels
|
||||
head_dim_qk = self.qk_nope_head_dim + self.qk_rope_head_dim
|
||||
enable_ex2_emulation = tokenspeed_mla.mla_prefill._enable_ex2_emulation()
|
||||
use_pdl = is_arch_support_pdl()
|
||||
for is_causal in (True, False):
|
||||
for return_lse in (True, False):
|
||||
# Non-causal is only entered from the chunked-prefix
|
||||
# branch, which always asks for the LSE.
|
||||
if is_causal is False and return_lse is False:
|
||||
continue
|
||||
# Runtime feeds fp8_e4m3fn q/k/v
|
||||
config = (
|
||||
torch.float8_e4m3fn,
|
||||
head_dim_qk,
|
||||
self.v_head_dim,
|
||||
is_causal,
|
||||
return_lse,
|
||||
use_pdl,
|
||||
enable_ex2_emulation,
|
||||
)
|
||||
if config in _compiled_kernels:
|
||||
continue
|
||||
_compiled_kernels[config] = _compile_prefill_kernel(
|
||||
torch.float8_e4m3fn,
|
||||
head_dim_qk,
|
||||
self.v_head_dim,
|
||||
is_causal,
|
||||
return_lse,
|
||||
use_pdl=use_pdl,
|
||||
enable_ex2_emulation=enable_ex2_emulation,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fused_rope_fp8_quantize(
|
||||
q_nope: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
k_nope: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
cos_sin_cache: Optional[torch.Tensor],
|
||||
positions: torch.Tensor,
|
||||
is_neox: bool,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Fused RoPE + FP8 quantize that also packs nope+pe along the last
|
||||
dim, so FMHA consumes contig FP8 Q/K without an extra concat or cast.
|
||||
|
||||
``cos_sin_cache`` is None for NoPE layers (``skip_rope``); they keep the
|
||||
FP8 quantize and the packed layout, only the rotation is dropped.
|
||||
"""
|
||||
num_heads = q_nope.shape[1]
|
||||
seq_len = q_nope.shape[0]
|
||||
q_fp8 = torch.empty(
|
||||
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=q_nope.device,
|
||||
)
|
||||
k_fp8 = torch.empty(
|
||||
(seq_len, num_heads, qk_nope_head_dim + qk_rope_head_dim),
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=k_nope.device,
|
||||
)
|
||||
if seq_len == 0:
|
||||
return q_fp8, k_fp8
|
||||
|
||||
# Broadcast the shared latent k_pe across heads — RoPE is position-only
|
||||
# so per-head outputs are identical, and the cache write below reuses
|
||||
# head 0.
|
||||
if k_pe.dim() == 3 and k_pe.shape[1] == 1:
|
||||
k_pe_expanded = k_pe.expand(-1, num_heads, -1)
|
||||
else:
|
||||
k_pe_expanded = k_pe
|
||||
|
||||
if cos_sin_cache is None:
|
||||
# NoPE layer: quantize straight into the packed buffers.
|
||||
q_fp8[..., :qk_nope_head_dim].copy_(q_nope)
|
||||
q_fp8[..., qk_nope_head_dim:].copy_(q_pe)
|
||||
k_fp8[..., :qk_nope_head_dim].copy_(k_nope)
|
||||
k_fp8[..., qk_nope_head_dim:].copy_(k_pe_expanded)
|
||||
return q_fp8, k_fp8
|
||||
|
||||
_flashinfer_rope.mla_rope_quantize_fp8(
|
||||
q_rope=q_pe,
|
||||
k_rope=k_pe_expanded,
|
||||
q_nope=q_nope,
|
||||
k_nope=k_nope,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
pos_ids=positions,
|
||||
is_neox=is_neox,
|
||||
quantize_dtype=torch.float8_e4m3fn,
|
||||
q_rope_out=q_fp8[..., qk_nope_head_dim:],
|
||||
k_rope_out=k_fp8[..., qk_nope_head_dim:],
|
||||
q_nope_out=q_fp8[..., :qk_nope_head_dim],
|
||||
k_nope_out=k_fp8[..., :qk_nope_head_dim],
|
||||
quant_scale_q=1.0,
|
||||
quant_scale_kv=1.0,
|
||||
enable_pdl=is_arch_support_pdl(),
|
||||
)
|
||||
return q_fp8, k_fp8
|
||||
|
||||
def prepare_prefill_qkv(
|
||||
self,
|
||||
*,
|
||||
q: torch.Tensor,
|
||||
q_pe: torch.Tensor,
|
||||
kv_a: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
layer: DeepseekV2AttentionMLA,
|
||||
forward_batch: ForwardBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build FP8 (Q, K, V) for the FMHA kernel and write FP8 KV cache."""
|
||||
kv = layer.kv_b_proj(kv_a)[0]
|
||||
kv = kv.view(
|
||||
-1, layer.num_local_heads, layer.qk_nope_head_dim + layer.v_head_dim
|
||||
)
|
||||
k_nope = kv[..., : layer.qk_nope_head_dim]
|
||||
v_bf16 = kv[..., layer.qk_nope_head_dim :]
|
||||
q_nope = q[..., : layer.qk_nope_head_dim]
|
||||
|
||||
rotary_emb = layer.rotary_emb
|
||||
q_fp8, k_fp8 = self._fused_rope_fp8_quantize(
|
||||
q_nope=q_nope,
|
||||
q_pe=q_pe,
|
||||
k_nope=k_nope,
|
||||
k_pe=k_pe,
|
||||
cos_sin_cache=None if rotary_emb is None else rotary_emb.cos_sin_cache,
|
||||
positions=positions,
|
||||
is_neox=True if rotary_emb is None else rotary_emb.is_neox_style,
|
||||
qk_nope_head_dim=layer.qk_nope_head_dim,
|
||||
qk_rope_head_dim=layer.qk_rope_head_dim,
|
||||
)
|
||||
v_fp8 = fp8_quantize(v_bf16, enable_pdl=is_arch_support_pdl())
|
||||
|
||||
# k_pe is shared across heads (RoPE is position-only), so head 0
|
||||
# reproduces the original [tokens, 1, qk_rope] latent layout.
|
||||
kv_a_fp8 = fp8_quantize(kv_a, enable_pdl=is_arch_support_pdl())
|
||||
k_pe_fp8 = k_fp8[:, 0:1, layer.qk_nope_head_dim :]
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer.attn_mha,
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
kv_a_fp8.unsqueeze(1),
|
||||
k_pe_fp8,
|
||||
)
|
||||
return q_fp8, k_fp8, v_fp8
|
||||
|
||||
def pack_prefix_chunk_kv(
|
||||
self,
|
||||
k_nope: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pack strided ``k_nope``+``k_pe`` into contig FP8 K and quantize
|
||||
strided ``v`` into contig FP8 V in a single kernel.
|
||||
"""
|
||||
return mla_kv_pack_quantize_fp8(
|
||||
k_nope, k_pe, v, enable_pdl=is_arch_support_pdl()
|
||||
)
|
||||
|
||||
def _run_decode_kernel(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
layer: RadixAttention,
|
||||
*,
|
||||
causal_seqs: Optional[torch.Tensor] = None,
|
||||
cp_world: int = 1,
|
||||
cp_rank: int = 0,
|
||||
return_lse: bool = False,
|
||||
):
|
||||
k_scale = getattr(layer, "k_scale_float", None)
|
||||
if k_scale is None:
|
||||
k_scale = 1.0
|
||||
softmax_scale = float(layer.scaling) * float(k_scale)
|
||||
output_scale = float(k_scale)
|
||||
|
||||
seq_lens_i32 = (
|
||||
seq_lens if seq_lens.dtype == torch.int32 else seq_lens.to(torch.int32)
|
||||
)
|
||||
return tokenspeed_mla.tokenspeed_mla_decode(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
workspace_buffer=self._tokenspeed_workspace,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens_i32,
|
||||
max_seq_len=int(max_seq_len),
|
||||
softmax_scale=softmax_scale,
|
||||
output_scale=output_scale,
|
||||
enable_pdl=is_arch_support_pdl(),
|
||||
return_lse=return_lse,
|
||||
causal_seqs=causal_seqs,
|
||||
cp_world=cp_world,
|
||||
cp_rank=cp_rank,
|
||||
)
|
||||
|
||||
def forward_decode(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
forward_batch: ForwardBatch,
|
||||
save_kv_cache: bool = True,
|
||||
q_rope: Optional[torch.Tensor] = None,
|
||||
k_rope: Optional[torch.Tensor] = None,
|
||||
cos_sin_cache: Optional[torch.Tensor] = None,
|
||||
is_neox: Optional[bool] = False,
|
||||
llama_4_scaling: Optional[torch.Tensor] = None,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
if parallel.dcp_enabled and get_in_autotune_dummy_run():
|
||||
return self._dummy_dcp_decode_for_autotune(q, layer)
|
||||
|
||||
if not parallel.dcp_enabled:
|
||||
return super().forward_decode(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
layer,
|
||||
forward_batch,
|
||||
save_kv_cache,
|
||||
q_rope,
|
||||
k_rope,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
llama_4_scaling,
|
||||
)
|
||||
|
||||
assert q_rope is not None and k_rope is not None
|
||||
if cos_sin_cache is None:
|
||||
q, k, k_rope = mla_quantize_without_rope_for_fp8(
|
||||
q, q_rope, k.squeeze(1), k_rope.squeeze(1)
|
||||
)
|
||||
else:
|
||||
q, k, k_rope = mla_quantize_and_rope_for_fp8(
|
||||
q,
|
||||
q_rope,
|
||||
k.squeeze(1),
|
||||
k_rope.squeeze(1),
|
||||
forward_batch.positions,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
self.kv_lora_rank,
|
||||
self.qk_rope_head_dim,
|
||||
)
|
||||
|
||||
if save_kv_cache:
|
||||
self.token_to_kv_pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
KVWriteLoc.for_batch(forward_batch),
|
||||
k,
|
||||
k_rope,
|
||||
)
|
||||
|
||||
query = q.view(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
if llama_4_scaling is not None:
|
||||
query = (query.to(self.q_data_type) * llama_4_scaling).to(self.data_type)
|
||||
if query.dim() == 3:
|
||||
query = query.unsqueeze(1)
|
||||
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
|
||||
metadata = (
|
||||
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
|
||||
or self.forward_decode_metadata
|
||||
)
|
||||
metadata_batch_size = getattr(metadata, "batch_size", None)
|
||||
if (
|
||||
metadata_batch_size is not None
|
||||
and metadata_batch_size < forward_batch.batch_size
|
||||
):
|
||||
self.init_forward_metadata(forward_batch)
|
||||
metadata = forward_batch.decode_trtllm_mla_metadata
|
||||
|
||||
global_seq_lens = forward_batch.seq_lens[: forward_batch.batch_size]
|
||||
local_seq_lens = self._get_dcp_local_seq_lens(global_seq_lens)
|
||||
raw_out, lse = self._run_decode_kernel(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_tables=metadata.block_kv_indices,
|
||||
seq_lens=local_seq_lens,
|
||||
max_seq_len=metadata.max_seq_len_k,
|
||||
layer=layer,
|
||||
causal_seqs=global_seq_lens,
|
||||
cp_world=parallel.dcp_size,
|
||||
cp_rank=parallel.dcp_rank,
|
||||
return_lse=True,
|
||||
)
|
||||
|
||||
output = raw_out.view(-1, layer.tp_q_head_num, layer.v_head_dim)
|
||||
lse = lse.view(-1, layer.tp_q_head_num)
|
||||
fixup_zero_kv_rows(
|
||||
output,
|
||||
lse,
|
||||
local_seq_lens,
|
||||
self.q_indptr_decode[: forward_batch.batch_size + 1],
|
||||
1,
|
||||
)
|
||||
return output.flatten(1), lse
|
||||
|
||||
def _run_prefill_kernel(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
layer: RadixAttention,
|
||||
batch_size: int,
|
||||
cum_seq_lens_q: torch.Tensor,
|
||||
max_q_len: int,
|
||||
seq_lens_kv: torch.Tensor,
|
||||
cum_seq_lens_kv: torch.Tensor,
|
||||
max_kv_len: int,
|
||||
is_causal: bool,
|
||||
return_lse: bool,
|
||||
out_buffer: torch.Tensor,
|
||||
o_sf_scale: float = 1.0,
|
||||
): # Q/K/V arrive already in FP8 via the model-side fused path
|
||||
# (prepare_prefill_qkv / pack_prefix_chunk_kv); no quantize here.
|
||||
# Hybrid MLA models resolve the model-side hook through the outer
|
||||
# HybridLinearAttnBackend, so their fallback MHA path can pass V as a
|
||||
# last-dimension slice of kv_b_proj (stride(-2) > size(-1)). The
|
||||
# TokenSpeed prefill kernel requires dense Q/K/V layouts even though
|
||||
# the public wrapper accepts arbitrary torch tensors.
|
||||
q = q.contiguous()
|
||||
k = k.contiguous()
|
||||
v = v.contiguous()
|
||||
return tokenspeed_mla.tokenspeed_mla_prefill(
|
||||
query=q,
|
||||
key=k,
|
||||
value=v,
|
||||
seq_lens=seq_lens_kv,
|
||||
cum_seq_lens=cum_seq_lens_kv,
|
||||
max_seq_len=int(max_kv_len),
|
||||
batch_size=int(batch_size),
|
||||
softmax_scale=float(layer.scaling),
|
||||
is_causal=is_causal,
|
||||
return_lse=return_lse,
|
||||
cum_seq_lens_q=cum_seq_lens_q,
|
||||
max_seq_len_q=int(max_q_len),
|
||||
enable_pdl=is_arch_support_pdl(),
|
||||
)
|
||||
|
||||
|
||||
class TokenspeedMLAMultiStepDraftBackend(TRTLLMMLAMultiStepDraftBackend):
|
||||
"""Multi-step draft backend for tokenspeed_mla used by EAGLE."""
|
||||
|
||||
def __init__(
|
||||
self, model_runner: ModelRunner, topk: int, speculative_num_steps: int
|
||||
):
|
||||
super().__init__(model_runner, topk, speculative_num_steps)
|
||||
# Parent populates self.attn_backends with TRT-LLM instances; replace
|
||||
# them with tokenspeed instances sharing the parent's index buffers.
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
self.attn_backends[i] = TokenspeedMLABackend(
|
||||
model_runner,
|
||||
skip_prefill=True,
|
||||
kv_indptr_buf=self.kv_indptr[i],
|
||||
q_indptr_decode_buf=self.q_indptr_decode,
|
||||
)
|
||||
@@ -1017,8 +1017,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
raise NotImplementedError(
|
||||
"trtllm_mla cannot forward a global causal bound to its decode "
|
||||
"kernel, which is required for DCP with q_len > 1 (speculative "
|
||||
"target-verify / draft-extend); select cutedsl_mla or "
|
||||
"tokenspeed_mla for a DCP speculative run"
|
||||
"target-verify / draft-extend); select cutedsl_mla "
|
||||
"for a DCP speculative run"
|
||||
)
|
||||
|
||||
# Scale computation for TRTLLM MLA kernel BMM1 operation:
|
||||
@@ -1073,10 +1073,11 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
o_sf_scale: float = 1.0,
|
||||
):
|
||||
"""Hook for subclasses to swap the ragged prefill kernel. Q/K/V arrive
|
||||
in model-native dtype; subclasses do any kernel-specific quantization.
|
||||
in model-native dtype, or already FP8 from a model-side prefill hook;
|
||||
subclasses do any kernel-specific quantization.
|
||||
Returns the output tensor or (output, lse) if return_lse."""
|
||||
q_scale = k_scale = v_scale = 1.0
|
||||
if self.data_type == torch.float8_e4m3fn:
|
||||
if self.data_type == torch.float8_e4m3fn and q.dtype != torch.float8_e4m3fn:
|
||||
q, k, v, k_scale, v_scale = _quantize_fp8_qkv(q, k, v, layer)
|
||||
return flashinfer.prefill.trtllm_ragged_attention_deepseek(
|
||||
query=q,
|
||||
|
||||
@@ -181,12 +181,6 @@ def handle_attention_trtllm_mla(attn, forward_batch):
|
||||
return _dispatch_mla_subtype(attn, forward_batch)
|
||||
|
||||
|
||||
def handle_attention_tokenspeed_mla(attn, forward_batch):
|
||||
# tokenspeed_mla shares the trtllm_mla dispatch pattern: pure prefill goes
|
||||
# via MHA chunked KV (TRT-LLM ragged), spec decode / decode goes via MLA.
|
||||
return handle_attention_trtllm_mla(attn, forward_batch)
|
||||
|
||||
|
||||
def handle_attention_aiter(attn, forward_batch):
|
||||
# During PCG/BCG capture on ROCm, aiter fp8 MLA prefill has no capture
|
||||
# kernels; route through the MHA path (radix_attention swaps attn_mqa for
|
||||
@@ -268,7 +262,6 @@ AttentionBackendRegistry.register("fa3", handle_attention_fa3)
|
||||
AttentionBackendRegistry.register("flashmla", handle_attention_flashmla)
|
||||
AttentionBackendRegistry.register("fa4", handle_attention_fa4)
|
||||
AttentionBackendRegistry.register("trtllm_mla", handle_attention_trtllm_mla)
|
||||
AttentionBackendRegistry.register("tokenspeed_mla", handle_attention_tokenspeed_mla)
|
||||
AttentionBackendRegistry.register("aiter", handle_attention_aiter)
|
||||
AttentionBackendRegistry.register("dsa", handle_attention_dsa)
|
||||
AttentionBackendRegistry.register(
|
||||
|
||||
@@ -942,8 +942,7 @@ class DeepseekMLAForwardMixin:
|
||||
|
||||
return (
|
||||
self.rotary_emb is not None
|
||||
and self.current_attention_backend
|
||||
in ("trtllm_mla", "tokenspeed_mla", "cutedsl_mla")
|
||||
and self.current_attention_backend in ("trtllm_mla", "cutedsl_mla")
|
||||
and (
|
||||
forward_batch.forward_mode.is_decode_or_idle()
|
||||
or forward_batch.forward_mode.is_target_verify()
|
||||
|
||||
@@ -70,7 +70,6 @@ FORWARD_ABSORB_CORE_ATTENTION_BACKENDS = [
|
||||
"flashinfer",
|
||||
"trtllm_mla",
|
||||
"cutedsl_mla",
|
||||
"tokenspeed_mla",
|
||||
"ascend",
|
||||
"intel_xpu",
|
||||
]
|
||||
|
||||
@@ -105,7 +105,6 @@ class DraftBackendFactory:
|
||||
"trtllm_mha": self._create_trtllm_mha_decode_backend,
|
||||
"trtllm_mla": self._create_trtllm_mla_decode_backend,
|
||||
"cutedsl_mla": self._create_cutedsl_mla_decode_backend,
|
||||
"tokenspeed_mla": self._create_tokenspeed_mla_decode_backend,
|
||||
"dsa": self._create_dsa_decode_backend,
|
||||
"nsa": self._create_dsa_decode_backend, # Deprecated alias for "dsa"
|
||||
"ascend": self._create_ascend_decode_backend,
|
||||
@@ -137,7 +136,6 @@ class DraftBackendFactory:
|
||||
"trtllm_mla": self._create_trtllm_mla_prefill_backend,
|
||||
# cute-dsl MLA only supports decode; draft-extend falls back to trtllm-gen.
|
||||
"cutedsl_mla": self._create_trtllm_mla_prefill_backend,
|
||||
"tokenspeed_mla": self._create_tokenspeed_mla_prefill_backend,
|
||||
"dsa": self._create_dsa_prefill_backend,
|
||||
"nsa": self._create_dsa_prefill_backend, # Deprecated alias for "dsa"
|
||||
"ascend": self._create_ascend_prefill_backend,
|
||||
@@ -394,23 +392,6 @@ class DraftBackendFactory:
|
||||
),
|
||||
)
|
||||
|
||||
def _create_tokenspeed_mla_decode_backend(self):
|
||||
if not self.draft_model_runner.use_mla_backend:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
|
||||
)
|
||||
|
||||
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
|
||||
TokenspeedMLAMultiStepDraftBackend,
|
||||
)
|
||||
|
||||
return (
|
||||
"tokenspeed_mla",
|
||||
TokenspeedMLAMultiStepDraftBackend(
|
||||
self.draft_model_runner, self.topk, self.speculative_num_steps
|
||||
),
|
||||
)
|
||||
|
||||
def _create_ascend_decode_backend(self):
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
|
||||
AscendAttnMultiStepDraftBackend,
|
||||
@@ -541,21 +522,6 @@ class DraftBackendFactory:
|
||||
TRTLLMMLABackend(self.draft_model_runner, skip_prefill=False),
|
||||
)
|
||||
|
||||
def _create_tokenspeed_mla_prefill_backend(self):
|
||||
if not self.draft_model_runner.use_mla_backend:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires MLA model (use_mla_backend=True)."
|
||||
)
|
||||
|
||||
from sglang.srt.layers.attention.tokenspeed_mla_backend import (
|
||||
TokenspeedMLABackend,
|
||||
)
|
||||
|
||||
return (
|
||||
"tokenspeed_mla",
|
||||
TokenspeedMLABackend(self.draft_model_runner, skip_prefill=False),
|
||||
)
|
||||
|
||||
def _create_ascend_prefill_backend(self):
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
|
||||
AscendAttnBackend,
|
||||
|
||||
@@ -27,7 +27,6 @@ from sglang.srt.layers.attention.qwen_sparse_attn_backend import (
|
||||
QwenSparseAttnBackend,
|
||||
QwenSparseMultiStepDraftBackend,
|
||||
)
|
||||
from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend
|
||||
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
|
||||
from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend
|
||||
from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
@@ -536,7 +535,6 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
TritonAttnBackend,
|
||||
TRTLLMMLABackend,
|
||||
TRTLLMHAAttnBackend,
|
||||
TokenspeedMLABackend,
|
||||
FlashInferAttnBackend,
|
||||
QwenSparseAttnBackend,
|
||||
]
|
||||
|
||||
@@ -394,18 +394,6 @@ def is_flashinfer_available():
|
||||
return importlib.util.find_spec("flashinfer") is not None and is_cuda()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def is_tokenspeed_mla_available():
|
||||
"""
|
||||
Check whether the tokenspeed_mla CuTe DSL kernels are available.
|
||||
Only available on NVIDIA Blackwell (SM100) at the moment.
|
||||
"""
|
||||
return (
|
||||
importlib.util.find_spec("tokenspeed_mla") is not None
|
||||
and is_blackwell_supported()
|
||||
)
|
||||
|
||||
|
||||
def is_nvidia_cublas_version_ge_12_9():
|
||||
"""
|
||||
temporary fix for issue #11272 (cublas 12.9+)
|
||||
|
||||
@@ -228,8 +228,8 @@ class MockMLAModelRunner(ModelRunner):
|
||||
self.device = device
|
||||
self.dtype = dtype
|
||||
# `kv_cache_dtype` is the dtype the *storage* uses. For FP8 KV
|
||||
# cache (the production deployment dtype for tokenspeed_mla and
|
||||
# some trtllm_mla configs), the pool stores quantized bytes
|
||||
# cache (the production deployment dtype for some trtllm_mla
|
||||
# configs), the pool stores quantized bytes
|
||||
# while the model still projects K/V in bf16; `set_mla_kv_buffer`
|
||||
# does the BF16->FP8 cast on the way in.
|
||||
self.kv_cache_dtype = torch.float8_e4m3fn if fp8_kv_cache else dtype
|
||||
|
||||
Reference in New Issue
Block a user