From 93dbe88183df35a36da5618642fd6f4bc37d283a Mon Sep 17 00:00:00 2001 From: leonardHONG <2695316095@qq.com> Date: Thu, 15 Oct 2026 16:15:24 +0800 Subject: [PATCH] Fuse MiMo verification RoPE and KV cache writes (#43106) Co-authored-by: lianghougong Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../kernels/jit/csrc/elementwise/pos_enc.cuh | 133 ++++++++++++++++++ python/sglang/kernels/ops/attention/rope.py | 30 ++++ .../srt/layers/attention/base_attn_backend.py | 6 + .../attention/flashattention_backend.py | 35 +++++ .../layers/attention/hybrid_attn_backend.py | 5 + .../srt/layers/rotary_embedding/base.py | 44 ++++++ python/sglang/srt/models/mimo_v2.py | 30 ++++ 7 files changed, 283 insertions(+) diff --git a/python/sglang/kernels/jit/csrc/elementwise/pos_enc.cuh b/python/sglang/kernels/jit/csrc/elementwise/pos_enc.cuh index 13880865f4..8dd5d22043 100644 --- a/python/sglang/kernels/jit/csrc/elementwise/pos_enc.cuh +++ b/python/sglang/kernels/jit/csrc/elementwise/pos_enc.cuh @@ -8,6 +8,7 @@ #include +#include #include #include @@ -310,4 +311,136 @@ struct RotaryEmbeddingKernel { } }; +template +__global__ void bf16_rope_scale_store_kernel( + const nv_bfloat16* qkv, + nv_bfloat16* query, + nv_bfloat16* key_cache, + nv_bfloat16* value_cache, + const nv_bfloat16* cos_sin, + const int64_t* positions, + const Index* locations, + int64_t input_stride, + int64_t query_stride, + int64_t key_stride, + int64_t value_stride, + int query_heads, + int kv_heads, + int head_dim, + int value_dim, + int rotary_dim, + float value_scale) { + const int token = blockIdx.x; + const int query_size = query_heads * head_dim; + const int key_size = kv_heads * head_dim; + const int value_size = kv_heads * value_dim; + const int64_t slot = locations[token]; + const auto* input = qkv + int64_t(token) * input_stride; + auto* q = query + int64_t(token) * query_stride; + // Slot 0 is reserved for CUDA graph padding. + auto* k = slot > 0 ? key_cache + slot * key_stride : nullptr; + auto* v = slot > 0 ? value_cache + slot * value_stride : nullptr; + for (int i = threadIdx.x; i < query_size; i += blockDim.x) + q[i] = input[i]; + if (slot > 0) { + for (int i = threadIdx.x; i < key_size; i += blockDim.x) + k[i] = input[query_size + i]; + for (int i = threadIdx.x; i < value_size; i += blockDim.x) { + auto x = input[query_size + key_size + i]; + v[i] = value_scale == 1.f ? x : __float2bfloat16_rn(static_cast(x) * value_scale); + } + } + __syncthreads(); + // Preserve the standalone RoPE's BF16 multiply/subtract/add sequence. + apply_rotary_embedding( + q, k, cos_sin + positions[token] * rotary_dim, head_dim, query_heads, kv_heads, rotary_dim, 0, 0, 0, head_dim); +} + +struct BF16RopeScaleStoreKernel { + static void + run(tvm::ffi::TensorView qkv, + tvm::ffi::TensorView query, + tvm::ffi::TensorView key_cache, + tvm::ffi::TensorView value_cache, + tvm::ffi::TensorView cos_sin, + tvm::ffi::TensorView positions, + tvm::ffi::TensorView locations, + double value_scale) { + using namespace host; + auto tokens = SymbolicSize{"tokens"}; + auto input_width = SymbolicSize{"input_width"}; + auto query_width = SymbolicSize{"query_width"}; + auto capacity = SymbolicSize{"capacity"}; + auto kv_heads = SymbolicSize{"kv_heads"}; + auto head_dim = SymbolicSize{"head_dim"}; + auto value_dim = SymbolicSize{"value_dim"}; + auto rotary_dim = SymbolicSize{"rotary_dim"}; + auto input_stride = SymbolicSize{"input_stride"}; + auto query_stride = SymbolicSize{"query_stride"}; + auto key_stride = SymbolicSize{"key_stride"}; + auto value_stride = SymbolicSize{"value_stride"}; + auto device = SymbolicDevice{}; + auto index_type = SymbolicDType{}; + device.set_options(); + TensorMatcher({tokens, input_width}) + .with_strides({input_stride, 1}) + .with_dtype() + .with_device(device) + .verify(qkv); + TensorMatcher({tokens, query_width}) + .with_strides({query_stride, 1}) + .with_dtype() + .with_device(device) + .verify(query); + TensorMatcher({capacity, kv_heads, head_dim}) + .with_strides({key_stride, head_dim, 1}) + .with_dtype() + .with_device(device) + .verify(key_cache); + TensorMatcher({capacity, kv_heads, value_dim}) + .with_strides({value_stride, value_dim, 1}) + .with_dtype() + .with_device(device) + .verify(value_cache); + TensorMatcher({-1, rotary_dim}).with_dtype().with_device(device).verify(cos_sin); + TensorMatcher({tokens}).with_dtype().with_device(device).verify(positions); + TensorMatcher({tokens}).with_dtype(index_type).with_device(device).verify(locations); + RuntimeCheck(query_width.unwrap() % head_dim.unwrap() == 0, "Query width must contain whole heads"); + RuntimeCheck( + input_width.unwrap() == query_width.unwrap() + kv_heads.unwrap() * (head_dim.unwrap() + value_dim.unwrap()), + "Invalid fused QKV width"); + RuntimeCheck( + rotary_dim.unwrap() > 0 && rotary_dim.unwrap() % 2 == 0 && rotary_dim.unwrap() <= head_dim.unwrap(), + "Invalid rotary dimension"); + const int query_heads = query_width.unwrap() / head_dim.unwrap(); + const int threads = std::min(query_heads * static_cast(rotary_dim.unwrap()) / 2, 512); + auto launch = [&]() { + LaunchKernel(tokens.unwrap(), threads, device.unwrap())( + bf16_rope_scale_store_kernel, + static_cast(qkv.data_ptr()), + static_cast(query.data_ptr()), + static_cast(key_cache.data_ptr()), + static_cast(value_cache.data_ptr()), + static_cast(cos_sin.data_ptr()), + static_cast(positions.data_ptr()), + static_cast(locations.data_ptr()), + input_stride.unwrap(), + query_stride.unwrap(), + key_stride.unwrap(), + value_stride.unwrap(), + query_heads, + static_cast(kv_heads.unwrap()), + static_cast(head_dim.unwrap()), + static_cast(value_dim.unwrap()), + static_cast(rotary_dim.unwrap()), + static_cast(value_scale)); + }; + if (index_type.is_type()) { + launch.template operator()(); + } else { + launch.template operator()(); + } + } +}; + } // namespace sglang diff --git a/python/sglang/kernels/ops/attention/rope.py b/python/sglang/kernels/ops/attention/rope.py index a5f6be632f..1d5d94956e 100644 --- a/python/sglang/kernels/ops/attention/rope.py +++ b/python/sglang/kernels/ops/attention/rope.py @@ -26,6 +26,36 @@ def _jit_rotary_embedding_module() -> Module: ) +@cache_once +def _jit_rope_scale_store_module() -> Module: + return load_jit( + "bf16_rope_scale_store", + cuda_files=["elementwise/pos_enc.cuh"], + cuda_wrappers=[("run", "BF16RopeScaleStoreKernel::run")], + extra_cuda_cflags=["--ftz=false"], + ) + + +@register_custom_op( + op_name="bf16_rope_scale_store", + mutates_args=["query", "key_cache", "value_cache"], +) +def bf16_rope_scale_store( + qkv: torch.Tensor, + query: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + cos_sin: torch.Tensor, + positions: torch.Tensor, + locations: torch.Tensor, + value_scale: float, +) -> None: + """NeoX RoPE on packed BF16 QKV, with contiguous Q and scaled V cache writes.""" + _jit_rope_scale_store_module().run( + qkv, query, key_cache, value_cache, cos_sin, positions, locations, value_scale + ) + + @cache_once def _jit_fused_rope_module( is_neox: bool, rope_dim: int, dtype: torch.dtype, q_dtype: torch.dtype diff --git a/python/sglang/srt/layers/attention/base_attn_backend.py b/python/sglang/srt/layers/attention/base_attn_backend.py index fa79775581..76c88b6ba5 100644 --- a/python/sglang/srt/layers/attention/base_attn_backend.py +++ b/python/sglang/srt/layers/attention/base_attn_backend.py @@ -88,6 +88,12 @@ class AttentionBackend(ABC): # allow-list enforces. kv_index_translator = None + def get_fused_kv_write_buffers( + self, layer: RadixAttention, forward_batch: ForwardBatch + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None: + """Return token-major K/V buffers and physical slots for a fused writer.""" + return None + def init_forward_metadata(self, forward_batch: ForwardBatch): """Eager entry point. Default = ``_out_graph(fb) + _in_graph(fb)``. diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 8e18842499..abb0a8cf4a 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -1389,6 +1389,41 @@ class FlashAttentionBackend(AttentionBackend): k_rope = k_rope.to(self.kv_cache_dtype) if k_rope is not None else None return q, q_rope, k_rope, k_descale, v_descale + def get_fused_kv_write_buffers(self, layer, forward_batch): + if ( + self.kv_cache_dtype != torch.bfloat16 + or self.use_mla + or layer.is_cross_attention + or self.kv_index_translator.is_translating + or self._kv_shard_pool is not None + or self.fa_skip_kv_cache + or layer.tp_k_head_num != layer.tp_v_head_num + or is_cp_active(forward_batch) + or forward_batch.dcp_kv_mask is not None + ): + return None + write_loc = KVWriteLoc.for_layer( + forward_batch, + layer, + swa_loc=self.forward_metadata.swa_out_cache_loc, + ) + locations = ( + write_loc.swa_loc + if self.use_sliding_window_kv_pool and layer.sliding_window_size >= 0 + else write_loc.loc + ) + if locations is None: + return None + buffers = self.token_to_kv_pool.get_kv_buffer(layer.layer_id) + shapes = ( + (layer.tp_k_head_num, layer.head_dim), + (layer.tp_v_head_num, layer.v_head_dim), + ) + for buffer, shape in zip(buffers, shapes): + if buffer.shape[1:] != shape or buffer.stride()[-2:] != (shape[-1], 1): + return None + return (*buffers, locations) + def forward_extend( self, q: torch.Tensor, diff --git a/python/sglang/srt/layers/attention/hybrid_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_attn_backend.py index 3056235f6b..4ebb4846d1 100644 --- a/python/sglang/srt/layers/attention/hybrid_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_attn_backend.py @@ -90,6 +90,11 @@ class HybridAttnBackend(AttentionBackend): def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds: return self._select_backend(fm).shared_read_ends(fm) + def get_fused_kv_write_buffers(self, layer, forward_batch): + return self._select_backend( + forward_batch.forward_mode + ).get_fused_kv_write_buffers(layer, forward_batch) + @property def supports_full_cuda_graph_chunked_prefix(self) -> bool: return self.prefill_backend.supports_full_cuda_graph_chunked_prefix diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index d70388bf2a..2a9a74992b 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -139,6 +139,14 @@ class RotaryEmbedding(BaseFusedOp): else: self.use_fallback_kernel = False + self._supports_bf16_rope_store = ( + _is_cuda + and type(self) is RotaryEmbedding + and self.use_fallback_kernel + and self.is_neox_style + and not self._force_native + ) + self.cos_sin_cache: torch.Tensor self.register_buffer("cos_sin_cache", cache, persistent=False) @@ -153,6 +161,42 @@ class RotaryEmbedding(BaseFusedOp): )(apply_rotary_emb) self.position_cos, self.position_sin = None, None + def try_bf16_rope_store( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + kv_target: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None, + value_scale: float, + ) -> torch.Tensor | None: + """Return contiguous rotated Q and write K/V; leave packed QKV unchanged.""" + if ( + not self._supports_bf16_rope_store + or kv_target is None + or qkv.dtype != torch.bfloat16 + or qkv.shape[0] == 0 + ): + return None + from sglang.kernels.ops.attention.rope import bf16_rope_scale_store + + key_cache, value_cache, locations = kv_target + query_size = qkv.shape[-1] - ( + key_cache.shape[1] * key_cache.shape[2] + + value_cache.shape[1] * value_cache.shape[2] + ) + query = qkv.new_empty((qkv.shape[0], query_size)) + self._match_cos_sin_cache_dtype(qkv) + bf16_rope_scale_store( + qkv, + query, + key_cache, + value_cache, + self.cos_sin_cache, + positions, + locations, + value_scale, + ) + return query + def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None: # __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`) # is expensive, so avoid calling it if possible diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 33393d623a..f6bb7434ea 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -78,6 +78,7 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( ) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope +from sglang.srt.layers.rotary_embedding.base import RotaryEmbedding from sglang.srt.layers.utils import PPMissingLayer, get_layer_id from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -801,6 +802,26 @@ class MiMoV2Attention(nn.Module): output, _ = self.o_proj(attn_output) return output + def _forward_rope_store(self, qkv, positions, forward_batch): + from sglang.srt.model_executor.forward_context import get_attn_backend + + query = self.rotary_emb.try_bf16_rope_store( + qkv, + positions, + get_attn_backend().get_fused_kv_write_buffers(self.attn, forward_batch), + value_scale=1.0 if self.v_scale is None else self.v_scale, + ) + if query is None: + return None + return self.attn( + query, + None, + None, + forward_batch, + save_kv_cache=False, + sinks=self.attention_sink_bias, + ) + def forward( self, positions: torch.Tensor, @@ -808,6 +829,15 @@ class MiMoV2Attention(nn.Module): forward_batch: ForwardBatch, ) -> torch.Tensor: qkv, _ = self.qkv_proj(hidden_states) + if ( + forward_batch.forward_mode.is_target_verify() + and qkv.shape[0] > 0 + and isinstance(self.rotary_emb, RotaryEmbedding) + and not torch.compiler.is_compiling() + ): + fused = self._forward_rope_store(qkv, positions, forward_batch) + if fused is not None: + return self.o_proj(fused)[0] q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) # [t, h, dr]