Fuse MiMo verification RoPE and KV cache writes (#43106)

Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
leonardHONG
2026-10-15 16:15:24 +08:00
committed by GitHub
co-authored by lianghougong Xiaoyu Zhang
parent 28703575f9
commit 93dbe88183
7 changed files with 283 additions and 0 deletions
@@ -8,6 +8,7 @@
#include <tvm/ffi/container/tensor.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cuda_runtime.h>
@@ -310,4 +311,136 @@ struct RotaryEmbeddingKernel {
}
};
template <typename Index>
__global__ void bf16_rope_scale_store_kernel(
const nv_bfloat16* qkv,
nv_bfloat16* query,
nv_bfloat16* key_cache,
nv_bfloat16* value_cache,
const nv_bfloat16* cos_sin,
const int64_t* positions,
const Index* locations,
int64_t input_stride,
int64_t query_stride,
int64_t key_stride,
int64_t value_stride,
int query_heads,
int kv_heads,
int head_dim,
int value_dim,
int rotary_dim,
float value_scale) {
const int token = blockIdx.x;
const int query_size = query_heads * head_dim;
const int key_size = kv_heads * head_dim;
const int value_size = kv_heads * value_dim;
const int64_t slot = locations[token];
const auto* input = qkv + int64_t(token) * input_stride;
auto* q = query + int64_t(token) * query_stride;
// Slot 0 is reserved for CUDA graph padding.
auto* k = slot > 0 ? key_cache + slot * key_stride : nullptr;
auto* v = slot > 0 ? value_cache + slot * value_stride : nullptr;
for (int i = threadIdx.x; i < query_size; i += blockDim.x)
q[i] = input[i];
if (slot > 0) {
for (int i = threadIdx.x; i < key_size; i += blockDim.x)
k[i] = input[query_size + i];
for (int i = threadIdx.x; i < value_size; i += blockDim.x) {
auto x = input[query_size + key_size + i];
v[i] = value_scale == 1.f ? x : __float2bfloat16_rn(static_cast<float>(x) * value_scale);
}
}
__syncthreads();
// Preserve the standalone RoPE's BF16 multiply/subtract/add sequence.
apply_rotary_embedding<nv_bfloat16, true>(
q, k, cos_sin + positions[token] * rotary_dim, head_dim, query_heads, kv_heads, rotary_dim, 0, 0, 0, head_dim);
}
struct BF16RopeScaleStoreKernel {
static void
run(tvm::ffi::TensorView qkv,
tvm::ffi::TensorView query,
tvm::ffi::TensorView key_cache,
tvm::ffi::TensorView value_cache,
tvm::ffi::TensorView cos_sin,
tvm::ffi::TensorView positions,
tvm::ffi::TensorView locations,
double value_scale) {
using namespace host;
auto tokens = SymbolicSize{"tokens"};
auto input_width = SymbolicSize{"input_width"};
auto query_width = SymbolicSize{"query_width"};
auto capacity = SymbolicSize{"capacity"};
auto kv_heads = SymbolicSize{"kv_heads"};
auto head_dim = SymbolicSize{"head_dim"};
auto value_dim = SymbolicSize{"value_dim"};
auto rotary_dim = SymbolicSize{"rotary_dim"};
auto input_stride = SymbolicSize{"input_stride"};
auto query_stride = SymbolicSize{"query_stride"};
auto key_stride = SymbolicSize{"key_stride"};
auto value_stride = SymbolicSize{"value_stride"};
auto device = SymbolicDevice{};
auto index_type = SymbolicDType{};
device.set_options<kDLCUDA>();
TensorMatcher({tokens, input_width})
.with_strides({input_stride, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(qkv);
TensorMatcher({tokens, query_width})
.with_strides({query_stride, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(query);
TensorMatcher({capacity, kv_heads, head_dim})
.with_strides({key_stride, head_dim, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(key_cache);
TensorMatcher({capacity, kv_heads, value_dim})
.with_strides({value_stride, value_dim, 1})
.with_dtype<bf16_t>()
.with_device(device)
.verify(value_cache);
TensorMatcher({-1, rotary_dim}).with_dtype<bf16_t>().with_device(device).verify(cos_sin);
TensorMatcher({tokens}).with_dtype<int64_t>().with_device(device).verify(positions);
TensorMatcher({tokens}).with_dtype<int32_t, int64_t>(index_type).with_device(device).verify(locations);
RuntimeCheck(query_width.unwrap() % head_dim.unwrap() == 0, "Query width must contain whole heads");
RuntimeCheck(
input_width.unwrap() == query_width.unwrap() + kv_heads.unwrap() * (head_dim.unwrap() + value_dim.unwrap()),
"Invalid fused QKV width");
RuntimeCheck(
rotary_dim.unwrap() > 0 && rotary_dim.unwrap() % 2 == 0 && rotary_dim.unwrap() <= head_dim.unwrap(),
"Invalid rotary dimension");
const int query_heads = query_width.unwrap() / head_dim.unwrap();
const int threads = std::min(query_heads * static_cast<int>(rotary_dim.unwrap()) / 2, 512);
auto launch = [&]<typename Index>() {
LaunchKernel(tokens.unwrap(), threads, device.unwrap())(
bf16_rope_scale_store_kernel<Index>,
static_cast<const nv_bfloat16*>(qkv.data_ptr()),
static_cast<nv_bfloat16*>(query.data_ptr()),
static_cast<nv_bfloat16*>(key_cache.data_ptr()),
static_cast<nv_bfloat16*>(value_cache.data_ptr()),
static_cast<const nv_bfloat16*>(cos_sin.data_ptr()),
static_cast<const int64_t*>(positions.data_ptr()),
static_cast<const Index*>(locations.data_ptr()),
input_stride.unwrap(),
query_stride.unwrap(),
key_stride.unwrap(),
value_stride.unwrap(),
query_heads,
static_cast<int>(kv_heads.unwrap()),
static_cast<int>(head_dim.unwrap()),
static_cast<int>(value_dim.unwrap()),
static_cast<int>(rotary_dim.unwrap()),
static_cast<float>(value_scale));
};
if (index_type.is_type<int64_t>()) {
launch.template operator()<int64_t>();
} else {
launch.template operator()<int32_t>();
}
}
};
} // namespace sglang
@@ -26,6 +26,36 @@ def _jit_rotary_embedding_module() -> Module:
)
@cache_once
def _jit_rope_scale_store_module() -> Module:
return load_jit(
"bf16_rope_scale_store",
cuda_files=["elementwise/pos_enc.cuh"],
cuda_wrappers=[("run", "BF16RopeScaleStoreKernel::run")],
extra_cuda_cflags=["--ftz=false"],
)
@register_custom_op(
op_name="bf16_rope_scale_store",
mutates_args=["query", "key_cache", "value_cache"],
)
def bf16_rope_scale_store(
qkv: torch.Tensor,
query: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
cos_sin: torch.Tensor,
positions: torch.Tensor,
locations: torch.Tensor,
value_scale: float,
) -> None:
"""NeoX RoPE on packed BF16 QKV, with contiguous Q and scaled V cache writes."""
_jit_rope_scale_store_module().run(
qkv, query, key_cache, value_cache, cos_sin, positions, locations, value_scale
)
@cache_once
def _jit_fused_rope_module(
is_neox: bool, rope_dim: int, dtype: torch.dtype, q_dtype: torch.dtype
@@ -88,6 +88,12 @@ class AttentionBackend(ABC):
# allow-list enforces.
kv_index_translator = None
def get_fused_kv_write_buffers(
self, layer: RadixAttention, forward_batch: ForwardBatch
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
"""Return token-major K/V buffers and physical slots for a fused writer."""
return None
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Eager entry point. Default = ``_out_graph(fb) + _in_graph(fb)``.
@@ -1389,6 +1389,41 @@ class FlashAttentionBackend(AttentionBackend):
k_rope = k_rope.to(self.kv_cache_dtype) if k_rope is not None else None
return q, q_rope, k_rope, k_descale, v_descale
def get_fused_kv_write_buffers(self, layer, forward_batch):
if (
self.kv_cache_dtype != torch.bfloat16
or self.use_mla
or layer.is_cross_attention
or self.kv_index_translator.is_translating
or self._kv_shard_pool is not None
or self.fa_skip_kv_cache
or layer.tp_k_head_num != layer.tp_v_head_num
or is_cp_active(forward_batch)
or forward_batch.dcp_kv_mask is not None
):
return None
write_loc = KVWriteLoc.for_layer(
forward_batch,
layer,
swa_loc=self.forward_metadata.swa_out_cache_loc,
)
locations = (
write_loc.swa_loc
if self.use_sliding_window_kv_pool and layer.sliding_window_size >= 0
else write_loc.loc
)
if locations is None:
return None
buffers = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
shapes = (
(layer.tp_k_head_num, layer.head_dim),
(layer.tp_v_head_num, layer.v_head_dim),
)
for buffer, shape in zip(buffers, shapes):
if buffer.shape[1:] != shape or buffer.stride()[-2:] != (shape[-1], 1):
return None
return (*buffers, locations)
def forward_extend(
self,
q: torch.Tensor,
@@ -90,6 +90,11 @@ class HybridAttnBackend(AttentionBackend):
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
return self._select_backend(fm).shared_read_ends(fm)
def get_fused_kv_write_buffers(self, layer, forward_batch):
return self._select_backend(
forward_batch.forward_mode
).get_fused_kv_write_buffers(layer, forward_batch)
@property
def supports_full_cuda_graph_chunked_prefix(self) -> bool:
return self.prefill_backend.supports_full_cuda_graph_chunked_prefix
@@ -139,6 +139,14 @@ class RotaryEmbedding(BaseFusedOp):
else:
self.use_fallback_kernel = False
self._supports_bf16_rope_store = (
_is_cuda
and type(self) is RotaryEmbedding
and self.use_fallback_kernel
and self.is_neox_style
and not self._force_native
)
self.cos_sin_cache: torch.Tensor
self.register_buffer("cos_sin_cache", cache, persistent=False)
@@ -153,6 +161,42 @@ class RotaryEmbedding(BaseFusedOp):
)(apply_rotary_emb)
self.position_cos, self.position_sin = None, None
def try_bf16_rope_store(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
kv_target: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None,
value_scale: float,
) -> torch.Tensor | None:
"""Return contiguous rotated Q and write K/V; leave packed QKV unchanged."""
if (
not self._supports_bf16_rope_store
or kv_target is None
or qkv.dtype != torch.bfloat16
or qkv.shape[0] == 0
):
return None
from sglang.kernels.ops.attention.rope import bf16_rope_scale_store
key_cache, value_cache, locations = kv_target
query_size = qkv.shape[-1] - (
key_cache.shape[1] * key_cache.shape[2]
+ value_cache.shape[1] * value_cache.shape[2]
)
query = qkv.new_empty((qkv.shape[0], query_size))
self._match_cos_sin_cache_dtype(qkv)
bf16_rope_scale_store(
qkv,
query,
key_cache,
value_cache,
self.cos_sin_cache,
positions,
locations,
value_scale,
)
return query
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
# __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`)
# is expensive, so avoid calling it if possible
+30
View File
@@ -78,6 +78,7 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.rotary_embedding.base import RotaryEmbedding
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
from sglang.srt.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -801,6 +802,26 @@ class MiMoV2Attention(nn.Module):
output, _ = self.o_proj(attn_output)
return output
def _forward_rope_store(self, qkv, positions, forward_batch):
from sglang.srt.model_executor.forward_context import get_attn_backend
query = self.rotary_emb.try_bf16_rope_store(
qkv,
positions,
get_attn_backend().get_fused_kv_write_buffers(self.attn, forward_batch),
value_scale=1.0 if self.v_scale is None else self.v_scale,
)
if query is None:
return None
return self.attn(
query,
None,
None,
forward_batch,
save_kv_cache=False,
sinks=self.attention_sink_bias,
)
def forward(
self,
positions: torch.Tensor,
@@ -808,6 +829,15 @@ class MiMoV2Attention(nn.Module):
forward_batch: ForwardBatch,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
if (
forward_batch.forward_mode.is_target_verify()
and qkv.shape[0] > 0
and isinstance(self.rotary_emb, RotaryEmbedding)
and not torch.compiler.is_compiling()
):
fused = self._forward_rope_store(qkv, positions, forward_batch)
if fused is not None:
return self.o_proj(fused)[0]
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
# [t, h, dr]