Fuse MiMo verification RoPE and KV cache writes (#43106)
Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
lianghougong
Xiaoyu Zhang
parent
28703575f9
commit
93dbe88183
@@ -8,6 +8,7 @@
|
||||
|
||||
#include <tvm/ffi/container/tensor.h>
|
||||
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
@@ -310,4 +311,136 @@ struct RotaryEmbeddingKernel {
|
||||
}
|
||||
};
|
||||
|
||||
template <typename Index>
|
||||
__global__ void bf16_rope_scale_store_kernel(
|
||||
const nv_bfloat16* qkv,
|
||||
nv_bfloat16* query,
|
||||
nv_bfloat16* key_cache,
|
||||
nv_bfloat16* value_cache,
|
||||
const nv_bfloat16* cos_sin,
|
||||
const int64_t* positions,
|
||||
const Index* locations,
|
||||
int64_t input_stride,
|
||||
int64_t query_stride,
|
||||
int64_t key_stride,
|
||||
int64_t value_stride,
|
||||
int query_heads,
|
||||
int kv_heads,
|
||||
int head_dim,
|
||||
int value_dim,
|
||||
int rotary_dim,
|
||||
float value_scale) {
|
||||
const int token = blockIdx.x;
|
||||
const int query_size = query_heads * head_dim;
|
||||
const int key_size = kv_heads * head_dim;
|
||||
const int value_size = kv_heads * value_dim;
|
||||
const int64_t slot = locations[token];
|
||||
const auto* input = qkv + int64_t(token) * input_stride;
|
||||
auto* q = query + int64_t(token) * query_stride;
|
||||
// Slot 0 is reserved for CUDA graph padding.
|
||||
auto* k = slot > 0 ? key_cache + slot * key_stride : nullptr;
|
||||
auto* v = slot > 0 ? value_cache + slot * value_stride : nullptr;
|
||||
for (int i = threadIdx.x; i < query_size; i += blockDim.x)
|
||||
q[i] = input[i];
|
||||
if (slot > 0) {
|
||||
for (int i = threadIdx.x; i < key_size; i += blockDim.x)
|
||||
k[i] = input[query_size + i];
|
||||
for (int i = threadIdx.x; i < value_size; i += blockDim.x) {
|
||||
auto x = input[query_size + key_size + i];
|
||||
v[i] = value_scale == 1.f ? x : __float2bfloat16_rn(static_cast<float>(x) * value_scale);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
// Preserve the standalone RoPE's BF16 multiply/subtract/add sequence.
|
||||
apply_rotary_embedding<nv_bfloat16, true>(
|
||||
q, k, cos_sin + positions[token] * rotary_dim, head_dim, query_heads, kv_heads, rotary_dim, 0, 0, 0, head_dim);
|
||||
}
|
||||
|
||||
struct BF16RopeScaleStoreKernel {
|
||||
static void
|
||||
run(tvm::ffi::TensorView qkv,
|
||||
tvm::ffi::TensorView query,
|
||||
tvm::ffi::TensorView key_cache,
|
||||
tvm::ffi::TensorView value_cache,
|
||||
tvm::ffi::TensorView cos_sin,
|
||||
tvm::ffi::TensorView positions,
|
||||
tvm::ffi::TensorView locations,
|
||||
double value_scale) {
|
||||
using namespace host;
|
||||
auto tokens = SymbolicSize{"tokens"};
|
||||
auto input_width = SymbolicSize{"input_width"};
|
||||
auto query_width = SymbolicSize{"query_width"};
|
||||
auto capacity = SymbolicSize{"capacity"};
|
||||
auto kv_heads = SymbolicSize{"kv_heads"};
|
||||
auto head_dim = SymbolicSize{"head_dim"};
|
||||
auto value_dim = SymbolicSize{"value_dim"};
|
||||
auto rotary_dim = SymbolicSize{"rotary_dim"};
|
||||
auto input_stride = SymbolicSize{"input_stride"};
|
||||
auto query_stride = SymbolicSize{"query_stride"};
|
||||
auto key_stride = SymbolicSize{"key_stride"};
|
||||
auto value_stride = SymbolicSize{"value_stride"};
|
||||
auto device = SymbolicDevice{};
|
||||
auto index_type = SymbolicDType{};
|
||||
device.set_options<kDLCUDA>();
|
||||
TensorMatcher({tokens, input_width})
|
||||
.with_strides({input_stride, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(qkv);
|
||||
TensorMatcher({tokens, query_width})
|
||||
.with_strides({query_stride, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(query);
|
||||
TensorMatcher({capacity, kv_heads, head_dim})
|
||||
.with_strides({key_stride, head_dim, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(key_cache);
|
||||
TensorMatcher({capacity, kv_heads, value_dim})
|
||||
.with_strides({value_stride, value_dim, 1})
|
||||
.with_dtype<bf16_t>()
|
||||
.with_device(device)
|
||||
.verify(value_cache);
|
||||
TensorMatcher({-1, rotary_dim}).with_dtype<bf16_t>().with_device(device).verify(cos_sin);
|
||||
TensorMatcher({tokens}).with_dtype<int64_t>().with_device(device).verify(positions);
|
||||
TensorMatcher({tokens}).with_dtype<int32_t, int64_t>(index_type).with_device(device).verify(locations);
|
||||
RuntimeCheck(query_width.unwrap() % head_dim.unwrap() == 0, "Query width must contain whole heads");
|
||||
RuntimeCheck(
|
||||
input_width.unwrap() == query_width.unwrap() + kv_heads.unwrap() * (head_dim.unwrap() + value_dim.unwrap()),
|
||||
"Invalid fused QKV width");
|
||||
RuntimeCheck(
|
||||
rotary_dim.unwrap() > 0 && rotary_dim.unwrap() % 2 == 0 && rotary_dim.unwrap() <= head_dim.unwrap(),
|
||||
"Invalid rotary dimension");
|
||||
const int query_heads = query_width.unwrap() / head_dim.unwrap();
|
||||
const int threads = std::min(query_heads * static_cast<int>(rotary_dim.unwrap()) / 2, 512);
|
||||
auto launch = [&]<typename Index>() {
|
||||
LaunchKernel(tokens.unwrap(), threads, device.unwrap())(
|
||||
bf16_rope_scale_store_kernel<Index>,
|
||||
static_cast<const nv_bfloat16*>(qkv.data_ptr()),
|
||||
static_cast<nv_bfloat16*>(query.data_ptr()),
|
||||
static_cast<nv_bfloat16*>(key_cache.data_ptr()),
|
||||
static_cast<nv_bfloat16*>(value_cache.data_ptr()),
|
||||
static_cast<const nv_bfloat16*>(cos_sin.data_ptr()),
|
||||
static_cast<const int64_t*>(positions.data_ptr()),
|
||||
static_cast<const Index*>(locations.data_ptr()),
|
||||
input_stride.unwrap(),
|
||||
query_stride.unwrap(),
|
||||
key_stride.unwrap(),
|
||||
value_stride.unwrap(),
|
||||
query_heads,
|
||||
static_cast<int>(kv_heads.unwrap()),
|
||||
static_cast<int>(head_dim.unwrap()),
|
||||
static_cast<int>(value_dim.unwrap()),
|
||||
static_cast<int>(rotary_dim.unwrap()),
|
||||
static_cast<float>(value_scale));
|
||||
};
|
||||
if (index_type.is_type<int64_t>()) {
|
||||
launch.template operator()<int64_t>();
|
||||
} else {
|
||||
launch.template operator()<int32_t>();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace sglang
|
||||
|
||||
@@ -26,6 +26,36 @@ def _jit_rotary_embedding_module() -> Module:
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_rope_scale_store_module() -> Module:
|
||||
return load_jit(
|
||||
"bf16_rope_scale_store",
|
||||
cuda_files=["elementwise/pos_enc.cuh"],
|
||||
cuda_wrappers=[("run", "BF16RopeScaleStoreKernel::run")],
|
||||
extra_cuda_cflags=["--ftz=false"],
|
||||
)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="bf16_rope_scale_store",
|
||||
mutates_args=["query", "key_cache", "value_cache"],
|
||||
)
|
||||
def bf16_rope_scale_store(
|
||||
qkv: torch.Tensor,
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
cos_sin: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
locations: torch.Tensor,
|
||||
value_scale: float,
|
||||
) -> None:
|
||||
"""NeoX RoPE on packed BF16 QKV, with contiguous Q and scaled V cache writes."""
|
||||
_jit_rope_scale_store_module().run(
|
||||
qkv, query, key_cache, value_cache, cos_sin, positions, locations, value_scale
|
||||
)
|
||||
|
||||
|
||||
@cache_once
|
||||
def _jit_fused_rope_module(
|
||||
is_neox: bool, rope_dim: int, dtype: torch.dtype, q_dtype: torch.dtype
|
||||
|
||||
@@ -88,6 +88,12 @@ class AttentionBackend(ABC):
|
||||
# allow-list enforces.
|
||||
kv_index_translator = None
|
||||
|
||||
def get_fused_kv_write_buffers(
|
||||
self, layer: RadixAttention, forward_batch: ForwardBatch
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None:
|
||||
"""Return token-major K/V buffers and physical slots for a fused writer."""
|
||||
return None
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
"""Eager entry point. Default = ``_out_graph(fb) + _in_graph(fb)``.
|
||||
|
||||
|
||||
@@ -1389,6 +1389,41 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
k_rope = k_rope.to(self.kv_cache_dtype) if k_rope is not None else None
|
||||
return q, q_rope, k_rope, k_descale, v_descale
|
||||
|
||||
def get_fused_kv_write_buffers(self, layer, forward_batch):
|
||||
if (
|
||||
self.kv_cache_dtype != torch.bfloat16
|
||||
or self.use_mla
|
||||
or layer.is_cross_attention
|
||||
or self.kv_index_translator.is_translating
|
||||
or self._kv_shard_pool is not None
|
||||
or self.fa_skip_kv_cache
|
||||
or layer.tp_k_head_num != layer.tp_v_head_num
|
||||
or is_cp_active(forward_batch)
|
||||
or forward_batch.dcp_kv_mask is not None
|
||||
):
|
||||
return None
|
||||
write_loc = KVWriteLoc.for_layer(
|
||||
forward_batch,
|
||||
layer,
|
||||
swa_loc=self.forward_metadata.swa_out_cache_loc,
|
||||
)
|
||||
locations = (
|
||||
write_loc.swa_loc
|
||||
if self.use_sliding_window_kv_pool and layer.sliding_window_size >= 0
|
||||
else write_loc.loc
|
||||
)
|
||||
if locations is None:
|
||||
return None
|
||||
buffers = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
|
||||
shapes = (
|
||||
(layer.tp_k_head_num, layer.head_dim),
|
||||
(layer.tp_v_head_num, layer.v_head_dim),
|
||||
)
|
||||
for buffer, shape in zip(buffers, shapes):
|
||||
if buffer.shape[1:] != shape or buffer.stride()[-2:] != (shape[-1], 1):
|
||||
return None
|
||||
return (*buffers, locations)
|
||||
|
||||
def forward_extend(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
|
||||
@@ -90,6 +90,11 @@ class HybridAttnBackend(AttentionBackend):
|
||||
def shared_read_ends(self, fm: ForwardMode) -> SharedReadEnds:
|
||||
return self._select_backend(fm).shared_read_ends(fm)
|
||||
|
||||
def get_fused_kv_write_buffers(self, layer, forward_batch):
|
||||
return self._select_backend(
|
||||
forward_batch.forward_mode
|
||||
).get_fused_kv_write_buffers(layer, forward_batch)
|
||||
|
||||
@property
|
||||
def supports_full_cuda_graph_chunked_prefix(self) -> bool:
|
||||
return self.prefill_backend.supports_full_cuda_graph_chunked_prefix
|
||||
|
||||
@@ -139,6 +139,14 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
else:
|
||||
self.use_fallback_kernel = False
|
||||
|
||||
self._supports_bf16_rope_store = (
|
||||
_is_cuda
|
||||
and type(self) is RotaryEmbedding
|
||||
and self.use_fallback_kernel
|
||||
and self.is_neox_style
|
||||
and not self._force_native
|
||||
)
|
||||
|
||||
self.cos_sin_cache: torch.Tensor
|
||||
self.register_buffer("cos_sin_cache", cache, persistent=False)
|
||||
|
||||
@@ -153,6 +161,42 @@ class RotaryEmbedding(BaseFusedOp):
|
||||
)(apply_rotary_emb)
|
||||
self.position_cos, self.position_sin = None, None
|
||||
|
||||
def try_bf16_rope_store(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
kv_target: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None,
|
||||
value_scale: float,
|
||||
) -> torch.Tensor | None:
|
||||
"""Return contiguous rotated Q and write K/V; leave packed QKV unchanged."""
|
||||
if (
|
||||
not self._supports_bf16_rope_store
|
||||
or kv_target is None
|
||||
or qkv.dtype != torch.bfloat16
|
||||
or qkv.shape[0] == 0
|
||||
):
|
||||
return None
|
||||
from sglang.kernels.ops.attention.rope import bf16_rope_scale_store
|
||||
|
||||
key_cache, value_cache, locations = kv_target
|
||||
query_size = qkv.shape[-1] - (
|
||||
key_cache.shape[1] * key_cache.shape[2]
|
||||
+ value_cache.shape[1] * value_cache.shape[2]
|
||||
)
|
||||
query = qkv.new_empty((qkv.shape[0], query_size))
|
||||
self._match_cos_sin_cache_dtype(qkv)
|
||||
bf16_rope_scale_store(
|
||||
qkv,
|
||||
query,
|
||||
key_cache,
|
||||
value_cache,
|
||||
self.cos_sin_cache,
|
||||
positions,
|
||||
locations,
|
||||
value_scale,
|
||||
)
|
||||
return query
|
||||
|
||||
def _match_cos_sin_cache_dtype(self, query: torch.Tensor) -> None:
|
||||
# __setattr__ in nn.Module (called by `self.cos_sin_cache = ...`)
|
||||
# is expensive, so avoid calling it if possible
|
||||
|
||||
@@ -78,6 +78,7 @@ from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.rotary_embedding.base import RotaryEmbedding
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -801,6 +802,26 @@ class MiMoV2Attention(nn.Module):
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
def _forward_rope_store(self, qkv, positions, forward_batch):
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
|
||||
query = self.rotary_emb.try_bf16_rope_store(
|
||||
qkv,
|
||||
positions,
|
||||
get_attn_backend().get_fused_kv_write_buffers(self.attn, forward_batch),
|
||||
value_scale=1.0 if self.v_scale is None else self.v_scale,
|
||||
)
|
||||
if query is None:
|
||||
return None
|
||||
return self.attn(
|
||||
query,
|
||||
None,
|
||||
None,
|
||||
forward_batch,
|
||||
save_kv_cache=False,
|
||||
sinks=self.attention_sink_bias,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -808,6 +829,15 @@ class MiMoV2Attention(nn.Module):
|
||||
forward_batch: ForwardBatch,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
if (
|
||||
forward_batch.forward_mode.is_target_verify()
|
||||
and qkv.shape[0] > 0
|
||||
and isinstance(self.rotary_emb, RotaryEmbedding)
|
||||
and not torch.compiler.is_compiling()
|
||||
):
|
||||
fused = self._forward_rope_store(qkv, positions, forward_batch)
|
||||
if fused is not None:
|
||||
return self.o_proj(fused)[0]
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
|
||||
# [t, h, dr]
|
||||
|
||||
Reference in New Issue
Block a user