[Refactor] Simplify GDN, vision RoPE and MLA kernel dispatch (#42506)

This commit is contained in:
Xiaoyu Zhang
2026-10-05 09:37:53 +08:00
committed by GitHub
parent 734cf3cf3b
commit f385390be5
6 changed files with 55 additions and 152 deletions
@@ -81,12 +81,11 @@ def covered(
q_nope: torch.Tensor,
q_rope: torch.Tensor,
) -> bool:
"""Per-call gate mirroring the launcher's alignment/layout tripwires, so
uncovered layouts fall back to the two-kernel path instead of faulting.
"""Select the fused path for supported dtypes, widths and alignment.
Expects the already-flattened views the wrapper launches with:
kv_buffer [pages, D], k_nope/k_rope [B, d], q_nope/q_rope [B, H, d],
loc [B].
loc [B]. The C++ launcher validates matching tensor shapes.
"""
if not (
kv_buffer.dtype == torch.bfloat16
@@ -98,22 +97,10 @@ def covered(
return False
if loc.dtype not in (torch.int32, torch.int64):
return False
if loc.dim() != 1 or not loc.is_contiguous():
return False
if not (
k_nope.shape[0]
== k_rope.shape[0]
== loc.shape[0]
== q_nope.shape[0]
== q_rope.shape[0]
):
return False
if q_nope.shape[1] != q_rope.shape[1]:
if not loc.is_contiguous():
return False
nope_bytes = k_nope.shape[-1] * 2
rope_bytes = k_rope.shape[-1] * 2
if q_nope.shape[-1] * 2 != nope_bytes or q_rope.shape[-1] * 2 != rope_bytes:
return False
if not can_use_set_mla_kv_concat_q(nope_bytes, rope_bytes):
return False
# Last dims must be dense for the vectorised row accesses.
@@ -212,10 +199,12 @@ def covered_fp8(
q_nope: torch.Tensor,
q_rope: torch.Tensor,
) -> bool:
"""Per-call gate for the fused fp8 quantize+scatter+concat kernel,
mirroring the launcher tripwires. Expects flattened views: kv_buffer
"""Select the fused FP8 path for supported dtypes, widths and alignment.
Expects flattened views: kv_buffer
[pages, 576] fp8/uint8, k halves [B, 512/64] bf16, q halves
[B, H, 512/64] bf16, loc [B]."""
[B, H, 512/64] bf16, loc [B]. The C++ launcher validates matching shapes.
"""
if not (
kv_buffer.dtype in (torch.float8_e4m3fn, torch.uint8)
and k_nope.dtype == torch.bfloat16
@@ -226,25 +215,9 @@ def covered_fp8(
return False
if loc.dtype not in (torch.int32, torch.int64):
return False
if loc.dim() != 1 or not loc.is_contiguous():
if not loc.is_contiguous():
return False
if not (
k_nope.shape[0]
== k_rope.shape[0]
== loc.shape[0]
== q_nope.shape[0]
== q_rope.shape[0]
):
return False
if q_nope.shape[1] != q_rope.shape[1]:
return False
if (
k_nope.shape[-1] != 512
or k_rope.shape[-1] != 64
or q_nope.shape[-1] != 512
or q_rope.shape[-1] != 64
or kv_buffer.shape[-1] < 576
):
if k_nope.shape[-1] != 512 or k_rope.shape[-1] != 64:
return False
if not can_use_set_mla_kv_concat_q_fp8():
return False
@@ -511,41 +511,15 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous(
conv_state: torch.Tensor,
conv_weight: torch.Tensor,
conv_bias: torch.Tensor | None,
conv_state_indices: torch.Tensor,
*,
qkv_dim: int,
v_dim: int,
num_v_heads: int,
activation: str | None,
) -> tuple[bool, str]:
"""Return an explicit eligibility decision for the decode fusion."""
tensors = (mixed_qkvz, mixed_ba, conv_state, conv_weight, conv_state_indices)
if not all(isinstance(tensor, torch.Tensor) for tensor in tensors):
return False, "all inputs must be torch.Tensor instances"
if not all(tensor.is_cuda for tensor in tensors):
"""Select the fused decode path for supported dtypes and layouts."""
if not mixed_qkvz.is_cuda:
return False, "CUDA tensors are required"
if mixed_qkvz.ndim != 2 or mixed_ba.ndim != 2:
return False, "projection outputs must be rank-2"
if conv_state.ndim != 3 or conv_weight.ndim != 2:
return False, "Conv1D state/weight ranks must be 3/2"
if conv_state_indices.ndim != 1:
return False, "conv_state_indices must be rank-1"
batch = mixed_qkvz.shape[0]
if mixed_ba.shape[0] != batch or conv_state_indices.shape[0] != batch:
return False, "batch dimensions must match"
if qkv_dim <= 0 or v_dim <= 0 or num_v_heads <= 0:
return False, "TP-local dimensions must be positive"
if mixed_qkvz.shape[1] != qkv_dim + v_dim:
return False, "qkvz layout is not contiguous [Q|K|V|Z]"
if mixed_ba.shape[1] != 2 * num_v_heads:
return False, "ba layout is not contiguous [B|A]"
if conv_state.shape[1] != qkv_dim or conv_weight.shape[0] != qkv_dim:
return False, "Conv1D feature dimension does not match packed QKV"
width = conv_weight.shape[1]
width = conv_weight.shape[-1]
if width < 2 or width > 4:
return False, "only Conv1D widths 2 through 4 are supported"
if conv_state.shape[2] < width - 1:
return False, "Conv1D state is shorter than width - 1"
supported_dtypes = (torch.float16, torch.bfloat16, torch.float32)
if mixed_qkvz.dtype not in supported_dtypes:
return False, "QKVZ activation dtype must be FP16, BF16, or FP32"
@@ -553,23 +527,14 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous(
return False, "QKVZ, Conv1D state, and weight dtypes must match"
if mixed_ba.dtype not in supported_dtypes:
return False, "BA activation dtype must be FP16, BF16, or FP32"
if conv_bias is not None:
if (
not isinstance(conv_bias, torch.Tensor)
or not conv_bias.is_cuda
or conv_bias.ndim != 1
or conv_bias.shape[0] != qkv_dim
or conv_bias.dtype != mixed_qkvz.dtype
):
return False, "Conv1D bias contract is incompatible"
if conv_bias is not None and conv_bias.dtype != mixed_qkvz.dtype:
return False, "Conv1D bias dtype must match QKVZ"
if activation not in (None, "silu", "swish"):
return False, "activation must be None, silu, or swish"
if mixed_qkvz.stride(1) != 1 or mixed_ba.stride(1) != 1:
if mixed_qkvz.stride(-1) != 1 or mixed_ba.stride(-1) != 1:
return False, "projection feature dimensions must be contiguous"
if conv_weight.stride(1) != 1:
if conv_weight.stride(-1) != 1:
return False, "Conv1D weight width dimension must be contiguous"
if conv_state_indices.dtype not in (torch.int32, torch.int64):
return False, "conv_state_indices must be int32 or int64"
return True, "eligible"
@@ -589,27 +554,34 @@ def fused_qkvzba_causal_conv1d_update_contiguous(
pad_slot_id: int = -1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Decode-only fused Qwen3.5 projection unpack and Conv1D state update."""
batch = mixed_qkvz.shape[0]
assert qkv_dim > 0 and v_dim > 0 and num_v_heads > 0
assert v_dim == num_v_heads * head_v_dim
assert mixed_qkvz.shape == (batch, qkv_dim + v_dim)
assert mixed_ba.shape == (batch, 2 * num_v_heads)
assert conv_state.ndim == 3 and conv_state.shape[1] == qkv_dim
assert conv_weight.ndim == 2 and conv_weight.shape[0] == qkv_dim
assert conv_state.shape[2] >= conv_weight.shape[1] - 1
assert conv_state_indices.shape == (batch,)
assert conv_state_indices.dtype in (torch.int32, torch.int64)
assert all(
t.device == mixed_qkvz.device
for t in (mixed_ba, conv_state, conv_weight, conv_state_indices)
)
if conv_bias is not None:
assert conv_bias.shape == (qkv_dim,)
assert conv_bias.device == mixed_qkvz.device
eligible, reason = can_use_fused_qkvzba_causal_conv1d_update_contiguous(
mixed_qkvz,
mixed_ba,
conv_state,
conv_weight,
conv_bias,
conv_state_indices,
qkv_dim=qkv_dim,
v_dim=v_dim,
num_v_heads=num_v_heads,
activation=activation,
)
if not eligible:
raise ValueError(f"Ineligible fused GDN decode projection/Conv1D: {reason}")
if v_dim != num_v_heads * head_v_dim:
raise ValueError(
"Ineligible fused GDN decode projection/Conv1D: "
"v_dim must equal num_v_heads * head_v_dim"
)
batch = mixed_qkvz.shape[0]
mixed_qkv = torch.empty(
(batch, qkv_dim), dtype=mixed_qkvz.dtype, device=mixed_qkvz.device
)
@@ -74,28 +74,21 @@ def _fused_qk_complex_rope_kernel(
def can_use_fused_qk_complex_rope(
q: torch.Tensor,
k: torch.Tensor,
*,
dtype: torch.dtype,
device: torch.device,
freqs_cis: torch.Tensor,
) -> bool:
"""Whether the NVIDIA fused path supports these vision RoPE tensors."""
"""Select the fused path before projecting the vision Q/K tensors."""
if not (
q.is_cuda
and k.is_cuda
device.type == "cuda"
and freqs_cis.is_cuda
and q.device == k.device == freqs_cis.device
and device == freqs_cis.device
and dtype in (torch.bfloat16, torch.float16)
and freqs_cis.dtype == torch.complex64
):
return False
if q.dtype != k.dtype or q.dtype not in (torch.bfloat16, torch.float16):
return False
if freqs_cis.dtype != torch.complex64 or q.shape != k.shape or q.ndim < 3:
return False
if q.shape[-1] % 2 != 0:
return False
if freqs_cis.shape != q.shape[:-2] + (q.shape[-1] // 2,):
return False
major, _ = torch.cuda.get_device_capability(q.device)
major, _ = torch.cuda.get_device_capability(device)
return major >= 9
@@ -112,7 +105,9 @@ def apply_fused_qk_complex_rope(
not create new Triton specializations.
"""
if not can_use_fused_qk_complex_rope(q, k, freqs_cis):
if not can_use_fused_qk_complex_rope(
dtype=q.dtype, device=q.device, freqs_cis=freqs_cis
):
raise ValueError(
"Unsupported fused vision RoPE inputs: "
f"q={q.shape}/{q.dtype}/{q.device}, "
@@ -120,6 +115,11 @@ def apply_fused_qk_complex_rope(
f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}"
)
assert q.device == k.device and q.dtype == k.dtype
assert q.shape == k.shape and q.ndim >= 3
assert q.shape[-1] % 2 == 0
assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,)
original_shape = q.shape
# Preserve the interleaved QKV token stride when the token dimension is 1.
# ``view(-1, ...)`` is otherwise free to collapse that singleton stride,
@@ -211,7 +211,7 @@ def precompile_fused_qk_complex_rope(
qkv = torch.empty((1, 3, num_heads, head_dim), dtype=dtype, device=device)
q, k, _ = torch.unbind(qkv, dim=1)
freqs = torch.ones((1, head_dim // 2), dtype=torch.complex64, device=device)
if not can_use_fused_qk_complex_rope(q, k, freqs):
if not can_use_fused_qk_complex_rope(dtype=dtype, device=q.device, freqs_cis=freqs):
return False
apply_fused_qk_complex_rope(q, k, freqs)
return True
@@ -687,10 +687,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
conv_states,
layer.conv_weights,
layer.bias,
cache_indices,
qkv_dim=layer.q_dim + layer.k_dim + layer.v_dim,
v_dim=layer.v_dim,
num_v_heads=layer.num_v_heads,
activation=layer.activation,
)
)
@@ -799,8 +795,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
)
conv_already_applied = True
else:
# Explicit correctness fallback for an unexpected runtime
# tensor/state contract. This still returns Z to the model.
if not _fused_decode_proj_conv_fallback_logged:
rank0_log(
"Falling back from fused GDN decode projection/Conv1D: "
+2 -33
View File
@@ -22,6 +22,7 @@ from torch import nn
from sglang.kernels.ops.attention.vision_rope import (
apply_fused_qk_complex_rope,
can_use_fused_qk_complex_rope,
precompile_fused_qk_complex_rope,
)
from sglang.srt.environ import envs
@@ -112,32 +113,6 @@ def apply_rope(
return xq_out.type_as(xq), xk_out.type_as(xk)
def _can_use_fused_rope(hidden_states: torch.Tensor, freqs_cis: torch.Tensor) -> bool:
return _can_use_fused_rope_for_shape(
dtype=hidden_states.dtype,
device=hidden_states.device,
freqs_cis=freqs_cis,
)
def _can_use_fused_rope_for_shape(
*,
dtype: torch.dtype,
device: torch.device,
freqs_cis: torch.Tensor,
) -> bool:
if not (
device.type == "cuda"
and freqs_cis.is_cuda
and device == freqs_cis.device
and dtype in (torch.bfloat16, torch.float16)
and freqs_cis.dtype == torch.complex64
):
return False
major, _ = torch.cuda.get_device_capability(device)
return major >= 9
def sdpa_varlen_attention(
q: torch.Tensor,
k: torch.Tensor,
@@ -681,7 +656,7 @@ class MoonViT3dEncoder(nn.Module):
segment_bounds=tuple(zip(cumulative_lengths[:-1], cumulative_lengths[1:])),
rope_freqs_cis=rope_freqs_cis,
attention=attention,
use_fused_rope=_can_use_fused_rope_for_shape(
use_fused_rope=can_use_fused_qk_complex_rope(
dtype=dtype,
device=device,
freqs_cis=rope_freqs_cis,
@@ -705,12 +680,6 @@ class MoonViT3dEncoder(nn.Module):
device=hidden_states.device,
grid_thw_list=grid_thw_list,
)
forward_metadata = replace(
forward_metadata,
use_fused_rope=_can_use_fused_rope(
hidden_states, forward_metadata.rope_freqs_cis
),
)
attention = forward_metadata.attention
cu_seqlens = attention.cu_seqlens
rope_freqs_cis = forward_metadata.rope_freqs_cis