[Refactor] Simplify GDN, vision RoPE and MLA kernel dispatch (#42506)
This commit is contained in:
@@ -81,12 +81,11 @@ def covered(
|
||||
q_nope: torch.Tensor,
|
||||
q_rope: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Per-call gate mirroring the launcher's alignment/layout tripwires, so
|
||||
uncovered layouts fall back to the two-kernel path instead of faulting.
|
||||
"""Select the fused path for supported dtypes, widths and alignment.
|
||||
|
||||
Expects the already-flattened views the wrapper launches with:
|
||||
kv_buffer [pages, D], k_nope/k_rope [B, d], q_nope/q_rope [B, H, d],
|
||||
loc [B].
|
||||
loc [B]. The C++ launcher validates matching tensor shapes.
|
||||
"""
|
||||
if not (
|
||||
kv_buffer.dtype == torch.bfloat16
|
||||
@@ -98,22 +97,10 @@ def covered(
|
||||
return False
|
||||
if loc.dtype not in (torch.int32, torch.int64):
|
||||
return False
|
||||
if loc.dim() != 1 or not loc.is_contiguous():
|
||||
return False
|
||||
if not (
|
||||
k_nope.shape[0]
|
||||
== k_rope.shape[0]
|
||||
== loc.shape[0]
|
||||
== q_nope.shape[0]
|
||||
== q_rope.shape[0]
|
||||
):
|
||||
return False
|
||||
if q_nope.shape[1] != q_rope.shape[1]:
|
||||
if not loc.is_contiguous():
|
||||
return False
|
||||
nope_bytes = k_nope.shape[-1] * 2
|
||||
rope_bytes = k_rope.shape[-1] * 2
|
||||
if q_nope.shape[-1] * 2 != nope_bytes or q_rope.shape[-1] * 2 != rope_bytes:
|
||||
return False
|
||||
if not can_use_set_mla_kv_concat_q(nope_bytes, rope_bytes):
|
||||
return False
|
||||
# Last dims must be dense for the vectorised row accesses.
|
||||
@@ -212,10 +199,12 @@ def covered_fp8(
|
||||
q_nope: torch.Tensor,
|
||||
q_rope: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Per-call gate for the fused fp8 quantize+scatter+concat kernel,
|
||||
mirroring the launcher tripwires. Expects flattened views: kv_buffer
|
||||
"""Select the fused FP8 path for supported dtypes, widths and alignment.
|
||||
|
||||
Expects flattened views: kv_buffer
|
||||
[pages, 576] fp8/uint8, k halves [B, 512/64] bf16, q halves
|
||||
[B, H, 512/64] bf16, loc [B]."""
|
||||
[B, H, 512/64] bf16, loc [B]. The C++ launcher validates matching shapes.
|
||||
"""
|
||||
if not (
|
||||
kv_buffer.dtype in (torch.float8_e4m3fn, torch.uint8)
|
||||
and k_nope.dtype == torch.bfloat16
|
||||
@@ -226,25 +215,9 @@ def covered_fp8(
|
||||
return False
|
||||
if loc.dtype not in (torch.int32, torch.int64):
|
||||
return False
|
||||
if loc.dim() != 1 or not loc.is_contiguous():
|
||||
if not loc.is_contiguous():
|
||||
return False
|
||||
if not (
|
||||
k_nope.shape[0]
|
||||
== k_rope.shape[0]
|
||||
== loc.shape[0]
|
||||
== q_nope.shape[0]
|
||||
== q_rope.shape[0]
|
||||
):
|
||||
return False
|
||||
if q_nope.shape[1] != q_rope.shape[1]:
|
||||
return False
|
||||
if (
|
||||
k_nope.shape[-1] != 512
|
||||
or k_rope.shape[-1] != 64
|
||||
or q_nope.shape[-1] != 512
|
||||
or q_rope.shape[-1] != 64
|
||||
or kv_buffer.shape[-1] < 576
|
||||
):
|
||||
if k_nope.shape[-1] != 512 or k_rope.shape[-1] != 64:
|
||||
return False
|
||||
if not can_use_set_mla_kv_concat_q_fp8():
|
||||
return False
|
||||
|
||||
@@ -511,41 +511,15 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous(
|
||||
conv_state: torch.Tensor,
|
||||
conv_weight: torch.Tensor,
|
||||
conv_bias: torch.Tensor | None,
|
||||
conv_state_indices: torch.Tensor,
|
||||
*,
|
||||
qkv_dim: int,
|
||||
v_dim: int,
|
||||
num_v_heads: int,
|
||||
activation: str | None,
|
||||
) -> tuple[bool, str]:
|
||||
"""Return an explicit eligibility decision for the decode fusion."""
|
||||
tensors = (mixed_qkvz, mixed_ba, conv_state, conv_weight, conv_state_indices)
|
||||
if not all(isinstance(tensor, torch.Tensor) for tensor in tensors):
|
||||
return False, "all inputs must be torch.Tensor instances"
|
||||
if not all(tensor.is_cuda for tensor in tensors):
|
||||
"""Select the fused decode path for supported dtypes and layouts."""
|
||||
if not mixed_qkvz.is_cuda:
|
||||
return False, "CUDA tensors are required"
|
||||
if mixed_qkvz.ndim != 2 or mixed_ba.ndim != 2:
|
||||
return False, "projection outputs must be rank-2"
|
||||
if conv_state.ndim != 3 or conv_weight.ndim != 2:
|
||||
return False, "Conv1D state/weight ranks must be 3/2"
|
||||
if conv_state_indices.ndim != 1:
|
||||
return False, "conv_state_indices must be rank-1"
|
||||
batch = mixed_qkvz.shape[0]
|
||||
if mixed_ba.shape[0] != batch or conv_state_indices.shape[0] != batch:
|
||||
return False, "batch dimensions must match"
|
||||
if qkv_dim <= 0 or v_dim <= 0 or num_v_heads <= 0:
|
||||
return False, "TP-local dimensions must be positive"
|
||||
if mixed_qkvz.shape[1] != qkv_dim + v_dim:
|
||||
return False, "qkvz layout is not contiguous [Q|K|V|Z]"
|
||||
if mixed_ba.shape[1] != 2 * num_v_heads:
|
||||
return False, "ba layout is not contiguous [B|A]"
|
||||
if conv_state.shape[1] != qkv_dim or conv_weight.shape[0] != qkv_dim:
|
||||
return False, "Conv1D feature dimension does not match packed QKV"
|
||||
width = conv_weight.shape[1]
|
||||
width = conv_weight.shape[-1]
|
||||
if width < 2 or width > 4:
|
||||
return False, "only Conv1D widths 2 through 4 are supported"
|
||||
if conv_state.shape[2] < width - 1:
|
||||
return False, "Conv1D state is shorter than width - 1"
|
||||
supported_dtypes = (torch.float16, torch.bfloat16, torch.float32)
|
||||
if mixed_qkvz.dtype not in supported_dtypes:
|
||||
return False, "QKVZ activation dtype must be FP16, BF16, or FP32"
|
||||
@@ -553,23 +527,14 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous(
|
||||
return False, "QKVZ, Conv1D state, and weight dtypes must match"
|
||||
if mixed_ba.dtype not in supported_dtypes:
|
||||
return False, "BA activation dtype must be FP16, BF16, or FP32"
|
||||
if conv_bias is not None:
|
||||
if (
|
||||
not isinstance(conv_bias, torch.Tensor)
|
||||
or not conv_bias.is_cuda
|
||||
or conv_bias.ndim != 1
|
||||
or conv_bias.shape[0] != qkv_dim
|
||||
or conv_bias.dtype != mixed_qkvz.dtype
|
||||
):
|
||||
return False, "Conv1D bias contract is incompatible"
|
||||
if conv_bias is not None and conv_bias.dtype != mixed_qkvz.dtype:
|
||||
return False, "Conv1D bias dtype must match QKVZ"
|
||||
if activation not in (None, "silu", "swish"):
|
||||
return False, "activation must be None, silu, or swish"
|
||||
if mixed_qkvz.stride(1) != 1 or mixed_ba.stride(1) != 1:
|
||||
if mixed_qkvz.stride(-1) != 1 or mixed_ba.stride(-1) != 1:
|
||||
return False, "projection feature dimensions must be contiguous"
|
||||
if conv_weight.stride(1) != 1:
|
||||
if conv_weight.stride(-1) != 1:
|
||||
return False, "Conv1D weight width dimension must be contiguous"
|
||||
if conv_state_indices.dtype not in (torch.int32, torch.int64):
|
||||
return False, "conv_state_indices must be int32 or int64"
|
||||
return True, "eligible"
|
||||
|
||||
|
||||
@@ -589,27 +554,34 @@ def fused_qkvzba_causal_conv1d_update_contiguous(
|
||||
pad_slot_id: int = -1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Decode-only fused Qwen3.5 projection unpack and Conv1D state update."""
|
||||
batch = mixed_qkvz.shape[0]
|
||||
assert qkv_dim > 0 and v_dim > 0 and num_v_heads > 0
|
||||
assert v_dim == num_v_heads * head_v_dim
|
||||
assert mixed_qkvz.shape == (batch, qkv_dim + v_dim)
|
||||
assert mixed_ba.shape == (batch, 2 * num_v_heads)
|
||||
assert conv_state.ndim == 3 and conv_state.shape[1] == qkv_dim
|
||||
assert conv_weight.ndim == 2 and conv_weight.shape[0] == qkv_dim
|
||||
assert conv_state.shape[2] >= conv_weight.shape[1] - 1
|
||||
assert conv_state_indices.shape == (batch,)
|
||||
assert conv_state_indices.dtype in (torch.int32, torch.int64)
|
||||
assert all(
|
||||
t.device == mixed_qkvz.device
|
||||
for t in (mixed_ba, conv_state, conv_weight, conv_state_indices)
|
||||
)
|
||||
if conv_bias is not None:
|
||||
assert conv_bias.shape == (qkv_dim,)
|
||||
assert conv_bias.device == mixed_qkvz.device
|
||||
|
||||
eligible, reason = can_use_fused_qkvzba_causal_conv1d_update_contiguous(
|
||||
mixed_qkvz,
|
||||
mixed_ba,
|
||||
conv_state,
|
||||
conv_weight,
|
||||
conv_bias,
|
||||
conv_state_indices,
|
||||
qkv_dim=qkv_dim,
|
||||
v_dim=v_dim,
|
||||
num_v_heads=num_v_heads,
|
||||
activation=activation,
|
||||
)
|
||||
if not eligible:
|
||||
raise ValueError(f"Ineligible fused GDN decode projection/Conv1D: {reason}")
|
||||
if v_dim != num_v_heads * head_v_dim:
|
||||
raise ValueError(
|
||||
"Ineligible fused GDN decode projection/Conv1D: "
|
||||
"v_dim must equal num_v_heads * head_v_dim"
|
||||
)
|
||||
|
||||
batch = mixed_qkvz.shape[0]
|
||||
mixed_qkv = torch.empty(
|
||||
(batch, qkv_dim), dtype=mixed_qkvz.dtype, device=mixed_qkvz.device
|
||||
)
|
||||
|
||||
@@ -74,28 +74,21 @@ def _fused_qk_complex_rope_kernel(
|
||||
|
||||
|
||||
def can_use_fused_qk_complex_rope(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
freqs_cis: torch.Tensor,
|
||||
) -> bool:
|
||||
"""Whether the NVIDIA fused path supports these vision RoPE tensors."""
|
||||
|
||||
"""Select the fused path before projecting the vision Q/K tensors."""
|
||||
if not (
|
||||
q.is_cuda
|
||||
and k.is_cuda
|
||||
device.type == "cuda"
|
||||
and freqs_cis.is_cuda
|
||||
and q.device == k.device == freqs_cis.device
|
||||
and device == freqs_cis.device
|
||||
and dtype in (torch.bfloat16, torch.float16)
|
||||
and freqs_cis.dtype == torch.complex64
|
||||
):
|
||||
return False
|
||||
if q.dtype != k.dtype or q.dtype not in (torch.bfloat16, torch.float16):
|
||||
return False
|
||||
if freqs_cis.dtype != torch.complex64 or q.shape != k.shape or q.ndim < 3:
|
||||
return False
|
||||
if q.shape[-1] % 2 != 0:
|
||||
return False
|
||||
if freqs_cis.shape != q.shape[:-2] + (q.shape[-1] // 2,):
|
||||
return False
|
||||
major, _ = torch.cuda.get_device_capability(q.device)
|
||||
major, _ = torch.cuda.get_device_capability(device)
|
||||
return major >= 9
|
||||
|
||||
|
||||
@@ -112,7 +105,9 @@ def apply_fused_qk_complex_rope(
|
||||
not create new Triton specializations.
|
||||
"""
|
||||
|
||||
if not can_use_fused_qk_complex_rope(q, k, freqs_cis):
|
||||
if not can_use_fused_qk_complex_rope(
|
||||
dtype=q.dtype, device=q.device, freqs_cis=freqs_cis
|
||||
):
|
||||
raise ValueError(
|
||||
"Unsupported fused vision RoPE inputs: "
|
||||
f"q={q.shape}/{q.dtype}/{q.device}, "
|
||||
@@ -120,6 +115,11 @@ def apply_fused_qk_complex_rope(
|
||||
f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}"
|
||||
)
|
||||
|
||||
assert q.device == k.device and q.dtype == k.dtype
|
||||
assert q.shape == k.shape and q.ndim >= 3
|
||||
assert q.shape[-1] % 2 == 0
|
||||
assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,)
|
||||
|
||||
original_shape = q.shape
|
||||
# Preserve the interleaved QKV token stride when the token dimension is 1.
|
||||
# ``view(-1, ...)`` is otherwise free to collapse that singleton stride,
|
||||
@@ -211,7 +211,7 @@ def precompile_fused_qk_complex_rope(
|
||||
qkv = torch.empty((1, 3, num_heads, head_dim), dtype=dtype, device=device)
|
||||
q, k, _ = torch.unbind(qkv, dim=1)
|
||||
freqs = torch.ones((1, head_dim // 2), dtype=torch.complex64, device=device)
|
||||
if not can_use_fused_qk_complex_rope(q, k, freqs):
|
||||
if not can_use_fused_qk_complex_rope(dtype=dtype, device=q.device, freqs_cis=freqs):
|
||||
return False
|
||||
apply_fused_qk_complex_rope(q, k, freqs)
|
||||
return True
|
||||
|
||||
@@ -687,10 +687,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
conv_states,
|
||||
layer.conv_weights,
|
||||
layer.bias,
|
||||
cache_indices,
|
||||
qkv_dim=layer.q_dim + layer.k_dim + layer.v_dim,
|
||||
v_dim=layer.v_dim,
|
||||
num_v_heads=layer.num_v_heads,
|
||||
activation=layer.activation,
|
||||
)
|
||||
)
|
||||
@@ -799,8 +795,6 @@ class GDNAttnBackend(MambaAttnBackendBase):
|
||||
)
|
||||
conv_already_applied = True
|
||||
else:
|
||||
# Explicit correctness fallback for an unexpected runtime
|
||||
# tensor/state contract. This still returns Z to the model.
|
||||
if not _fused_decode_proj_conv_fallback_logged:
|
||||
rank0_log(
|
||||
"Falling back from fused GDN decode projection/Conv1D: "
|
||||
|
||||
@@ -22,6 +22,7 @@ from torch import nn
|
||||
|
||||
from sglang.kernels.ops.attention.vision_rope import (
|
||||
apply_fused_qk_complex_rope,
|
||||
can_use_fused_qk_complex_rope,
|
||||
precompile_fused_qk_complex_rope,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
@@ -112,32 +113,6 @@ def apply_rope(
|
||||
return xq_out.type_as(xq), xk_out.type_as(xk)
|
||||
|
||||
|
||||
def _can_use_fused_rope(hidden_states: torch.Tensor, freqs_cis: torch.Tensor) -> bool:
|
||||
return _can_use_fused_rope_for_shape(
|
||||
dtype=hidden_states.dtype,
|
||||
device=hidden_states.device,
|
||||
freqs_cis=freqs_cis,
|
||||
)
|
||||
|
||||
|
||||
def _can_use_fused_rope_for_shape(
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
freqs_cis: torch.Tensor,
|
||||
) -> bool:
|
||||
if not (
|
||||
device.type == "cuda"
|
||||
and freqs_cis.is_cuda
|
||||
and device == freqs_cis.device
|
||||
and dtype in (torch.bfloat16, torch.float16)
|
||||
and freqs_cis.dtype == torch.complex64
|
||||
):
|
||||
return False
|
||||
major, _ = torch.cuda.get_device_capability(device)
|
||||
return major >= 9
|
||||
|
||||
|
||||
def sdpa_varlen_attention(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -681,7 +656,7 @@ class MoonViT3dEncoder(nn.Module):
|
||||
segment_bounds=tuple(zip(cumulative_lengths[:-1], cumulative_lengths[1:])),
|
||||
rope_freqs_cis=rope_freqs_cis,
|
||||
attention=attention,
|
||||
use_fused_rope=_can_use_fused_rope_for_shape(
|
||||
use_fused_rope=can_use_fused_qk_complex_rope(
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
freqs_cis=rope_freqs_cis,
|
||||
@@ -705,12 +680,6 @@ class MoonViT3dEncoder(nn.Module):
|
||||
device=hidden_states.device,
|
||||
grid_thw_list=grid_thw_list,
|
||||
)
|
||||
forward_metadata = replace(
|
||||
forward_metadata,
|
||||
use_fused_rope=_can_use_fused_rope(
|
||||
hidden_states, forward_metadata.rope_freqs_cis
|
||||
),
|
||||
)
|
||||
attention = forward_metadata.attention
|
||||
cu_seqlens = attention.cu_seqlens
|
||||
rope_freqs_cis = forward_metadata.rope_freqs_cis
|
||||
|
||||
Reference in New Issue
Block a user