diff --git a/python/sglang/kernels/ops/attention/set_mla_kv_concat_q.py b/python/sglang/kernels/ops/attention/set_mla_kv_concat_q.py index 729b1928ad..74f26143fa 100644 --- a/python/sglang/kernels/ops/attention/set_mla_kv_concat_q.py +++ b/python/sglang/kernels/ops/attention/set_mla_kv_concat_q.py @@ -81,12 +81,11 @@ def covered( q_nope: torch.Tensor, q_rope: torch.Tensor, ) -> bool: - """Per-call gate mirroring the launcher's alignment/layout tripwires, so - uncovered layouts fall back to the two-kernel path instead of faulting. + """Select the fused path for supported dtypes, widths and alignment. Expects the already-flattened views the wrapper launches with: kv_buffer [pages, D], k_nope/k_rope [B, d], q_nope/q_rope [B, H, d], - loc [B]. + loc [B]. The C++ launcher validates matching tensor shapes. """ if not ( kv_buffer.dtype == torch.bfloat16 @@ -98,22 +97,10 @@ def covered( return False if loc.dtype not in (torch.int32, torch.int64): return False - if loc.dim() != 1 or not loc.is_contiguous(): - return False - if not ( - k_nope.shape[0] - == k_rope.shape[0] - == loc.shape[0] - == q_nope.shape[0] - == q_rope.shape[0] - ): - return False - if q_nope.shape[1] != q_rope.shape[1]: + if not loc.is_contiguous(): return False nope_bytes = k_nope.shape[-1] * 2 rope_bytes = k_rope.shape[-1] * 2 - if q_nope.shape[-1] * 2 != nope_bytes or q_rope.shape[-1] * 2 != rope_bytes: - return False if not can_use_set_mla_kv_concat_q(nope_bytes, rope_bytes): return False # Last dims must be dense for the vectorised row accesses. @@ -212,10 +199,12 @@ def covered_fp8( q_nope: torch.Tensor, q_rope: torch.Tensor, ) -> bool: - """Per-call gate for the fused fp8 quantize+scatter+concat kernel, - mirroring the launcher tripwires. Expects flattened views: kv_buffer + """Select the fused FP8 path for supported dtypes, widths and alignment. + + Expects flattened views: kv_buffer [pages, 576] fp8/uint8, k halves [B, 512/64] bf16, q halves - [B, H, 512/64] bf16, loc [B].""" + [B, H, 512/64] bf16, loc [B]. The C++ launcher validates matching shapes. + """ if not ( kv_buffer.dtype in (torch.float8_e4m3fn, torch.uint8) and k_nope.dtype == torch.bfloat16 @@ -226,25 +215,9 @@ def covered_fp8( return False if loc.dtype not in (torch.int32, torch.int64): return False - if loc.dim() != 1 or not loc.is_contiguous(): + if not loc.is_contiguous(): return False - if not ( - k_nope.shape[0] - == k_rope.shape[0] - == loc.shape[0] - == q_nope.shape[0] - == q_rope.shape[0] - ): - return False - if q_nope.shape[1] != q_rope.shape[1]: - return False - if ( - k_nope.shape[-1] != 512 - or k_rope.shape[-1] != 64 - or q_nope.shape[-1] != 512 - or q_rope.shape[-1] != 64 - or kv_buffer.shape[-1] < 576 - ): + if k_nope.shape[-1] != 512 or k_rope.shape[-1] != 64: return False if not can_use_set_mla_kv_concat_q_fp8(): return False diff --git a/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py b/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py index e1f8fb7d4e..dba707f25e 100644 --- a/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py +++ b/python/sglang/kernels/ops/attention/triton_gdn_fused_proj.py @@ -511,41 +511,15 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous( conv_state: torch.Tensor, conv_weight: torch.Tensor, conv_bias: torch.Tensor | None, - conv_state_indices: torch.Tensor, *, - qkv_dim: int, - v_dim: int, - num_v_heads: int, activation: str | None, ) -> tuple[bool, str]: - """Return an explicit eligibility decision for the decode fusion.""" - tensors = (mixed_qkvz, mixed_ba, conv_state, conv_weight, conv_state_indices) - if not all(isinstance(tensor, torch.Tensor) for tensor in tensors): - return False, "all inputs must be torch.Tensor instances" - if not all(tensor.is_cuda for tensor in tensors): + """Select the fused decode path for supported dtypes and layouts.""" + if not mixed_qkvz.is_cuda: return False, "CUDA tensors are required" - if mixed_qkvz.ndim != 2 or mixed_ba.ndim != 2: - return False, "projection outputs must be rank-2" - if conv_state.ndim != 3 or conv_weight.ndim != 2: - return False, "Conv1D state/weight ranks must be 3/2" - if conv_state_indices.ndim != 1: - return False, "conv_state_indices must be rank-1" - batch = mixed_qkvz.shape[0] - if mixed_ba.shape[0] != batch or conv_state_indices.shape[0] != batch: - return False, "batch dimensions must match" - if qkv_dim <= 0 or v_dim <= 0 or num_v_heads <= 0: - return False, "TP-local dimensions must be positive" - if mixed_qkvz.shape[1] != qkv_dim + v_dim: - return False, "qkvz layout is not contiguous [Q|K|V|Z]" - if mixed_ba.shape[1] != 2 * num_v_heads: - return False, "ba layout is not contiguous [B|A]" - if conv_state.shape[1] != qkv_dim or conv_weight.shape[0] != qkv_dim: - return False, "Conv1D feature dimension does not match packed QKV" - width = conv_weight.shape[1] + width = conv_weight.shape[-1] if width < 2 or width > 4: return False, "only Conv1D widths 2 through 4 are supported" - if conv_state.shape[2] < width - 1: - return False, "Conv1D state is shorter than width - 1" supported_dtypes = (torch.float16, torch.bfloat16, torch.float32) if mixed_qkvz.dtype not in supported_dtypes: return False, "QKVZ activation dtype must be FP16, BF16, or FP32" @@ -553,23 +527,14 @@ def can_use_fused_qkvzba_causal_conv1d_update_contiguous( return False, "QKVZ, Conv1D state, and weight dtypes must match" if mixed_ba.dtype not in supported_dtypes: return False, "BA activation dtype must be FP16, BF16, or FP32" - if conv_bias is not None: - if ( - not isinstance(conv_bias, torch.Tensor) - or not conv_bias.is_cuda - or conv_bias.ndim != 1 - or conv_bias.shape[0] != qkv_dim - or conv_bias.dtype != mixed_qkvz.dtype - ): - return False, "Conv1D bias contract is incompatible" + if conv_bias is not None and conv_bias.dtype != mixed_qkvz.dtype: + return False, "Conv1D bias dtype must match QKVZ" if activation not in (None, "silu", "swish"): return False, "activation must be None, silu, or swish" - if mixed_qkvz.stride(1) != 1 or mixed_ba.stride(1) != 1: + if mixed_qkvz.stride(-1) != 1 or mixed_ba.stride(-1) != 1: return False, "projection feature dimensions must be contiguous" - if conv_weight.stride(1) != 1: + if conv_weight.stride(-1) != 1: return False, "Conv1D weight width dimension must be contiguous" - if conv_state_indices.dtype not in (torch.int32, torch.int64): - return False, "conv_state_indices must be int32 or int64" return True, "eligible" @@ -589,27 +554,34 @@ def fused_qkvzba_causal_conv1d_update_contiguous( pad_slot_id: int = -1, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """Decode-only fused Qwen3.5 projection unpack and Conv1D state update.""" + batch = mixed_qkvz.shape[0] + assert qkv_dim > 0 and v_dim > 0 and num_v_heads > 0 + assert v_dim == num_v_heads * head_v_dim + assert mixed_qkvz.shape == (batch, qkv_dim + v_dim) + assert mixed_ba.shape == (batch, 2 * num_v_heads) + assert conv_state.ndim == 3 and conv_state.shape[1] == qkv_dim + assert conv_weight.ndim == 2 and conv_weight.shape[0] == qkv_dim + assert conv_state.shape[2] >= conv_weight.shape[1] - 1 + assert conv_state_indices.shape == (batch,) + assert conv_state_indices.dtype in (torch.int32, torch.int64) + assert all( + t.device == mixed_qkvz.device + for t in (mixed_ba, conv_state, conv_weight, conv_state_indices) + ) + if conv_bias is not None: + assert conv_bias.shape == (qkv_dim,) + assert conv_bias.device == mixed_qkvz.device + eligible, reason = can_use_fused_qkvzba_causal_conv1d_update_contiguous( mixed_qkvz, mixed_ba, conv_state, conv_weight, conv_bias, - conv_state_indices, - qkv_dim=qkv_dim, - v_dim=v_dim, - num_v_heads=num_v_heads, activation=activation, ) if not eligible: raise ValueError(f"Ineligible fused GDN decode projection/Conv1D: {reason}") - if v_dim != num_v_heads * head_v_dim: - raise ValueError( - "Ineligible fused GDN decode projection/Conv1D: " - "v_dim must equal num_v_heads * head_v_dim" - ) - - batch = mixed_qkvz.shape[0] mixed_qkv = torch.empty( (batch, qkv_dim), dtype=mixed_qkvz.dtype, device=mixed_qkvz.device ) diff --git a/python/sglang/kernels/ops/attention/vision_rope.py b/python/sglang/kernels/ops/attention/vision_rope.py index b284075cbc..74bc0ba6d2 100644 --- a/python/sglang/kernels/ops/attention/vision_rope.py +++ b/python/sglang/kernels/ops/attention/vision_rope.py @@ -74,28 +74,21 @@ def _fused_qk_complex_rope_kernel( def can_use_fused_qk_complex_rope( - q: torch.Tensor, - k: torch.Tensor, + *, + dtype: torch.dtype, + device: torch.device, freqs_cis: torch.Tensor, ) -> bool: - """Whether the NVIDIA fused path supports these vision RoPE tensors.""" - + """Select the fused path before projecting the vision Q/K tensors.""" if not ( - q.is_cuda - and k.is_cuda + device.type == "cuda" and freqs_cis.is_cuda - and q.device == k.device == freqs_cis.device + and device == freqs_cis.device + and dtype in (torch.bfloat16, torch.float16) + and freqs_cis.dtype == torch.complex64 ): return False - if q.dtype != k.dtype or q.dtype not in (torch.bfloat16, torch.float16): - return False - if freqs_cis.dtype != torch.complex64 or q.shape != k.shape or q.ndim < 3: - return False - if q.shape[-1] % 2 != 0: - return False - if freqs_cis.shape != q.shape[:-2] + (q.shape[-1] // 2,): - return False - major, _ = torch.cuda.get_device_capability(q.device) + major, _ = torch.cuda.get_device_capability(device) return major >= 9 @@ -112,7 +105,9 @@ def apply_fused_qk_complex_rope( not create new Triton specializations. """ - if not can_use_fused_qk_complex_rope(q, k, freqs_cis): + if not can_use_fused_qk_complex_rope( + dtype=q.dtype, device=q.device, freqs_cis=freqs_cis + ): raise ValueError( "Unsupported fused vision RoPE inputs: " f"q={q.shape}/{q.dtype}/{q.device}, " @@ -120,6 +115,11 @@ def apply_fused_qk_complex_rope( f"freqs={freqs_cis.shape}/{freqs_cis.dtype}/{freqs_cis.device}" ) + assert q.device == k.device and q.dtype == k.dtype + assert q.shape == k.shape and q.ndim >= 3 + assert q.shape[-1] % 2 == 0 + assert freqs_cis.shape == q.shape[:-2] + (q.shape[-1] // 2,) + original_shape = q.shape # Preserve the interleaved QKV token stride when the token dimension is 1. # ``view(-1, ...)`` is otherwise free to collapse that singleton stride, @@ -211,7 +211,7 @@ def precompile_fused_qk_complex_rope( qkv = torch.empty((1, 3, num_heads, head_dim), dtype=dtype, device=device) q, k, _ = torch.unbind(qkv, dim=1) freqs = torch.ones((1, head_dim // 2), dtype=torch.complex64, device=device) - if not can_use_fused_qk_complex_rope(q, k, freqs): + if not can_use_fused_qk_complex_rope(dtype=dtype, device=q.device, freqs_cis=freqs): return False apply_fused_qk_complex_rope(q, k, freqs) return True diff --git a/python/sglang/srt/layers/attention/linear/gdn_backend.py b/python/sglang/srt/layers/attention/linear/gdn_backend.py index 1dfffdbb2c..4bf5aaec6e 100644 --- a/python/sglang/srt/layers/attention/linear/gdn_backend.py +++ b/python/sglang/srt/layers/attention/linear/gdn_backend.py @@ -687,10 +687,6 @@ class GDNAttnBackend(MambaAttnBackendBase): conv_states, layer.conv_weights, layer.bias, - cache_indices, - qkv_dim=layer.q_dim + layer.k_dim + layer.v_dim, - v_dim=layer.v_dim, - num_v_heads=layer.num_v_heads, activation=layer.activation, ) ) @@ -799,8 +795,6 @@ class GDNAttnBackend(MambaAttnBackendBase): ) conv_already_applied = True else: - # Explicit correctness fallback for an unexpected runtime - # tensor/state contract. This still returns Z to the model. if not _fused_decode_proj_conv_fallback_logged: rank0_log( "Falling back from fused GDN decode projection/Conv1D: " diff --git a/python/sglang/srt/models/kimi_k3_vl.py b/python/sglang/srt/models/kimi_k3_vl.py index c423027994..0c3c7fcaf9 100644 --- a/python/sglang/srt/models/kimi_k3_vl.py +++ b/python/sglang/srt/models/kimi_k3_vl.py @@ -22,6 +22,7 @@ from torch import nn from sglang.kernels.ops.attention.vision_rope import ( apply_fused_qk_complex_rope, + can_use_fused_qk_complex_rope, precompile_fused_qk_complex_rope, ) from sglang.srt.environ import envs @@ -112,32 +113,6 @@ def apply_rope( return xq_out.type_as(xq), xk_out.type_as(xk) -def _can_use_fused_rope(hidden_states: torch.Tensor, freqs_cis: torch.Tensor) -> bool: - return _can_use_fused_rope_for_shape( - dtype=hidden_states.dtype, - device=hidden_states.device, - freqs_cis=freqs_cis, - ) - - -def _can_use_fused_rope_for_shape( - *, - dtype: torch.dtype, - device: torch.device, - freqs_cis: torch.Tensor, -) -> bool: - if not ( - device.type == "cuda" - and freqs_cis.is_cuda - and device == freqs_cis.device - and dtype in (torch.bfloat16, torch.float16) - and freqs_cis.dtype == torch.complex64 - ): - return False - major, _ = torch.cuda.get_device_capability(device) - return major >= 9 - - def sdpa_varlen_attention( q: torch.Tensor, k: torch.Tensor, @@ -681,7 +656,7 @@ class MoonViT3dEncoder(nn.Module): segment_bounds=tuple(zip(cumulative_lengths[:-1], cumulative_lengths[1:])), rope_freqs_cis=rope_freqs_cis, attention=attention, - use_fused_rope=_can_use_fused_rope_for_shape( + use_fused_rope=can_use_fused_qk_complex_rope( dtype=dtype, device=device, freqs_cis=rope_freqs_cis, @@ -705,12 +680,6 @@ class MoonViT3dEncoder(nn.Module): device=hidden_states.device, grid_thw_list=grid_thw_list, ) - forward_metadata = replace( - forward_metadata, - use_fused_rope=_can_use_fused_rope( - hidden_states, forward_metadata.rope_freqs_cis - ), - ) attention = forward_metadata.attention cu_seqlens = attention.cu_seqlens rope_freqs_cis = forward_metadata.rope_freqs_cis diff --git a/test/registered/kernels/ops/attention/test_gdn_decode_fused_proj_conv.py b/test/registered/kernels/ops/attention/test_gdn_decode_fused_proj_conv.py index 7eda705639..d46045cfe9 100644 --- a/test/registered/kernels/ops/attention/test_gdn_decode_fused_proj_conv.py +++ b/test/registered/kernels/ops/attention/test_gdn_decode_fused_proj_conv.py @@ -371,17 +371,12 @@ class TestGDNDecodeFusedProjectionConv1D(unittest.TestCase): ba = torch.empty(1, 4, device="cuda", dtype=torch.bfloat16) state = torch.empty(2, 64, 3, device="cuda", dtype=torch.bfloat16) weight = torch.empty(64, 4, device="cuda", dtype=torch.bfloat16) - indices = torch.zeros(1, device="cuda", dtype=torch.int32) eligible, reason = can_use_fused_qkvzba_causal_conv1d_update_contiguous( qkvz, ba, state, weight, None, - indices, - qkv_dim=64, - v_dim=32, - num_v_heads=2, activation="silu", ) self.assertFalse(eligible)