diff --git a/python/sglang/kernels/ops/qwen4_ple.py b/python/sglang/kernels/ops/qwen4_ple.py index f91d9f2e9b..6749ae98ec 100644 --- a/python/sglang/kernels/ops/qwen4_ple.py +++ b/python/sglang/kernels/ops/qwen4_ple.py @@ -1,4 +1,4 @@ -"""Bitwise-exact fused kernels for decode-sized Qwen4 PLE paths.""" +"""Fused kernels for Qwen4 PLE decode and target verification.""" from __future__ import annotations @@ -307,3 +307,270 @@ def fused_qwen4_short_conv_state( num_warps=8, ) return conv_input + + +@triton.jit +def _qwen4_gate_reduce_kernel( + key_ptr, + query_ptr, + value_ptr, + output_ptr, + HIDDEN_SIZE: tl.constexpr, + HC_COUNT: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + group = tl.program_id(0) + hidden = tl.arange(0, BLOCK_SIZE) + mask = hidden < HIDDEN_SIZE + key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32) + query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to( + tl.float32 + ) + product = _round_bf16_to_fp32(key * query) + gate = _round_bf16_to_fp32(tl.sum(product, 0)) + gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5)) + magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6)) + root = _round_bf16_to_fp32(tl.sqrt(magnitude)) + sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0)) + activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign))) + value = tl.load( + value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0 + ).to(tl.float32) + tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask) + + +def can_fuse_qwen4_gate_reduce(key, query, value): + return ( + key.is_cuda + and key.dtype == torch.bfloat16 + and key.ndim == 3 + and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE) + and key.is_contiguous() + and query.shape == key.shape + and query.dtype == key.dtype + and query.device == key.device + and query.is_contiguous() + and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE) + and value.dtype == key.dtype + and value.device == key.device + and value.is_contiguous() + ) + + +def fused_qwen4_gate_reduce(key, query, value): + if not can_fuse_qwen4_gate_reduce(key, query, value): + raise ValueError("unsupported input for Qwen4 PLE gate reduction") + output = torch.empty_like(key) + if key.shape[0]: + _qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)]( + key, + query, + value, + output, + HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE, + HC_COUNT=_QWEN4_HC_COUNT, + BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE), + enable_fp_fusion=False, + ) + return output + + +@triton.jit +def _qwen4_verify_conv_prepare_kernel( + x_ptr, + state_ptr, + indices_ptr, + valid_ptr, + conv_ptr, + intermediate_ptr, + state_stride0, + state_stride1, + state_stride2, + index_stride, + cache_stride0, + cache_stride1, + cache_stride2, + cache_stride3, + CHANNELS: tl.constexpr, + WIDTH: tl.constexpr, + STATE_LEN: tl.constexpr, + HAS_INTERMEDIATE: tl.constexpr, + BLOCK: tl.constexpr, +): + req = tl.program_id(0) + offsets = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + slot = tl.load(indices_ptr + req * index_stride) + channel = offsets // (STATE_LEN + WIDTH) + col = offsets % (STATE_LEN + WIDTH) + mask = channel < CHANNELS + state = tl.load( + state_ptr + + slot * state_stride0 + + channel * state_stride1 + + col * state_stride2, + mask & (col < STATE_LEN), + other=0, + ).to(x_ptr.dtype.element_ty) + x = tl.load( + x_ptr + (req * WIDTH + col - STATE_LEN) * CHANNELS + channel, + mask & (col >= STATE_LEN), + other=0, + ) + tl.store( + conv_ptr + req * CHANNELS * (STATE_LEN + WIDTH) + offsets, + tl.where(col < STATE_LEN, state, x), + mask, + ) + if HAS_INTERMEDIATE and STATE_LEN > 0: + step = offsets // (CHANNELS * STATE_LEN) + channel = (offsets // STATE_LEN) % CHANNELS + state_col = offsets % STATE_LEN + col = step + 1 + state_col + mask = step < WIDTH + valid = tl.load(valid_ptr + req * WIDTH + step, mask, other=0) + state = tl.load( + state_ptr + + slot * state_stride0 + + channel * state_stride1 + + col * state_stride2, + mask & (col < STATE_LEN), + other=0, + ).to(x_ptr.dtype.element_ty) + x = tl.load( + x_ptr + (req * WIDTH + col - STATE_LEN) * CHANNELS + channel, + mask & (col >= STATE_LEN), + other=0, + ) + value = tl.where(valid, tl.where(col < STATE_LEN, state, x), 0) + tl.store( + intermediate_ptr + + req * cache_stride0 + + step * cache_stride1 + + channel * cache_stride2 + + state_col * cache_stride3, + value, + mask, + ) + + +@triton.jit +def _qwen4_verify_conv_finish_kernel( + conv_ptr, + residual_ptr, + valid_ptr, + output_ptr, + num_elements, + CHANNELS: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK: tl.constexpr, +): + offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + mask = offsets < num_elements + token = offsets // CHANNELS + channel = offsets % CHANNELS + req = token // WIDTH + step = token % WIDTH + conv = tl.load( + conv_ptr + (req * CHANNELS + channel) * WIDTH + step, mask, other=0 + ).to(tl.float32) + activated = _round_bf16_to_fp32(conv * tl.sigmoid(conv)) + residual = tl.load(residual_ptr + offsets, mask, other=0).to(tl.float32) + valid = tl.load(valid_ptr + token, mask, other=0) + tl.store(output_ptr + offsets, tl.where(valid, activated + residual, 0), mask) + + +def can_fuse_qwen4_verify_conv( + x, residual, weight, state, state_indices, valid, width, dilation, intermediate +): + return ( + x.is_cuda + and x.dtype == torch.bfloat16 + and x.ndim == 2 + and x.is_contiguous() + and residual.shape == x.shape + and residual.dtype == x.dtype + and residual.device == x.device + and residual.is_contiguous() + and 0 < width <= 32 + and x.shape[0] == state_indices.numel() * width + and state_indices.ndim == 1 + and state_indices.dtype in (torch.int32, torch.int64) + and state_indices.device == x.device + and valid.shape == (x.shape[0],) + and valid.dtype == torch.bool + and valid.device == x.device + and valid.is_contiguous() + and state.ndim == 3 + and state.shape[1] == x.shape[1] + and state.device == x.device + and state.dtype in (torch.bfloat16, torch.float32) + and 0 <= state.shape[2] <= _QWEN4_MAX_SHORT_CONV_STATE_LEN + and weight.ndim == 3 + and weight.shape[:2] == (x.shape[1], 1) + and weight.device == x.device + and weight.dtype == x.dtype + and dilation > 0 + and state.shape[2] == (weight.shape[2] - 1) * dilation + and ( + intermediate is None + or ( + intermediate.ndim == 4 + and intermediate.shape[0] >= state_indices.numel() + and intermediate.shape[1] >= width + and intermediate.shape[2:] == state.shape[1:] + and intermediate.device == x.device + and intermediate.dtype in (torch.bfloat16, torch.float32) + ) + ) + ) + + +def fused_qwen4_verify_conv( + x, residual, weight, state, state_indices, valid, width, dilation, intermediate +): + if not can_fuse_qwen4_verify_conv( + x, residual, weight, state, state_indices, valid, width, dilation, intermediate + ): + raise ValueError("unsupported input for Qwen4 PLE verify convolution") + output = torch.empty_like(x) + if x.shape[0] == 0: + return output + requests = state_indices.numel() + channels, state_len = state.shape[1:] + conv_input = torch.empty( + (requests, channels, state_len + width), dtype=x.dtype, device=x.device + ) + work = channels * max( + state_len + width, width * state_len if intermediate is not None else 0 + ) + _qwen4_verify_conv_prepare_kernel[(requests, triton.cdiv(work, 256))]( + x, + state, + state_indices, + valid, + conv_input, + intermediate if intermediate is not None else x, + *state.stride(), + state_indices.stride(0), + *(intermediate.stride() if intermediate is not None else (0, 0, 0, 0)), + CHANNELS=channels, + WIDTH=width, + STATE_LEN=state_len, + HAS_INTERMEDIATE=intermediate is not None, + BLOCK=256, + ) + conv = torch.nn.functional.conv1d( + conv_input, weight, dilation=dilation, groups=channels + ) + _qwen4_verify_conv_finish_kernel[(triton.cdiv(x.numel(), 256),)]( + conv, + residual, + valid, + output, + x.numel(), + CHANNELS=channels, + WIDTH=width, + BLOCK=256, + enable_fp_fusion=False, + ) + return output diff --git a/python/sglang/srt/models/qwen4_exp.py b/python/sglang/srt/models/qwen4_exp.py index 084ed01520..61866302e9 100644 --- a/python/sglang/srt/models/qwen4_exp.py +++ b/python/sglang/srt/models/qwen4_exp.py @@ -1236,25 +1236,59 @@ class Qwen4ExpPLELayer(nn.Module): query = hidden_states.reshape(token_count, hc_count, hidden_size) key_normed = self._apply_ple_norm(self.norm_key, key) query_normed = self._apply_ple_norm(self.norm_query, query) - gate = (key_normed * query_normed).sum(dim=-1, keepdim=True) - gate = gate / math.sqrt(hidden_size) - fused_gate_value = False - if batch.use_decode_fast_path: - from sglang.kernels.ops.qwen4_ple import ( - can_fuse_qwen4_gate_value, - fused_qwen4_gate_value, - ) + from sglang.kernels.ops.qwen4_ple import ( + can_fuse_qwen4_gate_reduce, + fused_qwen4_gate_reduce, + ) - fused_gate_value = can_fuse_qwen4_gate_value(gate, value) - if fused_gate_value: - gated_value = fused_qwen4_gate_value(gate, value) + fused_gate_reduce = ( + self.ple_embedding.enable_ple_fusion + and batch.mode.is_target_verify() + and can_fuse_qwen4_gate_reduce(key_normed, query_normed, value) + ) + if fused_gate_reduce: + gated_value = fused_qwen4_gate_reduce(key_normed, query_normed, value) else: - gate = gate.abs().clamp_min(1e-6).sqrt() * gate.sign() - gate = torch.sigmoid(gate) - gated_value = gate * value.unsqueeze(-2) + gate = (key_normed * query_normed).sum(dim=-1, keepdim=True) + gate = gate / math.sqrt(hidden_size) + fused_gate_value = False + if batch.use_decode_fast_path: + from sglang.kernels.ops.qwen4_ple import ( + can_fuse_qwen4_gate_value, + fused_qwen4_gate_value, + ) + + fused_gate_value = can_fuse_qwen4_gate_value(gate, value) + if fused_gate_value: + gated_value = fused_qwen4_gate_value(gate, value) + else: + gate = gate.abs().clamp_min(1e-6).sqrt() * gate.sign() + gate = torch.sigmoid(gate) + gated_value = gate * value.unsqueeze(-2) gated_value_normed = self._apply_ple_norm(self.norm_conv, gated_value) gated_value = gated_value.flatten(-2) gated_value_normed = gated_value_normed.flatten(-2) + if self.ple_embedding.enable_ple_fusion and batch.mode.is_target_verify(): + from sglang.kernels.ops.qwen4_ple import ( + can_fuse_qwen4_verify_conv, + fused_qwen4_verify_conv, + ) + + pool = get_req_to_token_pool() + conv_args = ( + gated_value_normed, + gated_value, + self.conv1d.weight, + pool.short_conv_layer_cache(self.layer_id), + batch.state_indices, + batch.valid_tokens, + batch.row_width, + self.short_conv_dilation, + pool.short_conv_layer_intermediate_cache(self.layer_id), + ) + if can_fuse_qwen4_verify_conv(*conv_args): + output = fused_qwen4_verify_conv(*conv_args) + return _pad_token_rows(output, batch.physical_tokens) conv_output = self._short_conv( gated_value_normed, forward_batch,