[qwen 3.8 next] Fuse Qwen PLE gate and convolution preparation for target verify (#40041)
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
"""Bitwise-exact fused kernels for decode-sized Qwen4 PLE paths."""
|
||||
"""Fused kernels for Qwen4 PLE decode and target verification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -307,3 +307,270 @@ def fused_qwen4_short_conv_state(
|
||||
num_warps=8,
|
||||
)
|
||||
return conv_input
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_gate_reduce_kernel(
|
||||
key_ptr,
|
||||
query_ptr,
|
||||
value_ptr,
|
||||
output_ptr,
|
||||
HIDDEN_SIZE: tl.constexpr,
|
||||
HC_COUNT: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
group = tl.program_id(0)
|
||||
hidden = tl.arange(0, BLOCK_SIZE)
|
||||
mask = hidden < HIDDEN_SIZE
|
||||
key = tl.load(key_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(tl.float32)
|
||||
query = tl.load(query_ptr + group * HIDDEN_SIZE + hidden, mask, other=0).to(
|
||||
tl.float32
|
||||
)
|
||||
product = _round_bf16_to_fp32(key * query)
|
||||
gate = _round_bf16_to_fp32(tl.sum(product, 0))
|
||||
gate = _round_bf16_to_fp32(gate * (HIDDEN_SIZE**-0.5))
|
||||
magnitude = _round_bf16_to_fp32(tl.maximum(tl.abs(gate), 1.0e-6))
|
||||
root = _round_bf16_to_fp32(tl.sqrt(magnitude))
|
||||
sign = tl.where(gate > 0, 1.0, tl.where(gate < 0, -1.0, 0.0))
|
||||
activated = _round_bf16_to_fp32(tl.sigmoid(_round_bf16_to_fp32(root * sign)))
|
||||
value = tl.load(
|
||||
value_ptr + (group // HC_COUNT) * HIDDEN_SIZE + hidden, mask, other=0
|
||||
).to(tl.float32)
|
||||
tl.store(output_ptr + group * HIDDEN_SIZE + hidden, activated * value, mask)
|
||||
|
||||
|
||||
def can_fuse_qwen4_gate_reduce(key, query, value):
|
||||
return (
|
||||
key.is_cuda
|
||||
and key.dtype == torch.bfloat16
|
||||
and key.ndim == 3
|
||||
and key.shape[1:] == (_QWEN4_HC_COUNT, _QWEN4_HIDDEN_SIZE)
|
||||
and key.is_contiguous()
|
||||
and query.shape == key.shape
|
||||
and query.dtype == key.dtype
|
||||
and query.device == key.device
|
||||
and query.is_contiguous()
|
||||
and value.shape == (key.shape[0], _QWEN4_HIDDEN_SIZE)
|
||||
and value.dtype == key.dtype
|
||||
and value.device == key.device
|
||||
and value.is_contiguous()
|
||||
)
|
||||
|
||||
|
||||
def fused_qwen4_gate_reduce(key, query, value):
|
||||
if not can_fuse_qwen4_gate_reduce(key, query, value):
|
||||
raise ValueError("unsupported input for Qwen4 PLE gate reduction")
|
||||
output = torch.empty_like(key)
|
||||
if key.shape[0]:
|
||||
_qwen4_gate_reduce_kernel[(key.shape[0] * _QWEN4_HC_COUNT,)](
|
||||
key,
|
||||
query,
|
||||
value,
|
||||
output,
|
||||
HIDDEN_SIZE=_QWEN4_HIDDEN_SIZE,
|
||||
HC_COUNT=_QWEN4_HC_COUNT,
|
||||
BLOCK_SIZE=triton.next_power_of_2(_QWEN4_HIDDEN_SIZE),
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_verify_conv_prepare_kernel(
|
||||
x_ptr,
|
||||
state_ptr,
|
||||
indices_ptr,
|
||||
valid_ptr,
|
||||
conv_ptr,
|
||||
intermediate_ptr,
|
||||
state_stride0,
|
||||
state_stride1,
|
||||
state_stride2,
|
||||
index_stride,
|
||||
cache_stride0,
|
||||
cache_stride1,
|
||||
cache_stride2,
|
||||
cache_stride3,
|
||||
CHANNELS: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
HAS_INTERMEDIATE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
req = tl.program_id(0)
|
||||
offsets = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
slot = tl.load(indices_ptr + req * index_stride)
|
||||
channel = offsets // (STATE_LEN + WIDTH)
|
||||
col = offsets % (STATE_LEN + WIDTH)
|
||||
mask = channel < CHANNELS
|
||||
state = tl.load(
|
||||
state_ptr
|
||||
+ slot * state_stride0
|
||||
+ channel * state_stride1
|
||||
+ col * state_stride2,
|
||||
mask & (col < STATE_LEN),
|
||||
other=0,
|
||||
).to(x_ptr.dtype.element_ty)
|
||||
x = tl.load(
|
||||
x_ptr + (req * WIDTH + col - STATE_LEN) * CHANNELS + channel,
|
||||
mask & (col >= STATE_LEN),
|
||||
other=0,
|
||||
)
|
||||
tl.store(
|
||||
conv_ptr + req * CHANNELS * (STATE_LEN + WIDTH) + offsets,
|
||||
tl.where(col < STATE_LEN, state, x),
|
||||
mask,
|
||||
)
|
||||
if HAS_INTERMEDIATE and STATE_LEN > 0:
|
||||
step = offsets // (CHANNELS * STATE_LEN)
|
||||
channel = (offsets // STATE_LEN) % CHANNELS
|
||||
state_col = offsets % STATE_LEN
|
||||
col = step + 1 + state_col
|
||||
mask = step < WIDTH
|
||||
valid = tl.load(valid_ptr + req * WIDTH + step, mask, other=0)
|
||||
state = tl.load(
|
||||
state_ptr
|
||||
+ slot * state_stride0
|
||||
+ channel * state_stride1
|
||||
+ col * state_stride2,
|
||||
mask & (col < STATE_LEN),
|
||||
other=0,
|
||||
).to(x_ptr.dtype.element_ty)
|
||||
x = tl.load(
|
||||
x_ptr + (req * WIDTH + col - STATE_LEN) * CHANNELS + channel,
|
||||
mask & (col >= STATE_LEN),
|
||||
other=0,
|
||||
)
|
||||
value = tl.where(valid, tl.where(col < STATE_LEN, state, x), 0)
|
||||
tl.store(
|
||||
intermediate_ptr
|
||||
+ req * cache_stride0
|
||||
+ step * cache_stride1
|
||||
+ channel * cache_stride2
|
||||
+ state_col * cache_stride3,
|
||||
value,
|
||||
mask,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _qwen4_verify_conv_finish_kernel(
|
||||
conv_ptr,
|
||||
residual_ptr,
|
||||
valid_ptr,
|
||||
output_ptr,
|
||||
num_elements,
|
||||
CHANNELS: tl.constexpr,
|
||||
WIDTH: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offsets < num_elements
|
||||
token = offsets // CHANNELS
|
||||
channel = offsets % CHANNELS
|
||||
req = token // WIDTH
|
||||
step = token % WIDTH
|
||||
conv = tl.load(
|
||||
conv_ptr + (req * CHANNELS + channel) * WIDTH + step, mask, other=0
|
||||
).to(tl.float32)
|
||||
activated = _round_bf16_to_fp32(conv * tl.sigmoid(conv))
|
||||
residual = tl.load(residual_ptr + offsets, mask, other=0).to(tl.float32)
|
||||
valid = tl.load(valid_ptr + token, mask, other=0)
|
||||
tl.store(output_ptr + offsets, tl.where(valid, activated + residual, 0), mask)
|
||||
|
||||
|
||||
def can_fuse_qwen4_verify_conv(
|
||||
x, residual, weight, state, state_indices, valid, width, dilation, intermediate
|
||||
):
|
||||
return (
|
||||
x.is_cuda
|
||||
and x.dtype == torch.bfloat16
|
||||
and x.ndim == 2
|
||||
and x.is_contiguous()
|
||||
and residual.shape == x.shape
|
||||
and residual.dtype == x.dtype
|
||||
and residual.device == x.device
|
||||
and residual.is_contiguous()
|
||||
and 0 < width <= 32
|
||||
and x.shape[0] == state_indices.numel() * width
|
||||
and state_indices.ndim == 1
|
||||
and state_indices.dtype in (torch.int32, torch.int64)
|
||||
and state_indices.device == x.device
|
||||
and valid.shape == (x.shape[0],)
|
||||
and valid.dtype == torch.bool
|
||||
and valid.device == x.device
|
||||
and valid.is_contiguous()
|
||||
and state.ndim == 3
|
||||
and state.shape[1] == x.shape[1]
|
||||
and state.device == x.device
|
||||
and state.dtype in (torch.bfloat16, torch.float32)
|
||||
and 0 <= state.shape[2] <= _QWEN4_MAX_SHORT_CONV_STATE_LEN
|
||||
and weight.ndim == 3
|
||||
and weight.shape[:2] == (x.shape[1], 1)
|
||||
and weight.device == x.device
|
||||
and weight.dtype == x.dtype
|
||||
and dilation > 0
|
||||
and state.shape[2] == (weight.shape[2] - 1) * dilation
|
||||
and (
|
||||
intermediate is None
|
||||
or (
|
||||
intermediate.ndim == 4
|
||||
and intermediate.shape[0] >= state_indices.numel()
|
||||
and intermediate.shape[1] >= width
|
||||
and intermediate.shape[2:] == state.shape[1:]
|
||||
and intermediate.device == x.device
|
||||
and intermediate.dtype in (torch.bfloat16, torch.float32)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def fused_qwen4_verify_conv(
|
||||
x, residual, weight, state, state_indices, valid, width, dilation, intermediate
|
||||
):
|
||||
if not can_fuse_qwen4_verify_conv(
|
||||
x, residual, weight, state, state_indices, valid, width, dilation, intermediate
|
||||
):
|
||||
raise ValueError("unsupported input for Qwen4 PLE verify convolution")
|
||||
output = torch.empty_like(x)
|
||||
if x.shape[0] == 0:
|
||||
return output
|
||||
requests = state_indices.numel()
|
||||
channels, state_len = state.shape[1:]
|
||||
conv_input = torch.empty(
|
||||
(requests, channels, state_len + width), dtype=x.dtype, device=x.device
|
||||
)
|
||||
work = channels * max(
|
||||
state_len + width, width * state_len if intermediate is not None else 0
|
||||
)
|
||||
_qwen4_verify_conv_prepare_kernel[(requests, triton.cdiv(work, 256))](
|
||||
x,
|
||||
state,
|
||||
state_indices,
|
||||
valid,
|
||||
conv_input,
|
||||
intermediate if intermediate is not None else x,
|
||||
*state.stride(),
|
||||
state_indices.stride(0),
|
||||
*(intermediate.stride() if intermediate is not None else (0, 0, 0, 0)),
|
||||
CHANNELS=channels,
|
||||
WIDTH=width,
|
||||
STATE_LEN=state_len,
|
||||
HAS_INTERMEDIATE=intermediate is not None,
|
||||
BLOCK=256,
|
||||
)
|
||||
conv = torch.nn.functional.conv1d(
|
||||
conv_input, weight, dilation=dilation, groups=channels
|
||||
)
|
||||
_qwen4_verify_conv_finish_kernel[(triton.cdiv(x.numel(), 256),)](
|
||||
conv,
|
||||
residual,
|
||||
valid,
|
||||
output,
|
||||
x.numel(),
|
||||
CHANNELS=channels,
|
||||
WIDTH=width,
|
||||
BLOCK=256,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
|
||||
@@ -1236,25 +1236,59 @@ class Qwen4ExpPLELayer(nn.Module):
|
||||
query = hidden_states.reshape(token_count, hc_count, hidden_size)
|
||||
key_normed = self._apply_ple_norm(self.norm_key, key)
|
||||
query_normed = self._apply_ple_norm(self.norm_query, query)
|
||||
gate = (key_normed * query_normed).sum(dim=-1, keepdim=True)
|
||||
gate = gate / math.sqrt(hidden_size)
|
||||
fused_gate_value = False
|
||||
if batch.use_decode_fast_path:
|
||||
from sglang.kernels.ops.qwen4_ple import (
|
||||
can_fuse_qwen4_gate_value,
|
||||
fused_qwen4_gate_value,
|
||||
)
|
||||
from sglang.kernels.ops.qwen4_ple import (
|
||||
can_fuse_qwen4_gate_reduce,
|
||||
fused_qwen4_gate_reduce,
|
||||
)
|
||||
|
||||
fused_gate_value = can_fuse_qwen4_gate_value(gate, value)
|
||||
if fused_gate_value:
|
||||
gated_value = fused_qwen4_gate_value(gate, value)
|
||||
fused_gate_reduce = (
|
||||
self.ple_embedding.enable_ple_fusion
|
||||
and batch.mode.is_target_verify()
|
||||
and can_fuse_qwen4_gate_reduce(key_normed, query_normed, value)
|
||||
)
|
||||
if fused_gate_reduce:
|
||||
gated_value = fused_qwen4_gate_reduce(key_normed, query_normed, value)
|
||||
else:
|
||||
gate = gate.abs().clamp_min(1e-6).sqrt() * gate.sign()
|
||||
gate = torch.sigmoid(gate)
|
||||
gated_value = gate * value.unsqueeze(-2)
|
||||
gate = (key_normed * query_normed).sum(dim=-1, keepdim=True)
|
||||
gate = gate / math.sqrt(hidden_size)
|
||||
fused_gate_value = False
|
||||
if batch.use_decode_fast_path:
|
||||
from sglang.kernels.ops.qwen4_ple import (
|
||||
can_fuse_qwen4_gate_value,
|
||||
fused_qwen4_gate_value,
|
||||
)
|
||||
|
||||
fused_gate_value = can_fuse_qwen4_gate_value(gate, value)
|
||||
if fused_gate_value:
|
||||
gated_value = fused_qwen4_gate_value(gate, value)
|
||||
else:
|
||||
gate = gate.abs().clamp_min(1e-6).sqrt() * gate.sign()
|
||||
gate = torch.sigmoid(gate)
|
||||
gated_value = gate * value.unsqueeze(-2)
|
||||
gated_value_normed = self._apply_ple_norm(self.norm_conv, gated_value)
|
||||
gated_value = gated_value.flatten(-2)
|
||||
gated_value_normed = gated_value_normed.flatten(-2)
|
||||
if self.ple_embedding.enable_ple_fusion and batch.mode.is_target_verify():
|
||||
from sglang.kernels.ops.qwen4_ple import (
|
||||
can_fuse_qwen4_verify_conv,
|
||||
fused_qwen4_verify_conv,
|
||||
)
|
||||
|
||||
pool = get_req_to_token_pool()
|
||||
conv_args = (
|
||||
gated_value_normed,
|
||||
gated_value,
|
||||
self.conv1d.weight,
|
||||
pool.short_conv_layer_cache(self.layer_id),
|
||||
batch.state_indices,
|
||||
batch.valid_tokens,
|
||||
batch.row_width,
|
||||
self.short_conv_dilation,
|
||||
pool.short_conv_layer_intermediate_cache(self.layer_id),
|
||||
)
|
||||
if can_fuse_qwen4_verify_conv(*conv_args):
|
||||
output = fused_qwen4_verify_conv(*conv_args)
|
||||
return _pad_token_rows(output, batch.physical_tokens)
|
||||
conv_output = self._short_conv(
|
||||
gated_value_normed,
|
||||
forward_batch,
|
||||
|
||||
Reference in New Issue
Block a user