[diffusion] kernels: stop specializing on per-request sequence lengths (#41710)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
a977e3b9d5
commit
150f568900
@@ -171,7 +171,7 @@ def _push_vec4(
|
||||
return mean, m2, cnt
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["seq_len", "n_rows"])
|
||||
def _layernorm_modulate_kernel(
|
||||
y_ptr,
|
||||
y_q_ptr,
|
||||
@@ -353,7 +353,7 @@ def _qk_ln_head_one(
|
||||
tl.store(dst + row_base[:, None] + cols2[None, :], y, mask=out_mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["n_rows"])
|
||||
def _qk_ln_head_kernel(
|
||||
q_out_ptr,
|
||||
k_out_ptr,
|
||||
|
||||
@@ -5,7 +5,7 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["rows"])
|
||||
def _pack_qkv_destination_major_kernel(
|
||||
output_ptr,
|
||||
q_ptr,
|
||||
|
||||
@@ -22,7 +22,10 @@ import triton.language as tl # type: ignore
|
||||
_MAX_INT32 = 2**31 - 1
|
||||
|
||||
|
||||
@triton.jit
|
||||
# The per-request sizes only index. The channel count, ``total`` and the
|
||||
# strides stay specialized for the channel vectorization, and the per-layer
|
||||
# padding and cache length for their constant folding (1.5% of kernel time).
|
||||
@triton.jit(do_not_specialize=["T", "H", "W", "out_t", "out_h", "out_w"])
|
||||
def _cat_pad_cl3d_kernel(
|
||||
x_ptr,
|
||||
cache_ptr,
|
||||
@@ -207,7 +210,9 @@ def cat_pad_channels_last_3d(
|
||||
return out
|
||||
|
||||
|
||||
@triton.jit
|
||||
# ``out_w`` stays specialized: it is the contiguous dimension of the
|
||||
# channels-first traversal.
|
||||
@triton.jit(do_not_specialize=["out_t", "out_h", "t_offset"])
|
||||
def _dup_up3d_add_kernel(
|
||||
main_ptr,
|
||||
src_ptr,
|
||||
|
||||
@@ -71,7 +71,7 @@ def try_fused_scaled_residual_add_exact(
|
||||
return output
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["seq_len", "stride_i_b"])
|
||||
def _fused_layernorm_scale_shift_gate_select01_kernel(
|
||||
output_ptr,
|
||||
gate_out_ptr,
|
||||
@@ -165,7 +165,7 @@ def _fused_layernorm_scale_shift_gate_select01_kernel(
|
||||
tl.store(gate_row_ptr + cols, gate, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["seq_len", "stride_i_b"])
|
||||
def _fused_residual_layernorm_scale_shift_gate_select01_kernel(
|
||||
output_ptr,
|
||||
residual_out_ptr,
|
||||
@@ -273,7 +273,7 @@ def _fused_residual_layernorm_scale_shift_gate_select01_kernel(
|
||||
tl.store(gate_row_ptr + cols, gate, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["seq_len", "num_frames", "frame_seqlen"])
|
||||
def _fused_scale_shift_4d_kernel(
|
||||
output_ptr,
|
||||
normalized_ptr,
|
||||
@@ -322,7 +322,7 @@ def _fused_scale_shift_4d_kernel(
|
||||
tl.store(out_ptrs, output, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["B", "L"])
|
||||
def fuse_scale_shift_kernel_blc_opt(
|
||||
x_ptr,
|
||||
shift_ptr,
|
||||
|
||||
@@ -27,7 +27,7 @@ import triton.language as tl # type: ignore
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["rows"])
|
||||
def _temb_table_slices_kernel(
|
||||
out_ptr,
|
||||
temb_ptr,
|
||||
|
||||
Reference in New Issue
Block a user