862 lines
34 KiB
Python
862 lines
34 KiB
Python
"""Fused kernels for diffusion (multimodal-generation) models.
|
|
|
|
This module is the **only** supported import surface for these kernels::
|
|
|
|
from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact
|
|
|
|
Importing a submodule directly (``...diffusion.norm.norm_triton``) couples the
|
|
caller to the file layout. The one exception is a test that deliberately
|
|
exercises a single backend.
|
|
|
|
Layout -- ordinary implementations use one subpackage per **operator domain**
|
|
(``norm``, ``modulate``, ``rope``, ``activation``, ``attention``, ``gemm``, ``routing``,
|
|
``layout``).
|
|
Implementations generated by kernel-design agents live in
|
|
``sglang.kernels.kda_kernels`` and are still exported through this facade.
|
|
``common`` holds platform plumbing, ``sites`` the
|
|
request-scoped mount policy, and ``ext`` JIT C++/CUDA extensions that are not
|
|
kernels. Start from ``README.md``: several norms look interchangeable and are
|
|
not.
|
|
|
|
Resolution is lazy (PEP 562). The backends have disjoint, heavy dependencies
|
|
-- Triton, CUTLASS/CuTe-DSL, and FlyDSL (ROCm) -- so an eager
|
|
re-export would turn every one of them into a hard import-time requirement on
|
|
every platform. ``_EXPORTS`` maps a symbol to its relative or fully-qualified
|
|
owner module and the import happens on first attribute access.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from sglang.kernels.registry import register_kernel
|
|
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
|
|
|
|
_CUDA = frozenset({CapabilityRequirement.CUDA})
|
|
_CUDA_SM90_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))})
|
|
_CUDA_SM100_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))})
|
|
_HIP = frozenset({CapabilityRequirement.HIP})
|
|
_CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))})
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Registry inventory. Metadata only -- registering imports neither torch nor a
|
|
# backend and triggers no JIT build. Ops carrying several backends (e.g.
|
|
# ``scale_residual_norm_scale_shift`` in Triton, CuTe-DSL and FlyDSL) are
|
|
# inventory: callers name the one they want via ``select_kernel``.
|
|
# ---------------------------------------------------------------------------
|
|
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
|
|
(
|
|
"diffusion.residual_gate_fp32",
|
|
KernelBackend.TRITON,
|
|
"modulate.residual_gate_fp32:residual_gate_fp32",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.matrix_rope",
|
|
KernelBackend.TRITON,
|
|
"rope.matrix_rope:apply_matrix_rope",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.fp8_rowwise",
|
|
KernelBackend.TRITON,
|
|
"quantization.fp8_rowwise_triton:fp8_rowwise",
|
|
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
|
|
),
|
|
(
|
|
"diffusion.gelu_tanh_cat",
|
|
KernelBackend.JIT,
|
|
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_cast_before_weight",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_cast_before_weight_modulate",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_modulate_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_cast_before_weight_rope",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_rope_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.apply_group_norm_silu",
|
|
KernelBackend.TRITON,
|
|
"norm.group_norm_silu:apply_group_norm_silu",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.group_norm_silu_4d",
|
|
KernelBackend.TRITON,
|
|
"norm.group_norm_silu_twopass_triton:group_norm_silu_4d",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.group_norm_silu_rows",
|
|
KernelBackend.TRITON,
|
|
"norm.group_norm_silu_twopass_triton:group_norm_silu_rows",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.wan_rmsnorm_silu",
|
|
KernelBackend.TRITON,
|
|
"norm.wan_rmsnorm_silu_triton:wan_rmsnorm_silu",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.wan_norm_silu_post",
|
|
KernelBackend.JIT,
|
|
"norm.wan_norm_silu_post:wan_norm_silu_post",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_scale_shift",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_scale_shift_bitexact:fused_rmsnorm_scale_shift_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.scale_residual_norm_scale_shift",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_scale_residual_norm_scale_shift",
|
|
_CUDA_SM100_PLUS,
|
|
),
|
|
(
|
|
"diffusion.scale_residual_norm_scale_shift",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_scale_shift_bitexact:fused_scale_residual_rmsnorm_scale_shift_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.scale_residual_norm_scale_shift",
|
|
KernelBackend.CUTE_DSL,
|
|
"norm.scale_residual_norm_cutedsl:fused_scale_residual_norm_scale_shift",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.scale_residual_norm_scale_shift",
|
|
KernelBackend.FLYDSL,
|
|
"norm.fused_residual_norm_flydsl:flydsl_fused_residual_norm_scale_shift",
|
|
_HIP,
|
|
),
|
|
(
|
|
"diffusion.scale_residual_norm_scale_shift_nvfp4",
|
|
KernelBackend.JIT,
|
|
"sglang.kernels.kda_kernels.norm_scale_shift_jit:try_fused_scale_residual_norm_scale_shift_nvfp4",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.norm_scale_shift",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_norm_scale_shift",
|
|
_CUDA_SM100_PLUS,
|
|
),
|
|
(
|
|
"diffusion.norm_scale_shift",
|
|
KernelBackend.CUTE_DSL,
|
|
"norm.scale_residual_norm_cutedsl:fused_norm_scale_shift",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.norm_scale_shift",
|
|
KernelBackend.FLYDSL,
|
|
"norm.fused_residual_norm_flydsl:flydsl_norm_scale_shift",
|
|
_HIP,
|
|
),
|
|
(
|
|
"diffusion.layernorm_modulate",
|
|
KernelBackend.TRITON,
|
|
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.qk_head_layernorm",
|
|
KernelBackend.TRITON,
|
|
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_qk_head_layernorm",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.qk_rmsnorm_native",
|
|
KernelBackend.TRITON,
|
|
"norm.zimage_qk_rmsnorm_triton:zimage_qk_rmsnorm_native",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_scale",
|
|
KernelBackend.TRITON,
|
|
"norm.native_bf16_rmsnorm_triton:rmsnorm_scale",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_tanh_residual",
|
|
KernelBackend.TRITON,
|
|
"norm.native_bf16_rmsnorm_triton:rmsnorm_tanh_residual",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.modulate_scale_shift",
|
|
KernelBackend.JIT,
|
|
"modulate.modulate_scale_shift_jit:modulate_scale_shift",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.residual_gate_add",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.residual_gate_add_jit:residual_gate_add",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.residual_gate_add2",
|
|
KernelBackend.JIT,
|
|
"modulate.residual_gate_add2_jit:residual_gate_add2",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.flux3_fp8_linear",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_linear",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.flux3_fp8_linear_swiglu",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_fp8_prims_ts:flux3_fused_mlp_up",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.flux3_fp8_qkv_qknorm_rope",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_qkv_qknorm_rope",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.flux3_nvfp4_linear",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.flux3_nvfp4_linear_swiglu",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear_swiglu",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.flux3_nvfp4_qkv_qknorm_rope",
|
|
KernelBackend.FLASHINFER,
|
|
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_qkv_qknorm_rope",
|
|
_CUDA_SM10X,
|
|
),
|
|
(
|
|
"diffusion.timestep_embedding",
|
|
KernelBackend.JIT,
|
|
"modulate.timestep_embedding_jit:timestep_embedding",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.temb_table_slices",
|
|
KernelBackend.TRITON,
|
|
"modulate.wan_temb_table_slices_triton:fused_temb_table_slices",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.ltx2_ada_values",
|
|
KernelBackend.TRITON,
|
|
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.fused_inplace_qknorm_rope",
|
|
KernelBackend.JIT,
|
|
"rope.qknorm_rope_jit:fused_inplace_qknorm_rope",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vsa_block_sparse_sm100",
|
|
KernelBackend.JIT,
|
|
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
|
|
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
|
|
),
|
|
(
|
|
"diffusion.h3_vae_rmsnorm",
|
|
KernelBackend.TRITON,
|
|
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.h3_vae_scale_add_rmsnorm",
|
|
KernelBackend.TRITON,
|
|
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.flux2_layernorm_modulate_fp8_quant",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate_fp8_quant_raw",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.flux2_qkv_epilogue",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit:try_fused_flux2_qkv_epilogue",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.flux2_token_cat_fp8",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton:try_flux2_token_cat_fp8",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.qwen_qkv_epilogue",
|
|
KernelBackend.JIT,
|
|
"rope.qwen_qkv_epilogue_jit:try_fused_qwen_qkv_epilogue",
|
|
_CUDA_SM90_PLUS,
|
|
),
|
|
(
|
|
"diffusion.ltx2_qknorm_split_rope",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit:ltx2_qknorm_split_rope_cuda",
|
|
_CUDA_SM100_PLUS,
|
|
),
|
|
(
|
|
"diffusion.ltx25_decoder_rope",
|
|
KernelBackend.JIT,
|
|
"rope.ltx25_decoder_rope_jit:fused_ltx25_decoder_rope",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rope_rotate_half_fp32",
|
|
KernelBackend.TRITON,
|
|
"rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rope_rotate_half",
|
|
KernelBackend.TRITON,
|
|
"rope.rope_rotate_half_bitexact:fused_rope_rotate_half_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.interleaved_rope_fp64",
|
|
KernelBackend.JIT,
|
|
"rope.interleaved_rope_fp64_jit:fused_interleaved_rope_fp64",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.helios_qk_rope",
|
|
KernelBackend.JIT,
|
|
"rope.helios_qk_rope_jit:fused_inplace_helios_qk_rope",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.complex_rope",
|
|
KernelBackend.TRITON,
|
|
"rope.complex_rope_triton:fused_complex_rope",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.hunyuan_qkv_rope_pack",
|
|
KernelBackend.TRITON,
|
|
"rope.hunyuan_qkv_pack_triton:hunyuan_qkv_rope_pack",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.joint_qkv_cat",
|
|
KernelBackend.TRITON,
|
|
"layout.joint_qkv_cat_triton:joint_qkv_cat",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.rmsnorm_preserve_reduction",
|
|
KernelBackend.TRITON,
|
|
"norm.rmsnorm_preserve_reduction:rmsnorm_preserve_reduction",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.silu_mul",
|
|
KernelBackend.TRITON,
|
|
"activation.silu_mul_bitexact:fused_silu_mul_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.situ_softcap_mul",
|
|
KernelBackend.TRITON,
|
|
"activation.situ_softcap_mul_bitexact:fused_situ_softcap_mul_bitexact",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.bias_silu",
|
|
KernelBackend.TRITON,
|
|
"activation.sana_conv_post_triton:fused_bias_silu",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.bias_glu",
|
|
KernelBackend.TRITON,
|
|
"activation.sana_conv_post_triton:fused_bias_glu",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.linear_gelu_tanh",
|
|
KernelBackend.AOT,
|
|
"sites.fused_linear_gelu_site:fused_linear_gelu_tanh",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.sparse_linear_attn_fwd",
|
|
KernelBackend.TRITON,
|
|
"attention.sparse_linear_attn_triton:_attn_fwd",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.bigdn",
|
|
KernelBackend.TRITON,
|
|
"attention.sana_wm_gdn_triton:fused_bigdn_func",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.fused_qknorm_rope_out_of_place",
|
|
KernelBackend.JIT,
|
|
"rope.qknorm_rope_jit:fused_qknorm_rope_out_of_place",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.qknorm_complex_rope_cuda",
|
|
KernelBackend.JIT,
|
|
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.qknorm_complex_rope_pack_",
|
|
KernelBackend.JIT,
|
|
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.channel_rmsnorm_finish_silu",
|
|
KernelBackend.JIT,
|
|
"norm.channel_rmsnorm_finish_silu_jit:channel_rmsnorm_finish_silu",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.channel_rmsnorm_silu_nhwc",
|
|
KernelBackend.JIT,
|
|
"norm.channel_rmsnorm_silu_nhwc_jit:channel_rmsnorm_silu_nhwc",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.bias_residual_add",
|
|
KernelBackend.JIT,
|
|
"norm.bias_residual_add_jit:bias_residual_add",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_delta_factors",
|
|
KernelBackend.JIT,
|
|
"attention.vdn_delta_factors_jit:vdn_delta_factors",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_temporal_conv_act",
|
|
KernelBackend.TRITON,
|
|
"attention.vdn_linear_branch_triton:vdn_temporal_conv_act",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_silu_l2norm",
|
|
KernelBackend.TRITON,
|
|
"attention.vdn_linear_branch_triton:vdn_silu_l2norm",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_frame_stats_prep",
|
|
KernelBackend.TRITON,
|
|
"attention.vdn_linear_branch_triton:vdn_frame_stats_prep",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_gather_linear_state",
|
|
KernelBackend.TRITON,
|
|
"attention.vdn_linear_branch_triton:vdn_gather_linear_state",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.vdn_linear_epilogue",
|
|
KernelBackend.TRITON,
|
|
"attention.vdn_linear_branch_triton:vdn_linear_epilogue",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.mxfp8_quantize_swizzled",
|
|
KernelBackend.TRITON,
|
|
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.silu_mul_mxfp8",
|
|
KernelBackend.TRITON,
|
|
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.indexed_scale_shift_mxfp8_",
|
|
KernelBackend.TRITON,
|
|
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.group_limited_topk",
|
|
KernelBackend.TRITON,
|
|
"routing.group_limited_topk_triton:group_limited_topk",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.usp_merge_heads",
|
|
KernelBackend.JIT,
|
|
"layout.usp_relayout_jit:usp_merge_heads",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.pack_qkv_destination_major",
|
|
KernelBackend.TRITON,
|
|
"layout.ulysses_qkv_triton:pack_qkv_destination_major",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.varlen_pack_qkv",
|
|
KernelBackend.TRITON,
|
|
"layout.varlen_pack_pad_triton:fused_pack_qkv",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.varlen_pack_segmented_qkv",
|
|
KernelBackend.TRITON,
|
|
"layout.varlen_pack_pad_triton:fused_pack_segmented_qkv",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.varlen_scatter_to_padded",
|
|
KernelBackend.TRITON,
|
|
"layout.varlen_pack_pad_triton:fused_scatter_to_padded",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.causal_conv3d_cat_pad",
|
|
KernelBackend.KDA,
|
|
"sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit:fused_causal_conv3d_cat_pad_cuda",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.causal_conv3d_cat_pad",
|
|
KernelBackend.TRITON,
|
|
"layout.causal_conv3d_cat_pad_triton:fused_causal_conv3d_cat_pad",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.cat_pad_channels_last_3d",
|
|
KernelBackend.TRITON,
|
|
"layout.wan_causal_cache_triton:cat_pad_channels_last_3d",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.dup_up3d_add",
|
|
KernelBackend.TRITON,
|
|
"layout.wan_causal_cache_triton:dup_up3d_add",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.conv_bias_epilogue",
|
|
KernelBackend.TRITON,
|
|
"modulate.conv_bias_epilogue_triton:conv_bias_epilogue",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.nearest_upsample_nhwc",
|
|
KernelBackend.TRITON,
|
|
"layout.nearest_upsample_nhwc_triton:nearest_upsample_nhwc",
|
|
_CUDA,
|
|
),
|
|
(
|
|
"diffusion.flux2_token_cat_nvfp4",
|
|
KernelBackend.JIT,
|
|
"layout.flux2_token_cat_nvfp4_jit:try_flux2_token_cat_nvfp4",
|
|
_CUDA,
|
|
),
|
|
)
|
|
|
|
for _op, _backend, _target, _caps in _SPECS:
|
|
register_kernel(
|
|
KernelSpec(
|
|
op=_op,
|
|
backend=_backend,
|
|
target=(
|
|
_target
|
|
if _target.startswith("sglang.")
|
|
else f"sglang.kernels.ops.diffusion.{_target}"
|
|
),
|
|
capabilities=_caps,
|
|
)
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public export table: symbol -> owning submodule. Sorted by domain, module,
|
|
# then symbol; a new public kernel belongs here and nowhere else.
|
|
# ---------------------------------------------------------------------------
|
|
_EXPORTS: dict[str, str] = {
|
|
"apply_matrix_rope": "rope.matrix_rope",
|
|
"residual_gate_fp32": "modulate.residual_gate_fp32",
|
|
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
|
|
"fp8_rowwise": "quantization.fp8_rowwise_triton",
|
|
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
|
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
|
|
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
|
|
"can_defer_flux2_gated_residual": "norm.flux2_gated_resnorm_jit",
|
|
"can_use_flux2_gated_resnorm": "norm.flux2_gated_resnorm_jit",
|
|
"flux2_gated_resnorm_raw": "norm.flux2_gated_resnorm_jit",
|
|
"FLYDSL_NORM_MIN_ALIGNED_DIM": "norm.fused_residual_norm_flydsl",
|
|
"can_use_fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
|
|
"flydsl_fused_residual_norm_scale_shift": "norm.fused_residual_norm_flydsl",
|
|
"fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
|
|
"flydsl_norm_scale_shift": "norm.fused_residual_norm_flydsl",
|
|
"apply_group_norm_silu": "norm.group_norm_silu",
|
|
"triton_group_norm_silu": "norm.group_norm_silu_triton",
|
|
"can_use_group_norm_silu_4d": "norm.group_norm_silu_twopass_triton",
|
|
"can_use_group_norm_silu_rows": "norm.group_norm_silu_twopass_triton",
|
|
"group_norm_silu_4d": "norm.group_norm_silu_twopass_triton",
|
|
"group_norm_silu_rows": "norm.group_norm_silu_twopass_triton",
|
|
"can_use_fused_layernorm_modulate": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"can_use_fused_qk_head_layernorm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"fused_layernorm_modulate": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"fused_layernorm_modulate_fp8_quant_raw": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"fused_layernorm_modulate_raw": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"fused_qk_head_layernorm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"is_plain_layer_norm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"rmsnorm_scale": "norm.native_bf16_rmsnorm_triton",
|
|
"rmsnorm_tanh_residual": "norm.native_bf16_rmsnorm_triton",
|
|
"norm_infer": "norm.norm_triton",
|
|
"rms_norm_fn": "norm.norm_triton",
|
|
"try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
|
|
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
|
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
|
|
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
|
|
"can_use_channel_rmsnorm_finish_silu": "norm.channel_rmsnorm_finish_silu_jit",
|
|
"channel_rmsnorm_finish_silu": "norm.channel_rmsnorm_finish_silu_jit",
|
|
"can_use_channel_rmsnorm_silu_nhwc": "norm.channel_rmsnorm_silu_nhwc_jit",
|
|
"channel_rmsnorm_silu_nhwc": "norm.channel_rmsnorm_silu_nhwc_jit",
|
|
"is_channels_last_dense": "norm.channel_rmsnorm_silu_nhwc_jit",
|
|
"bias_residual_add": "norm.bias_residual_add_jit",
|
|
"can_use_bias_residual_add": "norm.bias_residual_add_jit",
|
|
"can_use_fused_rmsnorm_modulation": "norm.rmsnorm_scale_shift_bitexact",
|
|
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
|
|
"fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
|
|
"fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
|
|
"fused_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
|
|
"fused_scale_residual_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
|
|
"fused_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"try_fused_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
|
|
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
|
|
"can_use_rmsnorm_cast_before_weight": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"can_use_rmsnorm_cast_before_weight_modulate": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"can_use_rmsnorm_cast_before_weight_rope": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"rmsnorm_cast_before_weight_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"rmsnorm_cast_before_weight_modulate_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"rmsnorm_cast_before_weight_rope_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
|
|
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
|
|
"wan_norm_silu_post": "norm.wan_norm_silu_post",
|
|
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
|
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
|
|
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
|
"zimage_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
|
|
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
|
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
|
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
|
|
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
|
|
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
|
|
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
|
|
# adaLN modulation, gating and timestep conditioning
|
|
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
|
|
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
|
|
"indexed_scale_shift_bf16_": "modulate.indexed_modulation_triton",
|
|
"indexed_gate_fp32": "modulate.indexed_modulation_triton",
|
|
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
|
|
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
|
|
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
|
|
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
|
|
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
|
|
"modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
|
|
"can_use_residual_gate_add_cuda": "sglang.kernels.kda_kernels.residual_gate_add_jit",
|
|
"residual_gate_add": "sglang.kernels.kda_kernels.residual_gate_add_jit",
|
|
"residual_gate_add_cuda": "sglang.kernels.kda_kernels.residual_gate_add_jit",
|
|
"can_use_residual_gate_add2_cuda": "modulate.residual_gate_add2_jit",
|
|
"residual_gate_add2": "modulate.residual_gate_add2_jit",
|
|
"residual_gate_add2_cuda": "modulate.residual_gate_add2_jit",
|
|
"fuse_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
|
|
"fuse_residual_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
|
|
"fuse_scale_shift_kernel": "modulate.scale_shift_triton",
|
|
"try_fused_fp32_layernorm_bf16": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
|
|
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
|
|
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
|
|
"timestep_embedding": "modulate.timestep_embedding_jit",
|
|
"fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
|
|
# Rotary embeddings and the QK-norm chains fused around them
|
|
"try_fused_flux2_qkv_epilogue": "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit",
|
|
"hunyuan_qkv_rope_pack": "rope.hunyuan_qkv_pack_triton",
|
|
"joint_qkv_cat": "layout.joint_qkv_cat_triton",
|
|
"can_use_ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
|
|
"ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
|
|
"apply_ltx2_split_rotary_emb": "rope.ltx2_rotary_triton",
|
|
"fused_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
|
|
"can_use_fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
|
|
"fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
|
|
"fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit",
|
|
"try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit",
|
|
"can_use_fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
|
|
"fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
|
|
"fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact",
|
|
"fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
|
|
"fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit",
|
|
"apply_rotary_embedding": "rope.rotary_triton",
|
|
"fused_complex_rope": "rope.complex_rope_triton",
|
|
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
|
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
|
|
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
|
|
# Tensor layout transformations fused with downstream quantization
|
|
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
|
|
# Activation-function fusions
|
|
"fused_bias_glu": "activation.sana_conv_post_triton",
|
|
"fused_bias_silu": "activation.sana_conv_post_triton",
|
|
"fused_packed_silu_mul_bitexact": "activation.silu_mul_bitexact",
|
|
"fused_silu_mul_bitexact": "activation.silu_mul_bitexact",
|
|
"can_use_fused_situ_softcap_mul": "activation.situ_softcap_mul_bitexact",
|
|
"fused_situ_softcap_mul_bitexact": "activation.situ_softcap_mul_bitexact",
|
|
# Diffusion attention kernels
|
|
"cam_scan_bidi_chunkwise": "attention.sana_wm_gdn_chunkwise_triton",
|
|
"fused_bigdn_func": "attention.sana_wm_gdn_triton",
|
|
"fused_qk_inv_rms": "attention.sana_wm_gdn_triton",
|
|
"prepare_rope_tables": "attention.sana_wm_gdn_triton",
|
|
"_attn_fwd": "attention.sparse_linear_attn_triton",
|
|
"get_block_map": "attention.sparse_linear_attn_triton",
|
|
"can_use_vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
|
|
"vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
|
|
# MoE routing
|
|
"can_use_group_limited_topk": "routing.group_limited_topk_triton",
|
|
"group_limited_topk": "routing.group_limited_topk_triton",
|
|
# GEMMs with fused epilogues
|
|
"flux3_clustered_fp8_linear": "gemm.flux3_fp8_prims_ts",
|
|
"flux3_clustered_fp8_qkv_qknorm_rope": "gemm.flux3_fp8_prims_ts",
|
|
"flux3_fused_mlp_up": "gemm.flux3_fp8_prims_ts",
|
|
"pack_flux3_mlp_weight": "gemm.flux3_fp8_prims_ts",
|
|
"quantize_flux3_activation": "gemm.flux3_fp8_prims_ts",
|
|
"flux3_nvfp4_linear": "gemm.flux3_nvfp4_prims_ts",
|
|
"flux3_nvfp4_linear_from_packed_activation": "gemm.flux3_nvfp4_prims_ts",
|
|
"flux3_nvfp4_linear_swiglu": "gemm.flux3_nvfp4_prims_ts",
|
|
"flux3_nvfp4_linear_swiglu_fp4": "gemm.flux3_nvfp4_prims_ts",
|
|
"flux3_nvfp4_qkv_qknorm_rope": "gemm.flux3_nvfp4_prims_ts",
|
|
"quantize_flux3_nvfp4_activation": "gemm.flux3_nvfp4_prims_ts",
|
|
# Data movement: bitwise identical to the aten chains they replace
|
|
"can_use_fused_causal_conv3d_cat_pad_cuda": "sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit",
|
|
"fused_causal_conv3d_cat_pad_cuda": "sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit",
|
|
"fused_causal_conv3d_cat_pad": "layout.causal_conv3d_cat_pad_triton",
|
|
"pack_qkv_destination_major": "layout.ulysses_qkv_triton",
|
|
"fused_qknorm_rope_out_of_place": "rope.qknorm_rope_jit",
|
|
"vdn_delta_factors": "attention.vdn_delta_factors_jit",
|
|
"can_use_vdn_delta_factors": "attention.vdn_delta_factors_jit",
|
|
"vdn_temporal_conv_act": "attention.vdn_linear_branch_triton",
|
|
"can_use_vdn_temporal_conv_act": "attention.vdn_linear_branch_triton",
|
|
"can_use_vdn_silu_l2norm": "attention.vdn_linear_branch_triton",
|
|
"can_use_vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
|
|
"can_use_vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
|
|
"can_use_vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
|
|
"vdn_silu_l2norm": "attention.vdn_linear_branch_triton",
|
|
"vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
|
|
"vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
|
|
"vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
|
|
"can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
|
"can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
|
"indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
|
"mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
|
"silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
|
|
"can_use_usp_merge_heads": "layout.usp_relayout_jit",
|
|
"usp_merge_heads": "layout.usp_relayout_jit",
|
|
"build_inv_indices": "layout.varlen_pack_pad_triton",
|
|
"fused_pack_qkv": "layout.varlen_pack_pad_triton",
|
|
"fused_pack_segmented_qkv": "layout.varlen_pack_pad_triton",
|
|
"fused_scatter_to_padded": "layout.varlen_pack_pad_triton",
|
|
"cat_pad_channels_last_3d": "layout.wan_causal_cache_triton",
|
|
"dup_up3d_add": "layout.wan_causal_cache_triton",
|
|
"conv_bias_epilogue": "modulate.conv_bias_epilogue_triton",
|
|
"can_use_conv_bias_epilogue": "modulate.conv_bias_epilogue_triton",
|
|
"nearest_upsample_nhwc": "layout.nearest_upsample_nhwc_triton",
|
|
"can_use_nearest_upsample_nhwc": "layout.nearest_upsample_nhwc_triton",
|
|
"try_flux2_token_cat_nvfp4": "layout.flux2_token_cat_nvfp4_jit",
|
|
# Fusion-site policy: quality gate, first-sight verification, mount
|
|
"BitExactFusionGate": "sites.bitexact_gate",
|
|
"flashinfer_rmsnorm_diagnostic_hint": "sites.bitexact_gate",
|
|
"tensors_equal": "sites.bitexact_gate",
|
|
"FLUX2_NVFP4_SWIGLU_QUANT": "sites.flux2_nvfp4_swiglu_quant_site",
|
|
"FUSED_GATE_RMSNORM": "sites.fused_gate_rmsnorm_site",
|
|
"fused_rmsnorm_scale": "sites.fused_gate_rmsnorm_site",
|
|
"fused_rmsnorm_tanh_residual": "sites.fused_gate_rmsnorm_site",
|
|
"FUSED_LINEAR_GELU": "sites.fused_linear_gelu_site",
|
|
"can_use_linear_gelu": "sites.fused_linear_gelu_site",
|
|
"fused_linear_gelu_tanh": "sites.fused_linear_gelu_site",
|
|
"FUSED_LN_MODULATE": "sites.fused_ln_modulate_site",
|
|
"can_use_ln_modulate": "sites.fused_ln_modulate_site",
|
|
"fused_ln_modulate": "sites.fused_ln_modulate_site",
|
|
"HELIOS_GATED_RESIDUAL": "sites.gated_residual_site",
|
|
"LINGBOT_VIDEO_GATED_RESIDUAL": "sites.gated_residual_site",
|
|
"try_gated_residual": "sites.gated_residual_site",
|
|
"HUNYUAN_QKNORM": "sites.hunyuan_qknorm_site",
|
|
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
|
|
"LINGBOT_VIDEO_RMSNORM": "sites.lingbot_video_rmsnorm_site",
|
|
"try_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
|
|
"LTX2_QKNORM_SPLIT_ROPE": "sites.ltx2_qknorm_split_rope_site",
|
|
"LTX2_RMS_NORM_MODULATE": "sites.ltx2_rmsnorm_modulate_site",
|
|
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
|
|
"MINIMAX_H3_NORM_MODULATE": "sites.minimax_h3_norm_modulate_site",
|
|
"NVFP4_BIAS_GELU": "sites.nvfp4_bias_gelu_site",
|
|
"QWEN_IMAGE_ADDED_QKV": "sites.qwen_image_added_qkv_site",
|
|
"SANA_VIDEO_LINEAR_ATTENTION": "sites.sana_video_linear_attention_site",
|
|
"try_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
|
|
"QualityGatedFusion": "sites.quality_gate",
|
|
# JIT C++/CUDA extensions (not kernels, not in the registry)
|
|
"interpolate": "ext.hunyuan3d_rasterizer",
|
|
"rasterize": "ext.hunyuan3d_rasterizer",
|
|
"meshVerticeInpaint": "ext.mesh_processor",
|
|
"load_mesh_processor": "ext.mesh_processor",
|
|
}
|
|
|
|
|
|
def __getattr__(name: str) -> Any:
|
|
"""Resolve a public symbol to its submodule on first access (PEP 562)."""
|
|
module = _EXPORTS.get(name)
|
|
if module is None:
|
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
from importlib import import_module
|
|
|
|
owner = module if module.startswith("sglang.") else f"{__name__}.{module}"
|
|
value = getattr(import_module(owner), name)
|
|
globals()[name] = value # cache; later lookups skip __getattr__ entirely
|
|
return value
|
|
|
|
|
|
def __dir__() -> list[str]:
|
|
return sorted(set(globals()) | set(_EXPORTS))
|
|
|
|
|
|
__all__ = sorted(_EXPORTS)
|