Files
sglang/python/sglang/kernels/ops/diffusion/__init__.py
T

862 lines
34 KiB
Python

"""Fused kernels for diffusion (multimodal-generation) models.
This module is the **only** supported import surface for these kernels::
from sglang.kernels.ops.diffusion import fused_rmsnorm_scale_shift_bitexact
Importing a submodule directly (``...diffusion.norm.norm_triton``) couples the
caller to the file layout. The one exception is a test that deliberately
exercises a single backend.
Layout -- ordinary implementations use one subpackage per **operator domain**
(``norm``, ``modulate``, ``rope``, ``activation``, ``attention``, ``gemm``, ``routing``,
``layout``).
Implementations generated by kernel-design agents live in
``sglang.kernels.kda_kernels`` and are still exported through this facade.
``common`` holds platform plumbing, ``sites`` the
request-scoped mount policy, and ``ext`` JIT C++/CUDA extensions that are not
kernels. Start from ``README.md``: several norms look interchangeable and are
not.
Resolution is lazy (PEP 562). The backends have disjoint, heavy dependencies
-- Triton, CUTLASS/CuTe-DSL, and FlyDSL (ROCm) -- so an eager
re-export would turn every one of them into a hard import-time requirement on
every platform. ``_EXPORTS`` maps a symbol to its relative or fully-qualified
owner module and the import happens on first attribute access.
"""
from __future__ import annotations
from typing import Any
from sglang.kernels.registry import register_kernel
from sglang.kernels.spec import CapabilityRequirement, KernelBackend, KernelSpec
_CUDA = frozenset({CapabilityRequirement.CUDA})
_CUDA_SM90_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(9, 0))})
_CUDA_SM100_PLUS = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0))})
_HIP = frozenset({CapabilityRequirement.HIP})
_CUDA_SM10X = frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 9))})
# ---------------------------------------------------------------------------
# Registry inventory. Metadata only -- registering imports neither torch nor a
# backend and triggers no JIT build. Ops carrying several backends (e.g.
# ``scale_residual_norm_scale_shift`` in Triton, CuTe-DSL and FlyDSL) are
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset], ...] = (
(
"diffusion.residual_gate_fp32",
KernelBackend.TRITON,
"modulate.residual_gate_fp32:residual_gate_fp32",
_CUDA,
),
(
"diffusion.matrix_rope",
KernelBackend.TRITON,
"rope.matrix_rope:apply_matrix_rope",
_CUDA,
),
(
"diffusion.fp8_rowwise",
KernelBackend.TRITON,
"quantization.fp8_rowwise_triton:fp8_rowwise",
frozenset({CapabilityRequirement.cuda(min_sm=(8, 9))}),
),
(
"diffusion.gelu_tanh_cat",
KernelBackend.JIT,
"activation.gelu_tanh_cat_jit:fused_gelu_tanh_cat",
_CUDA,
),
(
"diffusion.rmsnorm_cast_before_weight",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_bitexact",
_CUDA,
),
(
"diffusion.rmsnorm_cast_before_weight_modulate",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_modulate_bitexact",
_CUDA,
),
(
"diffusion.rmsnorm_cast_before_weight_rope",
KernelBackend.TRITON,
"norm.rmsnorm_cast_before_weight_bitexact:rmsnorm_cast_before_weight_rope_bitexact",
_CUDA,
),
(
"diffusion.apply_group_norm_silu",
KernelBackend.TRITON,
"norm.group_norm_silu:apply_group_norm_silu",
_CUDA,
),
(
"diffusion.group_norm_silu_4d",
KernelBackend.TRITON,
"norm.group_norm_silu_twopass_triton:group_norm_silu_4d",
_CUDA,
),
(
"diffusion.group_norm_silu_rows",
KernelBackend.TRITON,
"norm.group_norm_silu_twopass_triton:group_norm_silu_rows",
_CUDA,
),
(
"diffusion.wan_rmsnorm_silu",
KernelBackend.TRITON,
"norm.wan_rmsnorm_silu_triton:wan_rmsnorm_silu",
_CUDA,
),
(
"diffusion.wan_norm_silu_post",
KernelBackend.JIT,
"norm.wan_norm_silu_post:wan_norm_silu_post",
_CUDA,
),
(
"diffusion.rmsnorm_scale_shift",
KernelBackend.TRITON,
"norm.rmsnorm_scale_shift_bitexact:fused_rmsnorm_scale_shift_bitexact",
_CUDA,
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_scale_residual_norm_scale_shift",
_CUDA_SM100_PLUS,
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.TRITON,
"norm.rmsnorm_scale_shift_bitexact:fused_scale_residual_rmsnorm_scale_shift_bitexact",
_CUDA,
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.CUTE_DSL,
"norm.scale_residual_norm_cutedsl:fused_scale_residual_norm_scale_shift",
_CUDA,
),
(
"diffusion.scale_residual_norm_scale_shift",
KernelBackend.FLYDSL,
"norm.fused_residual_norm_flydsl:flydsl_fused_residual_norm_scale_shift",
_HIP,
),
(
"diffusion.scale_residual_norm_scale_shift_nvfp4",
KernelBackend.JIT,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:try_fused_scale_residual_norm_scale_shift_nvfp4",
_CUDA,
),
(
"diffusion.norm_scale_shift",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.norm_scale_shift_jit:kda_norm_scale_shift",
_CUDA_SM100_PLUS,
),
(
"diffusion.norm_scale_shift",
KernelBackend.CUTE_DSL,
"norm.scale_residual_norm_cutedsl:fused_norm_scale_shift",
_CUDA,
),
(
"diffusion.norm_scale_shift",
KernelBackend.FLYDSL,
"norm.fused_residual_norm_flydsl:flydsl_norm_scale_shift",
_HIP,
),
(
"diffusion.layernorm_modulate",
KernelBackend.TRITON,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate",
_CUDA,
),
(
"diffusion.qk_head_layernorm",
KernelBackend.TRITON,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_qk_head_layernorm",
_CUDA,
),
(
"diffusion.qk_rmsnorm_native",
KernelBackend.TRITON,
"norm.zimage_qk_rmsnorm_triton:zimage_qk_rmsnorm_native",
_CUDA,
),
(
"diffusion.rmsnorm_scale",
KernelBackend.TRITON,
"norm.native_bf16_rmsnorm_triton:rmsnorm_scale",
_CUDA,
),
(
"diffusion.rmsnorm_tanh_residual",
KernelBackend.TRITON,
"norm.native_bf16_rmsnorm_triton:rmsnorm_tanh_residual",
_CUDA,
),
(
"diffusion.modulate_scale_shift",
KernelBackend.JIT,
"modulate.modulate_scale_shift_jit:modulate_scale_shift",
_CUDA,
),
(
"diffusion.residual_gate_add",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.residual_gate_add_jit:residual_gate_add",
_CUDA,
),
(
"diffusion.residual_gate_add2",
KernelBackend.JIT,
"modulate.residual_gate_add2_jit:residual_gate_add2",
_CUDA,
),
(
"diffusion.flux3_fp8_linear",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_linear",
_CUDA_SM10X,
),
(
"diffusion.flux3_fp8_linear_swiglu",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_fused_mlp_up",
_CUDA_SM10X,
),
(
"diffusion.flux3_fp8_qkv_qknorm_rope",
KernelBackend.FLASHINFER,
"gemm.flux3_fp8_prims_ts:flux3_clustered_fp8_qkv_qknorm_rope",
_CUDA_SM10X,
),
(
"diffusion.flux3_nvfp4_linear",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear",
_CUDA_SM10X,
),
(
"diffusion.flux3_nvfp4_linear_swiglu",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_linear_swiglu",
_CUDA_SM10X,
),
(
"diffusion.flux3_nvfp4_qkv_qknorm_rope",
KernelBackend.FLASHINFER,
"gemm.flux3_nvfp4_prims_ts:flux3_nvfp4_qkv_qknorm_rope",
_CUDA_SM10X,
),
(
"diffusion.timestep_embedding",
KernelBackend.JIT,
"modulate.timestep_embedding_jit:timestep_embedding",
_CUDA,
),
(
"diffusion.temb_table_slices",
KernelBackend.TRITON,
"modulate.wan_temb_table_slices_triton:fused_temb_table_slices",
_CUDA,
),
(
"diffusion.ltx2_ada_values",
KernelBackend.TRITON,
"modulate.ltx2_ada_values_triton:ltx2_ada_values9",
_CUDA,
),
(
"diffusion.fused_inplace_qknorm_rope",
KernelBackend.JIT,
"rope.qknorm_rope_jit:fused_inplace_qknorm_rope",
_CUDA,
),
(
"diffusion.vsa_block_sparse_sm100",
KernelBackend.JIT,
"attention.vsa_block_sparse_sm100_jit:vsa_block_sparse_sm100",
frozenset({CapabilityRequirement.cuda(min_sm=(10, 0), max_sm=(10, 3))}),
),
(
"diffusion.h3_vae_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_rmsnorm",
_CUDA,
),
(
"diffusion.h3_vae_scale_add_rmsnorm",
KernelBackend.TRITON,
"norm.h3_vae_fused_norm:h3_vae_scale_add_rmsnorm",
_CUDA,
),
(
"diffusion.flux2_layernorm_modulate_fp8_quant",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.layernorm_modulate_triton:fused_layernorm_modulate_fp8_quant_raw",
_CUDA,
),
(
"diffusion.flux2_qkv_epilogue",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit:try_fused_flux2_qkv_epilogue",
_CUDA,
),
(
"diffusion.flux2_token_cat_fp8",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton:try_flux2_token_cat_fp8",
_CUDA,
),
(
"diffusion.qwen_qkv_epilogue",
KernelBackend.JIT,
"rope.qwen_qkv_epilogue_jit:try_fused_qwen_qkv_epilogue",
_CUDA_SM90_PLUS,
),
(
"diffusion.ltx2_qknorm_split_rope",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit:ltx2_qknorm_split_rope_cuda",
_CUDA_SM100_PLUS,
),
(
"diffusion.ltx25_decoder_rope",
KernelBackend.JIT,
"rope.ltx25_decoder_rope_jit:fused_ltx25_decoder_rope",
_CUDA,
),
(
"diffusion.rope_rotate_half_fp32",
KernelBackend.TRITON,
"rope.rope_rotate_half_fp32:fused_rope_rotate_half_fp32",
_CUDA,
),
(
"diffusion.rope_rotate_half",
KernelBackend.TRITON,
"rope.rope_rotate_half_bitexact:fused_rope_rotate_half_bitexact",
_CUDA,
),
(
"diffusion.interleaved_rope_fp64",
KernelBackend.JIT,
"rope.interleaved_rope_fp64_jit:fused_interleaved_rope_fp64",
_CUDA,
),
(
"diffusion.helios_qk_rope",
KernelBackend.JIT,
"rope.helios_qk_rope_jit:fused_inplace_helios_qk_rope",
_CUDA,
),
(
"diffusion.complex_rope",
KernelBackend.TRITON,
"rope.complex_rope_triton:fused_complex_rope",
_CUDA,
),
(
"diffusion.hunyuan_qkv_rope_pack",
KernelBackend.TRITON,
"rope.hunyuan_qkv_pack_triton:hunyuan_qkv_rope_pack",
_CUDA,
),
(
"diffusion.joint_qkv_cat",
KernelBackend.TRITON,
"layout.joint_qkv_cat_triton:joint_qkv_cat",
_CUDA,
),
(
"diffusion.rmsnorm_preserve_reduction",
KernelBackend.TRITON,
"norm.rmsnorm_preserve_reduction:rmsnorm_preserve_reduction",
_CUDA,
),
(
"diffusion.silu_mul",
KernelBackend.TRITON,
"activation.silu_mul_bitexact:fused_silu_mul_bitexact",
_CUDA,
),
(
"diffusion.situ_softcap_mul",
KernelBackend.TRITON,
"activation.situ_softcap_mul_bitexact:fused_situ_softcap_mul_bitexact",
_CUDA,
),
(
"diffusion.bias_silu",
KernelBackend.TRITON,
"activation.sana_conv_post_triton:fused_bias_silu",
_CUDA,
),
(
"diffusion.bias_glu",
KernelBackend.TRITON,
"activation.sana_conv_post_triton:fused_bias_glu",
_CUDA,
),
(
"diffusion.linear_gelu_tanh",
KernelBackend.AOT,
"sites.fused_linear_gelu_site:fused_linear_gelu_tanh",
_CUDA,
),
(
"diffusion.sparse_linear_attn_fwd",
KernelBackend.TRITON,
"attention.sparse_linear_attn_triton:_attn_fwd",
_CUDA,
),
(
"diffusion.bigdn",
KernelBackend.TRITON,
"attention.sana_wm_gdn_triton:fused_bigdn_func",
_CUDA,
),
(
"diffusion.fused_qknorm_rope_out_of_place",
KernelBackend.JIT,
"rope.qknorm_rope_jit:fused_qknorm_rope_out_of_place",
_CUDA,
),
(
"diffusion.qknorm_complex_rope_cuda",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_cuda",
_CUDA,
),
(
"diffusion.qknorm_complex_rope_pack_",
KernelBackend.JIT,
"rope.qknorm_complex_rope_jit:qknorm_complex_rope_pack_",
_CUDA,
),
(
"diffusion.channel_rmsnorm_finish_silu",
KernelBackend.JIT,
"norm.channel_rmsnorm_finish_silu_jit:channel_rmsnorm_finish_silu",
_CUDA,
),
(
"diffusion.channel_rmsnorm_silu_nhwc",
KernelBackend.JIT,
"norm.channel_rmsnorm_silu_nhwc_jit:channel_rmsnorm_silu_nhwc",
_CUDA,
),
(
"diffusion.bias_residual_add",
KernelBackend.JIT,
"norm.bias_residual_add_jit:bias_residual_add",
_CUDA,
),
(
"diffusion.vdn_delta_factors",
KernelBackend.JIT,
"attention.vdn_delta_factors_jit:vdn_delta_factors",
_CUDA,
),
(
"diffusion.vdn_temporal_conv_act",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_temporal_conv_act",
_CUDA,
),
(
"diffusion.vdn_silu_l2norm",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_silu_l2norm",
_CUDA,
),
(
"diffusion.vdn_frame_stats_prep",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_frame_stats_prep",
_CUDA,
),
(
"diffusion.vdn_gather_linear_state",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_gather_linear_state",
_CUDA,
),
(
"diffusion.vdn_linear_epilogue",
KernelBackend.TRITON,
"attention.vdn_linear_branch_triton:vdn_linear_epilogue",
_CUDA,
),
(
"diffusion.mxfp8_quantize_swizzled",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:mxfp8_quantize_swizzled",
_CUDA,
),
(
"diffusion.silu_mul_mxfp8",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:silu_mul_mxfp8",
_CUDA,
),
(
"diffusion.indexed_scale_shift_mxfp8_",
KernelBackend.TRITON,
"sglang.kernels.ops.quantization.mxfp8_swizzled_triton:indexed_scale_shift_mxfp8_",
_CUDA,
),
(
"diffusion.group_limited_topk",
KernelBackend.TRITON,
"routing.group_limited_topk_triton:group_limited_topk",
_CUDA,
),
(
"diffusion.usp_merge_heads",
KernelBackend.JIT,
"layout.usp_relayout_jit:usp_merge_heads",
_CUDA,
),
(
"diffusion.pack_qkv_destination_major",
KernelBackend.TRITON,
"layout.ulysses_qkv_triton:pack_qkv_destination_major",
_CUDA,
),
(
"diffusion.varlen_pack_qkv",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_pack_qkv",
_CUDA,
),
(
"diffusion.varlen_pack_segmented_qkv",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_pack_segmented_qkv",
_CUDA,
),
(
"diffusion.varlen_scatter_to_padded",
KernelBackend.TRITON,
"layout.varlen_pack_pad_triton:fused_scatter_to_padded",
_CUDA,
),
(
"diffusion.causal_conv3d_cat_pad",
KernelBackend.KDA,
"sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit:fused_causal_conv3d_cat_pad_cuda",
_CUDA,
),
(
"diffusion.causal_conv3d_cat_pad",
KernelBackend.TRITON,
"layout.causal_conv3d_cat_pad_triton:fused_causal_conv3d_cat_pad",
_CUDA,
),
(
"diffusion.cat_pad_channels_last_3d",
KernelBackend.TRITON,
"layout.wan_causal_cache_triton:cat_pad_channels_last_3d",
_CUDA,
),
(
"diffusion.dup_up3d_add",
KernelBackend.TRITON,
"layout.wan_causal_cache_triton:dup_up3d_add",
_CUDA,
),
(
"diffusion.conv_bias_epilogue",
KernelBackend.TRITON,
"modulate.conv_bias_epilogue_triton:conv_bias_epilogue",
_CUDA,
),
(
"diffusion.nearest_upsample_nhwc",
KernelBackend.TRITON,
"layout.nearest_upsample_nhwc_triton:nearest_upsample_nhwc",
_CUDA,
),
(
"diffusion.flux2_token_cat_nvfp4",
KernelBackend.JIT,
"layout.flux2_token_cat_nvfp4_jit:try_flux2_token_cat_nvfp4",
_CUDA,
),
)
for _op, _backend, _target, _caps in _SPECS:
register_kernel(
KernelSpec(
op=_op,
backend=_backend,
target=(
_target
if _target.startswith("sglang.")
else f"sglang.kernels.ops.diffusion.{_target}"
),
capabilities=_caps,
)
)
# ---------------------------------------------------------------------------
# Public export table: symbol -> owning submodule. Sorted by domain, module,
# then symbol; a new public kernel belongs here and nowhere else.
# ---------------------------------------------------------------------------
_EXPORTS: dict[str, str] = {
"apply_matrix_rope": "rope.matrix_rope",
"residual_gate_fp32": "modulate.residual_gate_fp32",
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
"fp8_rowwise": "quantization.fp8_rowwise_triton",
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
"load_extension_with_recovery": "sglang.srt.utils.cpp_extension_loader",
# Normalization: RMSNorm / LayerNorm / GroupNorm and their fused epilogues
"can_defer_flux2_gated_residual": "norm.flux2_gated_resnorm_jit",
"can_use_flux2_gated_resnorm": "norm.flux2_gated_resnorm_jit",
"flux2_gated_resnorm_raw": "norm.flux2_gated_resnorm_jit",
"FLYDSL_NORM_MIN_ALIGNED_DIM": "norm.fused_residual_norm_flydsl",
"can_use_fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
"flydsl_fused_residual_norm_scale_shift": "norm.fused_residual_norm_flydsl",
"fused_scale_residual_norm_scale_shift_triton": "norm.scale_residual_norm_scale_shift_triton",
"flydsl_norm_scale_shift": "norm.fused_residual_norm_flydsl",
"apply_group_norm_silu": "norm.group_norm_silu",
"triton_group_norm_silu": "norm.group_norm_silu_triton",
"can_use_group_norm_silu_4d": "norm.group_norm_silu_twopass_triton",
"can_use_group_norm_silu_rows": "norm.group_norm_silu_twopass_triton",
"group_norm_silu_4d": "norm.group_norm_silu_twopass_triton",
"group_norm_silu_rows": "norm.group_norm_silu_twopass_triton",
"can_use_fused_layernorm_modulate": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"can_use_fused_qk_head_layernorm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"fused_layernorm_modulate": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"fused_layernorm_modulate_fp8_quant_raw": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"fused_layernorm_modulate_raw": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"fused_qk_head_layernorm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"is_plain_layer_norm": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"rmsnorm_scale": "norm.native_bf16_rmsnorm_triton",
"rmsnorm_tanh_residual": "norm.native_bf16_rmsnorm_triton",
"norm_infer": "norm.norm_triton",
"rms_norm_fn": "norm.norm_triton",
"try_fused_bias_mul_add": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"try_fused_bias_scale_residual_norm_scale_shift": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"triton_one_pass_rms_norm": "norm.rmsnorm_onepass_triton",
"can_use_flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"flux2_strided_qknorm_rope": "rope.flux2_qknorm_rope_triton",
"rmsnorm_preserve_reduction": "norm.rmsnorm_preserve_reduction",
"can_use_channel_rmsnorm_finish_silu": "norm.channel_rmsnorm_finish_silu_jit",
"channel_rmsnorm_finish_silu": "norm.channel_rmsnorm_finish_silu_jit",
"can_use_channel_rmsnorm_silu_nhwc": "norm.channel_rmsnorm_silu_nhwc_jit",
"channel_rmsnorm_silu_nhwc": "norm.channel_rmsnorm_silu_nhwc_jit",
"is_channels_last_dense": "norm.channel_rmsnorm_silu_nhwc_jit",
"bias_residual_add": "norm.bias_residual_add_jit",
"can_use_bias_residual_add": "norm.bias_residual_add_jit",
"can_use_fused_rmsnorm_modulation": "norm.rmsnorm_scale_shift_bitexact",
"can_use_fused_rmsnorm_scale_shift": "norm.rmsnorm_scale_shift_bitexact",
"fused_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
"fused_scale_residual_rmsnorm_scale_shift_bitexact": "norm.rmsnorm_scale_shift_bitexact",
"fused_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
"fused_scale_residual_norm_scale_shift": "norm.scale_residual_norm_cutedsl",
"fused_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"try_fused_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"try_fused_scale_residual_norm_scale_shift_fp8": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"validate_scale_shift": "norm.scale_residual_norm_cutedsl",
"try_fused_scale_residual_norm_scale_shift_nvfp4": "sglang.kernels.kda_kernels.norm_scale_shift_jit",
"can_use_rmsnorm_cast_before_weight": "norm.rmsnorm_cast_before_weight_bitexact",
"can_use_rmsnorm_cast_before_weight_modulate": "norm.rmsnorm_cast_before_weight_bitexact",
"can_use_rmsnorm_cast_before_weight_rope": "norm.rmsnorm_cast_before_weight_bitexact",
"rmsnorm_cast_before_weight_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
"rmsnorm_cast_before_weight_modulate_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
"rmsnorm_cast_before_weight_rope_bitexact": "norm.rmsnorm_cast_before_weight_bitexact",
"can_use_wan_norm_silu_post": "norm.wan_norm_silu_post",
"wan_norm_silu_post": "norm.wan_norm_silu_post",
"can_use_wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"wan_rmsnorm_silu": "norm.wan_rmsnorm_silu_triton",
"can_use_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
"zimage_qk_rmsnorm_native": "norm.zimage_qk_rmsnorm_triton",
"can_use_rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"can_use_rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_indexed_scale_shift": "norm.rmsnorm_scale_shift_triton",
"rmsnorm_scale_shift_per_token": "norm.rmsnorm_scale_shift_triton",
"h3_vae_rmsnorm": "norm.h3_vae_fused_norm",
"h3_vae_scale_add_rmsnorm": "norm.h3_vae_fused_norm",
# adaLN modulation, gating and timestep conditioning
"indexed_gate_bf16": "modulate.indexed_modulation_triton",
"indexed_gate_bf16_": "modulate.indexed_modulation_triton",
"indexed_scale_shift_bf16_": "modulate.indexed_modulation_triton",
"indexed_gate_fp32": "modulate.indexed_modulation_triton",
"indexed_gate_fp32_": "modulate.indexed_modulation_triton",
"indexed_scale_shift_fp32_": "modulate.indexed_modulation_triton",
"ltx2_ada_values9": "modulate.ltx2_ada_values_triton",
"can_use_modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
"modulate_scale_shift": "modulate.modulate_scale_shift_jit",
"modulate_scale_shift_cuda": "modulate.modulate_scale_shift_jit",
"can_use_residual_gate_add_cuda": "sglang.kernels.kda_kernels.residual_gate_add_jit",
"residual_gate_add": "sglang.kernels.kda_kernels.residual_gate_add_jit",
"residual_gate_add_cuda": "sglang.kernels.kda_kernels.residual_gate_add_jit",
"can_use_residual_gate_add2_cuda": "modulate.residual_gate_add2_jit",
"residual_gate_add2": "modulate.residual_gate_add2_jit",
"residual_gate_add2_cuda": "modulate.residual_gate_add2_jit",
"fuse_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
"fuse_residual_layernorm_scale_shift_gate_select01_kernel": "modulate.scale_shift_triton",
"fuse_scale_shift_kernel": "modulate.scale_shift_triton",
"try_fused_fp32_layernorm_bf16": "sglang.kernels.kda_kernels.layernorm_modulate_triton",
"try_fused_scaled_residual_bf16": "modulate.scale_shift_triton",
"try_fused_scaled_residual_add_exact": "modulate.scale_shift_triton",
"timestep_embedding": "modulate.timestep_embedding_jit",
"fused_temb_table_slices": "modulate.wan_temb_table_slices_triton",
# Rotary embeddings and the QK-norm chains fused around them
"try_fused_flux2_qkv_epilogue": "sglang.kernels.kda_kernels.flux2_qkv_epilogue_jit",
"hunyuan_qkv_rope_pack": "rope.hunyuan_qkv_pack_triton",
"joint_qkv_cat": "layout.joint_qkv_cat_triton",
"can_use_ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
"ltx2_qknorm_split_rope_cuda": "sglang.kernels.kda_kernels.ltx2_qknorm_split_rope_jit",
"apply_ltx2_split_rotary_emb": "rope.ltx2_rotary_triton",
"fused_ltx25_decoder_rope": "rope.ltx25_decoder_rope_jit",
"can_use_fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
"fused_inplace_qknorm_rope": "rope.qknorm_rope_jit",
"fused_qknorm_rope_pack_kv": "rope.qknorm_rope_jit",
"try_fused_qwen_qkv_epilogue": "rope.qwen_qkv_epilogue_jit",
"can_use_fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
"fused_rope_rotate_half_fp32": "rope.rope_rotate_half_fp32",
"fused_rope_rotate_half_bitexact": "rope.rope_rotate_half_bitexact",
"fused_interleaved_rope_fp64": "rope.interleaved_rope_fp64_jit",
"fused_inplace_helios_qk_rope": "rope.helios_qk_rope_jit",
"apply_rotary_embedding": "rope.rotary_triton",
"fused_complex_rope": "rope.complex_rope_triton",
"can_use_qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_cuda": "rope.qknorm_complex_rope_jit",
"qknorm_complex_rope_pack_": "rope.qknorm_complex_rope_jit",
# Tensor layout transformations fused with downstream quantization
"try_flux2_token_cat_fp8": "sglang.kernels.kda_kernels.flux2_token_cat_fp8_triton",
# Activation-function fusions
"fused_bias_glu": "activation.sana_conv_post_triton",
"fused_bias_silu": "activation.sana_conv_post_triton",
"fused_packed_silu_mul_bitexact": "activation.silu_mul_bitexact",
"fused_silu_mul_bitexact": "activation.silu_mul_bitexact",
"can_use_fused_situ_softcap_mul": "activation.situ_softcap_mul_bitexact",
"fused_situ_softcap_mul_bitexact": "activation.situ_softcap_mul_bitexact",
# Diffusion attention kernels
"cam_scan_bidi_chunkwise": "attention.sana_wm_gdn_chunkwise_triton",
"fused_bigdn_func": "attention.sana_wm_gdn_triton",
"fused_qk_inv_rms": "attention.sana_wm_gdn_triton",
"prepare_rope_tables": "attention.sana_wm_gdn_triton",
"_attn_fwd": "attention.sparse_linear_attn_triton",
"get_block_map": "attention.sparse_linear_attn_triton",
"can_use_vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
"vsa_block_sparse_sm100": "attention.vsa_block_sparse_sm100_jit",
# MoE routing
"can_use_group_limited_topk": "routing.group_limited_topk_triton",
"group_limited_topk": "routing.group_limited_topk_triton",
# GEMMs with fused epilogues
"flux3_clustered_fp8_linear": "gemm.flux3_fp8_prims_ts",
"flux3_clustered_fp8_qkv_qknorm_rope": "gemm.flux3_fp8_prims_ts",
"flux3_fused_mlp_up": "gemm.flux3_fp8_prims_ts",
"pack_flux3_mlp_weight": "gemm.flux3_fp8_prims_ts",
"quantize_flux3_activation": "gemm.flux3_fp8_prims_ts",
"flux3_nvfp4_linear": "gemm.flux3_nvfp4_prims_ts",
"flux3_nvfp4_linear_from_packed_activation": "gemm.flux3_nvfp4_prims_ts",
"flux3_nvfp4_linear_swiglu": "gemm.flux3_nvfp4_prims_ts",
"flux3_nvfp4_linear_swiglu_fp4": "gemm.flux3_nvfp4_prims_ts",
"flux3_nvfp4_qkv_qknorm_rope": "gemm.flux3_nvfp4_prims_ts",
"quantize_flux3_nvfp4_activation": "gemm.flux3_nvfp4_prims_ts",
# Data movement: bitwise identical to the aten chains they replace
"can_use_fused_causal_conv3d_cat_pad_cuda": "sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit",
"fused_causal_conv3d_cat_pad_cuda": "sglang.kernels.kda_kernels.causal_conv3d_cat_pad_jit",
"fused_causal_conv3d_cat_pad": "layout.causal_conv3d_cat_pad_triton",
"pack_qkv_destination_major": "layout.ulysses_qkv_triton",
"fused_qknorm_rope_out_of_place": "rope.qknorm_rope_jit",
"vdn_delta_factors": "attention.vdn_delta_factors_jit",
"can_use_vdn_delta_factors": "attention.vdn_delta_factors_jit",
"vdn_temporal_conv_act": "attention.vdn_linear_branch_triton",
"can_use_vdn_temporal_conv_act": "attention.vdn_linear_branch_triton",
"can_use_vdn_silu_l2norm": "attention.vdn_linear_branch_triton",
"can_use_vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
"can_use_vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
"can_use_vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
"vdn_silu_l2norm": "attention.vdn_linear_branch_triton",
"vdn_frame_stats_prep": "attention.vdn_linear_branch_triton",
"vdn_gather_linear_state": "attention.vdn_linear_branch_triton",
"vdn_linear_epilogue": "attention.vdn_linear_branch_triton",
"can_use_mxfp8_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"can_use_silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"indexed_scale_shift_mxfp8_": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"mxfp8_quantize_swizzled": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"silu_mul_mxfp8": "sglang.kernels.ops.quantization.mxfp8_swizzled_triton",
"can_use_usp_merge_heads": "layout.usp_relayout_jit",
"usp_merge_heads": "layout.usp_relayout_jit",
"build_inv_indices": "layout.varlen_pack_pad_triton",
"fused_pack_qkv": "layout.varlen_pack_pad_triton",
"fused_pack_segmented_qkv": "layout.varlen_pack_pad_triton",
"fused_scatter_to_padded": "layout.varlen_pack_pad_triton",
"cat_pad_channels_last_3d": "layout.wan_causal_cache_triton",
"dup_up3d_add": "layout.wan_causal_cache_triton",
"conv_bias_epilogue": "modulate.conv_bias_epilogue_triton",
"can_use_conv_bias_epilogue": "modulate.conv_bias_epilogue_triton",
"nearest_upsample_nhwc": "layout.nearest_upsample_nhwc_triton",
"can_use_nearest_upsample_nhwc": "layout.nearest_upsample_nhwc_triton",
"try_flux2_token_cat_nvfp4": "layout.flux2_token_cat_nvfp4_jit",
# Fusion-site policy: quality gate, first-sight verification, mount
"BitExactFusionGate": "sites.bitexact_gate",
"flashinfer_rmsnorm_diagnostic_hint": "sites.bitexact_gate",
"tensors_equal": "sites.bitexact_gate",
"FLUX2_NVFP4_SWIGLU_QUANT": "sites.flux2_nvfp4_swiglu_quant_site",
"FUSED_GATE_RMSNORM": "sites.fused_gate_rmsnorm_site",
"fused_rmsnorm_scale": "sites.fused_gate_rmsnorm_site",
"fused_rmsnorm_tanh_residual": "sites.fused_gate_rmsnorm_site",
"FUSED_LINEAR_GELU": "sites.fused_linear_gelu_site",
"can_use_linear_gelu": "sites.fused_linear_gelu_site",
"fused_linear_gelu_tanh": "sites.fused_linear_gelu_site",
"FUSED_LN_MODULATE": "sites.fused_ln_modulate_site",
"can_use_ln_modulate": "sites.fused_ln_modulate_site",
"fused_ln_modulate": "sites.fused_ln_modulate_site",
"HELIOS_GATED_RESIDUAL": "sites.gated_residual_site",
"LINGBOT_VIDEO_GATED_RESIDUAL": "sites.gated_residual_site",
"try_gated_residual": "sites.gated_residual_site",
"HUNYUAN_QKNORM": "sites.hunyuan_qknorm_site",
"try_hunyuan_qknorm": "sites.hunyuan_qknorm_site",
"LINGBOT_VIDEO_RMSNORM": "sites.lingbot_video_rmsnorm_site",
"try_lingbot_video_rmsnorm": "sites.lingbot_video_rmsnorm_site",
"LTX2_QKNORM_SPLIT_ROPE": "sites.ltx2_qknorm_split_rope_site",
"LTX2_RMS_NORM_MODULATE": "sites.ltx2_rmsnorm_modulate_site",
"fused_ltx2_rms_norm_modulate": "sites.ltx2_rmsnorm_modulate_site",
"MINIMAX_H3_NORM_MODULATE": "sites.minimax_h3_norm_modulate_site",
"NVFP4_BIAS_GELU": "sites.nvfp4_bias_gelu_site",
"QWEN_IMAGE_ADDED_QKV": "sites.qwen_image_added_qkv_site",
"SANA_VIDEO_LINEAR_ATTENTION": "sites.sana_video_linear_attention_site",
"try_sana_video_linear_attention": "sites.sana_video_linear_attention_site",
"QualityGatedFusion": "sites.quality_gate",
# JIT C++/CUDA extensions (not kernels, not in the registry)
"interpolate": "ext.hunyuan3d_rasterizer",
"rasterize": "ext.hunyuan3d_rasterizer",
"meshVerticeInpaint": "ext.mesh_processor",
"load_mesh_processor": "ext.mesh_processor",
}
def __getattr__(name: str) -> Any:
"""Resolve a public symbol to its submodule on first access (PEP 562)."""
module = _EXPORTS.get(name)
if module is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from importlib import import_module
owner = module if module.startswith("sglang.") else f"{__name__}.{module}"
value = getattr(import_module(owner), name)
globals()[name] = value # cache; later lookups skip __getattr__ entirely
return value
def __dir__() -> list[str]:
return sorted(set(globals()) | set(_EXPORTS))
__all__ = sorted(_EXPORTS)