From bf2d686a82a1fdb4bd7f1fa9ff18ad2562dd8314 Mon Sep 17 00:00:00 2001 From: elvischenv <219235043+elvischenv@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:25:37 +0800 Subject: [PATCH] Add triton autotune on the Mamba2 SSD kernels (#39130) Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> --- python/sglang/benchmark/one_batch.py | 16 +- .../kernels/ops/mamba/triton_ops/autotune.py | 44 ++++ .../kernels/ops/mamba/triton_ops/ssd_bmm.py | 62 ++++++ .../ops/mamba/triton_ops/ssd_chunk_scan.py | 142 +++++++++++++ .../ops/mamba/triton_ops/ssd_chunk_state.py | 192 ++++++++++++++++++ .../ops/mamba/triton_ops/ssd_state_passing.py | 17 ++ 6 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 python/sglang/kernels/ops/mamba/triton_ops/autotune.py diff --git a/python/sglang/benchmark/one_batch.py b/python/sglang/benchmark/one_batch.py index 7b95a6243a..320db074ed 100644 --- a/python/sglang/benchmark/one_batch.py +++ b/python/sglang/benchmark/one_batch.py @@ -115,6 +115,17 @@ from sglang.srt.utils import ( from sglang.srt.utils.hf_transformers_utils import get_tokenizer +def _init_process_global_configs() -> None: + """Process-global config both benchmark modes need before the model loads. + + The Mamba SSU backend is not here: `load_model` installs it for the models + that have one. + """ + initialize_moe_config() + initialize_fp8_gemm_config() + initialize_fp4_gemm_config() + + def start_profile( profile_activities, profile_record_shapes=False, @@ -686,6 +697,7 @@ def correctness_test( gpu_id=gpu_id, ), ) + _init_process_global_configs() # Configure the logger configure_logger(server_args, prefix=f" TP{tp_rank}") @@ -898,9 +910,7 @@ def latency_test( gpu_id=gpu_id, ), ) - initialize_moe_config() - initialize_fp8_gemm_config() - initialize_fp4_gemm_config() + _init_process_global_configs() if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"): set_gpu_proc_affinity(tp_rank) diff --git a/python/sglang/kernels/ops/mamba/triton_ops/autotune.py b/python/sglang/kernels/ops/mamba/triton_ops/autotune.py new file mode 100644 index 0000000000..174b98591d --- /dev/null +++ b/python/sglang/kernels/ops/mamba/triton_ops/autotune.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from typing import Callable, Mapping, Sequence + +import triton + +from sglang.kernels.ops.attention.fla.utils import autotune_cache_kwargs + +__all__ = ["autotune_cache_kwargs", "prune_oversized_tiles"] + + +def _tile_footprint(config: triton.Config) -> tuple[int, int, int]: + footprint = 1 + for block in config.kwargs.values(): + footprint *= block + return footprint, config.num_stages, config.num_warps + + +def prune_oversized_tiles( + tiled_dims: Mapping[str, Sequence[str]], +) -> Callable[..., list[triton.Config]]: + """Build an `early_config_prune` that drops tiles wider than what they cover. + + A block larger than the dimension it tiles computes a fully masked remainder + for the same grid, so it is dominated; `tiled_dims` maps each BLOCK_SIZE_* to + the kernel arguments it strides over, and a block is capped by the smallest. + """ + + def early_config_prune(configs, named_args, **kwargs): + # Triton splits the launch into positional args and kwargs; a dimension + # can arrive through either. + args = {**kwargs, **named_args} + caps = {} + for block, dims in tiled_dims.items(): + cap = triton.next_power_of_2(min(args[dim] for dim in dims)) + caps[block] = max(cap, min(c.kwargs[block] for c in configs)) + keep = [ + config + for config in configs + if all(config.kwargs[block] <= cap for block, cap in caps.items()) + ] + return keep or [min(configs, key=_tile_footprint)] + + return early_config_prune diff --git a/python/sglang/kernels/ops/mamba/triton_ops/ssd_bmm.py b/python/sglang/kernels/ops/mamba/triton_ops/ssd_bmm.py index ae9803a83c..825f7aa077 100644 --- a/python/sglang/kernels/ops/mamba/triton_ops/ssd_bmm.py +++ b/python/sglang/kernels/ops/mamba/triton_ops/ssd_bmm.py @@ -14,7 +14,69 @@ import torch import triton import triton.language as tl +from .autotune import autotune_cache_kwargs, prune_oversized_tiles + +@triton.autotune( + configs=[ + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=2, + ), + ], + key=["chunk_size", "K", "IS_CAUSAL"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles( + { + "BLOCK_SIZE_M": ("chunk_size",), + "BLOCK_SIZE_N": ("chunk_size",), + "BLOCK_SIZE_K": ("K",), + } + ) + }, + **autotune_cache_kwargs, +) @triton.jit def _bmm_chunk_fwd_kernel( # Pointers to matrices diff --git a/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_scan.py b/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_scan.py index 5fd4fb1385..83da049193 100644 --- a/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_scan.py +++ b/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_scan.py @@ -13,9 +13,151 @@ import triton import triton.language as tl from packaging import version +from .autotune import autotune_cache_kwargs, prune_oversized_tiles + TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0") +@triton.autotune( + configs=[ + # ================================================================= + # Higher warp count configs for better latency hiding + # More warps = more instructions in flight = better memory latency hiding + # ================================================================= + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=2, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=2, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=2, + num_warps=8, + ), + # Smaller tiles with more stages for software pipelining + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + # ================================================================= + # Low register pressure configs (num_stages=1) for large dstate + # ================================================================= + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64}, + num_stages=1, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=1, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=1, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=1, + num_warps=4, + ), + # num_stages=2 configs - moderate register pressure + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=2, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=2, + num_warps=4, + ), + # Original configs for larger dstate values + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=2, + ), + ], + key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles( + { + "BLOCK_SIZE_M": ("chunk_size",), + "BLOCK_SIZE_N": ("hdim",), + "BLOCK_SIZE_K": ("chunk_size",), + } + ) + }, + **autotune_cache_kwargs, +) @triton.jit def _chunk_scan_fwd_kernel( # Pointers to matrices diff --git a/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_state.py b/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_state.py index d33808a516..12054e915c 100644 --- a/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_state.py +++ b/python/sglang/kernels/ops/mamba/triton_ops/ssd_chunk_state.py @@ -14,9 +14,25 @@ import torch import triton import triton.language as tl +from .autotune import autotune_cache_kwargs, prune_oversized_tiles from .mamba_ssm import softplus +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE_H": 2}), + triton.Config({"BLOCK_SIZE_H": 4}), + triton.Config({"BLOCK_SIZE_H": 8}), + triton.Config({"BLOCK_SIZE_H": 16}), + triton.Config({"BLOCK_SIZE_H": 32}), + triton.Config({"BLOCK_SIZE_H": 64}), + ], + key=["chunk_size", "nheads"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles({"BLOCK_SIZE_H": ("nheads",)}) + }, + **autotune_cache_kwargs, +) @triton.jit def _chunk_cumsum_fwd_kernel( # Pointers to matrices @@ -109,6 +125,94 @@ def _chunk_cumsum_fwd_kernel( ) +@triton.autotune( + configs=[ + # Small headdim/dstate configs (hdim<=64, dstate<=128) - increased parallelism + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + # Low register pressure configs for large dstate (dstate=128) + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + # original configs for larger headdim/dstate values + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=2, + ), + ], + key=["hdim", "dstate", "chunk_size"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles( + { + "BLOCK_SIZE_M": ("hdim",), + "BLOCK_SIZE_N": ("dstate",), + "BLOCK_SIZE_K": ("chunk_size",), + } + ) + }, + **autotune_cache_kwargs, +) @triton.jit def _chunk_state_fwd_kernel( # Pointers to matrices @@ -259,6 +363,94 @@ def _chunk_state_fwd_kernel( tl.store(states_ptrs, states, mask=c_mask) +@triton.autotune( + configs=[ + # Small headdim/dstate configs (hdim<=64, dstate<=128) - increased parallelism + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=3, + num_warps=4, + ), + # Low register pressure configs for large dstate (dstate=128) + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64}, + num_stages=2, + num_warps=4, + ), + # original configs for larger headdim/dstate values + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=5, + num_warps=2, + ), + triton.Config( + {"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32}, + num_stages=4, + num_warps=2, + ), + ], + key=["hdim", "dstate", "chunk_size"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles( + { + "BLOCK_SIZE_M": ("hdim",), + "BLOCK_SIZE_N": ("dstate",), + "BLOCK_SIZE_K": ("chunk_size",), + } + ) + }, + **autotune_cache_kwargs, +) @triton.jit def _chunk_state_varlen_kernel( # Pointers to matrices diff --git a/python/sglang/kernels/ops/mamba/triton_ops/ssd_state_passing.py b/python/sglang/kernels/ops/mamba/triton_ops/ssd_state_passing.py index c25ea23a10..7a9ff59e59 100644 --- a/python/sglang/kernels/ops/mamba/triton_ops/ssd_state_passing.py +++ b/python/sglang/kernels/ops/mamba/triton_ops/ssd_state_passing.py @@ -12,7 +12,24 @@ import torch import triton import triton.language as tl +from .autotune import autotune_cache_kwargs, prune_oversized_tiles + +@triton.autotune( + configs=[ + triton.Config({"BLOCK_SIZE": 64}), + triton.Config({"BLOCK_SIZE": 128}), + triton.Config({"BLOCK_SIZE": 256}), + triton.Config({"BLOCK_SIZE": 512}), + triton.Config({"BLOCK_SIZE": 1024}), + triton.Config({"BLOCK_SIZE": 2048}), + ], + key=["dim"], + prune_configs_by={ + "early_config_prune": prune_oversized_tiles({"BLOCK_SIZE": ("dim",)}) + }, + **autotune_cache_kwargs, +) @triton.jit def _state_passing_fwd_kernel( # Pointers to matrices