Add triton autotune on the Mamba2 SSD kernels (#39130)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
elvischenv
2026-10-01 14:25:37 -07:00
committed by GitHub
co-authored by Claude Opus 5 Po-Han Huang
parent dcf4e18283
commit bf2d686a82
6 changed files with 470 additions and 3 deletions
+13 -3
View File
@@ -115,6 +115,17 @@ from sglang.srt.utils import (
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
def _init_process_global_configs() -> None:
"""Process-global config both benchmark modes need before the model loads.
The Mamba SSU backend is not here: `load_model` installs it for the models
that have one.
"""
initialize_moe_config()
initialize_fp8_gemm_config()
initialize_fp4_gemm_config()
def start_profile(
profile_activities,
profile_record_shapes=False,
@@ -686,6 +697,7 @@ def correctness_test(
gpu_id=gpu_id,
),
)
_init_process_global_configs()
# Configure the logger
configure_logger(server_args, prefix=f" TP{tp_rank}")
@@ -898,9 +910,7 @@ def latency_test(
gpu_id=gpu_id,
),
)
initialize_moe_config()
initialize_fp8_gemm_config()
initialize_fp4_gemm_config()
_init_process_global_configs()
if get_bool_env_var("SGLANG_SET_CPU_AFFINITY"):
set_gpu_proc_affinity(tp_rank)
@@ -0,0 +1,44 @@
from __future__ import annotations
from typing import Callable, Mapping, Sequence
import triton
from sglang.kernels.ops.attention.fla.utils import autotune_cache_kwargs
__all__ = ["autotune_cache_kwargs", "prune_oversized_tiles"]
def _tile_footprint(config: triton.Config) -> tuple[int, int, int]:
footprint = 1
for block in config.kwargs.values():
footprint *= block
return footprint, config.num_stages, config.num_warps
def prune_oversized_tiles(
tiled_dims: Mapping[str, Sequence[str]],
) -> Callable[..., list[triton.Config]]:
"""Build an `early_config_prune` that drops tiles wider than what they cover.
A block larger than the dimension it tiles computes a fully masked remainder
for the same grid, so it is dominated; `tiled_dims` maps each BLOCK_SIZE_* to
the kernel arguments it strides over, and a block is capped by the smallest.
"""
def early_config_prune(configs, named_args, **kwargs):
# Triton splits the launch into positional args and kwargs; a dimension
# can arrive through either.
args = {**kwargs, **named_args}
caps = {}
for block, dims in tiled_dims.items():
cap = triton.next_power_of_2(min(args[dim] for dim in dims))
caps[block] = max(cap, min(c.kwargs[block] for c in configs))
keep = [
config
for config in configs
if all(config.kwargs[block] <= cap for block, cap in caps.items())
]
return keep or [min(configs, key=_tile_footprint)]
return early_config_prune
@@ -14,7 +14,69 @@ import torch
import triton
import triton.language as tl
from .autotune import autotune_cache_kwargs, prune_oversized_tiles
@triton.autotune(
configs=[
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
key=["chunk_size", "K", "IS_CAUSAL"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles(
{
"BLOCK_SIZE_M": ("chunk_size",),
"BLOCK_SIZE_N": ("chunk_size",),
"BLOCK_SIZE_K": ("K",),
}
)
},
**autotune_cache_kwargs,
)
@triton.jit
def _bmm_chunk_fwd_kernel(
# Pointers to matrices
@@ -13,9 +13,151 @@ import triton
import triton.language as tl
from packaging import version
from .autotune import autotune_cache_kwargs, prune_oversized_tiles
TRITON_22 = version.parse(triton.__version__) >= version.parse("2.2.0")
@triton.autotune(
configs=[
# =================================================================
# Higher warp count configs for better latency hiding
# More warps = more instructions in flight = better memory latency hiding
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=8,
),
# Smaller tiles with more stages for software pipelining
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# =================================================================
# Low register pressure configs (num_stages=1) for large dstate
# =================================================================
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=1,
num_warps=4,
),
# num_stages=2 configs - moderate register pressure
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=2,
num_warps=4,
),
# Original configs for larger dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
key=["chunk_size", "hdim", "dstate", "IS_CAUSAL"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles(
{
"BLOCK_SIZE_M": ("chunk_size",),
"BLOCK_SIZE_N": ("hdim",),
"BLOCK_SIZE_K": ("chunk_size",),
}
)
},
**autotune_cache_kwargs,
)
@triton.jit
def _chunk_scan_fwd_kernel(
# Pointers to matrices
@@ -14,9 +14,25 @@ import torch
import triton
import triton.language as tl
from .autotune import autotune_cache_kwargs, prune_oversized_tiles
from .mamba_ssm import softplus
@triton.autotune(
configs=[
triton.Config({"BLOCK_SIZE_H": 2}),
triton.Config({"BLOCK_SIZE_H": 4}),
triton.Config({"BLOCK_SIZE_H": 8}),
triton.Config({"BLOCK_SIZE_H": 16}),
triton.Config({"BLOCK_SIZE_H": 32}),
triton.Config({"BLOCK_SIZE_H": 64}),
],
key=["chunk_size", "nheads"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles({"BLOCK_SIZE_H": ("nheads",)})
},
**autotune_cache_kwargs,
)
@triton.jit
def _chunk_cumsum_fwd_kernel(
# Pointers to matrices
@@ -109,6 +125,94 @@ def _chunk_cumsum_fwd_kernel(
)
@triton.autotune(
configs=[
# Small headdim/dstate configs (hdim<=64, dstate<=128) - increased parallelism
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
# Low register pressure configs for large dstate (dstate=128)
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# original configs for larger headdim/dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
key=["hdim", "dstate", "chunk_size"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles(
{
"BLOCK_SIZE_M": ("hdim",),
"BLOCK_SIZE_N": ("dstate",),
"BLOCK_SIZE_K": ("chunk_size",),
}
)
},
**autotune_cache_kwargs,
)
@triton.jit
def _chunk_state_fwd_kernel(
# Pointers to matrices
@@ -259,6 +363,94 @@ def _chunk_state_fwd_kernel(
tl.store(states_ptrs, states, mask=c_mask)
@triton.autotune(
configs=[
# Small headdim/dstate configs (hdim<=64, dstate<=128) - increased parallelism
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=3,
num_warps=4,
),
# Low register pressure configs for large dstate (dstate=128)
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 64},
num_stages=2,
num_warps=4,
),
# original configs for larger headdim/dstate values
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 64},
num_stages=3,
num_warps=8,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 256, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 128, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 128, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=4,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 32, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 32, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=5,
num_warps=2,
),
triton.Config(
{"BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, "BLOCK_SIZE_K": 32},
num_stages=4,
num_warps=2,
),
],
key=["hdim", "dstate", "chunk_size"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles(
{
"BLOCK_SIZE_M": ("hdim",),
"BLOCK_SIZE_N": ("dstate",),
"BLOCK_SIZE_K": ("chunk_size",),
}
)
},
**autotune_cache_kwargs,
)
@triton.jit
def _chunk_state_varlen_kernel(
# Pointers to matrices
@@ -12,7 +12,24 @@ import torch
import triton
import triton.language as tl
from .autotune import autotune_cache_kwargs, prune_oversized_tiles
@triton.autotune(
configs=[
triton.Config({"BLOCK_SIZE": 64}),
triton.Config({"BLOCK_SIZE": 128}),
triton.Config({"BLOCK_SIZE": 256}),
triton.Config({"BLOCK_SIZE": 512}),
triton.Config({"BLOCK_SIZE": 1024}),
triton.Config({"BLOCK_SIZE": 2048}),
],
key=["dim"],
prune_configs_by={
"early_config_prune": prune_oversized_tiles({"BLOCK_SIZE": ("dim",)})
},
**autotune_cache_kwargs,
)
@triton.jit
def _state_passing_fwd_kernel(
# Pointers to matrices