[Perf] moe: Extend FlashInfer CUTLASS MoE autotune coverage to prefill chunks (#43363)

This commit is contained in:
Ho-Ren (Jack) Chuang
2026-10-20 01:41:16 -07:00
committed by GitHub
parent 06314dc2e8
commit f8fd41c36e
5 changed files with 232 additions and 30 deletions
@@ -18,13 +18,21 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import is_allocation_symmetric
from sglang.srt.layers.dp_attention import (
deployment_attn_dp_size,
is_allocation_symmetric,
)
from sglang.srt.layers.moe.moe_runner.base import (
MoeQuantInfo,
MoeRunnerConfig,
register_fused_func,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import (
get_disagg,
get_parallel,
get_schedule,
max_prefill_buffer_tokens,
)
from sglang.srt.utils import is_flashinfer_available
from sglang.srt.utils.common import is_sm120, next_power_of_2
@@ -157,6 +165,22 @@ def _flashinfer_cutlass_fused_moe():
return cutlass_fused_moe, ActivationType
# Matches the default tune_max_num_tokens of FlashInfer's cutlass_fused_moe.
_FLASHINFER_DEFAULT_TUNE_MAX_NUM_TOKENS = 8192
# Tune range: the DP-gathered prefill chunk, capped at FlashInfer's default;
# PD decode servers run no prefill, so they size it per call.
def _cutlass_moe_tune_max_num_tokens(num_tokens: int) -> int:
if get_disagg().disaggregation_mode == "decode":
return next_power_of_2(num_tokens)
prefill_tokens = max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens
return min(
prefill_tokens * deployment_attn_dp_size(),
_FLASHINFER_DEFAULT_TUNE_MAX_NUM_TOKENS,
)
def _activation_type(runner_config: MoeRunnerConfig):
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import get_activation_type
@@ -362,7 +386,7 @@ def _run_flashinfer_cutlass(
ep_rank=runner_config.moe_ep_rank,
tp_size=runner_config.moe_tp_size,
tp_rank=runner_config.moe_tp_rank,
tune_max_num_tokens=next_power_of_2(x.shape[0]),
tune_max_num_tokens=_cutlass_moe_tune_max_num_tokens(x.shape[0]),
activation_type=_activation_type(runner_config),
enable_alltoall=enable_alltoall,
use_fused_finalize=envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get(),
@@ -584,7 +608,7 @@ def _fused_experts_flashinfer_mxfp4_cutlass(
if quant_info.use_swiglu_step
else ActivationType.Swiglu
),
tune_max_num_tokens=next_power_of_2(x.shape[0]),
tune_max_num_tokens=_cutlass_moe_tune_max_num_tokens(x.shape[0]),
output=out,
use_fused_finalize=envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get(),
**humming_kwargs,