diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py index 453247fb21..c98c9cfc1e 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py @@ -18,13 +18,21 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import is_allocation_symmetric +from sglang.srt.layers.dp_attention import ( + deployment_attn_dp_size, + is_allocation_symmetric, +) from sglang.srt.layers.moe.moe_runner.base import ( MoeQuantInfo, MoeRunnerConfig, register_fused_func, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import ( + get_disagg, + get_parallel, + get_schedule, + max_prefill_buffer_tokens, +) from sglang.srt.utils import is_flashinfer_available from sglang.srt.utils.common import is_sm120, next_power_of_2 @@ -157,6 +165,22 @@ def _flashinfer_cutlass_fused_moe(): return cutlass_fused_moe, ActivationType +# Matches the default tune_max_num_tokens of FlashInfer's cutlass_fused_moe. +_FLASHINFER_DEFAULT_TUNE_MAX_NUM_TOKENS = 8192 + + +# Tune range: the DP-gathered prefill chunk, capped at FlashInfer's default; +# PD decode servers run no prefill, so they size it per call. +def _cutlass_moe_tune_max_num_tokens(num_tokens: int) -> int: + if get_disagg().disaggregation_mode == "decode": + return next_power_of_2(num_tokens) + prefill_tokens = max_prefill_buffer_tokens() or get_schedule().max_prefill_tokens + return min( + prefill_tokens * deployment_attn_dp_size(), + _FLASHINFER_DEFAULT_TUNE_MAX_NUM_TOKENS, + ) + + def _activation_type(runner_config: MoeRunnerConfig): from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import get_activation_type @@ -362,7 +386,7 @@ def _run_flashinfer_cutlass( ep_rank=runner_config.moe_ep_rank, tp_size=runner_config.moe_tp_size, tp_rank=runner_config.moe_tp_rank, - tune_max_num_tokens=next_power_of_2(x.shape[0]), + tune_max_num_tokens=_cutlass_moe_tune_max_num_tokens(x.shape[0]), activation_type=_activation_type(runner_config), enable_alltoall=enable_alltoall, use_fused_finalize=envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get(), @@ -584,7 +608,7 @@ def _fused_experts_flashinfer_mxfp4_cutlass( if quant_info.use_swiglu_step else ActivationType.Swiglu ), - tune_max_num_tokens=next_power_of_2(x.shape[0]), + tune_max_num_tokens=_cutlass_moe_tune_max_num_tokens(x.shape[0]), output=out, use_fused_finalize=envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.get(), **humming_kwargs, diff --git a/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py b/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py index 6b3b46d2c3..d8dfb5b6f9 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py +++ b/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py @@ -1,13 +1,22 @@ -"""Unit tests for the FlashInfer CUTLASS MoE SwiGLU parameter materialization.""" +"""Unit tests for FlashInfer CUTLASS MoE runner helpers.""" +import inspect import unittest +from unittest import mock import torch from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + FlashInferCutlassMoeQuantInfo, + FlashInferCutlassMxfp4MoeQuantInfo, + _fused_experts_flashinfer_mxfp4_cutlass, + _run_flashinfer_cutlass, materialize_swiglu_params_for_cutlass, ) +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.runtime_context import get_context, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -16,6 +25,20 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu") GPT_OSS_ALPHA = 1.702 GPT_OSS_LIMIT = 7.0 NUM_LOCAL_EXPERTS = 4 +HIDDEN = 128 +INTERMEDIATE = 128 +TOP_K = 2 +STARTUP_AUTOTUNE_TOKENS = 256 +CUTLASS_FUSED_MOE_IMPORT = ( + "sglang.srt.layers.moe.moe_runner.flashinfer_cutlass._flashinfer_cutlass_fused_moe" +) + + +class _SingleRankMoeLayer(torch.nn.Module): + moe_tp_size = 1 + moe_tp_rank = 0 + moe_ep_size = 1 + moe_ep_rank = 0 class TestMaterializeSwigluParamsForCutlass(CustomTestCase): @@ -94,5 +117,151 @@ class TestMaterializeSwigluParamsForCutlass(CustomTestCase): self.assertEqual(tuple(tensor.shape), (NUM_LOCAL_EXPERTS,)) +def _dispatch_output(num_tokens): + return StandardDispatchOutput( + hidden_states=torch.empty(num_tokens, HIDDEN, dtype=torch.bfloat16), + hidden_states_scale=None, + topk_output=StandardTopKOutput( + topk_weights=torch.full(size=(num_tokens, TOP_K), fill_value=1.0 / TOP_K), + topk_ids=torch.zeros(num_tokens, TOP_K, dtype=torch.int32), + router_logits=torch.empty(num_tokens, NUM_LOCAL_EXPERTS), + ), + ) + + +def _run_bf16_cutlass(num_tokens): + _run_flashinfer_cutlass( + dispatch_output=_dispatch_output(num_tokens), + quant_info=FlashInferCutlassMoeQuantInfo( + quant_type="bf16", + w13_weight=torch.empty( + NUM_LOCAL_EXPERTS, 2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16 + ), + w2_weight=torch.empty( + NUM_LOCAL_EXPERTS, HIDDEN, INTERMEDIATE, dtype=torch.bfloat16 + ), + ), + runner_config=MoeRunnerConfig(layer=_SingleRankMoeLayer()), + ) + + +def _run_mxfp4_w4a16_cutlass(num_tokens): + _fused_experts_flashinfer_mxfp4_cutlass( + dispatch_output=_dispatch_output(num_tokens), + quant_info=FlashInferCutlassMxfp4MoeQuantInfo( + w13_weight=torch.empty( + NUM_LOCAL_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 2, dtype=torch.uint8 + ), + w2_weight=torch.empty( + NUM_LOCAL_EXPERTS, HIDDEN, INTERMEDIATE // 2, dtype=torch.uint8 + ), + w13_weight_scale=torch.empty( + NUM_LOCAL_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 32, dtype=torch.uint8 + ), + w2_weight_scale=torch.empty( + NUM_LOCAL_EXPERTS, HIDDEN, INTERMEDIATE // 32, dtype=torch.uint8 + ), + ), + runner_config=MoeRunnerConfig(layer=_SingleRankMoeLayer()), + ) + + +def _captured_tune_max_num_tokens(run_site, num_tokens): + from flashinfer.fused_moe.core import ActivationType + + kwargs_seen = {} + + def fused_moe_stub(**kwargs): + kwargs_seen.update(kwargs) + return [kwargs["output"]] + + with mock.patch( + target=CUTLASS_FUSED_MOE_IMPORT, + return_value=(fused_moe_stub, ActivationType), + ): + run_site(num_tokens) + return kwargs_seen["tune_max_num_tokens"] + + +class TestCutlassMoeTuneMaxNumTokens(CustomTestCase): + def test_cap_is_flashinfer_default(self): + """The cap must stay FlashInfer's own default tune range for this call.""" + from flashinfer.fused_moe import cutlass_fused_moe + + self.assertEqual( + inspect.signature(cutlass_fused_moe) + .parameters["tune_max_num_tokens"] + .default, + 8192, + ) + + def test_every_token_count_hits_a_startup_tuned_bucket(self): + """A prefill-sized MoE call must look up a bucket the decode-shaped startup + autotune tuned, up to the DP-gathered chunk capped at FlashInfer's default.""" + from flashinfer.fused_moe.utils import ( + get_hybrid_num_tokens_buckets, + map_to_hybrid_bucket, + ) + + dp_attention = dict(tp_size=4, attn_dp_size=4) + for fields, tuned_ceiling in ( + (dict(chunked_prefill_size=8192), 8192), + (dict(chunked_prefill_size=-1), 8192), + (dict(chunked_prefill_size=1024, **dp_attention), 4096), + (dict(chunked_prefill_size=4096, **dp_attention), 8192), + (dict(chunked_prefill_size=8192, disaggregation_mode="prefill"), 8192), + ): + with ( + get_context().override_server_args(**fields), + get_parallel().override(tp_group=None), + ): + for run_site in (_run_bf16_cutlass, _run_mxfp4_w4a16_cutlass): + tuned = get_hybrid_num_tokens_buckets( + _captured_tune_max_num_tokens( + run_site=run_site, num_tokens=STARTUP_AUTOTUNE_TOKENS + ) + ) + for num_tokens in (1, 257, 777, 2048, 4097, 8192, 16384): + with self.subTest( + site=run_site.__name__, num_tokens=num_tokens, **fields + ): + bucket = map_to_hybrid_bucket( + x=num_tokens, + max_num_tokens=_captured_tune_max_num_tokens( + run_site=run_site, num_tokens=num_tokens + ), + ) + self.assertIn(bucket, tuned) + self.assertGreaterEqual( + bucket, min(num_tokens, tuned_ceiling) + ) + self.assertLessEqual(bucket, tuned_ceiling) + + def test_pd_decode_server_sizes_each_call(self): + """A PD decode server runs no prefill, so it must not pay for prefill-sized + tuning: each call passes its own token count rounded up to a power of 2.""" + with ( + get_context().override_server_args( + chunked_prefill_size=8192, disaggregation_mode="decode" + ), + get_parallel().override(tp_group=None), + ): + for run_site in (_run_bf16_cutlass, _run_mxfp4_w4a16_cutlass): + for num_tokens, per_call in ( + (1, 1), + (STARTUP_AUTOTUNE_TOKENS, 256), + (257, 512), + (777, 1024), + (16384, 16384), + ): + with self.subTest(site=run_site.__name__, num_tokens=num_tokens): + self.assertEqual( + _captured_tune_max_num_tokens( + run_site=run_site, num_tokens=num_tokens + ), + per_call, + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/moe/test_moe_runner_config_placement.py b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py index 4ad03fd52e..6fa7992ed9 100644 --- a/test/registered/unit/layers/moe/test_moe_runner_config_placement.py +++ b/test/registered/unit/layers/moe/test_moe_runner_config_placement.py @@ -112,21 +112,22 @@ def _kernel_placement(monkeypatch, runner_config): monkeypatch.setattr(flashinfer_cutlass, "_activation_type", lambda config: None) hidden_states = torch.zeros(2, 4, dtype=torch.bfloat16) - flashinfer_cutlass._run_flashinfer_cutlass( - dispatch_output=SimpleNamespace( - hidden_states=hidden_states, - hidden_states_scale=None, - topk_output=SimpleNamespace( - topk_weights=torch.ones(2, 1), - topk_ids=torch.zeros(2, 1, dtype=torch.int64), + with get_context().override_server_args(model_path="dummy"): + flashinfer_cutlass._run_flashinfer_cutlass( + dispatch_output=SimpleNamespace( + hidden_states=hidden_states, + hidden_states_scale=None, + topk_output=SimpleNamespace( + topk_weights=torch.ones(2, 1), + topk_ids=torch.zeros(2, 1, dtype=torch.int64), + ), ), - ), - quant_info=FlashInferCutlassMoeQuantInfo( - quant_type="bf16", w13_weight=torch.zeros(1), w2_weight=torch.zeros(1) - ), - runner_config=runner_config, - output=torch.empty_like(hidden_states), - ) + quant_info=FlashInferCutlassMoeQuantInfo( + quant_type="bf16", w13_weight=torch.zeros(1), w2_weight=torch.zeros(1) + ), + runner_config=runner_config, + output=torch.empty_like(hidden_states), + ) (call,) = calls return (call["tp_size"], call["tp_rank"], call["ep_size"], call["ep_rank"]) diff --git a/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py b/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py index 0b14a1a714..17f74042ed 100644 --- a/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py +++ b/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py @@ -224,7 +224,8 @@ def test_dsv4_sm120_matches_direct_flashinfer(monkeypatch, stated_tp_group): topk = StandardTopKOutput(topk_weights, topk_ids.to(torch.int32), logits) dispatch_output = StandardDispatchOutput(x, None, topk) - actual = method.apply(layer, dispatch_output).hidden_states + with get_context().override_server_args(): + actual = method.apply(layer, dispatch_output).hidden_states x_quant, x_scale = mxfp8_quantize( x, @@ -280,6 +281,7 @@ def test_gpt_oss_sm120_padding_layout_and_kernel(monkeypatch, stated_tp_group): from sglang.srt.layers.moe.topk import StandardTopKOutput from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.layers.quantization.mxfp4 import Mxfp4MoEMethod + from sglang.srt.runtime_context import get_context monkeypatch.setattr( runner_module, "use_symmetric_memory", lambda *args, **kwargs: nullcontext() @@ -412,7 +414,8 @@ def test_gpt_oss_sm120_padding_layout_and_kernel(monkeypatch, stated_tp_group): None, StandardTopKOutput(topk_weights, topk_ids.to(torch.int32), logits), ) - actual = method._apply_sm120_cutlass(layer, dispatch_output).hidden_states + with get_context().override_server_args(): + actual = method._apply_sm120_cutlass(layer, dispatch_output).hidden_states x_padded = torch.nn.functional.pad(x, (0, padded_hidden - hidden)) x_quant, x_scale = mxfp8_quantize( diff --git a/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py b/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py index 83e7b792ba..6eea6f9368 100644 --- a/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py +++ b/test/registered/unit/layers/quantization/test_mxfp4_sm90_cutlass.py @@ -356,6 +356,7 @@ def test_apply_sm90_cutlass_matches_flashinfer_direct( here we just verify that ``apply`` calls the kernel with the right arguments (incl. input padding + output trim).""" import sglang.srt.layers.moe.moe_runner.flashinfer_cutlass as fi_cutlass_mod + from sglang.srt.runtime_context import get_context # Bypass symmetric-memory / TP-group in the fused-func module, which is where # the kernel call lives. @@ -380,9 +381,10 @@ def test_apply_sm90_cutlass_matches_flashinfer_direct( method = _build_method(num_experts, hidden, inter, layer=layer) method._process_weights_for_sm90_cutlass(layer) - out_sglang = method._apply_sm90_cutlass( - layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) - ).hidden_states + with get_context().override_server_args(): + out_sglang = method._apply_sm90_cutlass( + layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) + ).hidden_states # ---- FlashInfer-direct reference using the same processed weights ---- K_pad = method._padded_hidden @@ -607,6 +609,7 @@ def test_apply_sm90_humming_matches_flashinfer_direct( ): """SGLang must forward the five Humming scales and enable the new kernel.""" import sglang.srt.layers.moe.moe_runner.flashinfer_cutlass as fi_cutlass_mod + from sglang.srt.runtime_context import get_context monkeypatch.setattr( fi_cutlass_mod, "use_symmetric_memory", lambda *a, **kw: nullcontext() @@ -631,9 +634,10 @@ def test_apply_sm90_humming_matches_flashinfer_direct( layer.moe_ep_rank = ep_rank method = _build_method(num_experts, hidden, inter, layer=layer, use_humming=True) method._process_weights_for_sm90_cutlass(layer) - out_sglang = method._apply_sm90_cutlass( - layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) - ).hidden_states + with get_context().override_server_args(): + out_sglang = method._apply_sm90_cutlass( + layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) + ).hidden_states padded_hidden = method._padded_hidden x_ref = ( @@ -790,9 +794,10 @@ def test_dsv4_apply_matches_flashinfer_direct( method.process_weights_after_loading(layer) - out_sglang = method.apply( - layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) - ).hidden_states + with get_context().override_server_args(): + out_sglang = method.apply( + layer, _MockDispatchOutput(x.clone(), topk_w, topk_i) + ).hidden_states # ---- Direct FlashInfer reference ---- w13_s_u8 = w31_s.view(torch.uint8)