diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index 13fd2b2a68..e34cd3373a 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -13,8 +13,8 @@ from sglang.srt.arg_groups.overrides import ( _a2a_fusion_adjustments, _moe_runner_backend_quant_constraints, _moe_runner_fusion_disable, - cutedsl_moe_max_num_tokens, declare_resolution, + flashinfer_a2a_max_dispatch_tokens_per_rank, max_prefill_buffer_tokens, max_speculative_num_draft_tokens, model_config_of, @@ -552,6 +552,29 @@ def required_deepep_v2_prefill_tokens_per_rank(server_args: Any) -> int: ) +def _decode_graph_dispatch_tokens_per_rank(server_args: Any) -> tuple[int, int]: + """(requests, tokens per request) of the largest decode/verify CUDA graph on + one DP rank; (0, 1) when this instance captures no decode graph.""" + view = resolved_view(server_args) + cg_config = view.cuda_graph_config + if ( + view.disaggregation_mode == "prefill" + or cg_config is None + or cg_config.decode.backend == Backend.DISABLED + ): + return 0, 1 + graph_bs = cg_config.decode.max_bs or 0 + if view.max_running_requests is not None: + per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size) + graph_bs = min(graph_bs, per_rank_pool_bs) + tokens_per_req = ( + max_speculative_num_draft_tokens(server_args) or 1 + if view.speculative_algorithm + else 1 + ) + return graph_bs, tokens_per_req + + def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None: """Check the configured prefill and decode-graph buffer bounds.""" view = resolved_view(server_args) @@ -570,21 +593,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None: "--max-prefill-tokens." ) - if view.disaggregation_mode == "prefill": - return - decode_config = getattr(view.cuda_graph_config, "decode", None) - if decode_config is None or decode_config.backend == Backend.DISABLED: - return - - graph_bs = decode_config.max_bs or 0 - if view.max_running_requests is not None: - per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size) - graph_bs = min(graph_bs, per_rank_pool_bs) - tokens_per_req = ( - max_speculative_num_draft_tokens(server_args) or 1 - if view.speculative_algorithm - else 1 - ) + graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args) graph_tokens = graph_bs * tokens_per_req if graph_tokens > capacity: raise ValueError( @@ -621,42 +630,46 @@ def validate_deepep_v2_model_architecture(server_args: Any) -> None: ) -def validate_cutedsl_a2a_token_budget(server_args: Any): - """Fail fast if the FlashInfer A2A dispatcher workspace cannot cover the - largest CuteDSL MoE forward. Runs after speculative decoding is resolved - so cutedsl_moe_max_num_tokens() sees the final num_tokens_per_req.""" - cfg = resolving_view(server_args) - +def _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args: Any) -> int: + """Largest token count one DP rank dispatches through FlashInfer A2A in one + forward: eager prefill chunks, prefill CUDA graphs, and decode/verify graphs. + """ view = resolved_view(server_args) - if not ( - view.moe_a2a_backend == "flashinfer" - and view.moe_runner_backend == "flashinfer_cutedsl" - and cfg.max_prefill_tokens > 0 - and cfg.disaggregation_mode != "decode" - ): + tokens = 0 + cg_config = view.cuda_graph_config + if view.disaggregation_mode != "decode": + tokens = max_prefill_buffer_tokens(server_args) + if cg_config is not None and cg_config.prefill.backend != Backend.DISABLED: + tokens = max(tokens, cg_config.prefill.max_bs or 0) + graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args) + return max(tokens, graph_bs * tokens_per_req) + + +def validate_flashinfer_a2a_token_budget(server_args: Any) -> None: + """Fail fast if the per-rank FlashInfer A2A workspace cannot hold the + largest forward; at runtime the dispatcher asserts on the first such batch.""" + view = resolved_view(server_args) + if view.moe_a2a_backend != "flashinfer": return - required_tokens = cutedsl_moe_max_num_tokens(server_args) - max_dispatch_tokens_per_rank = ( - envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() or 1024 - ) - max_cutedsl_tokens = max_dispatch_tokens_per_rank * view.ep_size - if max_cutedsl_tokens < required_tokens: - required_per_rank = (required_tokens + view.ep_size - 1) // view.ep_size + if view.disaggregation_mode != "decode" and view.chunked_prefill_size <= 0: raise ValueError( - "FlashInfer MoE A2A with flashinfer_cutedsl requires " - "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK * " - "ep_size to cover the largest CuteDSL MoE forward " - f"({required_tokens} tokens). Otherwise the FlashInfer " - "dispatcher can crash at runtime with " - "`ValueError: num_tokens (...) exceeds max_num_tokens (...)`. " - "Current values: " - f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK=" - f"{max_dispatch_tokens_per_rank}, ep_size={view.ep_size}, " - f"capacity={max_cutedsl_tokens}, required={required_tokens}. " - f"Set `export " - f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK=" - f"{required_per_rank}` or lower the relevant limit " - f"(e.g. --max-prefill-tokens) to <= {max_cutedsl_tokens}." + "FlashInfer MoE A2A requires chunked prefill: eager prefill " + "dispatches the whole per-rank extend batch through a fixed-size " + "workspace, so --chunked-prefill-size must be > 0." + ) + + required = _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args) + capacity = flashinfer_a2a_max_dispatch_tokens_per_rank( + max_prefill_buffer_tokens(server_args) + ) + if required > capacity: + raise ValueError( + "FlashInfer MoE A2A per-rank token budget exceeds its workspace: " + f"required={required} tokens per rank (prefill chunk, prefill " + f"graph, or decode/verify graph), capacity={capacity}. Set " + f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK>={required}, " + "or unset it to size from --chunked-prefill-size, or lower " + "--chunked-prefill-size / the CUDA graph max batch sizes." ) diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index fee2c4c53c..597c19f098 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1762,28 +1762,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool: return True -def cutedsl_moe_max_num_tokens(server_args: Any) -> int: - """Largest number of tokens a single forward routes through a CuteDSL - MoE layer on one (DP) rank. Single source of truth for both the - standard-allgather wrapper buffers and the FlashInfer A2A dispatcher - budget. Max over the prefill (max_prefill_tokens), piecewise-prefill - capture, and decode/verify bounds; num_tokens_per_req is - speculative_num_draft_tokens under speculative decoding, else 1. - """ - cfg = resolving_view(server_args) - if cfg.speculative_algorithm: - num_tokens_per_req = cfg.speculative_num_draft_tokens or 1 - else: - num_tokens_per_req = 1 - prefill_tokens = cfg.max_prefill_tokens - cg_config = cfg.cuda_graph_config - if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE: - prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0) - decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0 - decode_tokens = decode_max_bs * num_tokens_per_req - return max(prefill_tokens, decode_tokens) - - def max_prefill_buffer_tokens(server_args: Any) -> int: """Prefill-buffer ceiling: chunked_prefill_size, except PP dynamic chunking can grow chunks toward max_prefill_tokens and probe at 1.25x. @@ -1805,6 +1783,17 @@ def max_prefill_buffer_tokens(server_args: Any) -> int: return prefill_buffer_ceiling_of(record, tokens) +def flashinfer_a2a_max_dispatch_tokens_per_rank(prefill_buffer_tokens: int) -> int: + """Per-rank token capacity of the FlashInfer A2A workspace; the allocation + and the startup budget check must both read it from here.""" + configured = envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() + if configured is not None: + return configured + # max_running_requests is unresolved at model construction; 4096 covers the + # per-DP-worker cap resolve_max_num_reqs applies, and _dummy_run. + return max(prefill_buffer_tokens, 4096) + + def mamba_cache_chunk_size(server_args: Any) -> int: # For mamba cache with extra buffer, the chunk size is the max of FLA_CHUNK_SIZE # (or mamba_chunk_size if it is defined in the model's config) and page_size. diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index 32492d7a17..0ff314930f 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -269,9 +269,9 @@ def run_resolution_pipeline(server_args: Any) -> None: from sglang.srt.arg_groups.moe_hook import ( handle_a2a_moe, handle_moe_kernel_config, - validate_cutedsl_a2a_token_budget, validate_deepep_v2_dispatch_token_budget, validate_deepep_v2_speculative_draft, + validate_flashinfer_a2a_token_budget, ) run_hook(handle_moe_kernel_config, server_args) @@ -297,8 +297,8 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_layernorm_sp, server_args) - # Validate the CuteDSL A2A token budget now that num_tokens_per_req is final. - run_hook(validate_cutedsl_a2a_token_budget, server_args) + # Validate the FlashInfer A2A token budget now that draft-token counts are final. + run_hook(validate_flashinfer_a2a_token_budget, server_args) from sglang.srt.arg_groups.mega_moe_hook import ( validate_mega_moe_token_budget_for_model, diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index 4e1ba38f6d..cc401b45ca 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -77,7 +77,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( "validate_experimental_sgl_marlin", "handle_speculative_decoding", "handle_layernorm_sp", - "validate_cutedsl_a2a_token_budget", + "validate_flashinfer_a2a_token_budget", "validate_mega_moe_token_budget_for_model", "handle_load_format", "handle_encoder_disaggregation", diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index f25a9c5424..a0c27a51f5 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1073,7 +1073,7 @@ class Envs: # Default to the pick from flashinfer SGLANG_FLASHINFER_WORKSPACE_SIZE = EnvInt(384 * 1024 * 1024) # Per-rank dispatch capacity of the FlashInfer MoE A2A dispatcher. Unset - # means each call site keeps its own default. + # sizes it from the per-rank prefill chunk, with a 4096 floor. SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(None) # FlashInfer MegaMOE (generic moe_ep.MoEEpMegaLayer backend). Sizes the # per-rank symmetric workspace; must be >= the largest padded per-rank batch diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 7ed93a9942..312ce58095 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -222,7 +222,6 @@ def create_moe_dispatcher( num_experts=moe_runner_config.num_experts, num_local_experts=moe_runner_config.num_local_experts, hidden_size=moe_runner_config.hidden_size, - moe_runner_config=moe_runner_config, ) else: raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}") diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py index ff312750af..39a0ef0bd5 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutedsl.py @@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4( @register_fused_func("flashinfer", "flashinfer_cutedsl") def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4( - dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput, + dispatch_output: FlashinferDispatchOutput, quant_info: CuteDslFp4MoeQuantInfo, runner_config: MoeRunnerConfig, -) -> FlashinferCombineInput | StandardCombineInput: +) -> FlashinferCombineInput: """CuteDSL fused func for flashinfer alltoall dispatcher. Two cases depending on whether the dispatcher did FP4 quantization: @@ -527,10 +527,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4( from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferCombineInput, ) - from sglang.srt.layers.moe.token_dispatcher.standard import ( - StandardCombineInput, - StandardDispatchOutput, - ) from sglang.srt.layers.moe.topk import TopKOutputChecker from sglang.srt.layers.quantization.fp4_utils import fp4_quantize @@ -609,9 +605,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4( # Note: output contains routed expert results; shared_expert is handled separately - if isinstance(dispatch_output, StandardDispatchOutput): - return StandardCombineInput(hidden_states=output) - # Write into pre-allocated workspace buffer if available if dispatch_output.moe_output is not None: dispatch_output.moe_output.copy_(output) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 5895f2c786..46f5666cdf 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -1683,23 +1683,18 @@ def fused_experts_none_to_flashinfer_trtllm_routed( @register_fused_func("flashinfer", "flashinfer_trtllm") @register_fused_func("flashinfer", "flashinfer_trtllm_routed") def fused_experts_flashinfer_to_flashinfer_trtllm( - dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput, + dispatch_output: FlashinferDispatchOutput, quant_info: MoeQuantInfo, runner_config: MoeRunnerConfig, -) -> FlashinferCombineInput | StandardCombineInput: +) -> FlashinferCombineInput: """Fused function for FlashInfer A2A + TRT-LLM Gen MoE. - Both one-sided decode and AG+RS prefill materialize routing IDs and weights, - so the regular and explicitly-routed backend names enter TRT-LLM's routed - kernel. The dispatch formats share the fields consumed by the implementation; - only the combine wrapper differs. + A2A dispatch materializes routing IDs and weights, so the regular and + explicitly-routed backend names both enter TRT-LLM's routed kernel. """ from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( FlashinferCombineInput, ) - from sglang.srt.layers.moe.token_dispatcher.standard import ( - StandardDispatchOutput, - ) if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo): result = fused_experts_none_to_flashinfer_trtllm_fp4( @@ -1755,8 +1750,6 @@ def fused_experts_flashinfer_to_flashinfer_trtllm( "FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine " f"payload, got {result.hidden_states.dtype}." ) - if isinstance(dispatch_output, StandardDispatchOutput): - return result return FlashinferCombineInput(hidden_states=result.hidden_states) diff --git a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py index f248be4d78..7efbdc8407 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py @@ -6,10 +6,9 @@ from typing import NamedTuple, Optional import torch from sglang.kernels.kernel_api_logging import debug_kernel_api -from sglang.srt.environ import envs +from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank from sglang.srt.layers.dp_attention import ( get_dp_global_num_tokens, - get_is_extend_in_batch, is_dp_attention_enabled, ) from sglang.srt.layers.moe.token_dispatcher import ( @@ -22,11 +21,6 @@ from sglang.srt.layers.moe.token_dispatcher import ( from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import ( TorchDistributedCommBackend, ) -from sglang.srt.layers.moe.token_dispatcher.standard import ( - StandardCombineInput, - StandardDispatcher, - StandardDispatchOutput, -) from sglang.srt.layers.moe.topk import ( StandardTopKOutput, TopKOutput, @@ -37,7 +31,12 @@ from sglang.srt.layers.moe.utils import ( get_flashinfer_a2a_dispatch_type, get_moe_runner_backend, ) -from sglang.srt.runtime_context import get_flags, get_parallel, get_schedule, get_spec +from sglang.srt.runtime_context import ( + get_flags, + get_parallel, + get_spec, + max_prefill_buffer_tokens, +) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm try: @@ -67,19 +66,6 @@ def _max_tokens_per_scattered_source( return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size -def _scattered_source_token_counts( - dp_global_num_tokens: list[int], attn_tp_size: int -) -> list[int]: - assert attn_tp_size > 0 - counts = [] - for num_tokens in dp_global_num_tokens: - base, remainder = divmod(num_tokens, attn_tp_size) - counts.extend( - base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size) - ) - return counts - - def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int: slot = int(speculative) return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT @@ -127,7 +113,6 @@ class FlashinferDispatcher(BaseDispatcher): num_local_experts: int = None, # Unused hidden_size: int = None, params_dtype: torch.dtype = None, # Unused - moe_runner_config=None, ): super().__init__() if not use_flashinfer: @@ -155,38 +140,17 @@ class FlashinferDispatcher(BaseDispatcher): # TODO: Can other moe runners use payload_in_workspace too? self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass() self.dispatch_type = get_flashinfer_a2a_dispatch_type() - if moe_runner_config is None: - from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig - - moe_runner_config = MoeRunnerConfig( - num_experts=num_experts, - num_local_experts=num_local_experts, - hidden_size=hidden_size, - top_k=router_topk, - ) - self.prefill_dispatcher = StandardDispatcher(moe_runner_config) # FlashInfer sizes the workspace from the maximum dispatched tokens per # EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(), # which reserves ep_size * max_num_tokens * payload bytes, and the C++ # dispatch op's epSize * runtimeMaxTokensPerRank payload buffer. # - # The workspace must fit both: - # (a) the fattest prefill batch (bounded by chunked_prefill_size), and - # (b) the largest decode batch (bounded by max_running_requests, which - # resolve_max_num_reqs caps at 4096 per DP worker). - # max_running_requests is not yet resolved at model-construction time, - # so we use 4096 as a floor to cover decode batches and _dummy_run - # (which warms up at batch_size = req_to_token_pool.size). - cps = get_schedule().chunked_prefill_size - default_max_tokens = max(cps if cps and cps > 0 else 4096, 4096) - configured_max_tokens = ( - envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() - ) - self.max_num_tokens = ( - configured_max_tokens - if configured_max_tokens is not None - else default_max_tokens + # The workspace must fit both the largest eager prefill chunk on one rank + # (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest + # decode batch. + self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank( + max_prefill_buffer_tokens() ) speculative_algo = SpeculativeAlgorithm.from_string( get_spec().speculative_algorithm @@ -245,8 +209,7 @@ class FlashinferDispatcher(BaseDispatcher): is_speculative_model = get_flags().moe.speculative_context def make_moe_a2a() -> MoeAlltoAll: - # Target and draft decode graphs can coexist; prefill/mixed extend use - # AG+RS and do not lease MNNVL A2A workspaces. + # Target and draft decode graphs can coexist. workspace_size = _workspace_size_for_namespace( self.workspace_size, speculative=is_speculative_model, @@ -262,59 +225,6 @@ class FlashinferDispatcher(BaseDispatcher): self.moe_a2a = make_moe_a2a() - def set_quant_config(self, quant_config: dict) -> None: - super().set_quant_config(quant_config) - self.prefill_dispatcher.set_quant_config(quant_config) - - def _dispatch_prefill_allgather( - self, hidden_states: torch.Tensor, topk_output: TopKOutput - ) -> StandardDispatchOutput: - # Eager extend can overlap another stream, so use BF16 all-gatherv instead - # of reusing pure-decode A2A signal state across streams. - - if hidden_states.dtype != torch.bfloat16: - raise TypeError( - "FlashInfer WideEP prefill AG requires BF16 hidden states, got " - f"{hidden_states.dtype}." - ) - if TopKOutputChecker.format_is_bypassed(topk_output): - topk_output = topk_output.to_standard() - if not TopKOutputChecker.format_is_standard(topk_output): - raise TypeError( - "FlashInfer WideEP prefill AG requires materialized top-k " - f"routing, got {type(topk_output).__name__}." - ) - - dp_global = get_dp_global_num_tokens() - if dp_global is None: - source_sizes = [hidden_states.shape[0]] * self.ep_size - else: - source_sizes = _scattered_source_token_counts( - dp_global, get_parallel().attn_tp_size - ) - if len(source_sizes) != self.ep_size: - raise RuntimeError( - "FlashInfer WideEP prefill AG source geometry does not match " - f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}." - ) - if source_sizes[self.ep_rank] != hidden_states.shape[0]: - raise RuntimeError( - "FlashInfer WideEP prefill AG local source geometry mismatch: " - f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != " - f"hidden_states.shape[0]={hidden_states.shape[0]}." - ) - - topk_ids = topk_output.topk_ids.to(torch.int32) - hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv( - [hidden_states, topk_ids, topk_output.topk_weights], - sizes=source_sizes, - ) - self.prefill_source_sizes = source_sizes - return self.prefill_dispatcher.dispatch( - hidden_states, - StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits), - ) - def _effective_dispatch_type(self) -> FlashinferA2ADispatchType: if self.dispatch_type == FlashinferA2ADispatchType.NVFP4: global_scale = (self.quant_config or {}).get("input_global_scale", None) @@ -330,9 +240,7 @@ class FlashinferDispatcher(BaseDispatcher): @debug_kernel_api def dispatch( self, hidden_states: torch.Tensor, topk_output: TopKOutput - ) -> FlashinferDispatchOutput | StandardDispatchOutput: - if get_is_extend_in_batch(): - return self._dispatch_prefill_allgather(hidden_states, topk_output) + ) -> FlashinferDispatchOutput: self.active_moe_a2a = self.moe_a2a # Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16; # FP4 retains its packed wire path keyed by input_global_scale. @@ -509,23 +417,8 @@ class FlashinferDispatcher(BaseDispatcher): ) @debug_kernel_api - def combine( - self, combine_input: FlashinferCombineInput | StandardCombineInput - ) -> torch.Tensor: + def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor: hidden_states = combine_input.hidden_states - if combine_input.format == CombineInputFormat.STANDARD: - if hidden_states.dtype != torch.bfloat16: - raise TypeError( - "FlashInfer WideEP prefill RS requires BF16 expert output, " - f"got {hidden_states.dtype}." - ) - source_sizes = self.prefill_source_sizes - hidden_states = get_parallel().tp_group.reduce_scatterv( - hidden_states, sizes=source_sizes - ) - del self.prefill_source_sizes - return hidden_states - weight_dtype = self.quant_config.get("weight_dtype") runner_backend = get_moe_runner_backend() if ( diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 795a9c83b6..00ca5e8910 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -1991,13 +1991,12 @@ def exports_expert_balancedness_to_prometheus() -> bool: def cutedsl_moe_max_num_tokens() -> int: - """The CuteDSL A2A per-rank token budget. + """Largest token count one forward routes through a CuteDSL MoE layer on one + DP rank; sizes the standard-allgather wrapper, MegaMoE, and AR fusion buffers. Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so - this derives from the bags and follows a post-publish override; - ``overrides.cutedsl_moe_max_num_tokens`` is the pre-publish equivalent the - resolution pipeline uses. Max over the prefill bound, the piecewise-prefill - capture, and the decode/verify bound. + this follows a post-publish override. Max over the prefill bound, the + piecewise-prefill capture, and the decode/verify bound. """ from sglang.srt.model_executor.cuda_graph_config import Backend diff --git a/test/registered/unit/layers/moe/test_flashinfer_a2a_wide_ep.py b/test/registered/unit/layers/moe/test_flashinfer_a2a_wide_ep.py index 16197d8432..82f9ac39f2 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_a2a_wide_ep.py +++ b/test/registered/unit/layers/moe/test_flashinfer_a2a_wide_ep.py @@ -9,7 +9,6 @@ from sglang.srt.layers.moe.moe_runner.base import ( ) from sglang.srt.layers.moe.token_dispatcher.flashinfer import ( _max_tokens_per_scattered_source, - _scattered_source_token_counts, _workspace_size_for_namespace, ) from sglang.srt.layers.quantization import fp8 # noqa: F401 @@ -45,13 +44,6 @@ class TestFlashinferA2AWideEPPlumbing(CustomTestCase): } self.assertEqual(sizes, {4096, 4224}) - def test_prefill_ag_expands_dp_counts_to_physical_source_ranks(self): - self.assertEqual( - _scattered_source_token_counts([7, 3], 4), - [2, 2, 2, 1, 1, 1, 1, 0], - ) - self.assertEqual(_scattered_source_token_counts([4] * 16, 1), [4] * 16) - def test_deepgemm_dispatch_marks_empty_expert_lanes_invalid(self): topk_ids = torch.tensor([-1, 0, -1, 1], dtype=torch.int32, device="cuda") masked_m, src2dst = fused_moe_dispatch_index( diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 3dfd227254..be58ab769c 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -52,7 +52,6 @@ from sglang.srt.arg_groups.moe_hook import ( validate_deepep_v2_speculative_draft, ) from sglang.srt.arg_groups.overrides import ( - cutedsl_moe_max_num_tokens, max_speculative_num_draft_tokens, resolution_result, ) @@ -2886,67 +2885,6 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): ) -class TestCutedslMoeMaxNumTokens(CustomTestCase): - """The shared CuteDSL MoE per-forward token bound. Fields are set directly - to exercise the math independently of __post_init__ resolution. - - cg-refactor: the legacy disable_piecewise_cuda_graph / - piecewise_cuda_graph_max_tokens / cuda_graph_max_bs fields were - consolidated into cuda_graph_config; the helper accepts the legacy - kwarg names for test readability and translates them to the per-phase - dataclasses. - """ - - def _args(self, **overrides): - server_args = ServerArgs(model_path="dummy") - fields = dict( - speculative_algorithm=None, - speculative_num_draft_tokens=None, - max_prefill_tokens=16384, - disable_piecewise_cuda_graph=False, - piecewise_cuda_graph_max_tokens=2048, - cuda_graph_max_bs=512, - ) - fields.update(overrides) - disable_piecewise = fields.pop("disable_piecewise_cuda_graph") - piecewise_max = fields.pop("piecewise_cuda_graph_max_tokens") - cg_max_bs = fields.pop("cuda_graph_max_bs") - for key, value in fields.items(): - setattr(server_args, key, value) - server_args.cuda_graph_config = CudaGraphConfig( - decode=PhaseConfig(backend=Backend.FULL, max_bs=cg_max_bs), - prefill=PhaseConfig( - backend=( - Backend.DISABLED if disable_piecewise else Backend.TC_PIECEWISE - ), - max_bs=piecewise_max, - tc_compiler="eager", - ), - ) - return server_args - - def test_prefill_dominates_in_default_config(self): - self.assertEqual(cutedsl_moe_max_num_tokens(self._args()), 16384) - - def test_speculative_decoding_scales_decode_bound(self): - # decode bound 512 * 8 dominates the small prefill/piecewise bounds - args = self._args( - max_prefill_tokens=512, - piecewise_cuda_graph_max_tokens=512, - speculative_algorithm="EAGLE", - speculative_num_draft_tokens=8, - ) - self.assertEqual(cutedsl_moe_max_num_tokens(args), 4096) - - def test_piecewise_bound_excluded_when_disabled(self): - args = self._args( - max_prefill_tokens=512, - disable_piecewise_cuda_graph=True, - cuda_graph_max_bs=64, - ) - self.assertEqual(cutedsl_moe_max_num_tokens(args), 512) - - class TestSamplingBackendTokenOracleEnvGate(CustomTestCase): """The 'token_oracle' choice is gated on SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.