[MoE] Use FlashInfer A2A for prefill instead of AG+RS (#39818)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Brayden Zhong
2026-10-03 21:09:00 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent c12bc635ca
commit a3275295c4
12 changed files with 103 additions and 294 deletions
+62 -49
View File
@@ -13,8 +13,8 @@ from sglang.srt.arg_groups.overrides import (
_a2a_fusion_adjustments,
_moe_runner_backend_quant_constraints,
_moe_runner_fusion_disable,
cutedsl_moe_max_num_tokens,
declare_resolution,
flashinfer_a2a_max_dispatch_tokens_per_rank,
max_prefill_buffer_tokens,
max_speculative_num_draft_tokens,
model_config_of,
@@ -552,6 +552,29 @@ def required_deepep_v2_prefill_tokens_per_rank(server_args: Any) -> int:
)
def _decode_graph_dispatch_tokens_per_rank(server_args: Any) -> tuple[int, int]:
"""(requests, tokens per request) of the largest decode/verify CUDA graph on
one DP rank; (0, 1) when this instance captures no decode graph."""
view = resolved_view(server_args)
cg_config = view.cuda_graph_config
if (
view.disaggregation_mode == "prefill"
or cg_config is None
or cg_config.decode.backend == Backend.DISABLED
):
return 0, 1
graph_bs = cg_config.decode.max_bs or 0
if view.max_running_requests is not None:
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
graph_bs = min(graph_bs, per_rank_pool_bs)
tokens_per_req = (
max_speculative_num_draft_tokens(server_args) or 1
if view.speculative_algorithm
else 1
)
return graph_bs, tokens_per_req
def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
"""Check the configured prefill and decode-graph buffer bounds."""
view = resolved_view(server_args)
@@ -570,21 +593,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
"--max-prefill-tokens."
)
if view.disaggregation_mode == "prefill":
return
decode_config = getattr(view.cuda_graph_config, "decode", None)
if decode_config is None or decode_config.backend == Backend.DISABLED:
return
graph_bs = decode_config.max_bs or 0
if view.max_running_requests is not None:
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
graph_bs = min(graph_bs, per_rank_pool_bs)
tokens_per_req = (
max_speculative_num_draft_tokens(server_args) or 1
if view.speculative_algorithm
else 1
)
graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args)
graph_tokens = graph_bs * tokens_per_req
if graph_tokens > capacity:
raise ValueError(
@@ -621,42 +630,46 @@ def validate_deepep_v2_model_architecture(server_args: Any) -> None:
)
def validate_cutedsl_a2a_token_budget(server_args: Any):
"""Fail fast if the FlashInfer A2A dispatcher workspace cannot cover the
largest CuteDSL MoE forward. Runs after speculative decoding is resolved
so cutedsl_moe_max_num_tokens() sees the final num_tokens_per_req."""
cfg = resolving_view(server_args)
def _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args: Any) -> int:
"""Largest token count one DP rank dispatches through FlashInfer A2A in one
forward: eager prefill chunks, prefill CUDA graphs, and decode/verify graphs.
"""
view = resolved_view(server_args)
if not (
view.moe_a2a_backend == "flashinfer"
and view.moe_runner_backend == "flashinfer_cutedsl"
and cfg.max_prefill_tokens > 0
and cfg.disaggregation_mode != "decode"
):
tokens = 0
cg_config = view.cuda_graph_config
if view.disaggregation_mode != "decode":
tokens = max_prefill_buffer_tokens(server_args)
if cg_config is not None and cg_config.prefill.backend != Backend.DISABLED:
tokens = max(tokens, cg_config.prefill.max_bs or 0)
graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args)
return max(tokens, graph_bs * tokens_per_req)
def validate_flashinfer_a2a_token_budget(server_args: Any) -> None:
"""Fail fast if the per-rank FlashInfer A2A workspace cannot hold the
largest forward; at runtime the dispatcher asserts on the first such batch."""
view = resolved_view(server_args)
if view.moe_a2a_backend != "flashinfer":
return
required_tokens = cutedsl_moe_max_num_tokens(server_args)
max_dispatch_tokens_per_rank = (
envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() or 1024
)
max_cutedsl_tokens = max_dispatch_tokens_per_rank * view.ep_size
if max_cutedsl_tokens < required_tokens:
required_per_rank = (required_tokens + view.ep_size - 1) // view.ep_size
if view.disaggregation_mode != "decode" and view.chunked_prefill_size <= 0:
raise ValueError(
"FlashInfer MoE A2A with flashinfer_cutedsl requires "
"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK * "
"ep_size to cover the largest CuteDSL MoE forward "
f"({required_tokens} tokens). Otherwise the FlashInfer "
"dispatcher can crash at runtime with "
"`ValueError: num_tokens (...) exceeds max_num_tokens (...)`. "
"Current values: "
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK="
f"{max_dispatch_tokens_per_rank}, ep_size={view.ep_size}, "
f"capacity={max_cutedsl_tokens}, required={required_tokens}. "
f"Set `export "
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK="
f"{required_per_rank}` or lower the relevant limit "
f"(e.g. --max-prefill-tokens) to <= {max_cutedsl_tokens}."
"FlashInfer MoE A2A requires chunked prefill: eager prefill "
"dispatches the whole per-rank extend batch through a fixed-size "
"workspace, so --chunked-prefill-size must be > 0."
)
required = _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args)
capacity = flashinfer_a2a_max_dispatch_tokens_per_rank(
max_prefill_buffer_tokens(server_args)
)
if required > capacity:
raise ValueError(
"FlashInfer MoE A2A per-rank token budget exceeds its workspace: "
f"required={required} tokens per rank (prefill chunk, prefill "
f"graph, or decode/verify graph), capacity={capacity}. Set "
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK>={required}, "
"or unset it to size from --chunked-prefill-size, or lower "
"--chunked-prefill-size / the CUDA graph max batch sizes."
)
+11 -22
View File
@@ -1762,28 +1762,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
return True
def cutedsl_moe_max_num_tokens(server_args: Any) -> int:
"""Largest number of tokens a single forward routes through a CuteDSL
MoE layer on one (DP) rank. Single source of truth for both the
standard-allgather wrapper buffers and the FlashInfer A2A dispatcher
budget. Max over the prefill (max_prefill_tokens), piecewise-prefill
capture, and decode/verify bounds; num_tokens_per_req is
speculative_num_draft_tokens under speculative decoding, else 1.
"""
cfg = resolving_view(server_args)
if cfg.speculative_algorithm:
num_tokens_per_req = cfg.speculative_num_draft_tokens or 1
else:
num_tokens_per_req = 1
prefill_tokens = cfg.max_prefill_tokens
cg_config = cfg.cuda_graph_config
if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE:
prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0)
decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0
decode_tokens = decode_max_bs * num_tokens_per_req
return max(prefill_tokens, decode_tokens)
def max_prefill_buffer_tokens(server_args: Any) -> int:
"""Prefill-buffer ceiling: chunked_prefill_size, except PP dynamic
chunking can grow chunks toward max_prefill_tokens and probe at 1.25x.
@@ -1805,6 +1783,17 @@ def max_prefill_buffer_tokens(server_args: Any) -> int:
return prefill_buffer_ceiling_of(record, tokens)
def flashinfer_a2a_max_dispatch_tokens_per_rank(prefill_buffer_tokens: int) -> int:
"""Per-rank token capacity of the FlashInfer A2A workspace; the allocation
and the startup budget check must both read it from here."""
configured = envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get()
if configured is not None:
return configured
# max_running_requests is unresolved at model construction; 4096 covers the
# per-DP-worker cap resolve_max_num_reqs applies, and _dummy_run.
return max(prefill_buffer_tokens, 4096)
def mamba_cache_chunk_size(server_args: Any) -> int:
# For mamba cache with extra buffer, the chunk size is the max of FLA_CHUNK_SIZE
# (or mamba_chunk_size if it is defined in the model's config) and page_size.
+3 -3
View File
@@ -269,9 +269,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
handle_moe_kernel_config,
validate_cutedsl_a2a_token_budget,
validate_deepep_v2_dispatch_token_budget,
validate_deepep_v2_speculative_draft,
validate_flashinfer_a2a_token_budget,
)
run_hook(handle_moe_kernel_config, server_args)
@@ -297,8 +297,8 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_layernorm_sp, server_args)
# Validate the CuteDSL A2A token budget now that num_tokens_per_req is final.
run_hook(validate_cutedsl_a2a_token_budget, server_args)
# Validate the FlashInfer A2A token budget now that draft-token counts are final.
run_hook(validate_flashinfer_a2a_token_budget, server_args)
from sglang.srt.arg_groups.mega_moe_hook import (
validate_mega_moe_token_budget_for_model,
@@ -77,7 +77,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
"validate_experimental_sgl_marlin",
"handle_speculative_decoding",
"handle_layernorm_sp",
"validate_cutedsl_a2a_token_budget",
"validate_flashinfer_a2a_token_budget",
"validate_mega_moe_token_budget_for_model",
"handle_load_format",
"handle_encoder_disaggregation",
+1 -1
View File
@@ -1073,7 +1073,7 @@ class Envs:
# Default to the pick from flashinfer
SGLANG_FLASHINFER_WORKSPACE_SIZE = EnvInt(384 * 1024 * 1024)
# Per-rank dispatch capacity of the FlashInfer MoE A2A dispatcher. Unset
# means each call site keeps its own default.
# sizes it from the per-rank prefill chunk, with a 4096 floor.
SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(None)
# FlashInfer MegaMOE (generic moe_ep.MoEEpMegaLayer backend). Sizes the
# per-rank symmetric workspace; must be >= the largest padded per-rank batch
@@ -222,7 +222,6 @@ def create_moe_dispatcher(
num_experts=moe_runner_config.num_experts,
num_local_experts=moe_runner_config.num_local_experts,
hidden_size=moe_runner_config.hidden_size,
moe_runner_config=moe_runner_config,
)
else:
raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}")
@@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4(
@register_fused_func("flashinfer", "flashinfer_cutedsl")
def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
dispatch_output: FlashinferDispatchOutput,
quant_info: CuteDslFp4MoeQuantInfo,
runner_config: MoeRunnerConfig,
) -> FlashinferCombineInput | StandardCombineInput:
) -> FlashinferCombineInput:
"""CuteDSL fused func for flashinfer alltoall dispatcher.
Two cases depending on whether the dispatcher did FP4 quantization:
@@ -527,10 +527,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardCombineInput,
StandardDispatchOutput,
)
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.quantization.fp4_utils import fp4_quantize
@@ -609,9 +605,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
# Note: output contains routed expert results; shared_expert is handled separately
if isinstance(dispatch_output, StandardDispatchOutput):
return StandardCombineInput(hidden_states=output)
# Write into pre-allocated workspace buffer if available
if dispatch_output.moe_output is not None:
dispatch_output.moe_output.copy_(output)
@@ -1683,23 +1683,18 @@ def fused_experts_none_to_flashinfer_trtllm_routed(
@register_fused_func("flashinfer", "flashinfer_trtllm")
@register_fused_func("flashinfer", "flashinfer_trtllm_routed")
def fused_experts_flashinfer_to_flashinfer_trtllm(
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
dispatch_output: FlashinferDispatchOutput,
quant_info: MoeQuantInfo,
runner_config: MoeRunnerConfig,
) -> FlashinferCombineInput | StandardCombineInput:
) -> FlashinferCombineInput:
"""Fused function for FlashInfer A2A + TRT-LLM Gen MoE.
Both one-sided decode and AG+RS prefill materialize routing IDs and weights,
so the regular and explicitly-routed backend names enter TRT-LLM's routed
kernel. The dispatch formats share the fields consumed by the implementation;
only the combine wrapper differs.
A2A dispatch materializes routing IDs and weights, so the regular and
explicitly-routed backend names both enter TRT-LLM's routed kernel.
"""
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
FlashinferCombineInput,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardDispatchOutput,
)
if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo):
result = fused_experts_none_to_flashinfer_trtllm_fp4(
@@ -1755,8 +1750,6 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
"FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine "
f"payload, got {result.hidden_states.dtype}."
)
if isinstance(dispatch_output, StandardDispatchOutput):
return result
return FlashinferCombineInput(hidden_states=result.hidden_states)
@@ -6,10 +6,9 @@ from typing import NamedTuple, Optional
import torch
from sglang.kernels.kernel_api_logging import debug_kernel_api
from sglang.srt.environ import envs
from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank
from sglang.srt.layers.dp_attention import (
get_dp_global_num_tokens,
get_is_extend_in_batch,
is_dp_attention_enabled,
)
from sglang.srt.layers.moe.token_dispatcher import (
@@ -22,11 +21,6 @@ from sglang.srt.layers.moe.token_dispatcher import (
from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import (
TorchDistributedCommBackend,
)
from sglang.srt.layers.moe.token_dispatcher.standard import (
StandardCombineInput,
StandardDispatcher,
StandardDispatchOutput,
)
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
TopKOutput,
@@ -37,7 +31,12 @@ from sglang.srt.layers.moe.utils import (
get_flashinfer_a2a_dispatch_type,
get_moe_runner_backend,
)
from sglang.srt.runtime_context import get_flags, get_parallel, get_schedule, get_spec
from sglang.srt.runtime_context import (
get_flags,
get_parallel,
get_spec,
max_prefill_buffer_tokens,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
try:
@@ -67,19 +66,6 @@ def _max_tokens_per_scattered_source(
return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size
def _scattered_source_token_counts(
dp_global_num_tokens: list[int], attn_tp_size: int
) -> list[int]:
assert attn_tp_size > 0
counts = []
for num_tokens in dp_global_num_tokens:
base, remainder = divmod(num_tokens, attn_tp_size)
counts.extend(
base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size)
)
return counts
def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int:
slot = int(speculative)
return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT
@@ -127,7 +113,6 @@ class FlashinferDispatcher(BaseDispatcher):
num_local_experts: int = None, # Unused
hidden_size: int = None,
params_dtype: torch.dtype = None, # Unused
moe_runner_config=None,
):
super().__init__()
if not use_flashinfer:
@@ -155,38 +140,17 @@ class FlashinferDispatcher(BaseDispatcher):
# TODO: Can other moe runners use payload_in_workspace too?
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
if moe_runner_config is None:
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_size=hidden_size,
top_k=router_topk,
)
self.prefill_dispatcher = StandardDispatcher(moe_runner_config)
# FlashInfer sizes the workspace from the maximum dispatched tokens per
# EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(),
# which reserves ep_size * max_num_tokens * payload bytes, and the C++
# dispatch op's epSize * runtimeMaxTokensPerRank payload buffer.
#
# The workspace must fit both:
# (a) the fattest prefill batch (bounded by chunked_prefill_size), and
# (b) the largest decode batch (bounded by max_running_requests, which
# resolve_max_num_reqs caps at 4096 per DP worker).
# max_running_requests is not yet resolved at model-construction time,
# so we use 4096 as a floor to cover decode batches and _dummy_run
# (which warms up at batch_size = req_to_token_pool.size).
cps = get_schedule().chunked_prefill_size
default_max_tokens = max(cps if cps and cps > 0 else 4096, 4096)
configured_max_tokens = (
envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get()
)
self.max_num_tokens = (
configured_max_tokens
if configured_max_tokens is not None
else default_max_tokens
# The workspace must fit both the largest eager prefill chunk on one rank
# (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest
# decode batch.
self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank(
max_prefill_buffer_tokens()
)
speculative_algo = SpeculativeAlgorithm.from_string(
get_spec().speculative_algorithm
@@ -245,8 +209,7 @@ class FlashinferDispatcher(BaseDispatcher):
is_speculative_model = get_flags().moe.speculative_context
def make_moe_a2a() -> MoeAlltoAll:
# Target and draft decode graphs can coexist; prefill/mixed extend use
# AG+RS and do not lease MNNVL A2A workspaces.
# Target and draft decode graphs can coexist.
workspace_size = _workspace_size_for_namespace(
self.workspace_size,
speculative=is_speculative_model,
@@ -262,59 +225,6 @@ class FlashinferDispatcher(BaseDispatcher):
self.moe_a2a = make_moe_a2a()
def set_quant_config(self, quant_config: dict) -> None:
super().set_quant_config(quant_config)
self.prefill_dispatcher.set_quant_config(quant_config)
def _dispatch_prefill_allgather(
self, hidden_states: torch.Tensor, topk_output: TopKOutput
) -> StandardDispatchOutput:
# Eager extend can overlap another stream, so use BF16 all-gatherv instead
# of reusing pure-decode A2A signal state across streams.
if hidden_states.dtype != torch.bfloat16:
raise TypeError(
"FlashInfer WideEP prefill AG requires BF16 hidden states, got "
f"{hidden_states.dtype}."
)
if TopKOutputChecker.format_is_bypassed(topk_output):
topk_output = topk_output.to_standard()
if not TopKOutputChecker.format_is_standard(topk_output):
raise TypeError(
"FlashInfer WideEP prefill AG requires materialized top-k "
f"routing, got {type(topk_output).__name__}."
)
dp_global = get_dp_global_num_tokens()
if dp_global is None:
source_sizes = [hidden_states.shape[0]] * self.ep_size
else:
source_sizes = _scattered_source_token_counts(
dp_global, get_parallel().attn_tp_size
)
if len(source_sizes) != self.ep_size:
raise RuntimeError(
"FlashInfer WideEP prefill AG source geometry does not match "
f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}."
)
if source_sizes[self.ep_rank] != hidden_states.shape[0]:
raise RuntimeError(
"FlashInfer WideEP prefill AG local source geometry mismatch: "
f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != "
f"hidden_states.shape[0]={hidden_states.shape[0]}."
)
topk_ids = topk_output.topk_ids.to(torch.int32)
hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv(
[hidden_states, topk_ids, topk_output.topk_weights],
sizes=source_sizes,
)
self.prefill_source_sizes = source_sizes
return self.prefill_dispatcher.dispatch(
hidden_states,
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
)
def _effective_dispatch_type(self) -> FlashinferA2ADispatchType:
if self.dispatch_type == FlashinferA2ADispatchType.NVFP4:
global_scale = (self.quant_config or {}).get("input_global_scale", None)
@@ -330,9 +240,7 @@ class FlashinferDispatcher(BaseDispatcher):
@debug_kernel_api
def dispatch(
self, hidden_states: torch.Tensor, topk_output: TopKOutput
) -> FlashinferDispatchOutput | StandardDispatchOutput:
if get_is_extend_in_batch():
return self._dispatch_prefill_allgather(hidden_states, topk_output)
) -> FlashinferDispatchOutput:
self.active_moe_a2a = self.moe_a2a
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
# FP4 retains its packed wire path keyed by input_global_scale.
@@ -509,23 +417,8 @@ class FlashinferDispatcher(BaseDispatcher):
)
@debug_kernel_api
def combine(
self, combine_input: FlashinferCombineInput | StandardCombineInput
) -> torch.Tensor:
def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor:
hidden_states = combine_input.hidden_states
if combine_input.format == CombineInputFormat.STANDARD:
if hidden_states.dtype != torch.bfloat16:
raise TypeError(
"FlashInfer WideEP prefill RS requires BF16 expert output, "
f"got {hidden_states.dtype}."
)
source_sizes = self.prefill_source_sizes
hidden_states = get_parallel().tp_group.reduce_scatterv(
hidden_states, sizes=source_sizes
)
del self.prefill_source_sizes
return hidden_states
weight_dtype = self.quant_config.get("weight_dtype")
runner_backend = get_moe_runner_backend()
if (
+4 -5
View File
@@ -1991,13 +1991,12 @@ def exports_expert_balancedness_to_prometheus() -> bool:
def cutedsl_moe_max_num_tokens() -> int:
"""The CuteDSL A2A per-rank token budget.
"""Largest token count one forward routes through a CuteDSL MoE layer on one
DP rank; sizes the standard-allgather wrapper, MegaMoE, and AR fusion buffers.
Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so
this derives from the bags and follows a post-publish override;
``overrides.cutedsl_moe_max_num_tokens`` is the pre-publish equivalent the
resolution pipeline uses. Max over the prefill bound, the piecewise-prefill
capture, and the decode/verify bound.
this follows a post-publish override. Max over the prefill bound, the
piecewise-prefill capture, and the decode/verify bound.
"""
from sglang.srt.model_executor.cuda_graph_config import Backend