[MoE] Use FlashInfer A2A for prefill instead of AG+RS (#39818)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
c12bc635ca
commit
a3275295c4
@@ -13,8 +13,8 @@ from sglang.srt.arg_groups.overrides import (
|
||||
_a2a_fusion_adjustments,
|
||||
_moe_runner_backend_quant_constraints,
|
||||
_moe_runner_fusion_disable,
|
||||
cutedsl_moe_max_num_tokens,
|
||||
declare_resolution,
|
||||
flashinfer_a2a_max_dispatch_tokens_per_rank,
|
||||
max_prefill_buffer_tokens,
|
||||
max_speculative_num_draft_tokens,
|
||||
model_config_of,
|
||||
@@ -552,6 +552,29 @@ def required_deepep_v2_prefill_tokens_per_rank(server_args: Any) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _decode_graph_dispatch_tokens_per_rank(server_args: Any) -> tuple[int, int]:
|
||||
"""(requests, tokens per request) of the largest decode/verify CUDA graph on
|
||||
one DP rank; (0, 1) when this instance captures no decode graph."""
|
||||
view = resolved_view(server_args)
|
||||
cg_config = view.cuda_graph_config
|
||||
if (
|
||||
view.disaggregation_mode == "prefill"
|
||||
or cg_config is None
|
||||
or cg_config.decode.backend == Backend.DISABLED
|
||||
):
|
||||
return 0, 1
|
||||
graph_bs = cg_config.decode.max_bs or 0
|
||||
if view.max_running_requests is not None:
|
||||
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
|
||||
graph_bs = min(graph_bs, per_rank_pool_bs)
|
||||
tokens_per_req = (
|
||||
max_speculative_num_draft_tokens(server_args) or 1
|
||||
if view.speculative_algorithm
|
||||
else 1
|
||||
)
|
||||
return graph_bs, tokens_per_req
|
||||
|
||||
|
||||
def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
|
||||
"""Check the configured prefill and decode-graph buffer bounds."""
|
||||
view = resolved_view(server_args)
|
||||
@@ -570,21 +593,7 @@ def validate_deepep_v2_dispatch_token_budget(server_args: Any) -> None:
|
||||
"--max-prefill-tokens."
|
||||
)
|
||||
|
||||
if view.disaggregation_mode == "prefill":
|
||||
return
|
||||
decode_config = getattr(view.cuda_graph_config, "decode", None)
|
||||
if decode_config is None or decode_config.backend == Backend.DISABLED:
|
||||
return
|
||||
|
||||
graph_bs = decode_config.max_bs or 0
|
||||
if view.max_running_requests is not None:
|
||||
per_rank_pool_bs = max(1, view.max_running_requests // view.attn_dp_size)
|
||||
graph_bs = min(graph_bs, per_rank_pool_bs)
|
||||
tokens_per_req = (
|
||||
max_speculative_num_draft_tokens(server_args) or 1
|
||||
if view.speculative_algorithm
|
||||
else 1
|
||||
)
|
||||
graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args)
|
||||
graph_tokens = graph_bs * tokens_per_req
|
||||
if graph_tokens > capacity:
|
||||
raise ValueError(
|
||||
@@ -621,42 +630,46 @@ def validate_deepep_v2_model_architecture(server_args: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
def validate_cutedsl_a2a_token_budget(server_args: Any):
|
||||
"""Fail fast if the FlashInfer A2A dispatcher workspace cannot cover the
|
||||
largest CuteDSL MoE forward. Runs after speculative decoding is resolved
|
||||
so cutedsl_moe_max_num_tokens() sees the final num_tokens_per_req."""
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
def _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args: Any) -> int:
|
||||
"""Largest token count one DP rank dispatches through FlashInfer A2A in one
|
||||
forward: eager prefill chunks, prefill CUDA graphs, and decode/verify graphs.
|
||||
"""
|
||||
view = resolved_view(server_args)
|
||||
if not (
|
||||
view.moe_a2a_backend == "flashinfer"
|
||||
and view.moe_runner_backend == "flashinfer_cutedsl"
|
||||
and cfg.max_prefill_tokens > 0
|
||||
and cfg.disaggregation_mode != "decode"
|
||||
):
|
||||
tokens = 0
|
||||
cg_config = view.cuda_graph_config
|
||||
if view.disaggregation_mode != "decode":
|
||||
tokens = max_prefill_buffer_tokens(server_args)
|
||||
if cg_config is not None and cg_config.prefill.backend != Backend.DISABLED:
|
||||
tokens = max(tokens, cg_config.prefill.max_bs or 0)
|
||||
graph_bs, tokens_per_req = _decode_graph_dispatch_tokens_per_rank(server_args)
|
||||
return max(tokens, graph_bs * tokens_per_req)
|
||||
|
||||
|
||||
def validate_flashinfer_a2a_token_budget(server_args: Any) -> None:
|
||||
"""Fail fast if the per-rank FlashInfer A2A workspace cannot hold the
|
||||
largest forward; at runtime the dispatcher asserts on the first such batch."""
|
||||
view = resolved_view(server_args)
|
||||
if view.moe_a2a_backend != "flashinfer":
|
||||
return
|
||||
required_tokens = cutedsl_moe_max_num_tokens(server_args)
|
||||
max_dispatch_tokens_per_rank = (
|
||||
envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() or 1024
|
||||
)
|
||||
max_cutedsl_tokens = max_dispatch_tokens_per_rank * view.ep_size
|
||||
if max_cutedsl_tokens < required_tokens:
|
||||
required_per_rank = (required_tokens + view.ep_size - 1) // view.ep_size
|
||||
if view.disaggregation_mode != "decode" and view.chunked_prefill_size <= 0:
|
||||
raise ValueError(
|
||||
"FlashInfer MoE A2A with flashinfer_cutedsl requires "
|
||||
"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK * "
|
||||
"ep_size to cover the largest CuteDSL MoE forward "
|
||||
f"({required_tokens} tokens). Otherwise the FlashInfer "
|
||||
"dispatcher can crash at runtime with "
|
||||
"`ValueError: num_tokens (...) exceeds max_num_tokens (...)`. "
|
||||
"Current values: "
|
||||
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK="
|
||||
f"{max_dispatch_tokens_per_rank}, ep_size={view.ep_size}, "
|
||||
f"capacity={max_cutedsl_tokens}, required={required_tokens}. "
|
||||
f"Set `export "
|
||||
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK="
|
||||
f"{required_per_rank}` or lower the relevant limit "
|
||||
f"(e.g. --max-prefill-tokens) to <= {max_cutedsl_tokens}."
|
||||
"FlashInfer MoE A2A requires chunked prefill: eager prefill "
|
||||
"dispatches the whole per-rank extend batch through a fixed-size "
|
||||
"workspace, so --chunked-prefill-size must be > 0."
|
||||
)
|
||||
|
||||
required = _required_flashinfer_a2a_dispatch_tokens_per_rank(server_args)
|
||||
capacity = flashinfer_a2a_max_dispatch_tokens_per_rank(
|
||||
max_prefill_buffer_tokens(server_args)
|
||||
)
|
||||
if required > capacity:
|
||||
raise ValueError(
|
||||
"FlashInfer MoE A2A per-rank token budget exceeds its workspace: "
|
||||
f"required={required} tokens per rank (prefill chunk, prefill "
|
||||
f"graph, or decode/verify graph), capacity={capacity}. Set "
|
||||
f"SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK>={required}, "
|
||||
"or unset it to size from --chunked-prefill-size, or lower "
|
||||
"--chunked-prefill-size / the CUDA graph max batch sizes."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1762,28 +1762,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def cutedsl_moe_max_num_tokens(server_args: Any) -> int:
|
||||
"""Largest number of tokens a single forward routes through a CuteDSL
|
||||
MoE layer on one (DP) rank. Single source of truth for both the
|
||||
standard-allgather wrapper buffers and the FlashInfer A2A dispatcher
|
||||
budget. Max over the prefill (max_prefill_tokens), piecewise-prefill
|
||||
capture, and decode/verify bounds; num_tokens_per_req is
|
||||
speculative_num_draft_tokens under speculative decoding, else 1.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.speculative_algorithm:
|
||||
num_tokens_per_req = cfg.speculative_num_draft_tokens or 1
|
||||
else:
|
||||
num_tokens_per_req = 1
|
||||
prefill_tokens = cfg.max_prefill_tokens
|
||||
cg_config = cfg.cuda_graph_config
|
||||
if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE:
|
||||
prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0)
|
||||
decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0
|
||||
decode_tokens = decode_max_bs * num_tokens_per_req
|
||||
return max(prefill_tokens, decode_tokens)
|
||||
|
||||
|
||||
def max_prefill_buffer_tokens(server_args: Any) -> int:
|
||||
"""Prefill-buffer ceiling: chunked_prefill_size, except PP dynamic
|
||||
chunking can grow chunks toward max_prefill_tokens and probe at 1.25x.
|
||||
@@ -1805,6 +1783,17 @@ def max_prefill_buffer_tokens(server_args: Any) -> int:
|
||||
return prefill_buffer_ceiling_of(record, tokens)
|
||||
|
||||
|
||||
def flashinfer_a2a_max_dispatch_tokens_per_rank(prefill_buffer_tokens: int) -> int:
|
||||
"""Per-rank token capacity of the FlashInfer A2A workspace; the allocation
|
||||
and the startup budget check must both read it from here."""
|
||||
configured = envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get()
|
||||
if configured is not None:
|
||||
return configured
|
||||
# max_running_requests is unresolved at model construction; 4096 covers the
|
||||
# per-DP-worker cap resolve_max_num_reqs applies, and _dummy_run.
|
||||
return max(prefill_buffer_tokens, 4096)
|
||||
|
||||
|
||||
def mamba_cache_chunk_size(server_args: Any) -> int:
|
||||
# For mamba cache with extra buffer, the chunk size is the max of FLA_CHUNK_SIZE
|
||||
# (or mamba_chunk_size if it is defined in the model's config) and page_size.
|
||||
|
||||
@@ -269,9 +269,9 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
from sglang.srt.arg_groups.moe_hook import (
|
||||
handle_a2a_moe,
|
||||
handle_moe_kernel_config,
|
||||
validate_cutedsl_a2a_token_budget,
|
||||
validate_deepep_v2_dispatch_token_budget,
|
||||
validate_deepep_v2_speculative_draft,
|
||||
validate_flashinfer_a2a_token_budget,
|
||||
)
|
||||
|
||||
run_hook(handle_moe_kernel_config, server_args)
|
||||
@@ -297,8 +297,8 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_layernorm_sp, server_args)
|
||||
|
||||
# Validate the CuteDSL A2A token budget now that num_tokens_per_req is final.
|
||||
run_hook(validate_cutedsl_a2a_token_budget, server_args)
|
||||
# Validate the FlashInfer A2A token budget now that draft-token counts are final.
|
||||
run_hook(validate_flashinfer_a2a_token_budget, server_args)
|
||||
|
||||
from sglang.srt.arg_groups.mega_moe_hook import (
|
||||
validate_mega_moe_token_budget_for_model,
|
||||
|
||||
@@ -77,7 +77,7 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
"validate_experimental_sgl_marlin",
|
||||
"handle_speculative_decoding",
|
||||
"handle_layernorm_sp",
|
||||
"validate_cutedsl_a2a_token_budget",
|
||||
"validate_flashinfer_a2a_token_budget",
|
||||
"validate_mega_moe_token_budget_for_model",
|
||||
"handle_load_format",
|
||||
"handle_encoder_disaggregation",
|
||||
|
||||
@@ -1073,7 +1073,7 @@ class Envs:
|
||||
# Default to the pick from flashinfer
|
||||
SGLANG_FLASHINFER_WORKSPACE_SIZE = EnvInt(384 * 1024 * 1024)
|
||||
# Per-rank dispatch capacity of the FlashInfer MoE A2A dispatcher. Unset
|
||||
# means each call site keeps its own default.
|
||||
# sizes it from the per-rank prefill chunk, with a 4096 floor.
|
||||
SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(None)
|
||||
# FlashInfer MegaMOE (generic moe_ep.MoEEpMegaLayer backend). Sizes the
|
||||
# per-rank symmetric workspace; must be >= the largest padded per-rank batch
|
||||
|
||||
@@ -222,7 +222,6 @@ def create_moe_dispatcher(
|
||||
num_experts=moe_runner_config.num_experts,
|
||||
num_local_experts=moe_runner_config.num_local_experts,
|
||||
hidden_size=moe_runner_config.hidden_size,
|
||||
moe_runner_config=moe_runner_config,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(f"Unsupported a2a backend: {a2a_backend}")
|
||||
|
||||
@@ -514,10 +514,10 @@ def fused_experts_none_to_flashinfer_cutedsl_fp4(
|
||||
|
||||
@register_fused_func("flashinfer", "flashinfer_cutedsl")
|
||||
def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
quant_info: CuteDslFp4MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
) -> FlashinferCombineInput | StandardCombineInput:
|
||||
) -> FlashinferCombineInput:
|
||||
"""CuteDSL fused func for flashinfer alltoall dispatcher.
|
||||
|
||||
Two cases depending on whether the dispatcher did FP4 quantization:
|
||||
@@ -527,10 +527,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardCombineInput,
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import TopKOutputChecker
|
||||
from sglang.srt.layers.quantization.fp4_utils import fp4_quantize
|
||||
|
||||
@@ -609,9 +605,6 @@ def fused_experts_flashinfer_to_flashinfer_cutedsl_fp4(
|
||||
|
||||
# Note: output contains routed expert results; shared_expert is handled separately
|
||||
|
||||
if isinstance(dispatch_output, StandardDispatchOutput):
|
||||
return StandardCombineInput(hidden_states=output)
|
||||
|
||||
# Write into pre-allocated workspace buffer if available
|
||||
if dispatch_output.moe_output is not None:
|
||||
dispatch_output.moe_output.copy_(output)
|
||||
|
||||
@@ -1683,23 +1683,18 @@ def fused_experts_none_to_flashinfer_trtllm_routed(
|
||||
@register_fused_func("flashinfer", "flashinfer_trtllm")
|
||||
@register_fused_func("flashinfer", "flashinfer_trtllm_routed")
|
||||
def fused_experts_flashinfer_to_flashinfer_trtllm(
|
||||
dispatch_output: FlashinferDispatchOutput | StandardDispatchOutput,
|
||||
dispatch_output: FlashinferDispatchOutput,
|
||||
quant_info: MoeQuantInfo,
|
||||
runner_config: MoeRunnerConfig,
|
||||
) -> FlashinferCombineInput | StandardCombineInput:
|
||||
) -> FlashinferCombineInput:
|
||||
"""Fused function for FlashInfer A2A + TRT-LLM Gen MoE.
|
||||
|
||||
Both one-sided decode and AG+RS prefill materialize routing IDs and weights,
|
||||
so the regular and explicitly-routed backend names enter TRT-LLM's routed
|
||||
kernel. The dispatch formats share the fields consumed by the implementation;
|
||||
only the combine wrapper differs.
|
||||
A2A dispatch materializes routing IDs and weights, so the regular and
|
||||
explicitly-routed backend names both enter TRT-LLM's routed kernel.
|
||||
"""
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer import (
|
||||
FlashinferCombineInput,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
|
||||
if isinstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo):
|
||||
result = fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
@@ -1755,8 +1750,6 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
|
||||
"FlashInfer A2A + TRT-LLM Gen FP8 MoE must return a BF16 combine "
|
||||
f"payload, got {result.hidden_states.dtype}."
|
||||
)
|
||||
if isinstance(dispatch_output, StandardDispatchOutput):
|
||||
return result
|
||||
return FlashinferCombineInput(hidden_states=result.hidden_states)
|
||||
|
||||
|
||||
|
||||
@@ -6,10 +6,9 @@ from typing import NamedTuple, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.kernel_api_logging import debug_kernel_api
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.arg_groups.overrides import flashinfer_a2a_max_dispatch_tokens_per_rank
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
get_dp_global_num_tokens,
|
||||
get_is_extend_in_batch,
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher import (
|
||||
@@ -22,11 +21,6 @@ from sglang.srt.layers.moe.token_dispatcher import (
|
||||
from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import (
|
||||
TorchDistributedCommBackend,
|
||||
)
|
||||
from sglang.srt.layers.moe.token_dispatcher.standard import (
|
||||
StandardCombineInput,
|
||||
StandardDispatcher,
|
||||
StandardDispatchOutput,
|
||||
)
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutput,
|
||||
TopKOutput,
|
||||
@@ -37,7 +31,12 @@ from sglang.srt.layers.moe.utils import (
|
||||
get_flashinfer_a2a_dispatch_type,
|
||||
get_moe_runner_backend,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_flags, get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.runtime_context import (
|
||||
get_flags,
|
||||
get_parallel,
|
||||
get_spec,
|
||||
max_prefill_buffer_tokens,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
try:
|
||||
@@ -67,19 +66,6 @@ def _max_tokens_per_scattered_source(
|
||||
return (max_dp_tokens + attn_tp_size - 1) // attn_tp_size
|
||||
|
||||
|
||||
def _scattered_source_token_counts(
|
||||
dp_global_num_tokens: list[int], attn_tp_size: int
|
||||
) -> list[int]:
|
||||
assert attn_tp_size > 0
|
||||
counts = []
|
||||
for num_tokens in dp_global_num_tokens:
|
||||
base, remainder = divmod(num_tokens, attn_tp_size)
|
||||
counts.extend(
|
||||
base + int(attn_tp_rank < remainder) for attn_tp_rank in range(attn_tp_size)
|
||||
)
|
||||
return counts
|
||||
|
||||
|
||||
def _workspace_size_for_namespace(workspace_size: int, *, speculative: bool) -> int:
|
||||
slot = int(speculative)
|
||||
return workspace_size + slot * _WORKSPACE_NAMESPACE_ALIGNMENT
|
||||
@@ -127,7 +113,6 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
num_local_experts: int = None, # Unused
|
||||
hidden_size: int = None,
|
||||
params_dtype: torch.dtype = None, # Unused
|
||||
moe_runner_config=None,
|
||||
):
|
||||
super().__init__()
|
||||
if not use_flashinfer:
|
||||
@@ -155,38 +140,17 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
# TODO: Can other moe runners use payload_in_workspace too?
|
||||
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
|
||||
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
|
||||
if moe_runner_config is None:
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
|
||||
moe_runner_config = MoeRunnerConfig(
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_size=hidden_size,
|
||||
top_k=router_topk,
|
||||
)
|
||||
self.prefill_dispatcher = StandardDispatcher(moe_runner_config)
|
||||
|
||||
# FlashInfer sizes the workspace from the maximum dispatched tokens per
|
||||
# EP rank. See FlashInfer's moe_a2a_get_workspace_size_per_rank(),
|
||||
# which reserves ep_size * max_num_tokens * payload bytes, and the C++
|
||||
# dispatch op's epSize * runtimeMaxTokensPerRank payload buffer.
|
||||
#
|
||||
# The workspace must fit both:
|
||||
# (a) the fattest prefill batch (bounded by chunked_prefill_size), and
|
||||
# (b) the largest decode batch (bounded by max_running_requests, which
|
||||
# resolve_max_num_reqs caps at 4096 per DP worker).
|
||||
# max_running_requests is not yet resolved at model-construction time,
|
||||
# so we use 4096 as a floor to cover decode batches and _dummy_run
|
||||
# (which warms up at batch_size = req_to_token_pool.size).
|
||||
cps = get_schedule().chunked_prefill_size
|
||||
default_max_tokens = max(cps if cps and cps > 0 else 4096, 4096)
|
||||
configured_max_tokens = (
|
||||
envs.SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get()
|
||||
)
|
||||
self.max_num_tokens = (
|
||||
configured_max_tokens
|
||||
if configured_max_tokens is not None
|
||||
else default_max_tokens
|
||||
# The workspace must fit both the largest eager prefill chunk on one rank
|
||||
# (the prefill buffer ceiling, incl. PP dynamic chunking) and the largest
|
||||
# decode batch.
|
||||
self.max_num_tokens = flashinfer_a2a_max_dispatch_tokens_per_rank(
|
||||
max_prefill_buffer_tokens()
|
||||
)
|
||||
speculative_algo = SpeculativeAlgorithm.from_string(
|
||||
get_spec().speculative_algorithm
|
||||
@@ -245,8 +209,7 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
is_speculative_model = get_flags().moe.speculative_context
|
||||
|
||||
def make_moe_a2a() -> MoeAlltoAll:
|
||||
# Target and draft decode graphs can coexist; prefill/mixed extend use
|
||||
# AG+RS and do not lease MNNVL A2A workspaces.
|
||||
# Target and draft decode graphs can coexist.
|
||||
workspace_size = _workspace_size_for_namespace(
|
||||
self.workspace_size,
|
||||
speculative=is_speculative_model,
|
||||
@@ -262,59 +225,6 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
|
||||
self.moe_a2a = make_moe_a2a()
|
||||
|
||||
def set_quant_config(self, quant_config: dict) -> None:
|
||||
super().set_quant_config(quant_config)
|
||||
self.prefill_dispatcher.set_quant_config(quant_config)
|
||||
|
||||
def _dispatch_prefill_allgather(
|
||||
self, hidden_states: torch.Tensor, topk_output: TopKOutput
|
||||
) -> StandardDispatchOutput:
|
||||
# Eager extend can overlap another stream, so use BF16 all-gatherv instead
|
||||
# of reusing pure-decode A2A signal state across streams.
|
||||
|
||||
if hidden_states.dtype != torch.bfloat16:
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill AG requires BF16 hidden states, got "
|
||||
f"{hidden_states.dtype}."
|
||||
)
|
||||
if TopKOutputChecker.format_is_bypassed(topk_output):
|
||||
topk_output = topk_output.to_standard()
|
||||
if not TopKOutputChecker.format_is_standard(topk_output):
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill AG requires materialized top-k "
|
||||
f"routing, got {type(topk_output).__name__}."
|
||||
)
|
||||
|
||||
dp_global = get_dp_global_num_tokens()
|
||||
if dp_global is None:
|
||||
source_sizes = [hidden_states.shape[0]] * self.ep_size
|
||||
else:
|
||||
source_sizes = _scattered_source_token_counts(
|
||||
dp_global, get_parallel().attn_tp_size
|
||||
)
|
||||
if len(source_sizes) != self.ep_size:
|
||||
raise RuntimeError(
|
||||
"FlashInfer WideEP prefill AG source geometry does not match "
|
||||
f"EP: len(source_sizes)={len(source_sizes)}, ep_size={self.ep_size}."
|
||||
)
|
||||
if source_sizes[self.ep_rank] != hidden_states.shape[0]:
|
||||
raise RuntimeError(
|
||||
"FlashInfer WideEP prefill AG local source geometry mismatch: "
|
||||
f"source_sizes[{self.ep_rank}]={source_sizes[self.ep_rank]} != "
|
||||
f"hidden_states.shape[0]={hidden_states.shape[0]}."
|
||||
)
|
||||
|
||||
topk_ids = topk_output.topk_ids.to(torch.int32)
|
||||
hidden_states, topk_ids, topk_weights = get_parallel().tp_group.all_gatherv(
|
||||
[hidden_states, topk_ids, topk_output.topk_weights],
|
||||
sizes=source_sizes,
|
||||
)
|
||||
self.prefill_source_sizes = source_sizes
|
||||
return self.prefill_dispatcher.dispatch(
|
||||
hidden_states,
|
||||
StandardTopKOutput(topk_weights, topk_ids, topk_output.router_logits),
|
||||
)
|
||||
|
||||
def _effective_dispatch_type(self) -> FlashinferA2ADispatchType:
|
||||
if self.dispatch_type == FlashinferA2ADispatchType.NVFP4:
|
||||
global_scale = (self.quant_config or {}).get("input_global_scale", None)
|
||||
@@ -330,9 +240,7 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
@debug_kernel_api
|
||||
def dispatch(
|
||||
self, hidden_states: torch.Tensor, topk_output: TopKOutput
|
||||
) -> FlashinferDispatchOutput | StandardDispatchOutput:
|
||||
if get_is_extend_in_batch():
|
||||
return self._dispatch_prefill_allgather(hidden_states, topk_output)
|
||||
) -> FlashinferDispatchOutput:
|
||||
self.active_moe_a2a = self.moe_a2a
|
||||
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
|
||||
# FP4 retains its packed wire path keyed by input_global_scale.
|
||||
@@ -509,23 +417,8 @@ class FlashinferDispatcher(BaseDispatcher):
|
||||
)
|
||||
|
||||
@debug_kernel_api
|
||||
def combine(
|
||||
self, combine_input: FlashinferCombineInput | StandardCombineInput
|
||||
) -> torch.Tensor:
|
||||
def combine(self, combine_input: FlashinferCombineInput) -> torch.Tensor:
|
||||
hidden_states = combine_input.hidden_states
|
||||
if combine_input.format == CombineInputFormat.STANDARD:
|
||||
if hidden_states.dtype != torch.bfloat16:
|
||||
raise TypeError(
|
||||
"FlashInfer WideEP prefill RS requires BF16 expert output, "
|
||||
f"got {hidden_states.dtype}."
|
||||
)
|
||||
source_sizes = self.prefill_source_sizes
|
||||
hidden_states = get_parallel().tp_group.reduce_scatterv(
|
||||
hidden_states, sizes=source_sizes
|
||||
)
|
||||
del self.prefill_source_sizes
|
||||
return hidden_states
|
||||
|
||||
weight_dtype = self.quant_config.get("weight_dtype")
|
||||
runner_backend = get_moe_runner_backend()
|
||||
if (
|
||||
|
||||
@@ -1991,13 +1991,12 @@ def exports_expert_balancedness_to_prometheus() -> bool:
|
||||
|
||||
|
||||
def cutedsl_moe_max_num_tokens() -> int:
|
||||
"""The CuteDSL A2A per-rank token budget.
|
||||
"""Largest token count one forward routes through a CuteDSL MoE layer on one
|
||||
DP rank; sizes the standard-allgather wrapper, MegaMoE, and AR fusion buffers.
|
||||
|
||||
Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so
|
||||
this derives from the bags and follows a post-publish override;
|
||||
``overrides.cutedsl_moe_max_num_tokens`` is the pre-publish equivalent the
|
||||
resolution pipeline uses. Max over the prefill bound, the piecewise-prefill
|
||||
capture, and the decode/verify bound.
|
||||
this follows a post-publish override. Max over the prefill bound, the
|
||||
piecewise-prefill capture, and the decode/verify bound.
|
||||
"""
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
|
||||
|
||||
Reference in New Issue
Block a user