From 2f1acef3d8860ffe1c4744d99368772cb940f983 Mon Sep 17 00:00:00 2001 From: Huabin <47018520+R00Kie-Liu@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:14:44 +0800 Subject: [PATCH] [Model] Add LLaDA2.2 Block Routing MoE support (#31768) --- python/sglang/srt/dllm/config.py | 12 + python/sglang/srt/models/llada2.py | 571 +++++++++++++++++- .../kernels/ops/moe/test_block_topk_triton.py | 245 ++++++++ test/registered/unit/models/test_llada2.py | 348 +++++++++++ 4 files changed, 1167 insertions(+), 9 deletions(-) create mode 100644 test/registered/kernels/ops/moe/test_block_topk_triton.py create mode 100644 test/registered/unit/models/test_llada2.py diff --git a/python/sglang/srt/dllm/config.py b/python/sglang/srt/dllm/config.py index 3fa67674e2..574619b5c5 100644 --- a/python/sglang/srt/dllm/config.py +++ b/python/sglang/srt/dllm/config.py @@ -153,6 +153,18 @@ class DllmConfig: # Parse common algorithm configurations block_size = algorithm_config.get("block_size", block_size) + checkpoint_block_size = getattr(model_config.hf_config, "block_size", None) + if ( + getattr(model_config.hf_config, "expert_capacity", None) is not None + and block_size != checkpoint_block_size + ): + raise ValueError( + "LLaDA2 block routing requires the dLLM block size to match the " + f"checkpoint block_size ({checkpoint_block_size}), got {block_size}. " + "Remove the block_size override from --dllm-algorithm-config or " + "set it to the checkpoint value." + ) + return DllmConfig( algorithm=cfg.dllm_algorithm, algorithm_config=algorithm_config, diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index e0fa95b6be..e57db99096 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -24,9 +24,13 @@ from typing import Iterable, Optional, Tuple, Union import torch import torch.nn.functional as F +import triton +import triton.language as tl from torch import nn from transformers import PretrainedConfig +from triton.language.extra import libdevice +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -52,11 +56,17 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, + get_moe_runner_backend, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.token_dispatcher import DeepEPDispatcher -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import ( + StandardTopKOutput, + TopK, + TritonKernelTopKOutput, + capture_routed_experts_if_allowed, +) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import AttentionType, RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -89,6 +99,372 @@ logger = logging.getLogger(__name__) _is_cuda = is_cuda() _is_npu = is_npu() + +def _require_block_routing_ep1(moe_ep_size: int) -> None: + if moe_ep_size != 1: + raise ValueError( + "LLaDA2 block routing does not support expert parallelism; " + f"expected moe_ep_size=1, got moe_ep_size={moe_ep_size}" + ) + + +def _require_block_routing_simulated_experts_compatibility() -> None: + simulate_uniform_experts = envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get() + simulate_round_robin_experts = envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get() + if simulate_uniform_experts and simulate_round_robin_experts: + raise ValueError( + "SGLANG_SIMULATE_UNIFORM_EXPERTS and " + "SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS are mutually exclusive" + ) + if simulate_uniform_experts: + raise ValueError( + "LLaDA2 block routing does not support SGLANG_SIMULATE_UNIFORM_EXPERTS" + ) + if simulate_round_robin_experts: + raise ValueError( + "LLaDA2 block routing does not support SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS" + ) + + +def _get_effective_moe_runner_backend(experts): + """Return the runner selected by the quant method, including auto selection.""" + quant_method = getattr(experts, "quant_method", None) + + # UnquantizedFusedMoEMethod keeps a Triton fallback in ``runner`` but + # forward_cuda executes this AITER runner first whenever auto selection + # created it. Resolve the runner that actually executes, not the fallback. + aiter_runner = getattr(quant_method, "_aiter_runner", None) + backend = getattr(aiter_runner, "runner_backend", None) + if backend is not None: + return backend + + backend = getattr(quant_method, "_moe_runner_backend", None) + if backend is not None: + return backend + + runner = getattr(quant_method, "runner", None) + backend = getattr(runner, "runner_backend", None) + if backend is not None: + return backend + + runner = getattr(experts, "runner", None) + backend = getattr(runner, "runner_backend", None) + if backend is not None: + return backend + + return get_moe_runner_backend() + + +def _require_block_routing_runner_compatibility(experts) -> None: + backend = _get_effective_moe_runner_backend(experts) + if backend.is_flashinfer_trtllm(): + raise ValueError( + "LLaDA2 block routing does not support --moe-runner-backend " + "flashinfer_trtllm; use a compatible runner instead" + ) + + +def _make_block_routing_triton_output( + ragged_metadata, + combine_indx: torch.Tensor, + gate_scal: torch.Tensor, + n_expts_act: int, +) -> TritonKernelTopKOutput: + """Pack block-routing metadata using the current five-field ABI.""" + gather_indx = torch.div(combine_indx, n_expts_act, rounding_mode="trunc") + return TritonKernelTopKOutput( + a_ragged_metadata=ragged_metadata, + gather_indx=gather_indx, + scatter_indx=combine_indx, + gate_scal=gate_scal, + n_expts_act=n_expts_act, + ) + + +@triton.jit +def _block_aggregation_kernel( + router_logits_ptr, + correction_bias_ptr, + allowed_ids_ptr, + num_tokens, + num_experts_total: tl.constexpr, + block_size: tl.constexpr, + expert_capacity: tl.constexpr, + stride_logits_n, + stride_logits_e, + stride_bias, + BLOCK_SIZE_E: tl.constexpr, + BLOCK_SIZE_T: tl.constexpr, +): + """Select the block-level expert set and store compact expert IDs.""" + block_id = tl.program_id(0) + block_start_token = block_id * block_size + + expert_offs = tl.arange(0, BLOCK_SIZE_E) + expert_mask = expert_offs < num_experts_total + bias = tl.load( + correction_bias_ptr + expert_offs * stride_bias, + mask=expert_mask, + other=0.0, + ).to(tl.float32) + + token_in_block = tl.arange(0, BLOCK_SIZE_T) + token_offs = block_start_token + token_in_block[:, None] + valid_2d = ( + (token_in_block[:, None] < block_size) + & (token_offs < num_tokens) + & expert_mask[None, :] + ) + logits_ptrs = ( + router_logits_ptr + + token_offs * stride_logits_n + + expert_offs[None, :] * stride_logits_e + ) + logits = tl.load(logits_ptrs, mask=valid_2d, other=0.0) + # Keep this formula identical to _token_topk_kernel: a mismatch can change + # the selected expert at the block-capacity cutoff. + base_scores = tl.div_rn( + 1.0, + 1.0 + libdevice.exp(-logits.to(tl.float32)), + ) + corrected_scores = tl.where( + valid_2d, + base_scores + bias[None, :], + float("-inf"), + ) + block_scores = tl.max(corrected_scores, axis=0) + block_scores = tl.where(expert_mask, block_scores, float("-inf")) + + # Select on the unmodified score. Expert IDs participate only when scores + # are exactly tied, avoiding the old expert_id * 3e-7 ranking perturbation. + sorted_scores = tl.sort(block_scores, descending=True) + threshold = tl.sum( + tl.where( + expert_offs == expert_capacity - 1, + sorted_scores, + 0.0, + ), + axis=0, + ) + better = (block_scores > threshold) & expert_mask + tied = (block_scores == threshold) & expert_mask + num_better = tl.sum(better.to(tl.int32), axis=0) + ties_needed = expert_capacity - num_better + tie_rank = tl.cumsum(tied.to(tl.int32), axis=0) + selected = better | (tied & (tie_rank <= ties_needed)) + + # IDs are compact and ascending. Phase 2 consequently scans capacity + # candidates (normally 48, padded to 64) instead of every expert. + selected_rank = tl.cumsum(selected.to(tl.int32), axis=0) - 1 + safe_rank = tl.where(selected, selected_rank, 0) + tl.store( + allowed_ids_ptr + block_id * expert_capacity + safe_rank, + expert_offs, + mask=selected, + ) + + +@triton.jit +def _token_topk_kernel( + router_logits_ptr, + correction_bias_ptr, + allowed_ids_ptr, + topk_ids_ptr, + topk_weights_ptr, + num_tokens, + num_experts_total: tl.constexpr, + expert_capacity: tl.constexpr, + block_size: tl.constexpr, + top_k: tl.constexpr, + stride_logits_n, + stride_logits_e, + stride_bias, + BLOCK_SIZE_C: tl.constexpr, + TOPK_POW2: tl.constexpr, +): + """Ordered per-token top-k over the compact block-level candidates.""" + token_idx = tl.program_id(0) + if token_idx >= num_tokens: + return + + block_id = token_idx // block_size + candidate_offs = tl.arange(0, BLOCK_SIZE_C) + candidate_mask = candidate_offs < expert_capacity + expert_ids = tl.load( + allowed_ids_ptr + block_id * expert_capacity + candidate_offs, + mask=candidate_mask, + other=0, + ).to(tl.int32) + bias = tl.load( + correction_bias_ptr + expert_ids * stride_bias, + mask=candidate_mask, + other=0.0, + ).to(tl.float32) + logits = tl.load( + router_logits_ptr + token_idx * stride_logits_n + expert_ids * stride_logits_e, + mask=candidate_mask, + other=float("-inf"), + ) + + # Keep this formula identical to _block_aggregation_kernel. Use Triton's + # portable round-to-nearest division so this compiles on CUDA and HIP. + base_scores = tl.div_rn( + 1.0, + 1.0 + libdevice.exp(-logits.to(tl.float32)), + ) + remaining = tl.where( + candidate_mask, + base_scores + bias, + float("-inf"), + ) + + out_offs = tl.arange(0, TOPK_POW2) + out_ids = tl.zeros([TOPK_POW2], dtype=tl.int32) + out_raw = tl.zeros([TOPK_POW2], dtype=tl.float32) + raw_sum = 0.0 + for k in tl.static_range(0, top_k): + # Compact candidates are ascending expert IDs, so leftmost argmax gives + # the deterministic lower-ID tie break shared by the reference test. + selected_pos = tl.argmax(remaining, axis=0, tie_break_left=True) + selected_id = tl.sum( + tl.where(candidate_offs == selected_pos, expert_ids, 0), + axis=0, + ) + selected_raw = tl.sum( + tl.where(candidate_offs == selected_pos, base_scores, 0.0), + axis=0, + ) + out_ids = tl.where(out_offs == k, selected_id, out_ids) + out_raw = tl.where(out_offs == k, selected_raw, out_raw) + raw_sum += selected_raw + remaining = tl.where( + candidate_offs == selected_pos, + float("-inf"), + remaining, + ) + + out_mask = out_offs < top_k + if top_k > 1: + out_weights = tl.div_rn(out_raw, raw_sum + 1e-20) + else: + out_weights = out_raw + + tl.store( + topk_ids_ptr + token_idx * top_k + out_offs, + out_ids, + mask=out_mask, + ) + tl.store( + topk_weights_ptr + token_idx * top_k + out_offs, + out_weights, + mask=out_mask, + ) + + +def block_topk_triton( + router_logits: torch.Tensor, + correction_bias: torch.Tensor, + block_size: int, + expert_capacity: int, + top_k: int, +): + """Two-phase block routing with compact candidates and FP32 weights.""" + if not router_logits.is_cuda or not correction_bias.is_cuda: + raise ValueError("block_topk_triton requires CUDA tensors") + if router_logits.ndim != 2: + raise ValueError( + f"router_logits must be 2-D, got shape={tuple(router_logits.shape)}" + ) + + num_tokens, num_experts_total = router_logits.shape + if correction_bias.shape != (num_experts_total,): + raise ValueError( + "correction_bias must have shape " + f"({num_experts_total},), got {tuple(correction_bias.shape)}" + ) + if ( + isinstance(block_size, bool) + or not isinstance(block_size, int) + or block_size <= 0 + or block_size & (block_size - 1) + ): + raise ValueError( + f"block_size must be a positive power of two, got {block_size!r}" + ) + if not (0 < top_k <= expert_capacity <= num_experts_total): + raise ValueError( + "expected 0 < top_k <= expert_capacity <= num_experts, got " + f"top_k={top_k}, expert_capacity={expert_capacity}, " + f"num_experts={num_experts_total}" + ) + + device = router_logits.device + if num_tokens == 0: + return ( + torch.empty((0, top_k), dtype=torch.float32, device=device), + torch.empty((0, top_k), dtype=torch.int32, device=device), + ) + + # Tail blocks are valid; invalid token lanes are masked inside phase 1. + num_blocks = (num_tokens + block_size - 1) // block_size + allowed_ids = torch.empty( + (num_blocks, expert_capacity), + dtype=torch.int32, + device=device, + ) + topk_ids = torch.empty( + (num_tokens, top_k), + dtype=torch.int32, + device=device, + ) + topk_weights = torch.empty( + (num_tokens, top_k), + dtype=torch.float32, + device=device, + ) + + block_size_e = triton.next_power_of_2(num_experts_total) + block_size_c = triton.next_power_of_2(expert_capacity) + topk_pow2 = triton.next_power_of_2(top_k) + + _block_aggregation_kernel[(num_blocks,)]( + router_logits, + correction_bias, + allowed_ids, + num_tokens, + num_experts_total, + block_size, + expert_capacity, + router_logits.stride(0), + router_logits.stride(1), + correction_bias.stride(0), + BLOCK_SIZE_E=block_size_e, + BLOCK_SIZE_T=block_size, + num_warps=4, + num_stages=1, + ) + _token_topk_kernel[(num_tokens,)]( + router_logits, + correction_bias, + allowed_ids, + topk_ids, + topk_weights, + num_tokens, + num_experts_total, + expert_capacity, + block_size, + top_k, + router_logits.stride(0), + router_logits.stride(1), + correction_bias.stride(0), + BLOCK_SIZE_C=block_size_c, + TOPK_POW2=topk_pow2, + num_warps=1, + num_stages=1, + ) + return topk_weights, topk_ids + + split_qkv_rmsnorm_rope_pos_cache_half_npu = None if _is_npu: try: @@ -174,10 +550,10 @@ class LLaDA2MoeGate(nn.Module): self.expert_bias = None def forward(self, hidden_states): - logits = F.linear(hidden_states.to(self.weight.dtype), self.weight, None).to( - hidden_states.dtype - ) - return logits + # Keep router logits in the configured router/weight dtype. In + # particular, router_dtype="fp32" must not be truncated back to the + # BF16 activation dtype before block routing. + return F.linear(hidden_states.to(self.weight.dtype), self.weight, None) class LLaDA2MoeSparseMoeBlock(nn.Module): @@ -205,6 +581,18 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): if _is_npu: self.norm_topk_prob = False + self.use_block_routing = getattr(config, "expert_capacity", None) is not None + if self.use_block_routing: + _require_block_routing_simulated_experts_compatibility() + _require_block_routing_ep1(get_parallel().moe_ep_size) + if _is_npu: + raise ValueError( + "LLaDA2 block routing is not supported on NPU; " + "expert_capacity must be unset" + ) + self.block_size = config.block_size + self.expert_capacity = config.expert_capacity + if config.hidden_act != "silu": raise ValueError( f"Unsupported activation: {config.hidden_act}. " @@ -246,6 +634,47 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): self.gate.expert_bias.data if self.gate.expert_bias is not None else None ) + if self.use_block_routing: + if get_moe_a2a_backend().is_deepep(): + raise ValueError( + "LLaDA2 block routing does not support --moe-a2a-backend deepep" + ) + if self.correction_bias is None: + raise ValueError( + "block routing requires moe_router_enable_expert_bias=True" + ) + if self.score_function != "sigmoid": + raise ValueError( + "block routing requires score_function='sigmoid', got " + f"{self.score_function!r}" + ) + if ( + isinstance(self.block_size, bool) + or not isinstance(self.block_size, int) + or self.block_size <= 0 + or self.block_size & (self.block_size - 1) + ): + raise ValueError( + "block_size must be a positive power of two, got " + f"{self.block_size!r}" + ) + if not (0 < self.top_k <= self.expert_capacity <= self.num_experts): + raise ValueError( + "block routing requires " + "0 < num_experts_per_tok <= expert_capacity <= num_experts, " + f"got top_k={self.top_k}, " + f"expert_capacity={self.expert_capacity}, " + f"num_experts={self.num_experts}" + ) + if get_moe_runner_backend().is_triton_kernels() and self.top_k & ( + self.top_k - 1 + ): + raise ValueError( + "LLaDA2 block routing with --moe-runner-backend " + "triton_kernel requires num_experts_per_tok to be a " + f"power of two, got {self.top_k}" + ) + if self.score_function is not None: assert ( self.score_function == "softmax" and self.correction_bias is None @@ -279,6 +708,8 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): routed_scaling_factor=self.routed_scaling_factor, prefix=add_prefix("experts", prefix), ) + if self.use_block_routing: + _require_block_routing_runner_compatibility(self.experts) # shared expert if config.num_shared_experts is not None: if hasattr(config, "moe_shared_expert_intermediate_size"): @@ -335,12 +766,129 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): shared_output = self.shared_experts(hidden_states) return shared_output + def _block_topk(self, router_logits: torch.Tensor): + """Block top-k routing via two-phase Triton kernel.""" + return block_topk_triton( + router_logits, + self.correction_bias, + self.block_size, + self.expert_capacity, + self.top_k, + ) + + def _make_block_topk_output( + self, + router_logits: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ): + """Convert block-routing results to the selected MoE runner format.""" + # Block routing computes its own top-k and therefore bypasses TopK.forward, + # which normally invokes these diagnostics before producing runner output. + capture_routed_experts_if_allowed( + self.topk.topk_config, self.layer_id, topk_ids + ) + get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids) + + runner_backend = _get_effective_moe_runner_backend(self.experts) + if not runner_backend.is_triton_kernels(): + if ( + runner_backend.is_aiter() + or getattr( + self.experts, + "should_fuse_routed_scaling_factor_in_topk", + False, + ) + ) and self.routed_scaling_factor is not None: + topk_weights = topk_weights * self.routed_scaling_factor + return StandardTopKOutput( + topk_weights=topk_weights, + topk_ids=topk_ids, + router_logits=router_logits, + ) + + # triton_kernels needs expert-sorted routing metadata. Reuse the + # block router's selected expert ids via y_indx, but do not use the + # values produced by triton_kernels_topk: block routing uses sigmoid + # scores normalized over its own selected experts, not softmax scores. + from triton_kernels.tensor import make_ragged_tensor_metadata + from triton_kernels.topk import topk as triton_kernels_topk + + if topk_ids.numel() == 0: + # Avoid launching the Triton top-k kernel with a zero-row grid. + dispatch_indx = torch.empty((0,), dtype=torch.int32, device=topk_ids.device) + combine_indx = torch.empty_like(dispatch_indx) + expert_counts = torch.zeros( + (self.num_experts,), dtype=torch.int32, device=topk_ids.device + ) + else: + sparse_logits = triton_kernels_topk( + router_logits, + self.top_k, + apply_softmax=False, + y_indx=topk_ids, + ) + dispatch_indx = sparse_logits.mask_metadata.row_sorted_indx + combine_indx = sparse_logits.mask_metadata.col_sorted_indx + expert_counts = sparse_logits.mask_metadata.col_sum + + ragged_metadata = make_ragged_tensor_metadata( + expert_counts, + dispatch_indx.shape[0], + ) + + # combine_indx maps expert-major entries back to flattened token-major + # top-k entries. Preserve block routing's FP32 weights while packing the + # five-field triton_kernels output expected by the current MoE runner. + gate_scal = topk_weights.flatten()[combine_indx] + return _make_block_routing_triton_output( + ragged_metadata, + combine_indx, + gate_scal, + self.top_k, + ) + def _forward_router_experts(self, hidden_states: torch.Tensor): # router_logits: (num_tokens, n_experts) router_logits = self.gate(hidden_states) - topk_output = self.topk(hidden_states, router_logits) + + if self.use_block_routing: + topk_weights, topk_ids = self._block_topk(router_logits) + topk_output = self._make_block_topk_output( + router_logits, topk_weights, topk_ids + ) + else: + topk_output = self.topk(hidden_states, router_logits) + return self.experts(hidden_states, topk_output) + @torch.compiler.disable(recursive=False) + def forward_normal_block_routing_dual_stream( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + """Advanced dual stream: overlaps gate + block top-k with shared expert computation.""" + current_stream = torch.cuda.current_stream() + self.alt_stream.wait_stream(current_stream) + assert not (self.shared_experts.is_replicated and hidden_states.shape[0] == 0) + router_logits = self.gate(hidden_states) + with torch.cuda.stream(self.alt_stream): + shared_gate_up, _ = self.shared_experts.gate_up_proj(hidden_states) + current_stream.wait_stream(self.alt_stream) + + topk_weights, topk_ids = self._block_topk(router_logits) + + self.alt_stream.wait_stream(current_stream) + with torch.cuda.stream(self.alt_stream): + shared_hidden_states = self.shared_experts.act_fn(shared_gate_up) + shared_output, _ = self.shared_experts.down_proj(shared_hidden_states) + topk_output = self._make_block_topk_output( + router_logits, topk_weights, topk_ids + ) + router_output = self.experts(hidden_states, topk_output) + current_stream.wait_stream(self.alt_stream) + return router_output, shared_output + def forward_normal_dual_stream( self, hidden_states: torch.Tensor, @@ -367,9 +915,14 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): and hidden_states.shape[0] > 0 and get_is_capture_mode() ): - final_hidden_states, shared_output = self.forward_normal_dual_stream( - hidden_states - ) + if self.use_block_routing: + final_hidden_states, shared_output = ( + self.forward_normal_block_routing_dual_stream(hidden_states) + ) + else: + final_hidden_states, shared_output = self.forward_normal_dual_stream( + hidden_states + ) else: shared_output = self._forward_shared_experts(hidden_states) final_hidden_states = self._forward_router_experts(hidden_states) diff --git a/test/registered/kernels/ops/moe/test_block_topk_triton.py b/test/registered/kernels/ops/moe/test_block_topk_triton.py new file mode 100644 index 0000000000..2d1094085f --- /dev/null +++ b/test/registered/kernels/ops/moe/test_block_topk_triton.py @@ -0,0 +1,245 @@ +""" +Unit tests for block_topk_triton kernel (block-level expert routing). + +Verifies that the Triton implementation matches a pure PyTorch reference +across various configurations of num_tokens, num_experts, expert_capacity, +block_size, and top_k. +""" + +import unittest + +import torch + +from sglang.srt.models.llada2 import block_topk_triton +from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") +register_amd_ci(est_time=10, suite="jit-kernel-unit-test-amd") + + +def block_topk_reference( + router_logits: torch.Tensor, + correction_bias: torch.Tensor, + block_size: int, + expert_capacity: int, + top_k: int, +): + """Independent PyTorch reference for block-level expert routing. + + Selection uses the unmodified corrected score. Stable argsort defines the + exact-tie contract as lower expert ID first, without copying the Triton + kernel's threshold/cumsum implementation. Routing weights are unbiased + sigmoid scores normalized in FP32. + """ + num_tokens, num_experts = router_logits.shape + device = router_logits.device + if num_tokens == 0: + return ( + torch.empty((0, top_k), dtype=torch.float32, device=device), + torch.empty((0, top_k), dtype=torch.int32, device=device), + ) + + base_scores = torch.sigmoid(router_logits.float()) + corrected_scores = base_scores + correction_bias.float().unsqueeze(0) + num_blocks = (num_tokens + block_size - 1) // block_size + padded_tokens = num_blocks * block_size + if padded_tokens != num_tokens: + corrected_scores_padded = torch.nn.functional.pad( + corrected_scores, + (0, 0, 0, padded_tokens - num_tokens), + value=float("-inf"), + ) + else: + corrected_scores_padded = corrected_scores + + block_scores = corrected_scores_padded.view( + num_blocks, block_size, num_experts + ).amax(dim=1) + allowed_ids = torch.argsort(block_scores, dim=-1, descending=True, stable=True)[ + :, :expert_capacity + ] + allowed = torch.zeros((num_blocks, num_experts), dtype=torch.bool, device=device) + allowed.scatter_(1, allowed_ids, True) + token_allowed = allowed.repeat_interleave(block_size, dim=0)[:num_tokens] + + token_scores = corrected_scores.masked_fill(~token_allowed, float("-inf")) + topk_ids = torch.argsort(token_scores, dim=-1, descending=True, stable=True)[ + :, :top_k + ] + selected_base_scores = base_scores.gather(1, topk_ids) + if top_k > 1: + weight_sum = selected_base_scores.sum(dim=-1, keepdim=True) + topk_weights = selected_base_scores / (weight_sum + 1e-20) + else: + topk_weights = selected_base_scores + + return topk_weights.float(), topk_ids.to(torch.int32) + + +class TestBlockTopkTriton(CustomTestCase): + """Test block_topk_triton against PyTorch reference.""" + + def _run_case(self, num_tokens, num_experts, expert_capacity, block_size, top_k): + torch.manual_seed(42) + device = "cuda" + + router_logits = torch.randn( + num_tokens, num_experts, dtype=torch.float32, device=device + ) + correction_bias = ( + torch.randn(num_experts, dtype=torch.float32, device=device) * 0.1 + ) + + weights_triton, ids_triton = block_topk_triton( + router_logits, correction_bias, block_size, expert_capacity, top_k + ) + weights_ref, ids_ref = block_topk_reference( + router_logits, correction_bias, block_size, expert_capacity, top_k + ) + + self.assertTrue( + torch.equal(ids_triton, ids_ref), + f"IDs mismatch for config ({num_tokens}, {num_experts}, " + f"cap={expert_capacity}, bs={block_size}, k={top_k}).\n" + f" Triton: {ids_triton[:4]}\n" + f" Ref: {ids_ref[:4]}", + ) + self.assertEqual(weights_triton.dtype, torch.float32) + torch.testing.assert_close(weights_triton, weights_ref, rtol=0.0, atol=2e-6) + + def test_basic_small(self): + """32 tokens, 16 experts, capacity=8, block_size=8, top_k=2.""" + self._run_case( + num_tokens=32, num_experts=16, expert_capacity=8, block_size=8, top_k=2 + ) + + def test_single_expert_selection(self): + """top_k=1 path.""" + self._run_case( + num_tokens=32, num_experts=16, expert_capacity=8, block_size=8, top_k=1 + ) + + def test_large_expert_count(self): + """256 experts (production-like config).""" + self._run_case( + num_tokens=64, num_experts=256, expert_capacity=48, block_size=32, top_k=8 + ) + + def test_capacity_equals_experts(self): + """expert_capacity == num_experts means no routing restriction.""" + self._run_case( + num_tokens=32, num_experts=16, expert_capacity=16, block_size=8, top_k=4 + ) + + def test_tight_capacity(self): + """Very tight capacity (only top_k experts allowed per block).""" + self._run_case( + num_tokens=64, num_experts=64, expert_capacity=6, block_size=16, top_k=4 + ) + + def test_block_size_equals_one(self): + """block_size=1 degenerates to per-token routing.""" + self._run_case( + num_tokens=16, num_experts=32, expert_capacity=8, block_size=1, top_k=2 + ) + + def test_many_tokens(self): + """Stress test with more tokens.""" + self._run_case( + num_tokens=256, num_experts=128, expert_capacity=32, block_size=32, top_k=4 + ) + + def test_partial_block_and_non_power_of_two_experts(self): + """Tail tokens and non-power-of-two expert counts are supported.""" + self._run_case( + num_tokens=65, + num_experts=100, + expert_capacity=48, + block_size=32, + top_k=8, + ) + + def test_exact_ties_choose_lower_expert_id(self): + logits = torch.zeros((33, 100), dtype=torch.float32, device="cuda") + bias = torch.zeros((100,), dtype=torch.float32, device="cuda") + weights, ids = block_topk_triton(logits, bias, 32, 48, 8) + expected_ids = torch.arange(8, dtype=torch.int32, device="cuda").expand(33, -1) + self.assertTrue(torch.equal(ids, expected_ids)) + torch.testing.assert_close( + weights, torch.full_like(weights, 1.0 / 8), rtol=0.0, atol=0.0 + ) + + def test_very_negative_logits_use_epsilon_normalization(self): + """Very small sigmoid scores must not fall back to uniform weights.""" + logits = torch.full((32, 16), -90.0, dtype=torch.float32, device="cuda") + bias = torch.zeros((16,), dtype=torch.float32, device="cuda") + + weights, ids = block_topk_triton( + logits, bias, block_size=32, expert_capacity=16, top_k=8 + ) + + selected_scores = torch.sigmoid(logits.float()).gather(1, ids.long()) + expected = selected_scores / (selected_scores.sum(dim=-1, keepdim=True) + 1e-20) + torch.testing.assert_close(weights, expected, rtol=1e-5, atol=1e-25) + + # The old fallback returned 1 / top_k == 0.125 for this input. + self.assertTrue(torch.all(weights < 1e-10)) + + def test_close_unequal_scores_are_not_reordered(self): + logits = torch.full((32, 16), -20.0, dtype=torch.float32, device="cuda") + bias = torch.zeros((16,), dtype=torch.float32, device="cuda") + logits[:, 0] = 0.0 + logits[:, 1] = 0.0 + bias[1] = 2e-7 + _, ids = block_topk_triton(logits, bias, 32, 16, 1) + self.assertTrue(torch.equal(ids, torch.ones_like(ids))) + + def test_capacity_cutoff_near_tie(self): + """Phase 1 and phase 2 must use the same sigmoid at the cutoff.""" + logits = torch.full((8, 4), -20.0, dtype=torch.float32, device="cuda") + bias = torch.zeros((4,), dtype=torch.float32, device="cuda") + + # Keep the scores close while ensuring the device's FP32 sigmoid can + # represent their difference on both CUDA and ROCm. Derive the expected + # winner from that same device reference instead of assuming that two + # adjacent logits remain distinct after sigmoid. + logits[:, 0] = 0.4970 + logits[:, 1] = 0.4971 + reference_scores = torch.sigmoid(logits[0, :2].float()) + self.assertGreater(reference_scores[1].item(), reference_scores[0].item()) + expected_id = torch.argmax(reference_scores).to(torch.int32) + + weights, ids = block_topk_triton( + logits, bias, block_size=8, expert_capacity=1, top_k=1 + ) + + self.assertTrue(torch.equal(ids, torch.full_like(ids, expected_id))) + expected_weight = reference_scores[expected_id.long()] + torch.testing.assert_close( + weights, + torch.full_like(weights, expected_weight), + rtol=0.0, + atol=1e-7, + ) + + def test_rejects_non_power_of_two_block_size(self): + logits = torch.zeros((6, 4), dtype=torch.float32, device="cuda") + bias = torch.zeros((4,), dtype=torch.float32, device="cuda") + for block_size in (3, True): + with self.subTest(block_size=block_size): + with self.assertRaisesRegex(ValueError, "positive power of two"): + block_topk_triton(logits, bias, block_size, 2, 1) + + def test_empty_input_returns_fp32_weights(self): + logits = torch.empty((0, 16), dtype=torch.bfloat16, device="cuda") + bias = torch.zeros((16,), dtype=torch.float32, device="cuda") + weights, ids = block_topk_triton(logits, bias, 32, 8, 2) + self.assertEqual(weights.shape, (0, 2)) + self.assertEqual(ids.shape, (0, 2)) + self.assertEqual(weights.dtype, torch.float32) + self.assertEqual(ids.dtype, torch.int32) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/models/test_llada2.py b/test/registered/unit/models/test_llada2.py new file mode 100644 index 0000000000..ff92303fb8 --- /dev/null +++ b/test/registered/unit/models/test_llada2.py @@ -0,0 +1,348 @@ +"""Unit tests for LLaDA2 model validation and routing metadata.""" + +import unittest +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from torch import nn + +from sglang.srt.dllm.config import DllmConfig +from sglang.srt.environ import envs +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptNvFp4FusedMoEMethod, +) +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.models.llada2 import ( + LLaDA2MoeSparseMoeBlock, + _get_effective_moe_runner_backend, + _make_block_routing_triton_output, + _require_block_routing_ep1, + _require_block_routing_runner_compatibility, +) +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestLLaDA2BlockRoutingValidation(CustomTestCase): + def test_block_routing_accepts_ep1(self): + _require_block_routing_ep1(1) + + def test_block_routing_rejects_expert_parallelism(self): + config = SimpleNamespace( + num_experts_per_tok=8, + norm_topk_prob=True, + hidden_size=1024, + num_shared_experts=1, + expert_capacity=48, + ) + + with get_parallel().override(tp_size=4, moe_ep_size=4): + with self.assertRaisesRegex( + ValueError, + r"does not support expert parallelism.*moe_ep_size=4", + ): + LLaDA2MoeSparseMoeBlock(layer_id=0, config=config) + + def test_block_routing_uses_five_field_triton_output(self): + ragged_metadata = object() + combine_indx = torch.tensor([5, 0, 7, 2], dtype=torch.int32) + gate_scal = torch.tensor([0.4, 0.3, 0.2, 0.1], dtype=torch.float32) + + result = _make_block_routing_triton_output( + ragged_metadata, + combine_indx, + gate_scal, + n_expts_act=2, + ) + + self.assertIs(result.a_ragged_metadata, ragged_metadata) + torch.testing.assert_close( + result.gather_indx, + torch.tensor([2, 0, 3, 1], dtype=torch.int32), + ) + self.assertIs(result.scatter_indx, combine_indx) + self.assertIs(result.gate_scal, gate_scal) + self.assertEqual(result.n_expts_act, 2) + + @patch("sglang.srt.layers.quantization.modelopt_quant.MoeRunner") + @patch( + "sglang.srt.layers.quantization.modelopt_quant.get_device_capability", + return_value=(10, 0), + ) + @patch("sglang.srt.layers.quantization.modelopt_quant.is_cuda", return_value=True) + @patch("sglang.srt.layers.quantization.modelopt_quant.get_moe_runner_backend") + def test_block_routing_rejects_auto_selected_plain_flashinfer( + self, + backend, + _is_cuda, + _capability, + runner, + ): + backend.return_value = MoeRunnerBackend.AUTO + quant_method = ModelOptNvFp4FusedMoEMethod.__new__(ModelOptNvFp4FusedMoEMethod) + quant_method.create_moe_runner( + SimpleNamespace(), + SimpleNamespace(), + ) + experts = SimpleNamespace(quant_method=quant_method) + + self.assertIs( + _get_effective_moe_runner_backend(experts), + MoeRunnerBackend.FLASHINFER_TRTLLM, + ) + runner.assert_called_once_with( + MoeRunnerBackend.FLASHINFER_TRTLLM, + quant_method.moe_runner_config, + ) + with self.assertRaisesRegex(ValueError, r"does not support.*flashinfer_trtllm"): + _require_block_routing_runner_compatibility(experts) + + @patch("sglang.srt.models.llada2.get_moe_runner_backend") + def test_fused_scaling_contract_applies_scale_once(self, backend): + backend.return_value.is_triton_kernels.return_value = False + backend.return_value.is_aiter.return_value = False + + block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) + nn.Module.__init__(block) + block.layer_id = 0 + block.routed_scaling_factor = 2.5 + block.topk = SimpleNamespace( + topk_config=SimpleNamespace(allow_routed_experts_capture=False) + ) + block.experts = SimpleNamespace(should_fuse_routed_scaling_factor_in_topk=True) + + router_logits = torch.zeros((1, 4), dtype=torch.float32) + topk_weights = torch.tensor([[0.25, 0.75]], dtype=torch.float32) + topk_ids = torch.tensor([[0, 1]], dtype=torch.int32) + + result = block._make_block_topk_output( + router_logits, + topk_weights, + topk_ids, + ) + + torch.testing.assert_close( + result.topk_weights, + topk_weights * block.routed_scaling_factor, + ) + torch.testing.assert_close(topk_weights, torch.tensor([[0.25, 0.75]])) + self.assertIs(result.topk_ids, topk_ids) + self.assertIs(result.router_logits, router_logits) + + def test_aiter_block_routing_applies_scale_once(self): + with ( + patch( + "sglang.srt.layers.quantization.unquant._use_aiter", + True, + ), + patch( + "sglang.srt.layers.quantization.unquant.get_moe_runner_backend", + return_value=MoeRunnerBackend.AUTO, + ), + patch( + "sglang.srt.layers.quantization.unquant.get_moe_a2a_backend" + ) as get_a2a_backend, + patch( + "sglang.srt.layers.quantization.unquant.MoeRunner", + side_effect=lambda backend, config: SimpleNamespace( + runner_backend=backend + ), + ), + ): + get_a2a_backend.return_value.supports_aiter.return_value = True + quant_method = UnquantizedFusedMoEMethod() + quant_method.create_moe_runner( + SimpleNamespace(intermediate_size_per_partition=256), + SimpleNamespace(), + ) + + self.assertIs( + quant_method.runner.runner_backend, + MoeRunnerBackend.TRITON, + ) + self.assertIs( + quant_method._aiter_runner.runner_backend, + MoeRunnerBackend.AITER, + ) + + block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) + nn.Module.__init__(block) + block.layer_id = 0 + block.routed_scaling_factor = 2.5 + block.topk = SimpleNamespace( + topk_config=SimpleNamespace(allow_routed_experts_capture=False) + ) + block.experts = SimpleNamespace( + quant_method=quant_method, + runner=quant_method.runner, + should_fuse_routed_scaling_factor_in_topk=False, + ) + + self.assertIs( + _get_effective_moe_runner_backend(block.experts), + MoeRunnerBackend.AITER, + ) + + router_logits = torch.zeros((1, 4), dtype=torch.float32) + topk_weights = torch.tensor([[0.25, 0.75]], dtype=torch.float32) + topk_ids = torch.tensor([[0, 1]], dtype=torch.int32) + + result = block._make_block_topk_output( + router_logits, + topk_weights, + topk_ids, + ) + + torch.testing.assert_close( + result.topk_weights, + topk_weights * block.routed_scaling_factor, + ) + torch.testing.assert_close(topk_weights, torch.tensor([[0.25, 0.75]])) + self.assertIs(result.topk_ids, topk_ids) + self.assertIs(result.router_logits, router_logits) + + @patch("sglang.srt.models.llada2.get_moe_runner_backend") + @patch("sglang.srt.models.llada2.get_global_expert_distribution_recorder") + @patch("sglang.srt.models.llada2.capture_routed_experts_if_allowed") + def test_block_routing_runs_precomputed_route_hooks( + self, + capture_routed_experts, + get_recorder, + backend, + ): + backend.return_value.is_triton_kernels.return_value = False + backend.return_value.is_aiter.return_value = False + + block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) + nn.Module.__init__(block) + block.layer_id = 7 + block.routed_scaling_factor = 1.0 + block.topk = SimpleNamespace(topk_config=object()) + block.experts = SimpleNamespace(should_fuse_routed_scaling_factor_in_topk=False) + topk_ids = torch.tensor([[1, 3]], dtype=torch.int32) + + block._make_block_topk_output( + torch.zeros((1, 4)), + torch.tensor([[0.4, 0.6]]), + topk_ids, + ) + + capture_routed_experts.assert_called_once_with( + block.topk.topk_config, block.layer_id, topk_ids + ) + get_recorder.return_value.on_select_experts.assert_called_once_with( + topk_ids=topk_ids + ) + + def test_block_routing_rejects_uniform_expert_simulation(self): + with ( + get_parallel().override(tp_size=1, moe_ep_size=1), + envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.override(True), + envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.override(False), + self.assertRaisesRegex( + ValueError, r"does not support SGLANG_SIMULATE_UNIFORM_EXPERTS" + ), + ): + LLaDA2MoeSparseMoeBlock( + layer_id=0, + config=SimpleNamespace( + num_experts_per_tok=8, + norm_topk_prob=True, + hidden_size=1024, + num_shared_experts=1, + expert_capacity=48, + ), + ) + + def test_block_routing_rejects_round_robin_expert_simulation(self): + with ( + get_parallel().override(tp_size=1, moe_ep_size=1), + envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.override(False), + envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.override(True), + self.assertRaisesRegex( + ValueError, r"does not support SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS" + ), + ): + LLaDA2MoeSparseMoeBlock( + layer_id=0, + config=SimpleNamespace( + num_experts_per_tok=8, + norm_topk_prob=True, + hidden_size=1024, + num_shared_experts=1, + expert_capacity=48, + ), + ) + + def test_block_routing_rejects_conflicting_expert_simulations(self): + with ( + get_parallel().override(tp_size=1, moe_ep_size=1), + envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.override(True), + envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.override(True), + self.assertRaisesRegex(ValueError, r"mutually exclusive"), + ): + LLaDA2MoeSparseMoeBlock( + layer_id=0, + config=SimpleNamespace( + num_experts_per_tok=8, + norm_topk_prob=True, + hidden_size=1024, + num_shared_experts=1, + expert_capacity=48, + ), + ) + + +class TestLLaDA2DllmBlockSizeValidation(CustomTestCase): + @staticmethod + def _server_args(algorithm_config=None): + return SimpleNamespace( + dllm_algorithm="LowConfidence", + dllm_algorithm_config=algorithm_config, + max_running_requests=1, + model_path="unused", + revision=None, + dllm_fdfo=False, + ) + + @staticmethod + def _model_config(): + return SimpleNamespace( + hf_config=SimpleNamespace( + architectures=["LLaDA2MoeModelLM"], + block_size=32, + expert_capacity=48, + ) + ) + + @patch("sglang.srt.dllm.config.ModelConfig.from_server_args") + def test_block_routing_accepts_matching_dllm_block_size(self, from_server_args): + from_server_args.return_value = self._model_config() + + config = DllmConfig.from_server_args(self._server_args()) + + self.assertEqual(config.block_size, 32) + + @patch("sglang.srt.dllm.config.ModelConfig.from_server_args") + def test_block_routing_rejects_mismatched_dllm_block_size(self, from_server_args): + from_server_args.return_value = self._model_config() + with TemporaryDirectory() as tmpdir: + config_path = f"{tmpdir}/dllm.yaml" + with open(config_path, "w") as config_file: + config_file.write("block_size: 16\n") + + with self.assertRaisesRegex( + ValueError, + r"requires the dLLM block size to match.*\(32\), got 16", + ): + DllmConfig.from_server_args(self._server_args(config_path)) + + +if __name__ == "__main__": + unittest.main()