diff --git a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py index 9a15978bbe..4ed917db39 100644 --- a/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py +++ b/python/sglang/kernels/ops/moe/fused_moe_triton_kernels.py @@ -1398,8 +1398,8 @@ def _fused_append_remap_shared_experts_deepep_kernel( ): """Append shared experts AND apply the DeepEP interleaved remap in one pass. - Equivalent to fused_append_shared_experts() immediately followed by - topk._remap_topk_for_deepep(), but the remap math runs on the rows already + Equivalent to topk.remap_topk_for_per_rank_shared_slots() on the same + routed columns, but the remap math runs on the rows already loaded into registers, so it costs a few ALU ops instead of ~6 extra eager kernel launches (div_floor / add / arange / fill / copy) per MoE layer. @@ -1464,7 +1464,7 @@ def fused_append_remap_shared_experts_deepep( ): """Fused append + DeepEP remap (see kernel docstring). - Replaces the fused_append_shared_experts() + _remap_topk_for_deepep() pair on + Fused form of topk.remap_topk_for_per_rank_shared_slots() for the aiter/DeepEP-class path. Host computes the scalar remap params so the kernel stays branch-free. """ diff --git a/python/sglang/kernels/ops/moe/moe_fused_gate.py b/python/sglang/kernels/ops/moe/moe_fused_gate.py index 9cd0f6b864..feb3c47681 100644 --- a/python/sglang/kernels/ops/moe/moe_fused_gate.py +++ b/python/sglang/kernels/ops/moe/moe_fused_gate.py @@ -284,15 +284,6 @@ def _router_triton_kernel( :, None ] # [BLOCK_M, 1] - # Fill fused-shared-expert slots: weight = routed_sum / routed_scaling_factor, - # id = num_experts + (slot - K_ROUTED). - if K_ROUTED < K: - is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :] - shared_weight = routed_sum / routed_scaling_factor # [BLOCK_M, 1] - shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :] # [1, BLOCK_K] - selected_vals = tl.where(is_shared, shared_weight, selected_vals) - selected_idx = tl.where(is_shared, shared_idx, selected_idx) - if USE_PDL: tl.extra.cuda.gdc_launch_dependents() @@ -304,6 +295,13 @@ def _router_triton_kernel( selected_vals = selected_vals / norm if APPLY_SCALE: selected_vals = selected_vals * routed_scaling_factor + if K_ROUTED < K: + # The shared expert's contribution is independent of routed normalization. + is_shared = (offs_k[None, :] >= K_ROUTED) & mask_k_total[None, :] + shared_weight = 1.0 if APPLY_SCALE else 1.0 / routed_scaling_factor + shared_idx = (N + (offs_k - K_ROUTED)).to(tl.int32)[None, :] + selected_vals = tl.where(is_shared, shared_weight, selected_vals) + selected_idx = tl.where(is_shared, shared_idx, selected_idx) if HAS_PADDING: selected_vals = tl.where(live_m[:, None], selected_vals, 0.0) selected_idx = tl.where(live_m[:, None], selected_idx, -1) diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 3a037fecbb..7de6d941a4 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -266,12 +266,12 @@ class HashTopK(nn.Module): recorder_topk_ids = None if has_per_rank_fused_shared_slots(num_fused_shared_experts): - shared_cols = topk_ids[:, -num_fused_shared_experts:] - routed_cols = topk_ids[:, :-num_fused_shared_experts] + # The trailing columns are placeholders; the remap appends the real ones. routed_cols = topk_ids_logical_to_physical( - routed_cols, expert_location_dispatch_info, log2phy_prob + topk_ids[:, :-num_fused_shared_experts], + expert_location_dispatch_info, + log2phy_prob, ) - topk_ids = torch.cat([routed_cols, shared_cols], dim=-1) recorder_topk_ids = routed_cols num_physical_routed_experts = ( @@ -280,8 +280,8 @@ class HashTopK(nn.Module): else self.num_experts ) topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots( - topk_ids, - topk_weights, + routed_cols, + topk_weights[:, :-num_fused_shared_experts], num_fused_shared_experts, num_physical_routed_experts, TopKConfig( diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 17c2c2049b..00628c62bd 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -198,7 +198,7 @@ if _is_cuda: scaling_factor: float, topk_weights: torch.Tensor, topk_ids: torch.Tensor, - renormalize: bool, + launch_with_pdl: bool, ) -> None: _fused_topk_deepseek( gating_output, @@ -209,7 +209,7 @@ if _is_cuda: scaling_factor, topk_weights, topk_ids, - renormalize, + launch_with_pdl, ) except ImportError: @@ -1815,8 +1815,8 @@ def biased_grouped_topk_gpu( moe_fused_gate as jit_grouped_gate, ) - # The kernel wants the total width; select_experts passes a routed-only - # topk on the aiter path only (`num_routed_topk if _use_aiter else top_k`). + # The kernel wants the total width; off aiter, topk already counts the + # num_fused_shared_experts slots it writes. # # AITER folds this route even when no shared slots are requested. return jit_grouped_gate( @@ -1838,6 +1838,8 @@ def biased_grouped_topk_gpu( if ( _is_cuda and fused_topk_deepseek is not None + # flashinfer always renormalizes the selected weights + and renormalize and is_power_of_two(num_experts) # flashinfer constraints (applied to routed experts only) and topk_routed <= 8 @@ -1873,21 +1875,21 @@ def biased_grouped_topk_gpu( scaling_factor, topk_weights, topk_ids, - True, + launch_with_pdl=True, ) if num_fused_shared_experts > 0: - # Append shared expert columns: ID = num_experts (first shared slot), - # weight = sum(routed) / scaling_factor (matching biased_grouped_topk_impl). - # For DeepEP/MegaMOE per-rank shared-slot layout, post-process remaps - # this placeholder ID and overwrites the shared weight for the active scaling path. + # Append the global shared columns: ID = num_experts (first shared slot). + # On CUDA the routed weights carry the factor exactly when asked to. topk_ids = F.pad(topk_ids, (0, num_fused_shared_experts), value=num_experts) - topk_weights = F.pad(topk_weights, (0, num_fused_shared_experts)) - if routed_scaling_factor is not None: - topk_weights[:, topk_routed:] = ( - topk_weights[:, :topk_routed].sum(dim=-1, keepdim=True) - / routed_scaling_factor - ) + topk_weights = F.pad( + topk_weights, + (0, num_fused_shared_experts), + value=fused_shared_expert_weight( + routed_scaling_factor=routed_scaling_factor, + fused_in_topk=bool(apply_routed_scaling_factor_on_output), + ), + ) return topk_weights, topk_ids @@ -2240,10 +2242,7 @@ def remap_topk_for_per_rank_shared_slots( num_physical_routed_experts: int, topk_config: TopKConfig, ) -> tuple[torch.Tensor, torch.Tensor]: - """Map routed and shared columns into rank-major expert blocks.""" - if topk_ids.shape[0] == 0: - return topk_ids, topk_weights - + """Append shared columns to routed-only inputs in rank-major expert blocks.""" ep_size = get_parallel().moe_ep_size ep_rank = get_parallel().moe_ep_rank # Static EPLB may add redundant physical experts. At this point routed @@ -2251,31 +2250,36 @@ def remap_topk_for_per_rank_shared_slots( # per-rank shared-slot layout must use the physical routed count. num_local_routed = num_physical_routed_experts // ep_size num_local_experts = num_local_routed + num_fused_shared_experts + num_tokens = topk_ids.shape[0] # Remap routed IDs: insert gaps for shared expert slots (single fused op). # Each rank r == e // num_local_routed is preceded by r shared-slot blocks of # width num_fused_shared_experts, so shift by (e // num_local_routed) * S -- # a single-slot shift (S == 1) would let routed ids collide with an earlier # rank's shared slots once S > 1. - routed = topk_ids[:, :-num_fused_shared_experts] - topk_ids[:, :-num_fused_shared_experts] = ( - routed + (routed // num_local_routed) * num_fused_shared_experts - ) + routed_ids = topk_ids + (topk_ids // num_local_routed) * num_fused_shared_experts - # Set shared expert IDs to route to home rank (vectorized) - topk_ids[:, -num_fused_shared_experts:] = ( + # Shared expert IDs route to the home rank. + shared_ids = ( ep_rank * num_local_experts + num_local_routed - + torch.arange(num_fused_shared_experts, device=topk_ids.device) + + torch.arange( + num_fused_shared_experts, dtype=topk_ids.dtype, device=topk_ids.device + ) + ).expand(num_tokens, -1) + shared_weights = topk_weights.new_full( + (num_tokens, num_fused_shared_experts), + fused_shared_expert_weight( + routed_scaling_factor=topk_config.routed_scaling_factor, + fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config), + ), ) - topk_weights[:, -num_fused_shared_experts:] = fused_shared_expert_weight( - routed_scaling_factor=topk_config.routed_scaling_factor, - fused_in_topk=is_routed_scaling_factor_fused_in_topk(topk_config), + return ( + torch.cat([routed_ids, shared_ids], dim=-1), + torch.cat([topk_weights, shared_weights], dim=-1), ) - return topk_ids, topk_weights - def capture_routed_experts_if_allowed( topk_config: TopKConfig, @@ -2313,12 +2317,13 @@ def _post_process_topk_ids( fused_shared_experts_scaling_factor = ( topk_config.fused_shared_experts_scaling_factor ) + # False when select_experts asked the router for routed experts only. + router_wrote_shared_cols = ( + num_fused_shared_experts > 0 and topk_ids.shape[-1] == topk_config.top_k + ) # The router already wrote the aiter shared columns (see _aiter_append below). _gate_appended = ( - num_fused_shared_experts > 0 - and _use_aiter - and not use_per_rank_shared_slots - and topk_ids.shape[-1] == topk_config.top_k + router_wrote_shared_cols and _use_aiter and not use_per_rank_shared_slots ) routed_topk_ids = ( topk_ids[:, :-num_fused_shared_experts] if _gate_appended else topk_ids @@ -2352,7 +2357,7 @@ def _post_process_topk_ids( topk_ids, expert_location_dispatch_info, log2phy_prob ) _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) - elif num_fused_shared_experts > 0: + elif router_wrote_shared_cols: # Shared IDs are outside EPLB's routed-expert table for both global # and per-rank layouts, so remap only routed columns. shared_cols = topk_ids[:, -num_fused_shared_experts:] @@ -2504,6 +2509,10 @@ def _post_process_topk_ids( if expert_location_dispatch_info is not None else router_logits.shape[1] ) + if router_wrote_shared_cols: + # Placeholders; the remap appends the real shared columns. + topk_ids = topk_ids[:, :-num_fused_shared_experts] + topk_weights = topk_weights[:, :-num_fused_shared_experts] topk_ids, topk_weights = remap_topk_for_per_rank_shared_slots( topk_ids, topk_weights, @@ -2511,6 +2520,9 @@ def _post_process_topk_ids( num_physical_routed_experts, topk_config, ) + if _is_cuda: + # The remap shifts a padded row's -1 and gives it a live shared slot. + _mask_topk_ids_padded_region(topk_ids, num_token_non_padded) elif ( num_fused_shared_experts > 0 and fused_shared_experts_scaling_factor is not None ): @@ -2612,22 +2624,15 @@ def select_experts( ): correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype) - # DeepSeek V2/V3/R1 series models use grouped_top_k - # remove num_fused_shared_experts from grouped_topk/biased_grouped_topk num_routed_topk = top_k - num_fused_shared_experts - # On the per-rank shared-slot path the shared expert is appended afterwards by - # fused_append_remap_shared_experts_deepep, so the gate must not emit a shared - # marker as well. Doing both costs one routed expert (the gate is asked for - # K_routed = top_k - num_fused_shared_experts and then spends one of those - # slots on the marker) and places that marker at id num_experts, which the - # DeepEP remap shifts one past the end of the expert space -- 384 -> 392 for - # 384 routed experts on EP8, where the valid ids are 0..391. - # aiter appends the shared expert in _post_process_topk_ids, so the gate must not. - num_fused_shared_experts_for_gate = ( - 0 - if (has_per_rank_fused_shared_slots(num_fused_shared_experts) or _use_aiter) - else num_fused_shared_experts - ) + # On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends + # the shared columns, so the router selects and renormalizes routed experts only. + if _use_aiter or has_per_rank_fused_shared_slots(num_fused_shared_experts): + top_k_for_gate = num_routed_topk + num_fused_shared_experts_for_gate = 0 + else: + top_k_for_gate = top_k + num_fused_shared_experts_for_gate = num_fused_shared_experts if dynamic_expert_bias is not None: if scoring_func != "sigmoid": raise ValueError( @@ -2655,7 +2660,7 @@ def select_experts( topk_weights, topk_ids = grouped_topk( hidden_states=hidden_states, gating_output=router_logits, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, num_expert_group=num_expert_group, topk_group=topk_group, @@ -2669,7 +2674,7 @@ def select_experts( hidden_states=hidden_states, gating_output=router_logits, correction_bias=correction_bias, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, num_expert_group=num_expert_group, topk_group=topk_group, @@ -2686,7 +2691,7 @@ def select_experts( topk_weights, topk_ids = fused_topk_native( hidden_states=hidden_states, gating_output=router_logits, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, correction_bias=correction_bias, scoring_func=scoring_func, @@ -2731,7 +2736,7 @@ def select_experts( hidden_states=hidden_states, gating_output=router_logits, correction_bias=correction_bias, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, scoring_func=scoring_func, num_fused_shared_experts=num_fused_shared_experts_for_gate, @@ -2751,7 +2756,7 @@ def select_experts( topk_weights, topk_ids = fused_topk_softmax_torch_raw_logits( hidden_states=hidden_states, gating_output=router_logits, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, ) else: @@ -2794,7 +2799,7 @@ def select_experts( topk_weights, topk_ids = fused_topk( hidden_states=hidden_states, gating_output=router_logits, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, correction_bias=correction_bias, scoring_func=scoring_func, @@ -2812,7 +2817,7 @@ def select_experts( topk_weights, topk_ids = custom_routing_function( hidden_states=hidden_states, gating_output=router_logits, - topk=num_routed_topk if _use_aiter else top_k, + topk=top_k_for_gate, renormalize=renormalize, ) diff --git a/python/sglang/srt/state_capturer/routed_experts.py b/python/sglang/srt/state_capturer/routed_experts.py index ef989b822a..b7913a9f9e 100644 --- a/python/sglang/srt/state_capturer/routed_experts.py +++ b/python/sglang/srt/state_capturer/routed_experts.py @@ -3,6 +3,7 @@ from typing import Any, Optional import numpy as np import pybase64 import torch +import torch.nn.functional as F from sglang.srt.configs.model_config import ModelConfig from sglang.srt.layers.dp_attention import ( @@ -106,6 +107,18 @@ class RoutedExpertsCapturer(BaseTopkCapturer): ) def capture(self, layer_id: int, topk_indices: torch.Tensor): + device_width = self.device_cache.buffer.shape[2] + if topk_indices.shape[1] != device_width: + # Routes whose shared columns are appended after capture pass routed ids only. + if topk_indices.shape[1] != device_width - self.num_fused_shared_experts: + raise ValueError( + f"Captured top-k width {topk_indices.shape[1]} is neither the " + f"device width {device_width} nor its routed part " + f"{device_width - self.num_fused_shared_experts}" + ) + topk_indices = F.pad( + topk_indices, (0, self.num_fused_shared_experts), value=-1 + ) if _is_scattered_a2a_backend(): local_topk = topk_indices topk_indices = self.gather_buffer[ diff --git a/test/registered/kernels/ops/moe/test_moe_fused_gate.py b/test/registered/kernels/ops/moe/test_moe_fused_gate.py index e419acfe17..8bb6791587 100644 --- a/test/registered/kernels/ops/moe/test_moe_fused_gate.py +++ b/test/registered/kernels/ops/moe/test_moe_fused_gate.py @@ -1,16 +1,4 @@ -"""Correctness tests for the Triton :func:`moe_fused_gate` router. - -For the ungrouped case (``num_expert_group == 1``) we validate it two ways: - -* against an explicit, definition-based torch reference (documents the math), - at small shapes and at model routing shapes, and -* against the production ``biased_grouped_topk_impl`` for the sigmoid / no-shared - path that the kernel actually replaces in ``topk.py``. - -Comparisons are order-independent: weights are scattered back to a dense -``[M, num_experts + num_shared]`` layout so the per-row column order (and any -tie-break choice) does not matter. -""" +"""Triton routing must match independent math and supported production routes.""" from __future__ import annotations @@ -79,20 +67,24 @@ def _reference_gate( work.scatter_(1, winner.long().unsqueeze(1), float("-inf")) routed_sum = routed_wgt.sum(dim=1, keepdim=True) + if renormalize: + norm = torch.where(routed_sum > 0.0, routed_sum, torch.ones_like(routed_sum)) + routed_wgt = routed_wgt / norm + if apply_routed_scaling_factor_on_output: + routed_wgt = routed_wgt * routed_scaling_factor weights = torch.empty(bs, topk, dtype=torch.float32, device=scores.device) indices = torch.empty(bs, topk, dtype=torch.int32, device=scores.device) weights[:, :num_routed] = routed_wgt indices[:, :num_routed] = routed_idx if num_fused_shared_experts > 0: - weights[:, num_routed:] = routed_sum / routed_scaling_factor + weights[:, num_routed:] = ( + 1.0 + if apply_routed_scaling_factor_on_output + else 1.0 / routed_scaling_factor + ) for j in range(num_fused_shared_experts): indices[:, num_routed + j] = num_experts + j - if renormalize: - norm = torch.where(routed_sum > 0.0, routed_sum, torch.ones_like(routed_sum)) - weights = weights / norm - if apply_routed_scaling_factor_on_output: - weights = weights * routed_scaling_factor return weights, indices @@ -203,15 +195,14 @@ def test_moe_fused_gate_matches_reference_at_model_shapes( @pytest.mark.parametrize("num_experts,topk", [(256, 6), (384, 8), (512, 8)]) @pytest.mark.parametrize("apply_scale", [True, False]) +@pytest.mark.parametrize("renormalize", [True, False]) def test_moe_fused_gate_matches_production_impl( - num_experts: int, topk: int, apply_scale: bool + num_experts: int, topk: int, apply_scale: bool, renormalize: bool ) -> None: """Match production ``biased_grouped_topk_impl`` on the path it replaces. - The kernel supersedes the ungrouped sigmoid CUDA path in ``topk.py``; that - reference hardcodes sigmoid and no fused shared expert, and the production - path always renormalizes (the impl only applies the scaling factor when - ``renormalize`` is set), so we compare on the renormalized path. + Compare ungrouped sigmoid routing without shared slots, with normalization + and explicit routed scaling enabled independently. """ from sglang.srt.layers.moe.topk import biased_grouped_topk_impl @@ -224,7 +215,7 @@ def test_moe_fused_gate_matches_production_impl( bias, topk=topk, scoring_func="sigmoid", - renormalize=True, + renormalize=renormalize, routed_scaling_factor=scale, apply_routed_scaling_factor_on_output=apply_scale, ) @@ -234,7 +225,7 @@ def test_moe_fused_gate_matches_production_impl( scores, bias, topk, - True, + renormalize, num_expert_group=1, topk_group=1, routed_scaling_factor=scale, diff --git a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py index 3fef553852..79a0700d38 100644 --- a/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py +++ b/test/registered/moe/test_fused_append_remap_per_rank_shared_slots.py @@ -1,26 +1,28 @@ """Unit tests for fused append + per-rank shared-slot remap. -Covers ``fused_append_remap_shared_experts_deepep``, which collapses -``fused_append_shared_experts()`` followed by -``remap_topk_for_per_rank_shared_slots()`` into a -single Triton launch on the per-rank shared-slot path. The kernel is GPU-only -(Triton), so these tests are skipped when no accelerator is present. +Covers ``fused_append_remap_shared_experts_deepep``, the single Triton launch +form of ``remap_topk_for_per_rank_shared_slots()`` on the per-rank shared-slot +path, and ``select_experts`` on that path with the real routers. The kernels are +GPU-only (Triton), so these tests are skipped when no accelerator is present. """ import unittest +from unittest.mock import patch import torch from sglang.kernels.ops.moe.fused_moe_triton_kernels import ( fused_append_remap_shared_experts_deepep, - fused_append_shared_experts, ) +from sglang.srt.layers.moe import topk as topk_module from sglang.srt.layers.moe.topk import ( TopKConfig, + biased_grouped_topk_impl, remap_topk_for_per_rank_shared_slots, + select_experts, ) -from sglang.srt.runtime_context import get_parallel -from sglang.srt.utils import get_device +from sglang.srt.runtime_context import get_context, get_parallel +from sglang.srt.utils import get_device, is_cuda from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -176,7 +178,7 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): self.assertEqual(len(routed_set), npr) self.assertLess(max(routed_set), ep_size * num_local_experts) - def test_equivalence_with_eager_append_then_remap(self): + def test_equivalence_with_eager_remap(self): """Fused and eager append agree for folding and nonfolding routes.""" rsf = 2.5 for folded, scale_factor in ((False, 1.0 / rsf), (True, 1.0)): @@ -201,17 +203,10 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): with get_parallel().override( moe_ep_size=ep_size, moe_ep_rank=ep_rank ): - eager_ids, eager_w = fused_append_shared_experts( + eager_ids, eager_w = remap_topk_for_per_rank_shared_slots( topk_ids.clone(), topk_weights.clone(), s, - scale_factor, - npr, # shared-expert base id (overwritten by the remap) - ) - eager_ids, eager_w = remap_topk_for_per_rank_shared_slots( - eager_ids, - eager_w, - s, npr, TopKConfig( top_k=k, @@ -294,5 +289,160 @@ class TestFusedAppendRemapPerRankSharedSlots(CustomTestCase): self.assertTrue(torch.equal(got_w, topk_weights)) +@unittest.skipUnless(is_cuda(), "covers the CUDA routers") +class TestSelectExpertsPerRankSharedSlots(CustomTestCase): + """Shared slots must not alter routed selection or normalization.""" + + NUM_TOKENS, TOP_K, EP_SIZE, EP_RANK = 32, 8, 8, 3 + # (num_experts, num_expert_group, topk_group, scoring_func, biased); each row + # reaches a different router. + ROUTERS = [ + (256, 8, 4, "sigmoid", True), # DeepSeek-V3 grouped routing + (384, 8, 4, "sigmoid", True), # grouped, outside FlashInfer's shapes + (64, 8, 3, "softmax", False), # grouped, no correction bias + (256, None, None, "sqrtsoftplus", True), # DeepSeek-V4 routing + (128, None, None, "softmax", False), + ] + + def _route( + self, *, num_fused_shared_experts, logits, bias, router, renormalize=True + ): + _, num_expert_group, topk_group, scoring_func, _ = router + topk_config = TopKConfig( + top_k=self.TOP_K + num_fused_shared_experts, + renormalize=renormalize, + use_grouped_topk=num_expert_group is not None, + num_expert_group=num_expert_group, + topk_group=topk_group, + num_fused_shared_experts=num_fused_shared_experts, + correction_bias=bias, + scoring_func=scoring_func, + routed_scaling_factor=2.5, + allow_routed_experts_capture=False, + ) + with ( + patch.object( + topk_module, + "has_per_rank_fused_shared_slots", + side_effect=lambda num_fused: num_fused > 0, + ), + get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), + ): + out = select_experts( + torch.empty(logits.shape[0], 16, device=logits.device), + logits, + topk_config, + ) + return out.topk_ids.long(), out.topk_weights + + def test_routes_like_no_fusion(self): + k = self.TOP_K + for router in self.ROUTERS: + num_experts, _, _, _, biased = router + with self.subTest(router=router): + g = torch.Generator(device="cpu").manual_seed(num_experts) + logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g) + bias = 0.1 * torch.randn(num_experts, generator=g) if biased else None + logits = logits.to(get_device()) + bias = bias.to(get_device()) if biased else None + ref_ids, ref_weights = self._route( + num_fused_shared_experts=0, logits=logits, bias=bias, router=router + ) + ids, weights = self._route( + num_fused_shared_experts=1, logits=logits, bias=bias, router=router + ) + + self.assertEqual(ids.shape, (self.NUM_TOKENS, k + 1)) + torch.testing.assert_close( + weights[:, :k].sum(-1), torch.ones_like(weights[:, 0]) + ) + # Rank r's routed block starts at r * (num_local_routed + 1). + num_local_routed = num_experts // self.EP_SIZE + expected_ids = ref_ids + ref_ids // num_local_routed + order, ref_order = ids[:, :k].argsort(-1), expected_ids.argsort(-1) + self.assertTrue( + torch.equal( + ids[:, :k].gather(-1, order), expected_ids.gather(-1, ref_order) + ) + ) + torch.testing.assert_close( + weights[:, :k].gather(-1, order), ref_weights.gather(-1, ref_order) + ) + home_shared_id = ( + self.EP_RANK * (num_local_routed + 1) + num_local_routed + ) + self.assertTrue(torch.all(ids[:, k] == home_shared_id)) + + def test_unnormalized_global_shared_contribution_is_one(self): + """Raw routed scores must not multiply the global shared contribution.""" + logits = torch.zeros(4, 256, device=get_device()) + rsf = 2.5 + with get_context().override_server_args( + moe_a2a_backend="none", moe_runner_backend="triton" + ): + for folded in (False, True): + with self.subTest(folded=folded): + config = TopKConfig( + top_k=9, + use_grouped_topk=True, + num_expert_group=8, + topk_group=4, + renormalize=False, + num_fused_shared_experts=1, + correction_bias=torch.zeros(256, device=get_device()), + scoring_func="sigmoid", + routed_scaling_factor=rsf, + apply_routed_scaling_factor_on_output=folded, + allow_routed_experts_capture=False, + ) + output = select_experts(logits, logits, config) + torch.testing.assert_close( + output.topk_weights[:, :8], + torch.full_like( + output.topk_weights[:, :8], 0.5 * (rsf if folded else 1.0) + ), + ) + shared = output.topk_weights[:, -1] * (1.0 if folded else rsf) + torch.testing.assert_close(shared, torch.ones_like(shared)) + + def test_unnormalized_weights_stay_raw(self): + """Unnormalized routing must preserve raw selected sigmoid scores.""" + k, num_experts = self.TOP_K, 256 + router = (num_experts, 8, 4, "sigmoid", True) + g = torch.Generator(device="cpu").manual_seed(0) + logits = torch.randn(self.NUM_TOKENS, num_experts, generator=g).to(get_device()) + bias = (0.1 * torch.randn(num_experts, generator=g)).to(get_device()) + _, ref_ids = biased_grouped_topk_impl( + logits, + logits, + bias, + topk=k, + renormalize=False, + num_expert_group=8, + topk_group=4, + ) + num_local_routed = num_experts // self.EP_SIZE + for num_fused_shared_experts in (0, 1): + with self.subTest(num_fused_shared_experts=num_fused_shared_experts): + ids, weights = self._route( + num_fused_shared_experts=num_fused_shared_experts, + logits=logits, + bias=bias, + router=router, + renormalize=False, + ) + routed_ids = ids[:, :k] + if num_fused_shared_experts: + routed_ids = routed_ids - routed_ids // (num_local_routed + 1) + self.assertTrue( + torch.equal( + routed_ids.sort(-1).values, ref_ids.long().sort(-1).values + ) + ) + torch.testing.assert_close( + weights[:, :k], logits.sigmoid().gather(-1, routed_ids) + ) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/moe/test_topk_padded_region.py b/test/registered/moe/test_topk_padded_region.py index d3fc006a64..b6fb401c4f 100644 --- a/test/registered/moe/test_topk_padded_region.py +++ b/test/registered/moe/test_topk_padded_region.py @@ -1,4 +1,5 @@ import unittest +from unittest.mock import patch import torch @@ -11,6 +12,7 @@ from sglang.srt.layers.moe.topk import ( _post_process_topk_ids, _zero_topk_weights_padded_region, ) +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import is_hip from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -275,5 +277,57 @@ class TestSelectExpertsCustomRoutingPadMask(CustomTestCase): self.assertTrue(torch.all(out.topk_ids[:n_valid] < num_experts)) +@unittest.skipUnless(torch.cuda.is_available(), "padded-region masking needs a GPU") +@unittest.skipIf(_IS_HIP, "HIP marks padded rows with zero weights instead") +class TestSelectExpertsPerRankSharedSlotPadMask(CustomTestCase): + """With per-rank fused shared slots, padded rows must leave select_experts + fully -1, the appended shared column included. Masking only before the + append left them routed ids of -1 - S and a live home-rank shared slot.""" + + DEVICE = "cuda" + + def test_padded_tail_masked_after_shared_append(self): + num_tokens, num_experts, top_k, n_valid = 12, 256, 8, 9 + # EP8 rank 3: 32 routed slots per rank, then the rank's shared slot. + home_shared_id = 3 * 33 + 32 + # Grouped sigmoid is masked after routing, sqrtsoftplus by its router. + for scoring_func, num_expert_group in (("sigmoid", 8), ("sqrtsoftplus", None)): + with self.subTest(scoring_func=scoring_func): + g = torch.Generator(device="cpu").manual_seed(0) + logits = torch.randn(num_tokens, num_experts, generator=g) + bias = 0.1 * torch.randn(num_experts, generator=g) + topk_config = TopKConfig( + top_k=top_k + 1, + use_grouped_topk=num_expert_group is not None, + num_expert_group=num_expert_group, + topk_group=4 if num_expert_group else None, + num_fused_shared_experts=1, + correction_bias=bias.to(self.DEVICE), + scoring_func=scoring_func, + routed_scaling_factor=2.5, + allow_routed_experts_capture=False, + ) + with ( + patch.object( + topk_mod, + "has_per_rank_fused_shared_slots", + side_effect=lambda num_fused: num_fused > 0, + ), + get_parallel().override(moe_ep_size=8, moe_ep_rank=3), + ): + out = topk_mod.select_experts( + hidden_states=torch.empty(num_tokens, 16, device=self.DEVICE), + router_logits=logits.to(self.DEVICE), + topk_config=topk_config, + layer_id=0, + num_token_non_padded=torch.tensor( + n_valid, device=self.DEVICE, dtype=torch.int32 + ), + ) + self.assertTrue(torch.all(out.topk_ids[n_valid:] == -1)) + self.assertTrue(torch.all(out.topk_ids[:n_valid] >= 0)) + self.assertTrue(torch.all(out.topk_ids[:n_valid, -1] == home_shared_id)) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/eplb/test_waterfill_eplb.py b/test/registered/unit/eplb/test_waterfill_eplb.py index 99503d43e2..86c17d8ec2 100644 --- a/test/registered/unit/eplb/test_waterfill_eplb.py +++ b/test/registered/unit/eplb/test_waterfill_eplb.py @@ -98,6 +98,50 @@ class TestWaterfillEPLB(CustomTestCase): self.assertTrue(torch.equal(processed_ids, torch.tensor([[0, 34, 270, 271]]))) self.assertTrue(torch.equal(recorder_ids, torch.tensor([[0, 33, 263]]))) + def test_eplb_remaps_every_column_of_routed_only_per_rank_ids(self): + """EPLB must remap all routed columns before shared-slot append and capture.""" + topk_ids = torch.tensor([[0, 33, 262]], dtype=torch.int32) + topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) + topk_config = TopKConfig( + top_k=4, + num_fused_shared_experts=1, + routed_scaling_factor=1.0, + ) + dispatch_info = SimpleNamespace( + num_physical_experts=264, ep_dispatch_algorithm="static" + ) + + def fake_eplb_postprocess( + ids, expert_location_dispatch_info, num_token_non_padded + ): + return ids + 1 + + with ( + patch.object(topk_module, "_is_cuda", True), + patch.object(topk_module, "_use_aiter", False), + patch.object( + topk_module, "has_per_rank_fused_shared_slots", return_value=True + ), + get_parallel().override(moe_ep_size=8, moe_ep_rank=7), + patch.object( + topk_module, + "_biased_grouped_topk_postprocess", + side_effect=fake_eplb_postprocess, + ), + ): + processed_ids, _, recorder_ids = topk_module._post_process_topk_ids( + topk_ids=topk_ids.clone(), + topk_weights=topk_weights.clone(), + topk_config=topk_config, + router_logits=torch.empty((1, 256)), + layer_id=0, + expert_location_dispatch_info=dispatch_info, + ) + + # Physical ids 1, 34, 263 in the EP8 per-rank layout; 271 is rank 7's shared slot. + self.assertTrue(torch.equal(processed_ids, torch.tensor([[1, 35, 270, 271]]))) + self.assertTrue(torch.equal(recorder_ids, torch.tensor([[1, 34, 263]]))) + def test_topk_recorder_ids_exclude_global_fused_shared_slot(self): topk_ids = torch.tensor([[0, 33, 200, 256]], dtype=torch.int32) topk_weights = torch.ones_like(topk_ids, dtype=torch.float32) diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index c48016400b..d120bd8b8a 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -35,12 +35,10 @@ class TestFusedSharedExpertScaling(CustomTestCase): UNBIASED_ROUTING = {"use_grouped_topk": True, "correction_bias": None} def _run_remap(self, *, use_aiter, folded_in_topk=False): - # Layout: [routed, routed, routed, shared]; the trailing column is the - # fused shared expert. The shared weight starts at a sentinel to prove - # the function overwrites it. - topk_ids = torch.tensor([[5, 40, 100, 999]], dtype=torch.int32) + # The remap appends the fused shared column after the routed ones. + topk_ids = torch.tensor([[5, 40, 100]], dtype=torch.int32) routed_weights = torch.tensor([1.0, 0.5, 0.25], dtype=torch.float32) - topk_weights = torch.tensor([[1.0, 0.5, 0.25, -123.0]], dtype=torch.float32) + topk_weights = routed_weights.unsqueeze(0) topk_config = TopKConfig( top_k=4, @@ -441,8 +439,8 @@ class TestFusedSharedExpertScaling(CustomTestCase): def test_shared_expert_ids_route_to_home_rank(self): # Sanity: the shared slot id is placed at this rank's interleaved # position (ep_rank * num_local_experts + num_local_routed). - topk_ids = torch.tensor([[5, 40, 100, 999]], dtype=torch.int32) - topk_weights = torch.tensor([[1.0, 0.5, 0.25, 0.0]], dtype=torch.float32) + topk_ids = torch.tensor([[5, 40, 100]], dtype=torch.int32) + topk_weights = torch.tensor([[1.0, 0.5, 0.25]], dtype=torch.float32) topk_config = TopKConfig( top_k=4, num_fused_shared_experts=1, @@ -470,6 +468,61 @@ class TestFusedSharedExpertScaling(CustomTestCase): expected_shared_id = self.EP_RANK * num_local_experts + num_local_routed self.assertEqual(out_ids[0, -1].item(), expected_shared_id) + def _route_with_per_rank_slots(self, *, num_fused_shared_experts, logits, bias): + topk_config = TopKConfig( + top_k=8 + num_fused_shared_experts, + num_fused_shared_experts=num_fused_shared_experts, + renormalize=True, + correction_bias=bias, + torch_native=True, + scoring_func="sigmoid", + routed_scaling_factor=self.ROUTED_SCALING_FACTOR, + allow_routed_experts_capture=False, + ) + with ( + patch.object(topk_module, "_use_aiter", False), + patch.object( + topk_module, + "has_per_rank_fused_shared_slots", + side_effect=lambda num_fused: num_fused > 0, + ), + get_parallel().override(moe_ep_size=self.EP_SIZE, moe_ep_rank=self.EP_RANK), + ): + out = topk_module.select_experts( + torch.empty(logits.shape[0], 1), logits, topk_config + ) + return out.topk_ids, out.topk_weights + + def test_per_rank_slot_routes_like_no_fusion(self): + """With per-rank shared slots the router must pick and renormalize the + routed experts alone. Asked for top_k with the shared slot counted, it + renormalized over one routed expert too many, which the shared column + then overwrote, so the kept routed weights summed to about 0.9.""" + torch.manual_seed(0) + logits = torch.randn(16, self.NUM_PHYSICAL_ROUTED) + bias = 0.1 * torch.randn(self.NUM_PHYSICAL_ROUTED) + ref_ids, ref_weights = self._route_with_per_rank_slots( + num_fused_shared_experts=0, logits=logits, bias=bias + ) + ids, weights = self._route_with_per_rank_slots( + num_fused_shared_experts=1, logits=logits, bias=bias + ) + + self.assertEqual(ids.shape, (16, 9)) + torch.testing.assert_close(weights[:, :8].sum(-1), torch.ones(16)) + # Rank r's routed block starts at r * (num_local_routed + 1). + num_local_routed = self.NUM_PHYSICAL_ROUTED // self.EP_SIZE + expected_ids = ref_ids + ref_ids // num_local_routed + order, ref_order = ids[:, :8].argsort(-1), expected_ids.argsort(-1) + self.assertTrue( + torch.equal( + ids[:, :8].gather(-1, order), expected_ids.gather(-1, ref_order) + ) + ) + torch.testing.assert_close( + weights[:, :8].gather(-1, order), ref_weights.gather(-1, ref_order) + ) + def test_glm53_idle_rank_preserves_routed_topk_contract(self): """An idle rank must return the pre-append top-8 shape; returning top-9 here would make the AITER postprocessor append shared ID 288 twice.""" diff --git a/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py b/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py index 93e759899b..8ff6aa3e58 100644 --- a/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py +++ b/test/registered/unit/state_capturer/test_routed_experts_scattered_a2a.py @@ -1,4 +1,4 @@ -"""DeepEP-family backend recognition in RoutedExpertsCapturer.""" +"""RoutedExpertsCapturer: DeepEP-family backend recognition and capture width.""" import unittest from types import SimpleNamespace @@ -8,6 +8,7 @@ import torch from sglang.srt.layers.moe.utils import MoeA2ABackend from sglang.srt.state_capturer import routed_experts as re_mod +from sglang.srt.state_capturer.base import BaseDeviceCache from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -67,5 +68,55 @@ class TestGetLocalSliceBackendBranch(CustomTestCase): self.assertTrue(torch.equal(out, buf[6:10, :, : self.K])) +class TestCaptureRoutedIdsOnly(CustomTestCase): + TOPK_SIZE = 3 + + def _capturer(self): + cap = object.__new__(re_mod.RoutedExpertsCapturer) + cap.num_fused_shared_experts = 1 + cap.device_cache = BaseDeviceCache( + max_batch_size=8, + num_layers=2, + topk_size=self.TOPK_SIZE + 1, + device="cpu", + name="routed_experts", + ) + cap.gather_buffer = torch.empty((8, self.TOPK_SIZE + 1), dtype=torch.int32) + return cap + + def _capture(self, cap, ids, backend): + with ( + mock.patch.object( + re_mod, "get_moe_a2a_backend", return_value=MoeA2ABackend(backend) + ), + mock.patch.object( + re_mod, + "attn_tp_all_gather_into_tensor", + side_effect=lambda out, inp: out.copy_(inp), + ), + mock.patch.object( + re_mod, "get_parallel", return_value=SimpleNamespace(attn_tp_size=1) + ), + ): + cap.capture(layer_id=1, topk_indices=ids) + + def test_routed_only_ids_fill_the_routed_columns(self): + """Routed-only capture must fill the routed columns, including scattered a2a.""" + ids = torch.arange(4 * self.TOPK_SIZE, dtype=torch.int32).view(4, -1) + for backend in ("none", "deepep"): + with self.subTest(backend=backend): + cap = self._capturer() + self._capture(cap, ids, backend) + self.assertTrue( + torch.equal(cap.device_cache.buffer[:4, 1, : self.TOPK_SIZE], ids) + ) + + def test_other_widths_are_rejected(self): + """Only the routed-only width may be padded; any other width is a caller bug.""" + ids = torch.zeros((4, self.TOPK_SIZE - 1), dtype=torch.int32) + with self.assertRaises(ValueError): + self._capture(self._capturer(), ids, "none") + + if __name__ == "__main__": unittest.main()