From d5a8d76bf5061e0ecaae4e0db658b6cf66841cb5 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:17:26 -0700 Subject: [PATCH] [Refactor] Drop TBO op methods that no strategy ever schedules (#42300) --- .../srt/models/dots3_common/modeling.py | 176 ------------------ python/sglang/srt/models/glm4_moe.py | 137 -------------- python/sglang/srt/models/glm4_moe_lite.py | 150 --------------- python/sglang/srt/models/minimax_m2.py | 156 ---------------- python/sglang/srt/models/step3p5.py | 68 ------- 5 files changed, 687 deletions(-) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index ff9f43082e..8e391b83cf 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -129,7 +129,6 @@ from sglang.srt.utils import ( ceil_div, get_bool_env_var, is_cuda, - is_non_idle_and_non_empty, log_info_on_rank0, make_pp_layers, ) @@ -550,103 +549,6 @@ class Dots3MoE(nn.Module): else: return None - def op_gate(self, state): - if is_non_idle_and_non_empty( - state.forward_batch.forward_mode, state.hidden_states_mlp_input - ): - # router_logits: (num_tokens, n_experts) - state.router_logits = self.gate(state.hidden_states_mlp_input) - else: - state.router_logits = None - - def op_shared_experts(self, state): - hidden_states_mlp_input = state.pop("hidden_states_mlp_input") - if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty( - state.forward_batch.forward_mode, hidden_states_mlp_input - ): - state.shared_output = self.shared_experts(hidden_states_mlp_input) - else: - state.shared_output = None - - def op_select_experts(self, state): - router_logits = state.pop("router_logits") - hidden_states = state.hidden_states_mlp_input - - if router_logits is not None: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.topk_weights_local, state.topk_idx_local, _ = self.topk( - hidden_states=hidden_states, - router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), - ) - else: - state.topk_idx_local = torch.full( - (0, self.top_k), -1, dtype=torch.int, device=hidden_states.device - ) - state.topk_weights_local = torch.empty( - (0, self.top_k), dtype=torch.float32, device=hidden_states.device - ) - - def op_dispatch_a(self, state): - if self.ep_size > 1: - self.experts.deepep_dispatcher.dispatch_a( - hidden_states=state.hidden_states_mlp_input, - topk_idx=state.pop("topk_idx_local"), - topk_weights=state.pop("topk_weights_local"), - forward_batch=state.forward_batch, - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_dispatch_b(self, state): - if self.ep_size > 1: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_experts(self, state): - state.hidden_states_experts_output = self.experts.moe_impl( - dispatch_output=state.dispatch_output, - ) - - def op_combine_a(self, state): - if self.ep_size > 1: - self.experts.deepep_dispatcher.combine_a( - hidden_states=state.pop("hidden_states_experts_output"), - topk_idx=state.dispatch_output.topk_idx, - topk_weights=state.dispatch_output.topk_weights, - forward_batch=state.forward_batch, - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - state.pop("dispatch_output") - - def op_combine_b(self, state): - if self.ep_size > 1: - state.hidden_states_after_combine = ( - self.experts.deepep_dispatcher.combine_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - ) - - def op_output(self, state): - final_hidden_states = state.pop("hidden_states_after_combine") - - if (shared_output := state.pop("shared_output")) is not None: - x = shared_output - x.add_(final_hidden_states, alpha=self.routed_scaling_factor) - final_hidden_states = x - else: - final_hidden_states *= self.routed_scaling_factor - - state.hidden_states_mlp_output = final_hidden_states - # Aligned with HF's implementation, using sliding window inclusive with the last token. # SGLang assumes exclusive. @@ -1040,19 +942,6 @@ class Dots3AttentionMLA(nn.Module): ) return KVWriteLoc.for_batch(forward_batch, swa_loc=swa_loc) - def op_prepare(self, state): - state.attn_intermediate_state = self.forward_prepare( - positions=state.positions, - hidden_states=state.pop("hidden_states_after_comm_pre_attn"), - forward_batch=state.forward_batch, - zero_allocator=state.zero_allocator, - ) - - def op_core(self, state): - state.hidden_states_after_attn = self.forward_core( - state.pop("attn_intermediate_state") - ) - def forward( self, positions: torch.Tensor, @@ -1628,71 +1517,6 @@ class Dots3DecoderLayer(nn.Module): ) return ffn_exit.finish(hidden_states) - def op_comm_prepare_attn( - self, - state, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - tbo_subbatch_index: Optional[int] = None, - ): - state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( - hidden_states, forward_batch - ) - state.update( - dict( - forward_batch=forward_batch, - positions=positions, - zero_allocator=zero_allocator, - tbo_subbatch_index=tbo_subbatch_index, - ) - ) - - def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_boundary.finish( - state.pop("hidden_states_after_attn"), state.forward_batch - ) - state.hidden_states_mlp_input = self.ffn_boundary.prepare( - hidden_states, state.forward_batch - ) - - def op_mlp(self, state): - hidden_states = state.pop("hidden_states_mlp_input") - if not ( - is_dense_ffn_fully_dp() - and (not self.is_layer_sparse) - and hidden_states.shape[0] == 0 - ): - state.hidden_states_mlp_output = self.mlp( - hidden_states, state.forward_batch - ) - else: - state.hidden_states_mlp_output = hidden_states - - def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( - state.pop("hidden_states_mlp_output"), state.forward_batch - ) - - output = dict( - positions=state.positions, - hidden_states=hidden_states, - forward_batch=state.forward_batch, - zero_allocator=state.zero_allocator, - tbo_subbatch_index=state.tbo_subbatch_index, - ) - - state.clear( - expect_keys={ - "positions", - "forward_batch", - "zero_allocator", - "tbo_subbatch_index", - } - ) - return output - class Dots3Model(nn.Module): fall_back_to_pt_during_load = False diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index e47294bb51..c23263aa33 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -94,7 +94,6 @@ from sglang.srt.utils import ( is_cpu, is_cuda, is_hip, - is_non_idle_and_non_empty, is_npu, log_info_on_rank0, make_pp_layers, @@ -304,18 +303,6 @@ class Glm4MoeAttention(nn.Module): self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.alt_stream = alt_stream - def op_prepare(self, state): - state.attn_intermediate_state = self.forward_prepare( - positions=state.positions, - hidden_states=state.pop("hidden_states_after_comm_pre_attn"), - forward_batch=state.forward_batch, - ) - - def op_core(self, state): - state.hidden_states_after_attn = self.forward_core( - state.pop("attn_intermediate_state") - ) - def forward_prepare( self, positions: torch.Tensor, @@ -708,82 +695,6 @@ class Glm4MoeSparseMoeBlock(nn.Module): else: return None - def op_gate(self, state): - if is_non_idle_and_non_empty( - state.forward_batch.forward_mode, state.hidden_states_mlp_input - ): - # router_logits: (num_tokens, n_experts) - state.router_logits = self.gate(state.hidden_states_mlp_input) - else: - state.router_logits = None - - def op_select_experts(self, state): - router_logits = state.pop("router_logits") - hidden_states = state.hidden_states_mlp_input - - if router_logits is not None: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.topk_output = self.topk( - hidden_states=hidden_states, - router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), - ) - else: - state.topk_output = self.topk.empty_topk_output(hidden_states.device) - - def op_dispatch_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.dispatch_a( - hidden_states=state.hidden_states_mlp_input, - topk_output=state.pop("topk_output"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_dispatch_b(self, state): - if self.ep_size > 1: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.dispatch_output = self.experts.dispatcher.dispatch_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_experts(self, state): - state.combine_input = self.experts.run_moe_core( - dispatch_output=state.dispatch_output, - ) - - def op_combine_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.combine_a( - combine_input=state.pop("combine_input"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - state.pop("dispatch_output") - - def op_combine_b(self, state): - if self.ep_size > 1: - state.hidden_states_after_combine = self.experts.dispatcher.combine_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_output(self, state): - final_hidden_states = state.pop("hidden_states_after_combine") - - if (shared_output := state.pop("shared_output")) is not None: - x = shared_output - x.add_(final_hidden_states, alpha=self.routed_scaling_factor) - final_hidden_states = x - else: - final_hidden_states *= self.routed_scaling_factor - - state.hidden_states_mlp_output = final_hidden_states - class Glm4MoeDecoderLayer(nn.Module): def __init__( @@ -962,54 +873,6 @@ class Glm4MoeDecoderLayer(nn.Module): return hidden_states - def op_comm_prepare_attn( - self, - state, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - tbo_subbatch_index: Optional[int] = None, - ): - state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( - hidden_states, forward_batch - ) - state.update( - dict( - forward_batch=forward_batch, - positions=positions, - tbo_subbatch_index=tbo_subbatch_index, - ) - ) - - def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_boundary.finish( - state.pop("hidden_states_after_attn"), state.forward_batch - ) - state.hidden_states_mlp_input = self.ffn_boundary.prepare( - hidden_states, state.forward_batch - ) - - def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( - state.pop("hidden_states_mlp_output"), state.forward_batch - ) - - output = dict( - positions=state.positions, - hidden_states=hidden_states, - forward_batch=state.forward_batch, - tbo_subbatch_index=state.tbo_subbatch_index, - ) - - state.clear( - expect_keys={ - "positions", - "forward_batch", - "tbo_subbatch_index", - } - ) - return output - class Glm4MoeModel(nn.Module): def __init__( diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 84927139f9..b137ba968e 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -80,7 +80,6 @@ from sglang.srt.utils import ( BumpAllocator, LazyValue, add_prefix, - is_non_idle_and_non_empty, is_npu, make_pp_layers, ) @@ -411,101 +410,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): else: return None - def op_gate(self, state): - if is_non_idle_and_non_empty( - state.forward_batch.forward_mode, state.hidden_states_mlp_input - ): - # router_logits: (num_tokens, n_experts) - state.router_logits = self.gate(state.hidden_states_mlp_input) - else: - state.router_logits = None - - def op_shared_experts(self, state): - hidden_states_mlp_input = state.pop("hidden_states_mlp_input") - if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty( - state.forward_batch.forward_mode, hidden_states_mlp_input - ): - state.shared_output = self.shared_experts(hidden_states_mlp_input) - else: - state.shared_output = None - - def op_select_experts(self, state): - router_logits = state.pop("router_logits") - hidden_states = state.hidden_states_mlp_input - - if router_logits is not None: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.topk_output = self.topk( - hidden_states=hidden_states, - router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), - ) - else: - state.topk_output = self.topk.empty_topk_output(hidden_states.device) - - def op_dispatch_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.dispatch_a( - hidden_states=state.hidden_states_mlp_input, - topk_output=state.pop("topk_output"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_dispatch_b(self, state): - if self.ep_size > 1: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.dispatch_output = self.experts.dispatcher.dispatch_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_experts(self, state): - state.combine_input = self.experts.run_moe_core( - dispatch_output=state.dispatch_output, - ) - - def op_combine_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.combine_a( - combine_input=state.pop("combine_input"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - state.pop("dispatch_output") - - def op_combine_b(self, state): - if self.ep_size > 1: - state.hidden_states_after_combine = self.experts.dispatcher.combine_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_output(self, state): - final_hidden_states = state.pop("hidden_states_after_combine") - - if get_moe_a2a_backend().is_mori(): - num_tokens = state.pop("num_tokens") - final_hidden_states = final_hidden_states[:num_tokens] - - if (shared_output := state.pop("shared_output")) is not None: - x = shared_output - if _use_aiter: - x.add_(final_hidden_states) - else: - x.add_(final_hidden_states, alpha=self.routed_scaling_factor) - final_hidden_states = x - elif _use_aiter: - # fused in aiter_biased_grouped_topk so we can skip here - pass - else: - final_hidden_states *= self.routed_scaling_factor - - state.hidden_states_mlp_output = final_hidden_states - class Glm4MoeLiteDecoderLayer(nn.Module): def __init__( @@ -660,60 +564,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module): return hidden_states - def op_comm_prepare_attn( - self, - state, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - tbo_subbatch_index: Optional[int] = None, - ): - state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( - hidden_states, forward_batch - ) - if get_moe_a2a_backend().is_mori(): - state.num_tokens = hidden_states.shape[0] - state.update( - dict( - forward_batch=forward_batch, - positions=positions, - zero_allocator=zero_allocator, - tbo_subbatch_index=tbo_subbatch_index, - ) - ) - - def op_comm_prepare_mlp(self, state): - hidden_states = self.attn_boundary.finish( - state.pop("hidden_states_after_attn"), state.forward_batch - ) - state.hidden_states_mlp_input = self.ffn_boundary.prepare( - hidden_states, state.forward_batch - ) - - def op_comm_postprocess_layer(self, state): - hidden_states = self.ffn_boundary.finish_complete_output( - state.pop("hidden_states_mlp_output"), state.forward_batch - ) - - output = dict( - positions=state.positions, - hidden_states=hidden_states, - forward_batch=state.forward_batch, - zero_allocator=state.zero_allocator, - tbo_subbatch_index=state.tbo_subbatch_index, - ) - - state.clear( - expect_keys={ - "positions", - "forward_batch", - "zero_allocator", - "tbo_subbatch_index", - } - ) - return output - class Glm4MoeLiteModel(nn.Module): fall_back_to_pt_during_load = False diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 99e188b46b..9c5d8339ed 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -90,14 +90,12 @@ from sglang.srt.runtime_context import ( # transformers wrapper then crashes on config.rope_parameters (transformers v5 issue). # Other files (custom_all_reduce.py, hf_transformers_utils.py) also use sglang.srt.utils. from sglang.srt.utils import ( - BumpAllocator, add_prefix, cpu_has_amx_support, get_bool_env_var, get_compiler_backend, is_cpu, is_cuda, - is_non_idle_and_non_empty, is_npu, is_xpu, make_pp_layers, @@ -632,103 +630,6 @@ class MiniMaxM2MoE(nn.Module): return final_hidden_states # TBO Operations for MiniMax MoE - def op_gate(self, state): - """Gate operation for TBO - compute router logits""" - if is_non_idle_and_non_empty( - state.forward_batch.forward_mode, state.hidden_states_mlp_input - ): # router_logits: (num_tokens, num_experts) - state.router_logits, _ = self.gate(state.hidden_states_mlp_input) - else: - state.router_logits = None - - def op_select_experts(self, state): - """Expert selection operation for TBO""" - router_logits = state.pop("router_logits") - hidden_states = state.hidden_states_mlp_input - - if router_logits is not None: - ctx = ( - nullcontext() - if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) - else get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ) - ) - with ctx: - state.topk_weights_local, state.topk_idx_local, _ = self.topk( - hidden_states=hidden_states, - router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), - ) - else: - state.topk_idx_local = torch.full( - (0, self.top_k), -1, dtype=torch.int, device=hidden_states.device - ) - state.topk_weights_local = torch.empty( - (0, self.top_k), dtype=torch.float32, device=hidden_states.device - ) - - def op_dispatch_a(self, state): - """Dispatch A operation for TBO - start async dispatch""" - if self.ep_size > 1: - self.experts.deepep_dispatcher.dispatch_a( - hidden_states=state.pop("hidden_states_mlp_input"), - topk_idx=state.pop("topk_idx_local"), - topk_weights=state.pop("topk_weights_local"), - forward_batch=state.forward_batch, - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_dispatch_b(self, state): - """Dispatch B operation for TBO - complete async dispatch""" - if self.ep_size > 1: - ctx = ( - nullcontext() - if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) - else get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ) - ) - with ctx: - state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_experts(self, state): - """Expert computation for TBO""" - state.hidden_states_experts_output = self.experts.moe_impl( - dispatch_output=state.dispatch_output, - ) - - def op_combine_a(self, state): - """Combine A operation for TBO - start async combine""" - if self.ep_size > 1: - self.experts.deepep_dispatcher.combine_a( - hidden_states=state.pop("hidden_states_experts_output"), - topk_idx=state.dispatch_output.topk_idx, - topk_weights=state.dispatch_output.topk_weights, - forward_batch=state.forward_batch, - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - state.pop("dispatch_output") - - def op_combine_b(self, state): - """Combine B operation for TBO - complete async combine""" - if self.ep_size > 1: - state.hidden_states_after_combine = ( - self.experts.deepep_dispatcher.combine_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - ) - - def op_output(self, state): - """Output operation for TBO - final MLP output""" - final_hidden_states = state.pop("hidden_states_after_combine") - # MiniMax doesn't have shared experts like DeepSeek, so no need to add them - state.hidden_states_mlp_output = final_hidden_states class MiniMaxM2Attention(nn.Module): @@ -929,18 +830,6 @@ class MiniMaxM2Attention(nn.Module): ) return self.forward_core(s) - def op_prepare(self, state): - state.attn_intermediate_state = self.forward_prepare( - positions=state.positions, - hidden_states=state.pop("hidden_states_after_comm_pre_attn"), - forward_batch=state.forward_batch, - ) - - def op_core(self, state): - state.hidden_states_after_attn = self.forward_core( - state.pop("attn_intermediate_state") - ) - class MiniMaxM2DecoderLayer(nn.Module): """MiniMax Decoder Layer implementation with MoE support.""" @@ -1032,51 +921,6 @@ class MiniMaxM2DecoderLayer(nn.Module): return hidden_states # TBO Operations for MiniMax Decoder Layer - def op_comm_prepare_attn( - self, - state, - positions: torch.Tensor, - hidden_states: torch.Tensor, - forward_batch: ForwardBatch, - zero_allocator: BumpAllocator, - tbo_subbatch_index: Optional[int] = None, - ): - """Communication prepare for attention - TBO operation""" - state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare( - hidden_states, forward_batch - ) - state.update( - dict( - forward_batch=forward_batch, - positions=positions, - zero_allocator=zero_allocator, - tbo_subbatch_index=tbo_subbatch_index, - ) - ) - - def op_comm_prepare_mlp(self, state): - """Communication prepare for MLP - TBO operation""" - hidden_states = self.attn_boundary.finish( - state.pop("hidden_states_after_attn"), state.forward_batch - ) - state.hidden_states_mlp_input = self.ffn_boundary.prepare( - hidden_states, state.forward_batch - ) - - def op_comm_postprocess_layer(self, state): - """Communication postprocess for layer - TBO operation""" - hidden_states = self.ffn_boundary.finish_complete_output( - state.pop("hidden_states_mlp_output"), state.forward_batch - ) - - output = dict( - positions=state.positions, - hidden_states=hidden_states, - forward_batch=state.forward_batch, - zero_allocator=state.zero_allocator, - tbo_subbatch_index=state.tbo_subbatch_index, - ) - return output class MiniMaxM2Model(nn.Module): diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 9879fafdd3..73915bdec6 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -4,7 +4,6 @@ import torch import torch.nn.functional as F from torch import nn -from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul @@ -52,7 +51,6 @@ from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import ( add_prefix, is_cuda, - is_non_idle_and_non_empty, make_pp_layers, ) @@ -260,72 +258,6 @@ class Step3p5MoEMLP(nn.Module): final_hidden_states = shared_output return final_hidden_states - def op_gate(self, state): - if is_non_idle_and_non_empty( - state.forward_batch.forward_mode, state.hidden_states_mlp_input - ): - # router_logits: (num_tokens, n_experts) - state.router_logits, _ = self.gate(state.hidden_states_mlp_input) - else: - state.router_logits = None - - def op_select_experts(self, state): - router_logits = state.pop("router_logits") - hidden_states = state.hidden_states_mlp_input - if router_logits is not None: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.topk_output = self.topk( - hidden_states=hidden_states, - router_logits=router_logits, - num_token_non_padded=state.forward_batch.moe_num_token_non_padded(), - expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new( - layer_id=self.layer_id, - ), - ) - else: - state.topk_output = self.topk.empty_topk_output(hidden_states.device) - - def op_dispatch_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.dispatch_a( - hidden_states=state.pop("hidden_states_mlp_input"), - topk_output=state.pop("topk_output"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_dispatch_b(self, state): - if self.ep_size > 1: - with get_global_expert_distribution_recorder().with_current_layer( - self.layer_id - ): - state.dispatch_output = self.experts.dispatcher.dispatch_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_experts(self, state): - state.combine_input = self.experts.run_moe_core( - dispatch_output=state.dispatch_output, - ) - - def op_combine_a(self, state): - if self.ep_size > 1: - self.experts.dispatcher.combine_a( - combine_input=state.pop("combine_input"), - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - state.pop("dispatch_output") - - def op_combine_b(self, state): - if self.ep_size > 1: - state.hidden_states_after_combine = self.experts.dispatcher.combine_b( - tbo_subbatch_index=state.get("tbo_subbatch_index"), - ) - - def op_output(self, state): - state.hidden_states_mlp_output = state.pop("hidden_states_after_combine") - class Step3p5Attention(nn.Module): def __init__(