[Refactor] Drop TBO op methods that no strategy ever schedules (#42300)

This commit is contained in:
Cheng Wan
2026-10-03 02:17:26 -07:00
committed by GitHub
parent 122b59152c
commit d5a8d76bf5
5 changed files with 0 additions and 687 deletions
@@ -129,7 +129,6 @@ from sglang.srt.utils import (
ceil_div,
get_bool_env_var,
is_cuda,
is_non_idle_and_non_empty,
log_info_on_rank0,
make_pp_layers,
)
@@ -550,103 +549,6 @@ class Dots3MoE(nn.Module):
else:
return None
def op_gate(self, state):
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
):
# router_logits: (num_tokens, n_experts)
state.router_logits = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_shared_experts(self, state):
hidden_states_mlp_input = state.pop("hidden_states_mlp_input")
if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty(
state.forward_batch.forward_mode, hidden_states_mlp_input
):
state.shared_output = self.shared_experts(hidden_states_mlp_input)
else:
state.shared_output = None
def op_select_experts(self, state):
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.topk_weights_local, state.topk_idx_local, _ = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_idx_local = torch.full(
(0, self.top_k), -1, dtype=torch.int, device=hidden_states.device
)
state.topk_weights_local = torch.empty(
(0, self.top_k), dtype=torch.float32, device=hidden_states.device
)
def op_dispatch_a(self, state):
if self.ep_size > 1:
self.experts.deepep_dispatcher.dispatch_a(
hidden_states=state.hidden_states_mlp_input,
topk_idx=state.pop("topk_idx_local"),
topk_weights=state.pop("topk_weights_local"),
forward_batch=state.forward_batch,
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
if self.ep_size > 1:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
state.hidden_states_experts_output = self.experts.moe_impl(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
if self.ep_size > 1:
self.experts.deepep_dispatcher.combine_a(
hidden_states=state.pop("hidden_states_experts_output"),
topk_idx=state.dispatch_output.topk_idx,
topk_weights=state.dispatch_output.topk_weights,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
if self.ep_size > 1:
state.hidden_states_after_combine = (
self.experts.deepep_dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
)
def op_output(self, state):
final_hidden_states = state.pop("hidden_states_after_combine")
if (shared_output := state.pop("shared_output")) is not None:
x = shared_output
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
else:
final_hidden_states *= self.routed_scaling_factor
state.hidden_states_mlp_output = final_hidden_states
# Aligned with HF's implementation, using sliding window inclusive with the last token.
# SGLang assumes exclusive.
@@ -1040,19 +942,6 @@ class Dots3AttentionMLA(nn.Module):
)
return KVWriteLoc.for_batch(forward_batch, swa_loc=swa_loc)
def op_prepare(self, state):
state.attn_intermediate_state = self.forward_prepare(
positions=state.positions,
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
)
def op_core(self, state):
state.hidden_states_after_attn = self.forward_core(
state.pop("attn_intermediate_state")
)
def forward(
self,
positions: torch.Tensor,
@@ -1628,71 +1517,6 @@ class Dots3DecoderLayer(nn.Module):
)
return ffn_exit.finish(hidden_states)
def op_comm_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
zero_allocator=zero_allocator,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_mlp(self, state):
hidden_states = state.pop("hidden_states_mlp_input")
if not (
is_dense_ffn_fully_dp()
and (not self.is_layer_sparse)
and hidden_states.shape[0] == 0
):
state.hidden_states_mlp_output = self.mlp(
hidden_states, state.forward_batch
)
else:
state.hidden_states_mlp_output = hidden_states
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
)
state.clear(
expect_keys={
"positions",
"forward_batch",
"zero_allocator",
"tbo_subbatch_index",
}
)
return output
class Dots3Model(nn.Module):
fall_back_to_pt_during_load = False
-137
View File
@@ -94,7 +94,6 @@ from sglang.srt.utils import (
is_cpu,
is_cuda,
is_hip,
is_non_idle_and_non_empty,
is_npu,
log_info_on_rank0,
make_pp_layers,
@@ -304,18 +303,6 @@ class Glm4MoeAttention(nn.Module):
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.alt_stream = alt_stream
def op_prepare(self, state):
state.attn_intermediate_state = self.forward_prepare(
positions=state.positions,
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
forward_batch=state.forward_batch,
)
def op_core(self, state):
state.hidden_states_after_attn = self.forward_core(
state.pop("attn_intermediate_state")
)
def forward_prepare(
self,
positions: torch.Tensor,
@@ -708,82 +695,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
else:
return None
def op_gate(self, state):
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
):
# router_logits: (num_tokens, n_experts)
state.router_logits = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_select_experts(self, state):
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
def op_dispatch_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.dispatch_a(
hidden_states=state.hidden_states_mlp_input,
topk_output=state.pop("topk_output"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
if self.ep_size > 1:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.dispatch_output = self.experts.dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
state.combine_input = self.experts.run_moe_core(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.combine_a(
combine_input=state.pop("combine_input"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
if self.ep_size > 1:
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_output(self, state):
final_hidden_states = state.pop("hidden_states_after_combine")
if (shared_output := state.pop("shared_output")) is not None:
x = shared_output
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
else:
final_hidden_states *= self.routed_scaling_factor
state.hidden_states_mlp_output = final_hidden_states
class Glm4MoeDecoderLayer(nn.Module):
def __init__(
@@ -962,54 +873,6 @@ class Glm4MoeDecoderLayer(nn.Module):
return hidden_states
def op_comm_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.tbo_subbatch_index,
)
state.clear(
expect_keys={
"positions",
"forward_batch",
"tbo_subbatch_index",
}
)
return output
class Glm4MoeModel(nn.Module):
def __init__(
-150
View File
@@ -80,7 +80,6 @@ from sglang.srt.utils import (
BumpAllocator,
LazyValue,
add_prefix,
is_non_idle_and_non_empty,
is_npu,
make_pp_layers,
)
@@ -411,101 +410,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
else:
return None
def op_gate(self, state):
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
):
# router_logits: (num_tokens, n_experts)
state.router_logits = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_shared_experts(self, state):
hidden_states_mlp_input = state.pop("hidden_states_mlp_input")
if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty(
state.forward_batch.forward_mode, hidden_states_mlp_input
):
state.shared_output = self.shared_experts(hidden_states_mlp_input)
else:
state.shared_output = None
def op_select_experts(self, state):
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
def op_dispatch_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.dispatch_a(
hidden_states=state.hidden_states_mlp_input,
topk_output=state.pop("topk_output"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
if self.ep_size > 1:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.dispatch_output = self.experts.dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
state.combine_input = self.experts.run_moe_core(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.combine_a(
combine_input=state.pop("combine_input"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
if self.ep_size > 1:
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_output(self, state):
final_hidden_states = state.pop("hidden_states_after_combine")
if get_moe_a2a_backend().is_mori():
num_tokens = state.pop("num_tokens")
final_hidden_states = final_hidden_states[:num_tokens]
if (shared_output := state.pop("shared_output")) is not None:
x = shared_output
if _use_aiter:
x.add_(final_hidden_states)
else:
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
final_hidden_states = x
elif _use_aiter:
# fused in aiter_biased_grouped_topk so we can skip here
pass
else:
final_hidden_states *= self.routed_scaling_factor
state.hidden_states_mlp_output = final_hidden_states
class Glm4MoeLiteDecoderLayer(nn.Module):
def __init__(
@@ -660,60 +564,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
return hidden_states
def op_comm_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
if get_moe_a2a_backend().is_mori():
state.num_tokens = hidden_states.shape[0]
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
zero_allocator=zero_allocator,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_comm_prepare_mlp(self, state):
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
hidden_states = self.ffn_boundary.finish_complete_output(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
)
state.clear(
expect_keys={
"positions",
"forward_batch",
"zero_allocator",
"tbo_subbatch_index",
}
)
return output
class Glm4MoeLiteModel(nn.Module):
fall_back_to_pt_during_load = False
-156
View File
@@ -90,14 +90,12 @@ from sglang.srt.runtime_context import (
# transformers wrapper then crashes on config.rope_parameters (transformers v5 issue).
# Other files (custom_all_reduce.py, hf_transformers_utils.py) also use sglang.srt.utils.
from sglang.srt.utils import (
BumpAllocator,
add_prefix,
cpu_has_amx_support,
get_bool_env_var,
get_compiler_backend,
is_cpu,
is_cuda,
is_non_idle_and_non_empty,
is_npu,
is_xpu,
make_pp_layers,
@@ -632,103 +630,6 @@ class MiniMaxM2MoE(nn.Module):
return final_hidden_states
# TBO Operations for MiniMax MoE
def op_gate(self, state):
"""Gate operation for TBO - compute router logits"""
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
): # router_logits: (num_tokens, num_experts)
state.router_logits, _ = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_select_experts(self, state):
"""Expert selection operation for TBO"""
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
ctx = (
nullcontext()
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
else get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
)
)
with ctx:
state.topk_weights_local, state.topk_idx_local, _ = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_idx_local = torch.full(
(0, self.top_k), -1, dtype=torch.int, device=hidden_states.device
)
state.topk_weights_local = torch.empty(
(0, self.top_k), dtype=torch.float32, device=hidden_states.device
)
def op_dispatch_a(self, state):
"""Dispatch A operation for TBO - start async dispatch"""
if self.ep_size > 1:
self.experts.deepep_dispatcher.dispatch_a(
hidden_states=state.pop("hidden_states_mlp_input"),
topk_idx=state.pop("topk_idx_local"),
topk_weights=state.pop("topk_weights_local"),
forward_batch=state.forward_batch,
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
"""Dispatch B operation for TBO - complete async dispatch"""
if self.ep_size > 1:
ctx = (
nullcontext()
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
else get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
)
)
with ctx:
state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
"""Expert computation for TBO"""
state.hidden_states_experts_output = self.experts.moe_impl(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
"""Combine A operation for TBO - start async combine"""
if self.ep_size > 1:
self.experts.deepep_dispatcher.combine_a(
hidden_states=state.pop("hidden_states_experts_output"),
topk_idx=state.dispatch_output.topk_idx,
topk_weights=state.dispatch_output.topk_weights,
forward_batch=state.forward_batch,
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
"""Combine B operation for TBO - complete async combine"""
if self.ep_size > 1:
state.hidden_states_after_combine = (
self.experts.deepep_dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
)
def op_output(self, state):
"""Output operation for TBO - final MLP output"""
final_hidden_states = state.pop("hidden_states_after_combine")
# MiniMax doesn't have shared experts like DeepSeek, so no need to add them
state.hidden_states_mlp_output = final_hidden_states
class MiniMaxM2Attention(nn.Module):
@@ -929,18 +830,6 @@ class MiniMaxM2Attention(nn.Module):
)
return self.forward_core(s)
def op_prepare(self, state):
state.attn_intermediate_state = self.forward_prepare(
positions=state.positions,
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
forward_batch=state.forward_batch,
)
def op_core(self, state):
state.hidden_states_after_attn = self.forward_core(
state.pop("attn_intermediate_state")
)
class MiniMaxM2DecoderLayer(nn.Module):
"""MiniMax Decoder Layer implementation with MoE support."""
@@ -1032,51 +921,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
return hidden_states
# TBO Operations for MiniMax Decoder Layer
def op_comm_prepare_attn(
self,
state,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
zero_allocator: BumpAllocator,
tbo_subbatch_index: Optional[int] = None,
):
"""Communication prepare for attention - TBO operation"""
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
hidden_states, forward_batch
)
state.update(
dict(
forward_batch=forward_batch,
positions=positions,
zero_allocator=zero_allocator,
tbo_subbatch_index=tbo_subbatch_index,
)
)
def op_comm_prepare_mlp(self, state):
"""Communication prepare for MLP - TBO operation"""
hidden_states = self.attn_boundary.finish(
state.pop("hidden_states_after_attn"), state.forward_batch
)
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
hidden_states, state.forward_batch
)
def op_comm_postprocess_layer(self, state):
"""Communication postprocess for layer - TBO operation"""
hidden_states = self.ffn_boundary.finish_complete_output(
state.pop("hidden_states_mlp_output"), state.forward_batch
)
output = dict(
positions=state.positions,
hidden_states=hidden_states,
forward_batch=state.forward_batch,
zero_allocator=state.zero_allocator,
tbo_subbatch_index=state.tbo_subbatch_index,
)
return output
class MiniMaxM2Model(nn.Module):
-68
View File
@@ -4,7 +4,6 @@ import torch
import torch.nn.functional as F
from torch import nn
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
from sglang.srt.layers.activation import SiluAndMul
@@ -52,7 +51,6 @@ from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import (
add_prefix,
is_cuda,
is_non_idle_and_non_empty,
make_pp_layers,
)
@@ -260,72 +258,6 @@ class Step3p5MoEMLP(nn.Module):
final_hidden_states = shared_output
return final_hidden_states
def op_gate(self, state):
if is_non_idle_and_non_empty(
state.forward_batch.forward_mode, state.hidden_states_mlp_input
):
# router_logits: (num_tokens, n_experts)
state.router_logits, _ = self.gate(state.hidden_states_mlp_input)
else:
state.router_logits = None
def op_select_experts(self, state):
router_logits = state.pop("router_logits")
hidden_states = state.hidden_states_mlp_input
if router_logits is not None:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.topk_output = self.topk(
hidden_states=hidden_states,
router_logits=router_logits,
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
layer_id=self.layer_id,
),
)
else:
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
def op_dispatch_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.dispatch_a(
hidden_states=state.pop("hidden_states_mlp_input"),
topk_output=state.pop("topk_output"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_dispatch_b(self, state):
if self.ep_size > 1:
with get_global_expert_distribution_recorder().with_current_layer(
self.layer_id
):
state.dispatch_output = self.experts.dispatcher.dispatch_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_experts(self, state):
state.combine_input = self.experts.run_moe_core(
dispatch_output=state.dispatch_output,
)
def op_combine_a(self, state):
if self.ep_size > 1:
self.experts.dispatcher.combine_a(
combine_input=state.pop("combine_input"),
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
state.pop("dispatch_output")
def op_combine_b(self, state):
if self.ep_size > 1:
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
tbo_subbatch_index=state.get("tbo_subbatch_index"),
)
def op_output(self, state):
state.hidden_states_mlp_output = state.pop("hidden_states_after_combine")
class Step3p5Attention(nn.Module):
def __init__(