[Refactor] Drop TBO op methods that no strategy ever schedules (#42300)
This commit is contained in:
@@ -129,7 +129,6 @@ from sglang.srt.utils import (
|
||||
ceil_div,
|
||||
get_bool_env_var,
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
log_info_on_rank0,
|
||||
make_pp_layers,
|
||||
)
|
||||
@@ -550,103 +549,6 @@ class Dots3MoE(nn.Module):
|
||||
else:
|
||||
return None
|
||||
|
||||
def op_gate(self, state):
|
||||
if is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
||||
):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
state.router_logits = self.gate(state.hidden_states_mlp_input)
|
||||
else:
|
||||
state.router_logits = None
|
||||
|
||||
def op_shared_experts(self, state):
|
||||
hidden_states_mlp_input = state.pop("hidden_states_mlp_input")
|
||||
if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, hidden_states_mlp_input
|
||||
):
|
||||
state.shared_output = self.shared_experts(hidden_states_mlp_input)
|
||||
else:
|
||||
state.shared_output = None
|
||||
|
||||
def op_select_experts(self, state):
|
||||
router_logits = state.pop("router_logits")
|
||||
hidden_states = state.hidden_states_mlp_input
|
||||
|
||||
if router_logits is not None:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.topk_weights_local, state.topk_idx_local, _ = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_idx_local = torch.full(
|
||||
(0, self.top_k), -1, dtype=torch.int, device=hidden_states.device
|
||||
)
|
||||
state.topk_weights_local = torch.empty(
|
||||
(0, self.top_k), dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.deepep_dispatcher.dispatch_a(
|
||||
hidden_states=state.hidden_states_mlp_input,
|
||||
topk_idx=state.pop("topk_idx_local"),
|
||||
topk_weights=state.pop("topk_weights_local"),
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_dispatch_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_experts(self, state):
|
||||
state.hidden_states_experts_output = self.experts.moe_impl(
|
||||
dispatch_output=state.dispatch_output,
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.deepep_dispatcher.combine_a(
|
||||
hidden_states=state.pop("hidden_states_experts_output"),
|
||||
topk_idx=state.dispatch_output.topk_idx,
|
||||
topk_weights=state.dispatch_output.topk_weights,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
state.pop("dispatch_output")
|
||||
|
||||
def op_combine_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
state.hidden_states_after_combine = (
|
||||
self.experts.deepep_dispatcher.combine_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
)
|
||||
|
||||
def op_output(self, state):
|
||||
final_hidden_states = state.pop("hidden_states_after_combine")
|
||||
|
||||
if (shared_output := state.pop("shared_output")) is not None:
|
||||
x = shared_output
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
else:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
state.hidden_states_mlp_output = final_hidden_states
|
||||
|
||||
|
||||
# Aligned with HF's implementation, using sliding window inclusive with the last token.
|
||||
# SGLang assumes exclusive.
|
||||
@@ -1040,19 +942,6 @@ class Dots3AttentionMLA(nn.Module):
|
||||
)
|
||||
return KVWriteLoc.for_batch(forward_batch, swa_loc=swa_loc)
|
||||
|
||||
def op_prepare(self, state):
|
||||
state.attn_intermediate_state = self.forward_prepare(
|
||||
positions=state.positions,
|
||||
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
)
|
||||
|
||||
def op_core(self, state):
|
||||
state.hidden_states_after_attn = self.forward_core(
|
||||
state.pop("attn_intermediate_state")
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -1628,71 +1517,6 @@ class Dots3DecoderLayer(nn.Module):
|
||||
)
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
state,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
forward_batch=forward_batch,
|
||||
positions=positions,
|
||||
zero_allocator=zero_allocator,
|
||||
tbo_subbatch_index=tbo_subbatch_index,
|
||||
)
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_mlp(self, state):
|
||||
hidden_states = state.pop("hidden_states_mlp_input")
|
||||
if not (
|
||||
is_dense_ffn_fully_dp()
|
||||
and (not self.is_layer_sparse)
|
||||
and hidden_states.shape[0] == 0
|
||||
):
|
||||
state.hidden_states_mlp_output = self.mlp(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
else:
|
||||
state.hidden_states_mlp_output = hidden_states
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
|
||||
state.clear(
|
||||
expect_keys={
|
||||
"positions",
|
||||
"forward_batch",
|
||||
"zero_allocator",
|
||||
"tbo_subbatch_index",
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class Dots3Model(nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
@@ -94,7 +94,6 @@ from sglang.srt.utils import (
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
log_info_on_rank0,
|
||||
make_pp_layers,
|
||||
@@ -304,18 +303,6 @@ class Glm4MoeAttention(nn.Module):
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
def op_prepare(self, state):
|
||||
state.attn_intermediate_state = self.forward_prepare(
|
||||
positions=state.positions,
|
||||
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
|
||||
forward_batch=state.forward_batch,
|
||||
)
|
||||
|
||||
def op_core(self, state):
|
||||
state.hidden_states_after_attn = self.forward_core(
|
||||
state.pop("attn_intermediate_state")
|
||||
)
|
||||
|
||||
def forward_prepare(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
@@ -708,82 +695,6 @@ class Glm4MoeSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
return None
|
||||
|
||||
def op_gate(self, state):
|
||||
if is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
||||
):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
state.router_logits = self.gate(state.hidden_states_mlp_input)
|
||||
else:
|
||||
state.router_logits = None
|
||||
|
||||
def op_select_experts(self, state):
|
||||
router_logits = state.pop("router_logits")
|
||||
hidden_states = state.hidden_states_mlp_input
|
||||
|
||||
if router_logits is not None:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.dispatch_a(
|
||||
hidden_states=state.hidden_states_mlp_input,
|
||||
topk_output=state.pop("topk_output"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_dispatch_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.dispatch_output = self.experts.dispatcher.dispatch_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_experts(self, state):
|
||||
state.combine_input = self.experts.run_moe_core(
|
||||
dispatch_output=state.dispatch_output,
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.combine_a(
|
||||
combine_input=state.pop("combine_input"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
state.pop("dispatch_output")
|
||||
|
||||
def op_combine_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_output(self, state):
|
||||
final_hidden_states = state.pop("hidden_states_after_combine")
|
||||
|
||||
if (shared_output := state.pop("shared_output")) is not None:
|
||||
x = shared_output
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
else:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
state.hidden_states_mlp_output = final_hidden_states
|
||||
|
||||
|
||||
class Glm4MoeDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
@@ -962,54 +873,6 @@ class Glm4MoeDecoderLayer(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
state,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
forward_batch=forward_batch,
|
||||
positions=positions,
|
||||
tbo_subbatch_index=tbo_subbatch_index,
|
||||
)
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
|
||||
state.clear(
|
||||
expect_keys={
|
||||
"positions",
|
||||
"forward_batch",
|
||||
"tbo_subbatch_index",
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class Glm4MoeModel(nn.Module):
|
||||
def __init__(
|
||||
|
||||
@@ -80,7 +80,6 @@ from sglang.srt.utils import (
|
||||
BumpAllocator,
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
make_pp_layers,
|
||||
)
|
||||
@@ -411,101 +410,6 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
|
||||
else:
|
||||
return None
|
||||
|
||||
def op_gate(self, state):
|
||||
if is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
||||
):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
state.router_logits = self.gate(state.hidden_states_mlp_input)
|
||||
else:
|
||||
state.router_logits = None
|
||||
|
||||
def op_shared_experts(self, state):
|
||||
hidden_states_mlp_input = state.pop("hidden_states_mlp_input")
|
||||
if (self.num_fused_shared_experts == 0) and is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, hidden_states_mlp_input
|
||||
):
|
||||
state.shared_output = self.shared_experts(hidden_states_mlp_input)
|
||||
else:
|
||||
state.shared_output = None
|
||||
|
||||
def op_select_experts(self, state):
|
||||
router_logits = state.pop("router_logits")
|
||||
hidden_states = state.hidden_states_mlp_input
|
||||
|
||||
if router_logits is not None:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.dispatch_a(
|
||||
hidden_states=state.hidden_states_mlp_input,
|
||||
topk_output=state.pop("topk_output"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_dispatch_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.dispatch_output = self.experts.dispatcher.dispatch_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_experts(self, state):
|
||||
state.combine_input = self.experts.run_moe_core(
|
||||
dispatch_output=state.dispatch_output,
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.combine_a(
|
||||
combine_input=state.pop("combine_input"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
state.pop("dispatch_output")
|
||||
|
||||
def op_combine_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_output(self, state):
|
||||
final_hidden_states = state.pop("hidden_states_after_combine")
|
||||
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
num_tokens = state.pop("num_tokens")
|
||||
final_hidden_states = final_hidden_states[:num_tokens]
|
||||
|
||||
if (shared_output := state.pop("shared_output")) is not None:
|
||||
x = shared_output
|
||||
if _use_aiter:
|
||||
x.add_(final_hidden_states)
|
||||
else:
|
||||
x.add_(final_hidden_states, alpha=self.routed_scaling_factor)
|
||||
final_hidden_states = x
|
||||
elif _use_aiter:
|
||||
# fused in aiter_biased_grouped_topk so we can skip here
|
||||
pass
|
||||
else:
|
||||
final_hidden_states *= self.routed_scaling_factor
|
||||
|
||||
state.hidden_states_mlp_output = final_hidden_states
|
||||
|
||||
|
||||
class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
@@ -660,60 +564,6 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
state,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
if get_moe_a2a_backend().is_mori():
|
||||
state.num_tokens = hidden_states.shape[0]
|
||||
state.update(
|
||||
dict(
|
||||
forward_batch=forward_batch,
|
||||
positions=positions,
|
||||
zero_allocator=zero_allocator,
|
||||
tbo_subbatch_index=tbo_subbatch_index,
|
||||
)
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
|
||||
state.clear(
|
||||
expect_keys={
|
||||
"positions",
|
||||
"forward_batch",
|
||||
"zero_allocator",
|
||||
"tbo_subbatch_index",
|
||||
}
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class Glm4MoeLiteModel(nn.Module):
|
||||
fall_back_to_pt_during_load = False
|
||||
|
||||
@@ -90,14 +90,12 @@ from sglang.srt.runtime_context import (
|
||||
# transformers wrapper then crashes on config.rope_parameters (transformers v5 issue).
|
||||
# Other files (custom_all_reduce.py, hf_transformers_utils.py) also use sglang.srt.utils.
|
||||
from sglang.srt.utils import (
|
||||
BumpAllocator,
|
||||
add_prefix,
|
||||
cpu_has_amx_support,
|
||||
get_bool_env_var,
|
||||
get_compiler_backend,
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
make_pp_layers,
|
||||
@@ -632,103 +630,6 @@ class MiniMaxM2MoE(nn.Module):
|
||||
return final_hidden_states
|
||||
|
||||
# TBO Operations for MiniMax MoE
|
||||
def op_gate(self, state):
|
||||
"""Gate operation for TBO - compute router logits"""
|
||||
if is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
||||
): # router_logits: (num_tokens, num_experts)
|
||||
state.router_logits, _ = self.gate(state.hidden_states_mlp_input)
|
||||
else:
|
||||
state.router_logits = None
|
||||
|
||||
def op_select_experts(self, state):
|
||||
"""Expert selection operation for TBO"""
|
||||
router_logits = state.pop("router_logits")
|
||||
hidden_states = state.hidden_states_mlp_input
|
||||
|
||||
if router_logits is not None:
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
else get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
)
|
||||
)
|
||||
with ctx:
|
||||
state.topk_weights_local, state.topk_idx_local, _ = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_idx_local = torch.full(
|
||||
(0, self.top_k), -1, dtype=torch.int, device=hidden_states.device
|
||||
)
|
||||
state.topk_weights_local = torch.empty(
|
||||
(0, self.top_k), dtype=torch.float32, device=hidden_states.device
|
||||
)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
"""Dispatch A operation for TBO - start async dispatch"""
|
||||
if self.ep_size > 1:
|
||||
self.experts.deepep_dispatcher.dispatch_a(
|
||||
hidden_states=state.pop("hidden_states_mlp_input"),
|
||||
topk_idx=state.pop("topk_idx_local"),
|
||||
topk_weights=state.pop("topk_weights_local"),
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_dispatch_b(self, state):
|
||||
"""Dispatch B operation for TBO - complete async dispatch"""
|
||||
if self.ep_size > 1:
|
||||
ctx = (
|
||||
nullcontext()
|
||||
if check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE)
|
||||
else get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
)
|
||||
)
|
||||
with ctx:
|
||||
state.dispatch_output = self.experts.deepep_dispatcher.dispatch_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_experts(self, state):
|
||||
"""Expert computation for TBO"""
|
||||
state.hidden_states_experts_output = self.experts.moe_impl(
|
||||
dispatch_output=state.dispatch_output,
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
"""Combine A operation for TBO - start async combine"""
|
||||
if self.ep_size > 1:
|
||||
self.experts.deepep_dispatcher.combine_a(
|
||||
hidden_states=state.pop("hidden_states_experts_output"),
|
||||
topk_idx=state.dispatch_output.topk_idx,
|
||||
topk_weights=state.dispatch_output.topk_weights,
|
||||
forward_batch=state.forward_batch,
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
state.pop("dispatch_output")
|
||||
|
||||
def op_combine_b(self, state):
|
||||
"""Combine B operation for TBO - complete async combine"""
|
||||
if self.ep_size > 1:
|
||||
state.hidden_states_after_combine = (
|
||||
self.experts.deepep_dispatcher.combine_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
)
|
||||
|
||||
def op_output(self, state):
|
||||
"""Output operation for TBO - final MLP output"""
|
||||
final_hidden_states = state.pop("hidden_states_after_combine")
|
||||
# MiniMax doesn't have shared experts like DeepSeek, so no need to add them
|
||||
state.hidden_states_mlp_output = final_hidden_states
|
||||
|
||||
|
||||
class MiniMaxM2Attention(nn.Module):
|
||||
@@ -929,18 +830,6 @@ class MiniMaxM2Attention(nn.Module):
|
||||
)
|
||||
return self.forward_core(s)
|
||||
|
||||
def op_prepare(self, state):
|
||||
state.attn_intermediate_state = self.forward_prepare(
|
||||
positions=state.positions,
|
||||
hidden_states=state.pop("hidden_states_after_comm_pre_attn"),
|
||||
forward_batch=state.forward_batch,
|
||||
)
|
||||
|
||||
def op_core(self, state):
|
||||
state.hidden_states_after_attn = self.forward_core(
|
||||
state.pop("attn_intermediate_state")
|
||||
)
|
||||
|
||||
|
||||
class MiniMaxM2DecoderLayer(nn.Module):
|
||||
"""MiniMax Decoder Layer implementation with MoE support."""
|
||||
@@ -1032,51 +921,6 @@ class MiniMaxM2DecoderLayer(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
# TBO Operations for MiniMax Decoder Layer
|
||||
def op_comm_prepare_attn(
|
||||
self,
|
||||
state,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
zero_allocator: BumpAllocator,
|
||||
tbo_subbatch_index: Optional[int] = None,
|
||||
):
|
||||
"""Communication prepare for attention - TBO operation"""
|
||||
state.hidden_states_after_comm_pre_attn = self.attn_boundary.prepare(
|
||||
hidden_states, forward_batch
|
||||
)
|
||||
state.update(
|
||||
dict(
|
||||
forward_batch=forward_batch,
|
||||
positions=positions,
|
||||
zero_allocator=zero_allocator,
|
||||
tbo_subbatch_index=tbo_subbatch_index,
|
||||
)
|
||||
)
|
||||
|
||||
def op_comm_prepare_mlp(self, state):
|
||||
"""Communication prepare for MLP - TBO operation"""
|
||||
hidden_states = self.attn_boundary.finish(
|
||||
state.pop("hidden_states_after_attn"), state.forward_batch
|
||||
)
|
||||
state.hidden_states_mlp_input = self.ffn_boundary.prepare(
|
||||
hidden_states, state.forward_batch
|
||||
)
|
||||
|
||||
def op_comm_postprocess_layer(self, state):
|
||||
"""Communication postprocess for layer - TBO operation"""
|
||||
hidden_states = self.ffn_boundary.finish_complete_output(
|
||||
state.pop("hidden_states_mlp_output"), state.forward_batch
|
||||
)
|
||||
|
||||
output = dict(
|
||||
positions=state.positions,
|
||||
hidden_states=hidden_states,
|
||||
forward_batch=state.forward_batch,
|
||||
zero_allocator=state.zero_allocator,
|
||||
tbo_subbatch_index=state.tbo_subbatch_index,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
class MiniMaxM2Model(nn.Module):
|
||||
|
||||
@@ -4,7 +4,6 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
@@ -52,7 +51,6 @@ from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
@@ -260,72 +258,6 @@ class Step3p5MoEMLP(nn.Module):
|
||||
final_hidden_states = shared_output
|
||||
return final_hidden_states
|
||||
|
||||
def op_gate(self, state):
|
||||
if is_non_idle_and_non_empty(
|
||||
state.forward_batch.forward_mode, state.hidden_states_mlp_input
|
||||
):
|
||||
# router_logits: (num_tokens, n_experts)
|
||||
state.router_logits, _ = self.gate(state.hidden_states_mlp_input)
|
||||
else:
|
||||
state.router_logits = None
|
||||
|
||||
def op_select_experts(self, state):
|
||||
router_logits = state.pop("router_logits")
|
||||
hidden_states = state.hidden_states_mlp_input
|
||||
if router_logits is not None:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.topk_output = self.topk(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
num_token_non_padded=state.forward_batch.moe_num_token_non_padded(),
|
||||
expert_location_dispatch_info=ExpertLocationDispatchInfo.init_new(
|
||||
layer_id=self.layer_id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
state.topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
def op_dispatch_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.dispatch_a(
|
||||
hidden_states=state.pop("hidden_states_mlp_input"),
|
||||
topk_output=state.pop("topk_output"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_dispatch_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
with get_global_expert_distribution_recorder().with_current_layer(
|
||||
self.layer_id
|
||||
):
|
||||
state.dispatch_output = self.experts.dispatcher.dispatch_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_experts(self, state):
|
||||
state.combine_input = self.experts.run_moe_core(
|
||||
dispatch_output=state.dispatch_output,
|
||||
)
|
||||
|
||||
def op_combine_a(self, state):
|
||||
if self.ep_size > 1:
|
||||
self.experts.dispatcher.combine_a(
|
||||
combine_input=state.pop("combine_input"),
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
state.pop("dispatch_output")
|
||||
|
||||
def op_combine_b(self, state):
|
||||
if self.ep_size > 1:
|
||||
state.hidden_states_after_combine = self.experts.dispatcher.combine_b(
|
||||
tbo_subbatch_index=state.get("tbo_subbatch_index"),
|
||||
)
|
||||
|
||||
def op_output(self, state):
|
||||
state.hidden_states_mlp_output = state.pop("hidden_states_after_combine")
|
||||
|
||||
|
||||
class Step3p5Attention(nn.Module):
|
||||
def __init__(
|
||||
|
||||
Reference in New Issue
Block a user