From ea855f0daf9f104ad93f93056de79c6c376aacbe Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 29 Sep 2026 21:08:19 -0700 Subject: [PATCH] [Refactor] Leave decoder stacks through final_norm(skip_empty=True) (#41751) --- python/sglang/srt/models/gigachat35.py | 9 +++------ python/sglang/srt/models/interns2_mobius.py | 9 +++------ python/sglang/srt/models/laguna.py | 20 +++++++++----------- python/sglang/srt/models/minimax_m2.py | 9 +++------ python/sglang/srt/models/minimax_m3.py | 8 +++----- python/sglang/srt/models/qwen2_moe.py | 11 +++-------- python/sglang/srt/models/qwen3.py | 8 +++----- python/sglang/srt/models/qwen3_moe.py | 3 --- python/sglang/srt/models/qwen3_vl.py | 15 +++++++-------- python/sglang/srt/models/qwen3_vl_moe.py | 15 +++++++-------- python/sglang/srt/models/sarvam_moe.py | 15 +++------------ python/sglang/srt/models/step3_vl.py | 9 +++------ python/sglang/srt/models/xllm.py | 8 +++----- 13 files changed, 50 insertions(+), 89 deletions(-) diff --git a/python/sglang/srt/models/gigachat35.py b/python/sglang/srt/models/gigachat35.py index 22d77c3e1d..d8dd46a659 100644 --- a/python/sglang/srt/models/gigachat35.py +++ b/python/sglang/srt/models/gigachat35.py @@ -581,12 +581,9 @@ class GigaChat35Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return hidden_states diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index c6b52c08f7..331336351b 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -791,12 +791,9 @@ class InternS2MobiusForCausalLM(Qwen3_5ForCausalLM): input_deepstack_embeds[:, start : start + self.hidden_size], ) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return ( hidden_states if not aux_hidden_states diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index e970785619..fa641d1ee2 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -607,17 +607,15 @@ class LagunaModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, - forward_batch, - self.norm, - capture=aux_hidden_states.append - if self.end_layer in self.layers_to_capture - else None, - ) + hidden_states = residual_batch.final_norm( + hidden_states, + forward_batch, + self.norm, + capture=aux_hidden_states.append + if self.end_layer in self.layers_to_capture + else None, + skip_empty=True, + ) if len(aux_hidden_states) == 0: return hidden_states return hidden_states, aux_hidden_states diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 7a418a6a74..df0f148993 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -1177,12 +1177,9 @@ class MiniMaxM2Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 2cb86c4d5f..22345e1224 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -1517,11 +1517,9 @@ class MiniMaxM3Model(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index 3eff6d9de4..4c9dbdba9f 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -1214,11 +1214,9 @@ class Qwen2MoeModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) if len(aux_hidden_states) == 0: return hidden_states @@ -1310,9 +1308,6 @@ class Qwen2MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.complete_output( - forward_batch.hidden_states, forward_batch - ) # norm hidden_states = residual_batch.final_norm( forward_batch.hidden_states, forward_batch, self.model.norm diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 94ad78b95b..743232dc38 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -477,11 +477,9 @@ class Qwen3Model(Qwen2Model): ) if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return ( (hidden_states, aux_hidden_states) if aux_hidden_states else hidden_states ) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index ff033ea5dd..334ab2904b 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -1099,9 +1099,6 @@ class Qwen3MoeForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.complete_output( - forward_batch.hidden_states, forward_batch - ) # norm hidden_states = residual_batch.final_norm( forward_batch.hidden_states, forward_batch, self.model.norm diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index d3076ca6e6..f392eeefdf 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1267,14 +1267,13 @@ class Qwen3LLMModel(Qwen3Model): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, - forward_batch, - self.norm, - post_residual_addition=last_deepstack, - ) + hidden_states = residual_batch.final_norm( + hidden_states, + forward_batch, + self.norm, + post_residual_addition=last_deepstack, + skip_empty=True, + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/qwen3_vl_moe.py b/python/sglang/srt/models/qwen3_vl_moe.py index 5f4d31c6a2..9d9788989c 100644 --- a/python/sglang/srt/models/qwen3_vl_moe.py +++ b/python/sglang/srt/models/qwen3_vl_moe.py @@ -157,14 +157,13 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, - forward_batch, - self.norm, - post_residual_addition=last_deepstack, - ) + hidden_states = residual_batch.final_norm( + hidden_states, + forward_batch, + self.norm, + post_residual_addition=last_deepstack, + skip_empty=True, + ) if len(aux_hidden_states) == 0: return hidden_states diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 2a21eb3123..7225ffc3cc 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -1169,12 +1169,9 @@ class SarvamMLAModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return hidden_states @@ -1274,9 +1271,6 @@ class SarvamMLAForCausalLM(nn.Module): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.complete_output( - forward_batch.hidden_states, forward_batch - ) forward_batch.hidden_states = residual_batch.final_norm( forward_batch.hidden_states, forward_batch, self.model.norm ) @@ -1453,9 +1447,6 @@ class SarvamMoEForCausalLM(BailingMoEForCausalLM): ) if end == self.model.config.num_hidden_layers: - forward_batch.hidden_states = residual_batch.complete_output( - forward_batch.hidden_states, forward_batch - ) forward_batch.hidden_states = residual_batch.final_norm( forward_batch.hidden_states, forward_batch, self.model.norm ) diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 0d0fe098ba..64aef10e5a 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -479,12 +479,9 @@ class Step3TextModel(nn.Module): layer = self.layers[i] hidden_states = layer(positions, hidden_states, forward_batch) - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return hidden_states diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 79717bf6b8..1e749b851e 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -1646,11 +1646,9 @@ class XllmModel(nn.Module): if not self.pp_group.is_last_rank: return residual_batch.to_pp(hidden_states, forward_batch) else: - hidden_states = residual_batch.complete_output(hidden_states, forward_batch) - if hidden_states.shape[0] != 0: - hidden_states = residual_batch.final_norm( - hidden_states, forward_batch, self.norm - ) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm, skip_empty=True + ) return hidden_states