[Refactor] Build the Qwen2 decoders from stage boundaries (#42307)

This commit is contained in:
Cheng Wan
2026-10-03 02:19:56 -07:00
committed by GitHub
parent 2c739b225b
commit 740a4d5955
12 changed files with 143 additions and 187 deletions
@@ -19,7 +19,6 @@ from sglang.srt.layers.layer_boundary import (
tbo_split_moves,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.access import export_output
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.moe import (
get_deepep_mode,
@@ -1101,18 +1100,15 @@ def _model_forward_filter_inputs(
def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
stream_a, stream_b = output_a["residual"], output_b["residual"]
has_stream = isinstance(stream_a, ResidualStream)
assert has_stream == isinstance(stream_b, ResidualStream)
pending_a, pending_b = stream_a.pending, stream_b.pending
assert (pending_a is None) == (pending_b is None)
update = None
if has_stream:
pending_a, pending_b = stream_a.pending, stream_b.pending
assert (pending_a is None) == (pending_b is None)
if pending_a is not None:
assert pending_a.update is pending_b.update
update = pending_a.update
if pending_a is not None:
assert pending_a.update is pending_b.update
update = pending_a.update
for output in (output_a, output_b):
output["hidden_states"], output["residual"] = export_output(
output["hidden_states"], output["residual"], output["forward_batch"]
output["hidden_states"], output["residual"] = output["residual"].export(
output["hidden_states"]
)
def _handle_key(name):
@@ -1133,11 +1129,7 @@ def _model_forward_tbo_merge_outputs(output_a, output_b, original_len):
return res
hidden, residual = _handle_key("hidden_states"), _handle_key("residual")
return (
ResidualStream.from_handoff(hidden, residual, update)
if has_stream
else (hidden, residual)
)
return ResidualStream.from_handoff(hidden, residual, update)
# -------------------------------- Utilities and wrappers ---------------------------------------
@@ -17,14 +17,9 @@ from typing import Optional, Tuple, Union
import torch
from sglang.srt.distributed import GroupCoordinator
from sglang.srt.layers.layer_boundary.output import (
DeferredFinalize,
UnreducedOutput,
complete_owed,
)
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput, ResidualStream
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.layers.layer_boundary.output import DeferredFinalize, UnreducedOutput
from sglang.srt.layers.layer_boundary.residual.stream import OwedOutput
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
def buffer(
@@ -42,67 +37,6 @@ def buffer(
return hidden_states
def add_to_output(hidden_states, residual, extra: torch.Tensor) -> Tuple:
"""Complete the output before adding an extra contribution exactly once.
Leave its residual update for the next prepare call."""
hidden_states = (
residual.complete(hidden_states)
if isinstance(residual, ResidualStream)
else complete_owed(hidden_states)
)
hidden_states.add_(extra)
return hidden_states, residual
def fold(hidden_states, residual):
"""Finish a plain layer output and fold its residual into a complete value.
The following stage receives it with no outstanding residual addition."""
stream = residual if isinstance(residual, ResidualStream) else None
if stream is not None:
if stream.pending is not None and not stream.pending.update.is_plain_add:
raise NotImplementedError("fold requires a plain residual update")
hidden_states, residual = stream.export(hidden_states)
hidden_states = complete_owed(hidden_states)
if residual is not None:
hidden_states = hidden_states + residual
return (
(stream.write(hidden_states), stream)
if stream is not None
else (hidden_states, None)
)
def written(hidden_states):
"""Re-enter after a computation that already updated the full residual."""
return hidden_states, ResidualStream(hidden_states)
def export_output(
hidden_states: Union[torch.Tensor, UnreducedOutput, DeferredFinalize],
residual: Optional[torch.Tensor],
forward_batch: ForwardBatch,
*,
final_norm_takes_handoff: bool = False,
preserve_declared: bool = False,
) -> Tuple[Union[torch.Tensor, DeferredFinalize], Optional[torch.Tensor]]:
"""Complete what this layer left for a next layer, for callers that carry
an explicit residual tensor or merge TBO microbatches. Stage-boundary
decoders use residual_batch.final_norm(), to_pp() or take_output() instead:
to_pp() keeps a declared partial sum for the receiving from_pp(), which
this helper would complete early. A final norm that does a producer's
handoff together with its own work (``final_norm_takes_handoff``)
receives it as it is."""
if isinstance(residual, ResidualStream):
return residual.export(
hidden_states,
takes_handoff=final_norm_takes_handoff,
preserve_declared=preserve_declared,
)
if final_norm_takes_handoff and isinstance(hidden_states, DeferredFinalize):
return hidden_states, residual
return complete_owed(hidden_states), residual
def final_norm_pair(hidden_states, residual, norm, capture=None, **read_kwargs):
"""The final norm and an optional capture of the same updated residual.
@@ -136,20 +70,3 @@ def from_pp(
else:
residual = tensors["residual"]
return tensors["hidden_states"], residual
def snapshot(
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
*,
group: Optional[GroupCoordinator] = None,
) -> torch.Tensor:
"""Copy a complete output with its plain residual. A statically declared
sum is reduced on a copy; the main output and residual remain unchanged."""
if isinstance(residual, ResidualStream):
if group is not None:
raise ValueError("a residual stream carries its own reduction")
return residual.snapshot(hidden_states)
if group is not None:
hidden_states = group.all_reduce(hidden_states.clone())
return hidden_states.clone() if residual is None else hidden_states + residual
@@ -13,6 +13,7 @@
# ==============================================================================
"""Residual access for one layer-stack invocation or TBO microbatch."""
from sglang.srt.layers.layer_boundary.output import complete_owed
from sglang.srt.layers.layer_boundary.residual import access
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
@@ -135,13 +136,24 @@ def snapshot(hidden_states, forward_batch):
def add_to_output(hidden_states, forward_batch, extra):
output, _ = access.add_to_output(hidden_states, stream_of(forward_batch), extra)
return output
"""Complete the output before adding an extra contribution exactly once.
Leave its residual update for the next prepare call."""
hidden_states = stream_of(forward_batch).complete(hidden_states)
hidden_states.add_(extra)
return hidden_states
def fold(hidden_states, forward_batch):
output, _ = access.fold(hidden_states, stream_of(forward_batch))
return output
"""Finish a plain layer output and fold its residual into a complete value.
The following stage receives it with no outstanding residual addition."""
stream = stream_of(forward_batch)
if stream.pending is not None and not stream.pending.update.is_plain_add:
raise NotImplementedError("fold requires a plain residual update")
hidden_states, residual = stream.export(hidden_states)
hidden_states = complete_owed(hidden_states)
if residual is not None:
hidden_states = hidden_states + residual
return stream.write(hidden_states)
def set_written(hidden_states, forward_batch):
+11
View File
@@ -449,6 +449,9 @@ def _is_static_per_tensor_fp8_linear(quant_method, linear) -> bool:
class RMSNorm(BaseFusedOp):
# The projection that consumes this norm's output; see fuse_input_quant().
_quant_linear: Optional[nn.Module] = None
def __init__(
self,
hidden_size: int,
@@ -499,6 +502,12 @@ class RMSNorm(BaseFusedOp):
if force_native:
self._forward_method = self.forward_native
def fuse_input_quant(self, linear: nn.Module) -> None:
"""Fuse the static per-tensor FP8 input quantization of ``linear``, the
projection that consumes this norm's output, whenever forward_cuda can.
The projection stays out of this module's children."""
self.__dict__["_quant_linear"] = linear
def forward_cuda(
self,
x: torch.Tensor,
@@ -506,6 +515,8 @@ class RMSNorm(BaseFusedOp):
post_residual_addition: Optional[torch.Tensor] = None,
quant_linear: Optional[nn.Module] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
if quant_linear is None:
quant_linear = self._quant_linear
if x.numel() == 0:
if residual is not None:
if post_residual_addition is not None:
+25 -4
View File
@@ -466,8 +466,7 @@ class JetNemotronDecoderLayer(nn.Module):
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
) -> torch.Tensor:
# Self Attention
residual = hidden_states
@@ -490,7 +489,29 @@ class JetNemotronDecoderLayer(nn.Module):
hidden_states = residual + hidden_states
return hidden_states, None
return hidden_states
class JetNemotronModel(Qwen2Model):
"""Each decoder layer adds its own residual, so the stack runs without a
residual stream and ends with the final norm alone."""
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
input_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if input_embeds is None:
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
for i in range(self.start_layer, self.end_layer):
hidden_states = self.layers[i](positions, hidden_states, forward_batch)
if hidden_states.shape[0] != 0:
hidden_states = self.norm(hidden_states)
return hidden_states
class JetNemotronForCausalLM(nn.Module):
@@ -505,7 +526,7 @@ class JetNemotronForCausalLM(nn.Module):
self.config = config
self.quant_config = quant_config
self.model = Qwen2Model(
self.model = JetNemotronModel(
config,
quant_config=quant_config,
prefix=add_prefix("model", prefix),
+5 -3
View File
@@ -8,6 +8,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -69,13 +70,14 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
)
)
hidden_states, residual = self.mtp_block(
residual_batch.start(forward_batch)
hidden_states = self.mtp_block(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=None,
)
hidden_states = residual + hidden_states
hidden_states = residual_batch.fold(hidden_states, forward_batch)
hidden_states = residual_batch.take_output(hidden_states, forward_batch)
hidden_states = self.final_layernorm(hidden_states)
return hidden_states
+50 -46
View File
@@ -26,12 +26,15 @@ from sglang.srt.distributed import (
get_pp_indices,
)
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
from sglang.srt.layers.layer_boundary.residual.access import (
export_output,
from_pp,
snapshot,
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -183,6 +186,7 @@ class Qwen2Attention(nn.Module):
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=False,
prefix=add_prefix("o_proj", prefix),
)
@@ -280,29 +284,38 @@ class Qwen2DecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_readout(self, layer_id: int):
return NORM_QUANT_READOUT
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
capture_output=None,
) -> torch.Tensor:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(
hidden_states, quant_linear=self.self_attn.qkv_proj
)
else:
hidden_states, residual = self.input_layernorm(
hidden_states, residual, quant_linear=self.self_attn.qkv_proj
)
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture=capture_output
)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
@@ -310,11 +323,11 @@ class Qwen2DecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(
hidden_states, residual, quant_linear=self.mlp.gate_up_proj
)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class Qwen2Model(nn.Module):
@@ -410,37 +423,29 @@ class Qwen2Model(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states, residual = from_pp(pp_proxy_tensors)
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(snapshot(hidden_states, residual))
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = self.layers[i](
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
hidden_states, residual = export_output(hidden_states, residual, forward_batch)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
else:
if hidden_states.shape[0] != 0:
if residual is None:
hidden_states = self.norm(hidden_states)
else:
hidden_states, _ = self.norm(hidden_states, residual)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm, skip_empty=True
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -578,6 +583,7 @@ class Qwen2ForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -585,19 +591,17 @@ class Qwen2ForCausalLM(nn.Module):
# decoder layer
for i in range(start, end):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
forward_batch.hidden_states = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
)
if end == self.model.config.num_hidden_layers:
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
forward_batch.hidden_states = residual_batch.final_norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
+12 -11
View File
@@ -24,6 +24,8 @@ from typing import Iterable, Optional, Tuple
import torch
from torch import nn
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -51,7 +53,11 @@ class Qwen2DecoderLayer(Qwen2DecoderLayer):
# https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427
if layer_id == 0:
del self.input_layernorm
setattr(self, "input_layernorm", lambda x, quant_linear=None: x)
def _attn_readout(self, layer_id: int):
if layer_id == 0:
return PLAIN_READOUT
return super()._attn_readout(layer_id)
class Qwen2Model(nn.Module):
@@ -99,16 +105,11 @@ class Qwen2Model(nn.Module):
torch.cat((hidden_states, forward_batch.spec_info.hidden_states), dim=-1)
)
residual = None
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
return hidden_states + residual
residual_batch.start(forward_batch)
for layer in self.layers:
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states = residual_batch.fold(hidden_states, forward_batch)
return residual_batch.take_output(hidden_states, forward_batch)
class Qwen2ForCausalLMEagle(Qwen2ForCausalLM):