[Refactor] Build the Llama and Nemotron-NAS decoders from stage boundaries (#42308)

This commit is contained in:
Cheng Wan
2026-10-03 02:20:18 -07:00
committed by GitHub
parent 740a4d5955
commit e4554fd5e5
9 changed files with 173 additions and 98 deletions
@@ -85,10 +85,13 @@ def _reduce_update_read(
group: SumGroup = SumGroup.ATTN_TP,
read: ResidualReadout = NORM_READOUT,
update: ResidualUpdate = PLAIN_ADD,
quant_format: str = "",
post_residual_addition: Optional[torch.Tensor] = None,
):
"""Complete the sum the input owes over ``group`` on the rows it is on,
unless one of ``fusions`` does it with the residual add and the norm, then
write it into the residual and read the input."""
write it into the residual and read the input (an attention's read takes
``quant_format`` and ``post_residual_addition``)."""
if gathers_residual:
residual = update.gather_residual_attn_tp(residual)
for fused in fusions:
@@ -104,7 +107,14 @@ def _reduce_update_read(
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
if _is_npu and cache is not None:
_ = prepare_weight_cache(hidden_states, cache)
return read.update_and_read(update, hidden_states, residual, norm)
return read.update_and_read(
update,
hidden_states,
residual,
norm,
quant_format=quant_format,
post_residual_addition=post_residual_addition,
)
def _reduce_update_read_dp_gather(
+59 -40
View File
@@ -29,6 +29,14 @@ from sglang.srt.distributed import (
get_pp_indices,
)
from sglang.srt.layers.activation import SiluAndMul
from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
MergedColumnParallelLinear,
@@ -155,6 +163,7 @@ class LlamaAttention(nn.Module):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
bias: bool = False,
reduce_results: bool = True,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -199,6 +208,7 @@ class LlamaAttention(nn.Module):
hidden_size,
bias=bias,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=add_prefix("o_proj", prefix),
)
@@ -292,7 +302,11 @@ class LlamaDecoderLayer(nn.Module):
start_layer: int = 0,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
stage_boundaries: bool = True,
) -> None:
"""``stage_boundaries=False`` is for a subclass that drives its own
residual in its own forward(): the attention and the MLP then complete
their own sums and no boundaries are built."""
super().__init__()
self.hidden_size = config.hidden_size
rope_parameters = getattr(config, "rope_parameters", None)
@@ -329,6 +343,7 @@ class LlamaDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
bias=attention_bias,
reduce_results=not stage_boundaries,
)
self.mlp = LlamaMLP(
hidden_size=self.hidden_size,
@@ -336,29 +351,40 @@ class LlamaDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=not stage_boundaries,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
if not stage_boundaries:
return
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj)
self.attn_boundary, self.ffn_boundary = make_stages(
(declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm),
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
),
previous=declare_ffn() if layer_id != 0 else None,
terminal=layer_id == config.num_hidden_layers - 1,
)
def _attn_readout(self, layer_id: int):
return NORM_QUANT_READOUT
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
capture_output=None,
) -> torch.Tensor:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(
hidden_states, quant_linear=self.self_attn.qkv_proj
)
else:
hidden_states, residual = self.input_layernorm(
hidden_states, residual, quant_linear=self.self_attn.qkv_proj
)
hidden_states = self.attn_boundary.prepare(
hidden_states, forward_batch, capture=capture_output
)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
@@ -366,11 +392,11 @@ class LlamaDecoderLayer(nn.Module):
)
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(
hidden_states, residual, quant_linear=self.mlp.gate_up_proj
)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
return ffn_exit.finish(hidden_states)
class LlamaModel(nn.Module):
@@ -431,35 +457,29 @@ class LlamaModel(nn.Module):
hidden_states = self.embed_tokens(input_ids)
else:
hidden_states = input_embeds
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
# FIXME(@ying): reduce the number of proxy tensors by not fusing layer norms
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
deferred_norm = None
hidden_states = self.layers[self.start_layer].attn_boundary.from_pp(
pp_proxy_tensors, forward_batch
)
aux_hidden_states = []
aux_hidden_states = AuxHiddenStateList()
for i in range(self.start_layer, self.end_layer):
if i in self.layers_to_capture:
aux_hidden_states.append(hidden_states + residual)
layer = self.layers[i]
hidden_states, residual = layer(
hidden_states = self.layers[i](
positions,
hidden_states,
forward_batch,
residual,
capture_output=aux_hidden_states.capture
if i in self.layers_to_capture
else None,
)
if not self.pp_group.is_last_rank:
return PPProxyTensors(
{
"hidden_states": hidden_states,
"residual": residual,
}
)
else:
hidden_states, _ = self.norm(hidden_states, residual)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states = residual_batch.final_norm(
hidden_states, forward_batch, self.norm
)
if len(aux_hidden_states) == 0:
return hidden_states
@@ -609,6 +629,7 @@ class LlamaForCausalLM(nn.Module):
start, end = split_interval
# embed
if start == 0:
residual_batch.start(forward_batch)
if input_embeds is None:
forward_batch.hidden_states = self.model.embed_tokens(input_ids)
else:
@@ -616,19 +637,17 @@ class LlamaForCausalLM(nn.Module):
# decoder layer
for i in range(start, end):
layer = self.model.layers[i]
forward_batch.hidden_states, forward_batch.residual = layer(
forward_batch.hidden_states = layer(
positions,
forward_batch.hidden_states,
forward_batch,
forward_batch.residual,
)
if end == self.model.config.num_hidden_layers:
# norm
hidden_states, _ = self.model.norm(
forward_batch.hidden_states, forward_batch.residual
forward_batch.hidden_states = residual_batch.final_norm(
forward_batch.hidden_states, forward_batch, self.model.norm
)
forward_batch.hidden_states = hidden_states
# logits process
result = self.logits_processor(
input_ids, forward_batch.hidden_states, self.lm_head, forward_batch
+12 -11
View File
@@ -25,6 +25,8 @@ import torch
from torch import nn
from transformers import LlamaConfig
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.vocab_parallel_embedding import (
@@ -50,7 +52,11 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
# https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427
if layer_id == 0:
del self.input_layernorm
setattr(self, "input_layernorm", lambda x, quant_linear=None: x)
def _attn_readout(self, layer_id: int):
if layer_id == 0:
return PLAIN_READOUT
return super()._attn_readout(layer_id)
class LlamaModel(nn.Module):
@@ -98,16 +104,11 @@ class LlamaModel(nn.Module):
torch.cat((hidden_states, forward_batch.spec_info.hidden_states), dim=-1)
)
residual = None
for i in range(len(self.layers)):
layer = self.layers[i]
hidden_states, residual = layer(
positions,
hidden_states,
forward_batch,
residual,
)
return hidden_states + residual
residual_batch.start(forward_batch)
for layer in self.layers:
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states = residual_batch.fold(hidden_states, forward_batch)
return residual_batch.take_output(hidden_states, forward_batch)
class LlamaForCausalLMEagle(LlamaForCausalLM):
+7 -1
View File
@@ -49,7 +49,13 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
) -> None:
super().__init__(config, layer_id, quant_config=quant_config, prefix=prefix)
super().__init__(
config,
layer_id,
quant_config=quant_config,
prefix=prefix,
stage_boundaries=False,
)
# Input layer concats embeds + target_hidden before qkv (input dim 2x).
self.is_input_layer = layer_id == 0
+4
View File
@@ -39,6 +39,7 @@ class Ministral3Attention(LlamaAttention):
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
bias: bool = False,
reduce_results: bool = True,
) -> None:
super().__init__(
config=config,
@@ -54,6 +55,7 @@ class Ministral3Attention(LlamaAttention):
quant_config=quant_config,
prefix=prefix,
bias=bias,
reduce_results=reduce_results,
)
# Ministral3 specific: llama 4 style scaling beta
self.llama_4_scaling_beta = config.rope_parameters.get("llama_4_scaling_beta")
@@ -128,7 +130,9 @@ class Ministral3DecoderLayer(LlamaDecoderLayer):
prefix=add_prefix("self_attn", prefix),
bias=getattr(config, "attention_bias", False)
or getattr(config, "bias", False),
reduce_results=False,
)
self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj)
class Ministral3Model(LlamaModel):
+5 -5
View File
@@ -40,6 +40,7 @@ import torch
from torch import nn
from transformers import PretrainedConfig
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import RowParallelLinear
from sglang.srt.layers.quantization.base_config import QuantizationConfig
@@ -120,12 +121,11 @@ class MistralEagleModel(nn.Module):
)
)
residual = None
residual_batch.start(forward_batch)
for layer in self.layers:
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
return hidden_states + residual
hidden_states = layer(positions, hidden_states, forward_batch)
hidden_states = residual_batch.fold(hidden_states, forward_batch)
return residual_batch.take_output(hidden_states, forward_batch)
class MistralForCausalLMEagle(LlamaForCausalLMEagle):
+5 -3
View File
@@ -22,6 +22,7 @@ from transformers.models.mllama.modeling_mllama import (
from sglang.srt.layers.activation import get_act_fn
from sglang.srt.layers.attention.vision import VisionAttention
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
ColumnParallelLinear,
@@ -725,13 +726,14 @@ class MllamaTextModel(nn.Module):
forward_batch=forward_batch,
)
elif isinstance(decoder_layer, LlamaDecoderLayer):
hidden_states, residual = decoder_layer(
residual_batch.start(forward_batch)
hidden_states = decoder_layer(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
residual=None,
)
hidden_states = hidden_states + residual
hidden_states = residual_batch.fold(hidden_states, forward_batch)
hidden_states = residual_batch.take_output(hidden_states, forward_batch)
else:
raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}")
hidden_states = self.norm(hidden_states)
+68 -35
View File
@@ -23,6 +23,12 @@ import torch
from torch import nn
from transformers import LlamaConfig
from sglang.srt.layers.layer_boundary import (
declare_attn,
declare_ffn,
make_stages,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput
from sglang.srt.layers.pooler import Pooler, PoolingType
@@ -57,6 +63,21 @@ def _find_multiple(n: int, k: int) -> int:
return n + k - (n % k)
def _previous_stage(block_configs, layer_idx: int):
"""The declaration of the last attention or FFN before this layer, or None
when this layer's first stage starts the layer stack."""
for block in reversed(block_configs[:layer_idx]):
if not block.ffn.no_op:
return declare_ffn()
if not block.attention.no_op:
return declare_attn()
return None
def _has_stage(block) -> bool:
return not (block.attention.no_op and block.ffn.no_op)
class DeciLMDecoderLayer(nn.Module):
def __init__(
self,
@@ -107,6 +128,7 @@ class DeciLMDecoderLayer(nn.Module):
quant_config=quant_config,
prefix=add_prefix("self_attn", prefix),
bias=attention_bias,
reduce_results=False,
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -121,41 +143,61 @@ class DeciLMDecoderLayer(nn.Module):
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=add_prefix("mlp", prefix),
reduce_results=False,
)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
# A layer whose attention and FFN are both no-ops has no stages.
stages = []
if not self._is_no_op_attention:
stages.append((declare_attn(), self.input_layernorm))
if not self._is_no_op_ffn:
stages.append(
(
declare_ffn(sparse=False, next_layer_sparse=False),
self.post_attention_layernorm,
)
)
self.entry_boundary = None
if stages:
boundaries = make_stages(
*stages,
previous=_previous_stage(config.block_configs, layer_idx),
terminal=not any(
_has_stage(block) for block in config.block_configs[layer_idx + 1 :]
),
)
self.entry_boundary = boundaries[0]
if not self._is_no_op_attention:
self.attn_boundary = boundaries[0]
if not self._is_no_op_ffn:
self.ffn_boundary = boundaries[-1]
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
forward_batch: ForwardBatch,
residual: Optional[torch.Tensor],
) -> Tuple[torch.Tensor, torch.Tensor]:
) -> torch.Tensor:
# Self Attention
if self._is_no_op_attention:
pass
else:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
if not self._is_no_op_attention:
hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
forward_batch=forward_batch,
)
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
# Fully Connected
if not self._is_no_op_ffn:
hidden_states, residual = self.post_attention_layernorm(
hidden_states, residual
)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
class DeciModel(nn.Module):
@@ -223,32 +265,23 @@ class DeciModel(nn.Module):
hidden_states = inputs_embeds
else:
hidden_states = self.get_input_embeddings(input_ids)
residual = None
residual_batch.start(forward_batch)
else:
assert pp_proxy_tensors is not None
hidden_states = pp_proxy_tensors["hidden_states"]
residual = pp_proxy_tensors["residual"]
entry = next(
self.layers[i].entry_boundary
for i in range(self.start_layer, self.end_layer)
if self.layers[i].entry_boundary is not None
)
hidden_states = entry.from_pp(pp_proxy_tensors, forward_batch)
kv_cache_index = 0
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
if not layer._is_no_op_attention:
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
kv_cache_index += 1
else:
hidden_states, residual = layer(
positions, hidden_states, forward_batch, residual
)
hidden_states = self.layers[i](positions, hidden_states, forward_batch)
if not get_parallel().pp_group.is_last_rank:
return PPProxyTensors(
{"hidden_states": hidden_states, "residual": residual}
)
return residual_batch.to_pp(hidden_states, forward_batch)
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
return residual_batch.final_norm(hidden_states, forward_batch, self.norm)
class DeciLMForCausalLM(nn.Module):