diff --git a/python/sglang/srt/layers/layer_boundary/prepare.py b/python/sglang/srt/layers/layer_boundary/prepare.py index 562f3f6b78..1376c2e315 100644 --- a/python/sglang/srt/layers/layer_boundary/prepare.py +++ b/python/sglang/srt/layers/layer_boundary/prepare.py @@ -85,10 +85,13 @@ def _reduce_update_read( group: SumGroup = SumGroup.ATTN_TP, read: ResidualReadout = NORM_READOUT, update: ResidualUpdate = PLAIN_ADD, + quant_format: str = "", + post_residual_addition: Optional[torch.Tensor] = None, ): """Complete the sum the input owes over ``group`` on the rows it is on, unless one of ``fusions`` does it with the residual add and the norm, then - write it into the residual and read the input.""" + write it into the residual and read the input (an attention's read takes + ``quant_format`` and ``post_residual_addition``).""" if gathers_residual: residual = update.gather_residual_attn_tp(residual) for fused in fusions: @@ -104,7 +107,14 @@ def _reduce_update_read( hidden_states = tensor_model_parallel_all_reduce(hidden_states) if _is_npu and cache is not None: _ = prepare_weight_cache(hidden_states, cache) - return read.update_and_read(update, hidden_states, residual, norm) + return read.update_and_read( + update, + hidden_states, + residual, + norm, + quant_format=quant_format, + post_residual_addition=post_residual_addition, + ) def _reduce_update_read_dp_gather( diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 47670bfbda..8e74c9751e 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -29,6 +29,14 @@ from sglang.srt.distributed import ( get_pp_indices, ) from sglang.srt.layers.activation import SiluAndMul +from sglang.srt.layers.aux_hidden_states import AuxHiddenStateList +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -155,6 +163,7 @@ class LlamaAttention(nn.Module): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", bias: bool = False, + reduce_results: bool = True, ) -> None: super().__init__() self.hidden_size = hidden_size @@ -199,6 +208,7 @@ class LlamaAttention(nn.Module): hidden_size, bias=bias, quant_config=quant_config, + reduce_results=reduce_results, prefix=add_prefix("o_proj", prefix), ) @@ -292,7 +302,11 @@ class LlamaDecoderLayer(nn.Module): start_layer: int = 0, quant_config: Optional[QuantizationConfig] = None, prefix: str = "", + stage_boundaries: bool = True, ) -> None: + """``stage_boundaries=False`` is for a subclass that drives its own + residual in its own forward(): the attention and the MLP then complete + their own sums and no boundaries are built.""" super().__init__() self.hidden_size = config.hidden_size rope_parameters = getattr(config, "rope_parameters", None) @@ -329,6 +343,7 @@ class LlamaDecoderLayer(nn.Module): quant_config=quant_config, prefix=add_prefix("self_attn", prefix), bias=attention_bias, + reduce_results=not stage_boundaries, ) self.mlp = LlamaMLP( hidden_size=self.hidden_size, @@ -336,29 +351,40 @@ class LlamaDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=not stage_boundaries, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + if not stage_boundaries: + return + self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj) + self.post_attention_layernorm.fuse_input_quant(self.mlp.gate_up_proj) + self.attn_boundary, self.ffn_boundary = make_stages( + (declare_attn(read=self._attn_readout(layer_id)), self.input_layernorm), + ( + declare_ffn(sparse=False, next_layer_sparse=False), + self.post_attention_layernorm, + ), + previous=declare_ffn() if layer_id != 0 else None, + terminal=layer_id == config.num_hidden_layers - 1, + ) + + def _attn_readout(self, layer_id: int): + return NORM_QUANT_READOUT def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: + capture_output=None, + ) -> torch.Tensor: # Self Attention - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm( - hidden_states, quant_linear=self.self_attn.qkv_proj - ) - else: - hidden_states, residual = self.input_layernorm( - hidden_states, residual, quant_linear=self.self_attn.qkv_proj - ) + hidden_states = self.attn_boundary.prepare( + hidden_states, forward_batch, capture=capture_output + ) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, @@ -366,11 +392,11 @@ class LlamaDecoderLayer(nn.Module): ) # Fully Connected - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual, quant_linear=self.mlp.gate_up_proj - ) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + return ffn_exit.finish(hidden_states) class LlamaModel(nn.Module): @@ -431,35 +457,29 @@ class LlamaModel(nn.Module): hidden_states = self.embed_tokens(input_ids) else: hidden_states = input_embeds - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - # FIXME(@ying): reduce the number of proxy tensors by not fusing layer norms - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] - deferred_norm = None + hidden_states = self.layers[self.start_layer].attn_boundary.from_pp( + pp_proxy_tensors, forward_batch + ) - aux_hidden_states = [] + aux_hidden_states = AuxHiddenStateList() for i in range(self.start_layer, self.end_layer): - if i in self.layers_to_capture: - aux_hidden_states.append(hidden_states + residual) - layer = self.layers[i] - hidden_states, residual = layer( + hidden_states = self.layers[i]( positions, hidden_states, forward_batch, - residual, + capture_output=aux_hidden_states.capture + if i in self.layers_to_capture + else None, ) if not self.pp_group.is_last_rank: - return PPProxyTensors( - { - "hidden_states": hidden_states, - "residual": residual, - } - ) - else: - hidden_states, _ = self.norm(hidden_states, residual) + return residual_batch.to_pp(hidden_states, forward_batch) + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.norm + ) if len(aux_hidden_states) == 0: return hidden_states @@ -609,6 +629,7 @@ class LlamaForCausalLM(nn.Module): start, end = split_interval # embed if start == 0: + residual_batch.start(forward_batch) if input_embeds is None: forward_batch.hidden_states = self.model.embed_tokens(input_ids) else: @@ -616,19 +637,17 @@ class LlamaForCausalLM(nn.Module): # decoder layer for i in range(start, end): layer = self.model.layers[i] - forward_batch.hidden_states, forward_batch.residual = layer( + forward_batch.hidden_states = layer( positions, forward_batch.hidden_states, forward_batch, - forward_batch.residual, ) if end == self.model.config.num_hidden_layers: # norm - hidden_states, _ = self.model.norm( - forward_batch.hidden_states, forward_batch.residual + forward_batch.hidden_states = residual_batch.final_norm( + forward_batch.hidden_states, forward_batch, self.model.norm ) - forward_batch.hidden_states = hidden_states # logits process result = self.logits_processor( input_ids, forward_batch.hidden_states, self.lm_head, forward_batch diff --git a/python/sglang/srt/models/llama_eagle.py b/python/sglang/srt/models/llama_eagle.py index df64c3b515..400b431f11 100644 --- a/python/sglang/srt/models/llama_eagle.py +++ b/python/sglang/srt/models/llama_eagle.py @@ -25,6 +25,8 @@ import torch from torch import nn from transformers import LlamaConfig +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch +from sglang.srt.layers.layer_boundary.residual.post_norm import PLAIN_READOUT from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ( @@ -50,7 +52,11 @@ class LlamaDecoderLayer(LlamaDecoderLayer): # https://github.com/SafeAILab/EAGLE/blob/35c78f6cdc19a73e05cf5c330b4c358dad970c6a/eagle/model/cnets.py#L427 if layer_id == 0: del self.input_layernorm - setattr(self, "input_layernorm", lambda x, quant_linear=None: x) + + def _attn_readout(self, layer_id: int): + if layer_id == 0: + return PLAIN_READOUT + return super()._attn_readout(layer_id) class LlamaModel(nn.Module): @@ -98,16 +104,11 @@ class LlamaModel(nn.Module): torch.cat((hidden_states, forward_batch.spec_info.hidden_states), dim=-1) ) - residual = None - for i in range(len(self.layers)): - layer = self.layers[i] - hidden_states, residual = layer( - positions, - hidden_states, - forward_batch, - residual, - ) - return hidden_states + residual + residual_batch.start(forward_batch) + for layer in self.layers: + hidden_states = layer(positions, hidden_states, forward_batch) + hidden_states = residual_batch.fold(hidden_states, forward_batch) + return residual_batch.take_output(hidden_states, forward_batch) class LlamaForCausalLMEagle(LlamaForCausalLM): diff --git a/python/sglang/srt/models/llama_eagle3.py b/python/sglang/srt/models/llama_eagle3.py index 9f5ddceef8..4244ed1853 100644 --- a/python/sglang/srt/models/llama_eagle3.py +++ b/python/sglang/srt/models/llama_eagle3.py @@ -49,7 +49,13 @@ class LlamaDecoderLayer(LlamaDecoderLayer): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", ) -> None: - super().__init__(config, layer_id, quant_config=quant_config, prefix=prefix) + super().__init__( + config, + layer_id, + quant_config=quant_config, + prefix=prefix, + stage_boundaries=False, + ) # Input layer concats embeds + target_hidden before qkv (input dim 2x). self.is_input_layer = layer_id == 0 diff --git a/python/sglang/srt/models/ministral3.py b/python/sglang/srt/models/ministral3.py index 60e29d6644..51335242ef 100644 --- a/python/sglang/srt/models/ministral3.py +++ b/python/sglang/srt/models/ministral3.py @@ -39,6 +39,7 @@ class Ministral3Attention(LlamaAttention): quant_config: Optional[QuantizationConfig] = None, prefix: str = "", bias: bool = False, + reduce_results: bool = True, ) -> None: super().__init__( config=config, @@ -54,6 +55,7 @@ class Ministral3Attention(LlamaAttention): quant_config=quant_config, prefix=prefix, bias=bias, + reduce_results=reduce_results, ) # Ministral3 specific: llama 4 style scaling beta self.llama_4_scaling_beta = config.rope_parameters.get("llama_4_scaling_beta") @@ -128,7 +130,9 @@ class Ministral3DecoderLayer(LlamaDecoderLayer): prefix=add_prefix("self_attn", prefix), bias=getattr(config, "attention_bias", False) or getattr(config, "bias", False), + reduce_results=False, ) + self.input_layernorm.fuse_input_quant(self.self_attn.qkv_proj) class Ministral3Model(LlamaModel): diff --git a/python/sglang/srt/models/mistral_eagle.py b/python/sglang/srt/models/mistral_eagle.py index bdb8253ccc..f7dcf9dbe9 100644 --- a/python/sglang/srt/models/mistral_eagle.py +++ b/python/sglang/srt/models/mistral_eagle.py @@ -40,6 +40,7 @@ import torch from torch import nn from transformers import PretrainedConfig +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import RowParallelLinear from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -120,12 +121,11 @@ class MistralEagleModel(nn.Module): ) ) - residual = None + residual_batch.start(forward_batch) for layer in self.layers: - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual - ) - return hidden_states + residual + hidden_states = layer(positions, hidden_states, forward_batch) + hidden_states = residual_batch.fold(hidden_states, forward_batch) + return residual_batch.take_output(hidden_states, forward_batch) class MistralForCausalLMEagle(LlamaForCausalLMEagle): diff --git a/python/sglang/srt/models/mllama.py b/python/sglang/srt/models/mllama.py index c03c59b1aa..b3b4dd82d2 100644 --- a/python/sglang/srt/models/mllama.py +++ b/python/sglang/srt/models/mllama.py @@ -22,6 +22,7 @@ from transformers.models.mllama.modeling_mllama import ( from sglang.srt.layers.activation import get_act_fn from sglang.srt.layers.attention.vision import VisionAttention +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -725,13 +726,14 @@ class MllamaTextModel(nn.Module): forward_batch=forward_batch, ) elif isinstance(decoder_layer, LlamaDecoderLayer): - hidden_states, residual = decoder_layer( + residual_batch.start(forward_batch) + hidden_states = decoder_layer( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, - residual=None, ) - hidden_states = hidden_states + residual + hidden_states = residual_batch.fold(hidden_states, forward_batch) + hidden_states = residual_batch.take_output(hidden_states, forward_batch) else: raise ValueError(f"Unknown decoder layer type {type(decoder_layer)}") hidden_states = self.norm(hidden_states) diff --git a/python/sglang/srt/models/nemotron_nas.py b/python/sglang/srt/models/nemotron_nas.py index 8e59a70ac8..0f41989d8c 100644 --- a/python/sglang/srt/models/nemotron_nas.py +++ b/python/sglang/srt/models/nemotron_nas.py @@ -23,6 +23,12 @@ import torch from torch import nn from transformers import LlamaConfig +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor, LogitsProcessorOutput from sglang.srt.layers.pooler import Pooler, PoolingType @@ -57,6 +63,21 @@ def _find_multiple(n: int, k: int) -> int: return n + k - (n % k) +def _previous_stage(block_configs, layer_idx: int): + """The declaration of the last attention or FFN before this layer, or None + when this layer's first stage starts the layer stack.""" + for block in reversed(block_configs[:layer_idx]): + if not block.ffn.no_op: + return declare_ffn() + if not block.attention.no_op: + return declare_attn() + return None + + +def _has_stage(block) -> bool: + return not (block.attention.no_op and block.ffn.no_op) + + class DeciLMDecoderLayer(nn.Module): def __init__( self, @@ -107,6 +128,7 @@ class DeciLMDecoderLayer(nn.Module): quant_config=quant_config, prefix=add_prefix("self_attn", prefix), bias=attention_bias, + reduce_results=False, ) self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) @@ -121,41 +143,61 @@ class DeciLMDecoderLayer(nn.Module): hidden_act=config.hidden_act, quant_config=quant_config, prefix=add_prefix("mlp", prefix), + reduce_results=False, ) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps ) + # A layer whose attention and FFN are both no-ops has no stages. + stages = [] + if not self._is_no_op_attention: + stages.append((declare_attn(), self.input_layernorm)) + if not self._is_no_op_ffn: + stages.append( + ( + declare_ffn(sparse=False, next_layer_sparse=False), + self.post_attention_layernorm, + ) + ) + self.entry_boundary = None + if stages: + boundaries = make_stages( + *stages, + previous=_previous_stage(config.block_configs, layer_idx), + terminal=not any( + _has_stage(block) for block in config.block_configs[layer_idx + 1 :] + ), + ) + self.entry_boundary = boundaries[0] + if not self._is_no_op_attention: + self.attn_boundary = boundaries[0] + if not self._is_no_op_ffn: + self.ffn_boundary = boundaries[-1] + def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, forward_batch: ForwardBatch, - residual: Optional[torch.Tensor], - ) -> Tuple[torch.Tensor, torch.Tensor]: + ) -> torch.Tensor: # Self Attention - - if self._is_no_op_attention: - pass - else: - if residual is None: - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - else: - hidden_states, residual = self.input_layernorm(hidden_states, residual) + if not self._is_no_op_attention: + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.self_attn( positions=positions, hidden_states=hidden_states, forward_batch=forward_batch, ) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) # Fully Connected if not self._is_no_op_ffn: - hidden_states, residual = self.post_attention_layernorm( - hidden_states, residual - ) - hidden_states = self.mlp(hidden_states) - return hidden_states, residual + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states) + hidden_states = ffn_exit.finish(hidden_states) + return hidden_states class DeciModel(nn.Module): @@ -223,32 +265,23 @@ class DeciModel(nn.Module): hidden_states = inputs_embeds else: hidden_states = self.get_input_embeddings(input_ids) - residual = None + residual_batch.start(forward_batch) else: assert pp_proxy_tensors is not None - hidden_states = pp_proxy_tensors["hidden_states"] - residual = pp_proxy_tensors["residual"] + entry = next( + self.layers[i].entry_boundary + for i in range(self.start_layer, self.end_layer) + if self.layers[i].entry_boundary is not None + ) + hidden_states = entry.from_pp(pp_proxy_tensors, forward_batch) - kv_cache_index = 0 for i in range(self.start_layer, self.end_layer): - layer = self.layers[i] - if not layer._is_no_op_attention: - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual - ) - kv_cache_index += 1 - else: - hidden_states, residual = layer( - positions, hidden_states, forward_batch, residual - ) + hidden_states = self.layers[i](positions, hidden_states, forward_batch) if not get_parallel().pp_group.is_last_rank: - return PPProxyTensors( - {"hidden_states": hidden_states, "residual": residual} - ) + return residual_batch.to_pp(hidden_states, forward_batch) - hidden_states, _ = self.norm(hidden_states, residual) - return hidden_states + return residual_batch.final_norm(hidden_states, forward_batch, self.norm) class DeciLMForCausalLM(nn.Module): diff --git a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py index 5150d3b850..9eed39c8b4 100644 --- a/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py +++ b/test/registered/unit/layer_boundary/test_declared_decoder_boundary.py @@ -1973,7 +1973,7 @@ class TestTheFfnInputReduction(CustomTestCase): with self.subTest(is_plain_add=is_plain_add): update = SimpleNamespace(is_plain_add=is_plain_add) read = SimpleNamespace( - update_and_read=lambda update, h, r, norm: (h, r) + update_and_read=lambda update, h, r, norm, **read_kwargs: (h, r) ) calls = [] with (