diff --git a/python/sglang/srt/layers/layer_boundary/__init__.py b/python/sglang/srt/layers/layer_boundary/__init__.py index dd857d8c16..456e6c02af 100644 --- a/python/sglang/srt/layers/layer_boundary/__init__.py +++ b/python/sglang/srt/layers/layer_boundary/__init__.py @@ -72,6 +72,9 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import ( PLAIN_ADD, PLAIN_RESIDUAL_OPS, ) +from sglang.srt.layers.layer_boundary.residual.ihc import ( + IHCState, +) from sglang.srt.layers.layer_boundary.residual.mhc import ( MHCState, ) @@ -93,6 +96,7 @@ __all__ = [ "FfnExit", "FfnInputFusion", "DeferredFinalize", + "IHCState", "LayerResidualOps", "Layout", "MHCState", diff --git a/python/sglang/srt/layers/layer_boundary/residual/ihc.py b/python/sglang/srt/layers/layer_boundary/residual/ihc.py new file mode 100644 index 0000000000..b4027932e3 --- /dev/null +++ b/python/sglang/srt/layers/layer_boundary/residual/ihc.py @@ -0,0 +1,192 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gated hyper-connection residual streams.""" + +from dataclasses import dataclass +from typing import Callable, Optional + +import torch + +from sglang.srt.layers.layer_boundary.residual import LayerResidualOps +from sglang.srt.runtime_context import get_parallel + + +@dataclass +class IHCState: + """A layer's residual as hc_mult gated streams: a stage reads its input by + mixing the streams down with per-stream gates, and writes its output back + scaled by the post gates the same read produced. Unlike the hyper-connection + streams in mhc.py there is no combination matrix, so a read produces one + coefficient rather than a pair. ``residual_ops()`` gives the reads and + updates the layer's two stages declare. Parameters belong to the owning + layer's modules; this state only holds the coefficient shared between a + stage's read and its write-back. + + Args: + expand: Widens the layer stack's 2-D input into the stream + representation, and passes an already widened residual through. + attn_pre: The attention stage's read. + ffn_pre: The FFN stage's read. + attn_post: Writes the attention output back into the streams. + ffn_post: Writes the FFN output back into the streams. + post_pre: The attention write-back fused with the FFN read, or None to + run the two in sequence. + """ + + expand: Callable + attn_pre: Callable + ffn_pre: Callable + attn_post: Callable + ffn_post: Callable + post_pre: Optional[Callable] = None + # Produced by a stage's read and consumed by that same stage's write-back. + post_gate: Optional[torch.Tensor] = None + + def read_attn_input(self, residual, out_norm: Optional[torch.nn.Module] = None): + hidden_states, self.post_gate, residual = self.attn_pre(residual, out_norm) + return hidden_states, residual + + def update_and_read_ffn_input( + self, hidden_states, residual, out_norm: Optional[torch.nn.Module] = None + ): + if self.post_pre is not None: + hidden_states, self.post_gate, residual = self.post_pre( + hidden_states, residual, self.post_gate, out_norm + ) + return hidden_states, residual + residual = self.apply_attn_post(hidden_states, residual) + hidden_states, self.post_gate, residual = self.ffn_pre(residual, out_norm) + return hidden_states, residual + + def apply_attn_post(self, hidden_states, residual): + return self.attn_post(hidden_states, residual, self.post_gate) + + def apply_ffn_post(self, hidden_states, residual): + return self.ffn_post(hidden_states, residual, self.post_gate) + + def clear_coefficients(self): + self.post_gate = None + + def slice_residual_attn_tp(self, residual): + parallel = get_parallel() + rank, size = parallel.attn_tp_rank, parallel.attn_tp_size + self.post_gate = self.post_gate.tensor_split(size)[rank] + return residual.tensor_split(size)[rank] + + def gather_residual_attn_tp(self, residual): + raise NotImplementedError( + "Unsupported: iHC post-gate allgather is not implemented." + ) + + def residual_ops(self) -> LayerResidualOps: + return LayerResidualOps( + attn_readout=_AttnReadout(self), + attn_update=_AttnUpdate(self), + ffn_readout=_FfnReadout(self), + ffn_update=_FfnUpdate(self), + ) + + +class _AttnReadout: + """The gated mix of the streams and the input norm, from streams that + already hold the previous layer's output: an iHC layer takes its input + written back.""" + + is_plain_norm = False + reads_before_dp_gather = False + + def __init__(self, state: IHCState): + self.state = state + + def init_residual(self, hidden_states): + return self.state.expand(hidden_states) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + if quant_format: + raise NotImplementedError(f"an iHC attention input in {quant_format=}") + return self.state.read_attn_input(residual, out_norm=norm) + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + raise NotImplementedError("an iHC layer takes its input written back") + + +class _AttnUpdate: + """The gated write-back with the coefficient the attention's read produced. + It is not a plain add, so it runs only once the sum it writes in is + complete.""" + + is_plain_add = False + applied_at_exit = False + outlives_layer = False + + def __init__(self, state: IHCState): + self.state = state + + def update(self, hidden_states, residual): + return self.state.apply_attn_post(hidden_states, residual) + + def slice_residual_attn_tp(self, residual): + return self.state.slice_residual_attn_tp(residual) + + def gather_residual_attn_tp(self, residual): + return self.state.gather_residual_attn_tp(residual) + + +class _FfnReadout: + """The attention output's write-back and the FFN input's gated mix and + norm, fused in post_pre when the layer provides it.""" + + is_plain_norm = False + reads_before_dp_gather = False + + def __init__(self, state: IHCState): + self.state = state + + def init_residual(self, hidden_states): + return self.state.expand(hidden_states) + + def read(self, residual, norm, quant_format="", post_residual_addition=None): + raise NotImplementedError("an iHC FFN input read without its attention") + + def update_and_read(self, update, hidden_states, residual, norm, **kwargs): + if not isinstance(update, _AttnUpdate) or update.state is not self.state: + raise NotImplementedError(f"an iHC FFN input after {update=}") + return self.state.update_and_read_ffn_input( + hidden_states, residual, out_norm=norm + ) + + +class _FfnUpdate: + """The gated write-back with the coefficient the FFN input's read produced, + which this layer runs itself. The streams stay widened: unlike mhc.py the + layer stack's terminal read is a learned head rather than a contraction, + so it belongs to the final norm.""" + + is_plain_add = False + applied_at_exit = True + outlives_layer = False + + def __init__(self, state: IHCState): + self.state = state + + def update(self, hidden_states, residual): + hidden_states = self.state.apply_ffn_post(hidden_states, residual) + self.state.clear_coefficients() + return hidden_states + + def slice_residual_attn_tp(self, residual): + return self.state.slice_residual_attn_tp(residual) + + def gather_residual_attn_tp(self, residual): + return self.state.gather_residual_attn_tp(residual) diff --git a/python/sglang/srt/models/hunyuan_v4.py b/python/sglang/srt/models/hunyuan_v4.py index fce9541572..3ff4f3e840 100644 --- a/python/sglang/srt/models/hunyuan_v4.py +++ b/python/sglang/srt/models/hunyuan_v4.py @@ -7,7 +7,14 @@ from torch import nn from transformers import PretrainedConfig from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState -from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context +from sglang.srt.layers.layer_boundary import ( + IHCState, + declare_attn, + declare_ffn, + get_attn_tp_context, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear, ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -342,7 +349,8 @@ class HYV4Attention(DeepseekV2AttentionMLA): ), max_position_embeddings=config.max_position_embeddings, quant_config=quant_config, - reduce_results=True, + # The FFN input boundary completes this stage's sum. + reduce_results=False, layer_id=layer_id, prefix=prefix, alt_stream=alt_stream, @@ -535,6 +543,77 @@ class HYV4Attention(DeepseekV2AttentionMLA): return method +class _HeadNorm: + """The layer stack's terminal read: the learned head mixes the streams down + and fuses the final norm into the same kernel.""" + + def __init__(self, hc_head, norm): + self.hc_head = hc_head + self.norm = norm + + def __call__(self, hidden_states): + return self.hc_head(hidden_states, self.norm) + + +def _build_stages( + hc_attn_layer, + hc_mlp_layer, + input_layernorm, + post_attention_layernorm, + *, + qkv_latent_func, + layer_id, + config, +): + """The attention and FFN stage boundaries of one iHC decoder layer.""" + + def post_pre(hidden_states, residual, post_gate, out_norm): + return hc_attn_layer.post_pre( + hidden_states, residual, post_gate, hc_mlp_layer, out_norm + ) + + residual = IHCState( + expand=hc_attn_layer.prepare_input, + attn_pre=hc_attn_layer.pre, + ffn_pre=hc_mlp_layer.pre, + attn_post=hc_attn_layer.post, + ffn_post=hc_mlp_layer.post, + post_pre=post_pre, + ).residual_ops() + + layer_types = config.mlp_layer_types + is_sparse = layer_types[layer_id] != "dense" + next_id, previous_id = layer_id + 1, layer_id - 1 + return make_stages( + ( + declare_attn(read=residual.attn_readout, update=residual.attn_update), + input_layernorm, + {"qkv_latent_func": qkv_latent_func}, + ), + ( + declare_ffn( + sparse=is_sparse, + next_layer_sparse=( + next_id < len(layer_types) and layer_types[next_id] != "dense" + ), + read=residual.ffn_readout, + update=residual.ffn_update, + ), + post_attention_layernorm, + ), + previous=( + declare_ffn( + sparse=layer_types[previous_id] != "dense", + next_layer_sparse=is_sparse, + update=residual.ffn_update, + ) + if layer_id != 0 + else None + ), + terminal=layer_id == config.num_hidden_layers - 1, + ) + + class HYV4DecoderLayer(nn.Module): def __init__(self, config, layer_id, quant_config=None, prefix="", alt_stream=None): super().__init__() @@ -550,6 +629,8 @@ class HYV4DecoderLayer(nn.Module): config.hidden_act, quant_config=quant_config, prefix=f"{prefix}.mlp", + reduce_results=False, + allow_fused_down=False, ) else: self.mlp = DeepseekV2MoE( @@ -558,11 +639,21 @@ class HYV4DecoderLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.mlp", alt_stream=alt_stream, + reduce_results=False, ) if hasattr(self.mlp, "shared_experts"): self.mlp.shared_experts.swiglu_limit = None self.hc_attn_layer = HYV4HCLayer(config, f"{prefix}.hc_attn_layer") self.hc_mlp_layer = HYV4HCLayer(config, f"{prefix}.hc_mlp_layer") + self.attn_boundary, self.ffn_boundary = _build_stages( + self.hc_attn_layer, + self.hc_mlp_layer, + self.input_layernorm, + self.post_attention_layernorm, + qkv_latent_func=self.self_attn.prepare_qkv_latent, + layer_id=layer_id, + config=config, + ) def forward( self, @@ -572,15 +663,7 @@ class HYV4DecoderLayer(nn.Module): zero_allocator, prev_topk_indices=None, ): - hidden_states = self.hc_attn_layer.prepare_input(hidden_states) - hidden_states, post, residual = self.hc_attn_layer.pre( - hidden_states, self.input_layernorm - ) - get_attn_tp_context().set_attn_inputs( - AttentionInputs( - hidden_states, forward_batch, self.self_attn.prepare_qkv_latent - ) - ) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) try: hidden_states = self.self_attn( positions, @@ -595,18 +678,13 @@ class HYV4DecoderLayer(nn.Module): hidden_states, topk_indices = hidden_states else: topk_indices = None - hidden_states, post, residual = self.hc_attn_layer.post_pre( - hidden_states, - residual, - post, - self.hc_mlp_layer, - self.post_attention_layernorm, - ) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) if isinstance(self.mlp, DeepseekV2MoE): hidden_states = self.mlp(hidden_states, forward_batch) else: hidden_states = self.mlp(hidden_states) - hidden_states = self.hc_mlp_layer.post(hidden_states, residual, post) + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) return hidden_states, topk_indices @@ -639,11 +717,14 @@ class HYV4Model(nn.Module): ) self.hc_head = HYV4HCHeadLayer(config, f"{prefix}.hc_head") self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps) + # The head mixes the streams down and fuses the final norm into one kernel. + self._head_norm = _HeadNorm(self.hc_head, self.norm) def forward(self, input_ids, positions, forward_batch, input_embeds=None): hidden_states = ( self.embed_tokens(input_ids) if input_embeds is None else input_embeds ) + residual_batch.start(forward_batch) zero_allocator = BumpAllocator( buffer_size=2 * len(self.layers), dtype=torch.float32, @@ -660,7 +741,7 @@ class HYV4Model(nn.Module): ) topk_share.update(topk_indices) topk_share.publish() - return self.hc_head(hidden_states, self.norm) + return residual_batch.final_norm(hidden_states, forward_batch, self._head_norm) class HYV4ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): diff --git a/python/sglang/srt/models/hunyuan_v4_nextn.py b/python/sglang/srt/models/hunyuan_v4_nextn.py index 93198d6842..b5a9801d27 100644 --- a/python/sglang/srt/models/hunyuan_v4_nextn.py +++ b/python/sglang/srt/models/hunyuan_v4_nextn.py @@ -5,7 +5,13 @@ import torch from torch import nn from sglang.srt.layers.attention.index_topk_share import IndexTopKShareState -from sglang.srt.layers.layer_boundary import AttentionInputs, get_attn_tp_context +from sglang.srt.layers.layer_boundary import ( + declare_attn, + declare_ffn, + get_attn_tp_context, + make_stages, +) +from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.vocab_parallel_embedding import ( @@ -61,9 +67,19 @@ class HYV4MTPDecoderLayer(nn.Module): prefix=f"{prefix}.mlp", alt_stream=alt_stream, is_nextn=True, + reduce_results=False, ) if hasattr(self.mlp, "shared_experts"): self.mlp.shared_experts.swiglu_limit = None + self.attn_boundary, self.ffn_boundary = make_stages( + ( + declare_attn(), + self.input_layernorm, + {"qkv_latent_func": self.self_attn.prepare_qkv_latent}, + ), + (declare_ffn(sparse=True), self.post_attention_layernorm), + terminal=True, + ) def forward( self, @@ -73,13 +89,7 @@ class HYV4MTPDecoderLayer(nn.Module): zero_allocator, prev_topk_indices=None, ): - residual = hidden_states - hidden_states = self.input_layernorm(hidden_states) - get_attn_tp_context().set_attn_inputs( - AttentionInputs( - hidden_states, forward_batch, self.self_attn.prepare_qkv_latent - ) - ) + hidden_states = self.attn_boundary.prepare(hidden_states, forward_batch) try: hidden_states = self.self_attn( positions, @@ -94,9 +104,11 @@ class HYV4MTPDecoderLayer(nn.Module): hidden_states, topk_indices = hidden_states else: topk_indices = None - hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) + hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) hidden_states = self.mlp(hidden_states, forward_batch) - return hidden_states, residual, topk_indices + hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) + return hidden_states, topk_indices class HYV4ModelNextN(nn.Module): @@ -140,8 +152,9 @@ class HYV4ModelNextN(nn.Module): dtype=torch.float32, device=hidden_states.device, ) + residual_batch.start(forward_batch) topk_share = IndexTopKShareState.from_mtp_carry(forward_batch) - hidden_states, residual, topk_indices = self.decoder( + hidden_states, topk_indices = self.decoder( positions, hidden_states, forward_batch, @@ -150,9 +163,11 @@ class HYV4ModelNextN(nn.Module): ) topk_share.update(topk_indices) topk_share.publish() - if forward_batch.forward_mode.is_idle(): - return hidden_states - hidden_states, _ = self.shared_head.norm(hidden_states, residual) + hidden_states = residual_batch.complete_output(hidden_states, forward_batch) + if not forward_batch.forward_mode.is_idle(): + hidden_states = residual_batch.final_norm( + hidden_states, forward_batch, self.shared_head.norm + ) return hidden_states diff --git a/test/registered/unit/layer_boundary/test_ihc_residual_ops.py b/test/registered/unit/layer_boundary/test_ihc_residual_ops.py new file mode 100644 index 0000000000..3f3d77aa22 --- /dev/null +++ b/test/registered/unit/layer_boundary/test_ihc_residual_ops.py @@ -0,0 +1,205 @@ +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.layers.layer_boundary.residual.ihc import IHCState +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +HC_MULT = 3 +HIDDEN = 4 + + +def _expand(hidden_states): + """The layer stack's 2-D input widened into streams; already widened passes + through, as the gated write-back returns streams.""" + if hidden_states.ndim == 3: + return hidden_states + return hidden_states.unsqueeze(1).repeat(1, HC_MULT, 1) + + +def _gate(residual, seed): + """A per-stream coefficient standing in for the layer's learned gates.""" + rows = residual.shape[0] + return torch.arange(rows * HC_MULT, dtype=torch.float32).reshape( + rows, HC_MULT + ) * 0.1 + float(seed) + + +def _pre(seed): + def pre(residual, out_norm=None): + gate = _gate(residual, seed) + mixed = (gate.unsqueeze(-1) * residual).sum(dim=1) + if out_norm is not None: + mixed = out_norm(mixed) + return mixed, gate, residual + + return pre + + +def _post(output, residual, gate): + return gate.unsqueeze(-1) * output.unsqueeze(1) + residual + + +class _Norm: + """A stand-in whose effect is visible in the output.""" + + def __call__(self, hidden_states): + return hidden_states * 2.0 + + +def _state(post_pre=None): + return IHCState( + expand=_expand, + attn_pre=_pre(1), + ffn_pre=_pre(2), + attn_post=_post, + ffn_post=_post, + post_pre=post_pre, + ) + + +class TestIHCResidualOps(CustomTestCase): + def setUp(self): + self.hidden = torch.arange(2 * HIDDEN, dtype=torch.float32).reshape(2, HIDDEN) + + def test_entry_widens_the_stack_input(self): + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + self.assertEqual(residual.shape, (2, HC_MULT, HIDDEN)) + # Every stream starts from the same input row. + for stream in range(HC_MULT): + torch.testing.assert_close(residual[:, stream], self.hidden) + + def test_read_coefficient_is_consumed_by_the_same_stage(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + + self.assertIsNone(state.post_gate) + _, residual = ops.attn_readout.read(residual, None) + self.assertIsNotNone(state.post_gate) + # Whatever the read left is what the write-back scales by: a gate it + # recomputed would differ from this one. + carried = torch.full_like(state.post_gate, 3.0) + state.post_gate = carried + + output = torch.full((2, HIDDEN), 5.0) + updated = ops.attn_update.update(output, residual) + torch.testing.assert_close(updated, _post(output, residual, carried)) + + def test_ffn_write_back_clears_the_coefficient(self): + state = _state() + ops = state.residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + attn_out = torch.full((2, HIDDEN), 5.0) + _, residual = ops.ffn_readout.update_and_read( + ops.attn_update, attn_out, residual, None + ) + self.assertIsNotNone(state.post_gate) + + ffn_out = torch.full((2, HIDDEN), 7.0) + ops.ffn_update.update(ffn_out, residual) + self.assertIsNone(state.post_gate) + + def test_fused_post_pre_matches_the_unfused_chain(self): + unfused = _state().residual_ops() + residual = unfused.attn_readout.init_residual(self.hidden) + unfused.attn_readout.read(residual, None) + attn_out = torch.full((2, HIDDEN), 5.0) + want_input, want_residual = unfused.ffn_readout.update_and_read( + unfused.attn_update, attn_out, residual, _Norm() + ) + + def post_pre(hidden_states, res, gate, out_norm): + res = _post(hidden_states, res, gate) + mixed, next_gate, res = _pre(2)(res, out_norm) + return mixed, next_gate, res + + fused_state = _state(post_pre=post_pre) + fused = fused_state.residual_ops() + residual = fused.attn_readout.init_residual(self.hidden) + fused.attn_readout.read(residual, None) + got_input, got_residual = fused.ffn_readout.update_and_read( + fused.attn_update, attn_out, residual, _Norm() + ) + torch.testing.assert_close(got_input, want_input) + torch.testing.assert_close(got_residual, want_residual) + + def test_the_terminal_write_back_leaves_the_streams_widened(self): + """Unlike MHC, the stack's terminal read is a learned head, so the FFN + write-back must not contract.""" + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + attn_out = torch.full((2, HIDDEN), 5.0) + _, residual = ops.ffn_readout.update_and_read( + ops.attn_update, attn_out, residual, None + ) + written = ops.ffn_update.update(torch.full((2, HIDDEN), 7.0), residual) + self.assertEqual(written.shape, (2, HC_MULT, HIDDEN)) + + def test_declared_capabilities(self): + ops = _state().residual_ops() + # The gated write-back is not an add, so a sum it writes into must be + # complete first, and no add+norm fusion may claim it. + self.assertFalse(ops.attn_update.is_plain_add) + self.assertFalse(ops.ffn_update.is_plain_add) + self.assertFalse(ops.attn_readout.is_plain_norm) + self.assertFalse(ops.ffn_readout.is_plain_norm) + # The FFN writes at its exit; the attention leaves its update to the + # FFN input's read. + self.assertTrue(ops.ffn_update.applied_at_exit) + self.assertFalse(ops.attn_update.applied_at_exit) + + def test_rejects_reads_it_does_not_implement(self): + ops = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + with self.assertRaises(NotImplementedError): + ops.attn_readout.read(residual, None, quant_format="fp8") + with self.assertRaises(NotImplementedError): + ops.attn_readout.update_and_read( + ops.ffn_update, self.hidden, residual, None + ) + with self.assertRaises(NotImplementedError): + ops.ffn_readout.read(residual, None) + + def test_ffn_read_rejects_another_layers_update(self): + ops = _state().residual_ops() + other = _state().residual_ops() + residual = ops.attn_readout.init_residual(self.hidden) + ops.attn_readout.read(residual, None) + with self.assertRaises(NotImplementedError): + ops.ffn_readout.update_and_read( + other.attn_update, self.hidden, residual, None + ) + + def test_attn_tp_slice_moves_the_coefficient_with_the_streams(self): + state = _state() + ops = state.residual_ops() + residual = torch.arange(4 * HC_MULT * HIDDEN, dtype=torch.float32).reshape( + 4, HC_MULT, HIDDEN + ) + state.post_gate = _gate(residual, 1) + with patch( + "sglang.srt.layers.layer_boundary.residual.ihc.get_parallel" + ) as parallel: + parallel.return_value.attn_tp_rank = 1 + parallel.return_value.attn_tp_size = 2 + sliced = ops.attn_update.slice_residual_attn_tp(residual) + torch.testing.assert_close(sliced, residual[2:]) + # The gate rows must follow the residual rows they scale. + torch.testing.assert_close(state.post_gate, _gate(residual, 1)[2:]) + + def test_attn_tp_gather_is_rejected(self): + ops = _state().residual_ops() + with self.assertRaises(NotImplementedError): + ops.attn_update.gather_residual_attn_tp(self.hidden) + + +if __name__ == "__main__": + unittest.main()