diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 4767135a02..954abebe3a 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -32,7 +32,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_forward, get_parallel, get_stream +from sglang.srt.runtime_context import get_parallel, get_stream from sglang.srt.utils import add_prefix, is_cuda, make_layers logger = logging.getLogger(__name__) @@ -227,6 +227,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): input_layernorm=self.input_layernorm, post_attention_layernorm=self.pre_ff_layernorm, allow_reduce_scatter=True, + allow_deferred_ffn_reduction=False, ) self.alt_stream = alt_stream @@ -356,17 +357,9 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module): hidden_states, residual = self.layer_communicator.prepare_mlp( hidden_states, residual, forward_batch ) - mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter( - forward_batch - ) - with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter): + with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit: hidden_states = self.feed_forward(hidden_states, forward_batch) - - hidden_states, residual = self.layer_communicator.postprocess_layer( - hidden_states, residual, forward_batch - ) - - return hidden_states, residual + return ffn_exit.finish(hidden_states, residual) ALL_DECODER_LAYER_TYPES = { diff --git a/test/registered/unit/layers/test_communicator_ffn_exit.py b/test/registered/unit/layers/test_communicator_ffn_exit.py index f75f2f964b..c25628b045 100644 --- a/test/registered/unit/layers/test_communicator_ffn_exit.py +++ b/test/registered/unit/layers/test_communicator_ffn_exit.py @@ -447,7 +447,15 @@ class TestSelectFfnCompletion(CustomTestCase): SRT_DIR = Path(sglang.__file__).resolve().parent / "srt" # Attributes a decoder layer holds its FFN, or parts of it, in. A layer that is # one FFN stage holds it in ``mixer``. -FFN_ATTRS = {"mlp", "moe", "shared_expert", "shared_experts", "share_expert", "mixer"} +FFN_ATTRS = { + "mlp", + "moe", + "shared_expert", + "shared_experts", + "share_expert", + "mixer", + "feed_forward", +} def can_be_true(value, params):