[Refactor] Falcon-H1: complete the FFN's sum through ffn_exit (#41434)
This commit is contained in:
@@ -32,7 +32,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_forward, get_parallel, get_stream
|
||||
from sglang.srt.runtime_context import get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, make_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -227,6 +227,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
input_layernorm=self.input_layernorm,
|
||||
post_attention_layernorm=self.pre_ff_layernorm,
|
||||
allow_reduce_scatter=True,
|
||||
allow_deferred_ffn_reduction=False,
|
||||
)
|
||||
|
||||
self.alt_stream = alt_stream
|
||||
@@ -356,17 +357,9 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.layer_communicator.prepare_mlp(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
mlp_reduce_scatter = self.layer_communicator.should_use_reduce_scatter(
|
||||
forward_batch
|
||||
)
|
||||
with get_forward().scoped(mlp_reduce_scatter=mlp_reduce_scatter):
|
||||
with self.layer_communicator.ffn_exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.feed_forward(hidden_states, forward_batch)
|
||||
|
||||
hidden_states, residual = self.layer_communicator.postprocess_layer(
|
||||
hidden_states, residual, forward_batch
|
||||
)
|
||||
|
||||
return hidden_states, residual
|
||||
return ffn_exit.finish(hidden_states, residual)
|
||||
|
||||
|
||||
ALL_DECODER_LAYER_TYPES = {
|
||||
|
||||
Reference in New Issue
Block a user