[Fix] Make the Solar model constructible and runnable (#41810)
This commit is contained in:
@@ -257,6 +257,29 @@ class SolarDecoderLayer(nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _check_skips_stay_in_stage(config, start_layer: int, end_layer: int) -> None:
|
||||
"""Reject a pipeline split that cuts a backbone skip connection.
|
||||
|
||||
A layer in ``bskcn_3`` / ``bskcn_4`` mixes in the hidden states saved at the
|
||||
latest ``bskcn_1`` / ``bskcn_2`` layer before it, which must run on the same
|
||||
stage.
|
||||
"""
|
||||
for uses, saves in (
|
||||
(config.bskcn_3, config.bskcn_1),
|
||||
(config.bskcn_4, config.bskcn_2),
|
||||
):
|
||||
for layer in uses:
|
||||
if start_layer <= layer < end_layer and not any(
|
||||
start_layer <= saved < layer for saved in saves
|
||||
):
|
||||
raise ValueError(
|
||||
f"Solar layer {layer} reads a backbone skip connection saved "
|
||||
f"before this pipeline stage (layers [{start_layer}, "
|
||||
f"{end_layer})); choose a --pp-size or SGLANG_PP_LAYER_PARTITION "
|
||||
"that keeps each skip connection within one stage."
|
||||
)
|
||||
|
||||
|
||||
class SolarModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -279,7 +302,7 @@ class SolarModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SolarDecoderLayer(
|
||||
config=config,
|
||||
@@ -287,8 +310,11 @@ class SolarModel(nn.Module):
|
||||
layer_id=idx,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
_check_skips_stay_in_stage(config, self.start_layer, self.end_layer)
|
||||
if get_parallel().pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -305,7 +331,7 @@ class SolarModel(nn.Module):
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, List[torch.Tensor]], PPProxyTensors]:
|
||||
if self.pp_group().is_first_rank:
|
||||
if self.pp_group.is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
@@ -348,7 +374,7 @@ class SolarModel(nn.Module):
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
if not self.pp_group().is_last_rank:
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
@@ -433,13 +459,13 @@ class SolarForCausalLM(nn.Module):
|
||||
org_num_embeddings=config.vocab_size,
|
||||
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
|
||||
quant_config=quant_config,
|
||||
use_attn_tp_group=get_parallel().enable_dp_lm_head,
|
||||
)
|
||||
if config.tie_word_embeddings and self.pp_group.is_first_rank:
|
||||
self.lm_head.weight = self.model.embed_tokens.weight
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.unpadded_vocab_size, config.vocab_size, logit_scale
|
||||
config, logit_scale=getattr(config, "logit_scale", None)
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
@@ -450,16 +476,20 @@ class SolarForCausalLM(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, LogitsProcessorOutput]:
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, LogitsProcessorOutput, PPProxyTensors]:
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
inputs_embeds=inputs_embeds,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
|
||||
if self.pp_group().is_last_rank:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, forward_batch)
|
||||
if self.pp_group.is_last_rank:
|
||||
logits = self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
return logits
|
||||
|
||||
return hidden_states
|
||||
|
||||
Reference in New Issue
Block a user