[Fix] Make the Solar model constructible and runnable (#41810)
This commit is contained in:
@@ -257,6 +257,29 @@ class SolarDecoderLayer(nn.Module):
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
def _check_skips_stay_in_stage(config, start_layer: int, end_layer: int) -> None:
|
||||
"""Reject a pipeline split that cuts a backbone skip connection.
|
||||
|
||||
A layer in ``bskcn_3`` / ``bskcn_4`` mixes in the hidden states saved at the
|
||||
latest ``bskcn_1`` / ``bskcn_2`` layer before it, which must run on the same
|
||||
stage.
|
||||
"""
|
||||
for uses, saves in (
|
||||
(config.bskcn_3, config.bskcn_1),
|
||||
(config.bskcn_4, config.bskcn_2),
|
||||
):
|
||||
for layer in uses:
|
||||
if start_layer <= layer < end_layer and not any(
|
||||
start_layer <= saved < layer for saved in saves
|
||||
):
|
||||
raise ValueError(
|
||||
f"Solar layer {layer} reads a backbone skip connection saved "
|
||||
f"before this pipeline stage (layers [{start_layer}, "
|
||||
f"{end_layer})); choose a --pp-size or SGLANG_PP_LAYER_PARTITION "
|
||||
"that keeps each skip connection within one stage."
|
||||
)
|
||||
|
||||
|
||||
class SolarModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -279,7 +302,7 @@ class SolarModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SolarDecoderLayer(
|
||||
config=config,
|
||||
@@ -287,8 +310,11 @@ class SolarModel(nn.Module):
|
||||
layer_id=idx,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
_check_skips_stay_in_stage(config, self.start_layer, self.end_layer)
|
||||
if get_parallel().pp_group.is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
@@ -305,7 +331,7 @@ class SolarModel(nn.Module):
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, Tuple[torch.Tensor, List[torch.Tensor]], PPProxyTensors]:
|
||||
if self.pp_group().is_first_rank:
|
||||
if self.pp_group.is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
@@ -348,7 +374,7 @@ class SolarModel(nn.Module):
|
||||
residual=residual,
|
||||
)
|
||||
|
||||
if not self.pp_group().is_last_rank:
|
||||
if not self.pp_group.is_last_rank:
|
||||
return PPProxyTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
@@ -433,13 +459,13 @@ class SolarForCausalLM(nn.Module):
|
||||
org_num_embeddings=config.vocab_size,
|
||||
padding_size=DEFAULT_VOCAB_PADDING_SIZE,
|
||||
quant_config=quant_config,
|
||||
use_attn_tp_group=get_parallel().enable_dp_lm_head,
|
||||
)
|
||||
if config.tie_word_embeddings and self.pp_group.is_first_rank:
|
||||
self.lm_head.weight = self.model.embed_tokens.weight
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.unpadded_vocab_size, config.vocab_size, logit_scale
|
||||
config, logit_scale=getattr(config, "logit_scale", None)
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
@@ -450,16 +476,20 @@ class SolarForCausalLM(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
forward_batch: ForwardBatch,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> Union[torch.Tensor, LogitsProcessorOutput]:
|
||||
pp_proxy_tensors: Optional[PPProxyTensors] = None,
|
||||
) -> Union[torch.Tensor, LogitsProcessorOutput, PPProxyTensors]:
|
||||
hidden_states = self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
forward_batch=forward_batch,
|
||||
inputs_embeds=inputs_embeds,
|
||||
pp_proxy_tensors=pp_proxy_tensors,
|
||||
)
|
||||
|
||||
if self.pp_group().is_last_rank:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, forward_batch)
|
||||
if self.pp_group.is_last_rank:
|
||||
logits = self.logits_processor(
|
||||
input_ids, hidden_states, self.lm_head, forward_batch
|
||||
)
|
||||
return logits
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.distributed.utils import get_pp_indices
|
||||
from sglang.srt.models.solar import _check_skips_stay_in_stage
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
# The backbone skip connections of upstage/solar-pro-preview-instruct.
|
||||
CONFIG = SimpleNamespace(
|
||||
num_hidden_layers=64,
|
||||
bskcn_1=[12, 20, 32, 44],
|
||||
bskcn_2=[20, 32],
|
||||
bskcn_3=[16, 24, 36, 48],
|
||||
bskcn_4=[28, 40],
|
||||
)
|
||||
|
||||
|
||||
def _stages(pp_size):
|
||||
return [
|
||||
get_pp_indices(CONFIG.num_hidden_layers, pp_rank, pp_size)
|
||||
for pp_rank in range(pp_size)
|
||||
]
|
||||
|
||||
|
||||
class TestSolarPipelineSplit(CustomTestCase):
|
||||
def test_a_split_that_keeps_every_skip_in_one_stage_is_accepted(self):
|
||||
for pp_size in (1, 2):
|
||||
for start, end in _stages(pp_size):
|
||||
with self.subTest(pp_size=pp_size, stage=(start, end)):
|
||||
_check_skips_stay_in_stage(CONFIG, start, end)
|
||||
|
||||
def test_a_split_that_cuts_a_skip_is_rejected(self):
|
||||
# Four stages of 16 layers: layer 16 reads the state saved at layer 12.
|
||||
start, end = _stages(4)[1]
|
||||
with self.assertRaisesRegex(ValueError, "Solar layer 16"):
|
||||
_check_skips_stay_in_stage(CONFIG, start, end)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user