[Refactor] Add make_pp_layers so models stop handing their PP position down (#41816)
This commit is contained in:
@@ -57,7 +57,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -322,13 +322,11 @@ class ApertusModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: ApertusDecoderLayer(
|
||||
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -51,7 +51,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -296,13 +296,11 @@ class ArceeModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: ArceeDecoderLayer(
|
||||
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -83,7 +83,12 @@ from sglang.srt.models.utils import (
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
LoraConfig = None
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -780,7 +785,7 @@ class BailingMoEModel(nn.Module):
|
||||
|
||||
self.embedding_dropout = torch.nn.Dropout(config.embedding_dropout)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: BailingMoEBlock(
|
||||
layer_id=idx,
|
||||
@@ -789,8 +794,6 @@ class BailingMoEModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -78,7 +78,7 @@ from sglang.srt.utils import (
|
||||
is_gfx95_supported,
|
||||
is_hip,
|
||||
is_npu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.common import rank0_log
|
||||
|
||||
@@ -951,11 +951,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
self.num_layers,
|
||||
layer_fn,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_flashinfer_available,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
_is_fp8_fnuz = is_fp8_fnuz()
|
||||
@@ -1297,11 +1297,9 @@ class BailingMoELinearModel(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
self.num_layers,
|
||||
layer_fn,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_non_idle_and_non_empty,
|
||||
is_sm90_supported,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
use_intel_amx_backend,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
@@ -2923,7 +2923,7 @@ class DeepseekV2Model(nn.Module):
|
||||
else None
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: DeepseekV2DecoderLayer(
|
||||
config=config,
|
||||
@@ -2933,8 +2933,6 @@ class DeepseekV2Model(nn.Module):
|
||||
alt_stream=self.alt_stream,
|
||||
skip_rope=config.qk_rope_head_dim == 0,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
offloader_kwargs=dict(
|
||||
submodule_accessor=lambda layer: (
|
||||
|
||||
@@ -196,7 +196,7 @@ from sglang.srt.utils import (
|
||||
is_gfx942_supported,
|
||||
is_gfx1250_supported,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
@@ -4337,7 +4337,7 @@ class DeepseekV4Model(nn.Module):
|
||||
else None
|
||||
)
|
||||
self.engram_layout = EngramLayout.from_config(config)
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: DeepseekV4DecoderLayer(
|
||||
config=config,
|
||||
@@ -4349,8 +4349,6 @@ class DeepseekV4Model(nn.Module):
|
||||
hc_stats_stream=self.hc_stats_stream,
|
||||
moe_routed_quant_stream=self.moe_routed_quant_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -131,7 +131,7 @@ from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -1718,7 +1718,7 @@ class Dots3Model(nn.Module):
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.alt_stream = torch.cuda.Stream() if _is_cuda else None
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Dots3DecoderLayer(
|
||||
config=config,
|
||||
@@ -1727,8 +1727,6 @@ class Dots3Model(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -43,7 +43,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.models.deepseek_v2 import DeepseekV2MLP as Ernie4_5_VLMoeMLP
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils.hf_transformers.common import get_rope_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -485,7 +485,7 @@ class Ernie4_5_VLMoeModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Ernie4_5_VLMoeDecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -493,8 +493,6 @@ class Ernie4_5_VLMoeModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -38,7 +38,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.utils import get_exception_traceback, logger
|
||||
|
||||
|
||||
@@ -338,7 +338,7 @@ class Exaone4Model(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Exaone4DecoderLayer(
|
||||
config=config,
|
||||
@@ -346,8 +346,6 @@ class Exaone4Model(nn.Module):
|
||||
layer_id=idx,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -59,7 +59,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers
|
||||
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -537,7 +537,7 @@ class ExaoneMoEModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: ExaoneMoEDecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -546,8 +546,6 @@ class ExaoneMoEModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ from sglang.srt.models.utils import (
|
||||
create_fused_set_kv_buffer_arg,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -862,7 +862,7 @@ class Gemma4TextModel(PreTrainedModel):
|
||||
self.per_layer_input_scale = None
|
||||
self.per_layer_projection_scale = None
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Gemma4DecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -870,8 +870,6 @@ class Gemma4TextModel(PreTrainedModel):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ from sglang.srt.models.gemma4_mm import (
|
||||
)
|
||||
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -362,7 +362,7 @@ class DiffusionGemmaModel(nn.Module):
|
||||
prefix=add_prefix("self_conditioning", prefix),
|
||||
)
|
||||
self.pp_group = get_parallel().pp_group
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
text_config.num_hidden_layers,
|
||||
lambda idx, prefix: DiffusionGemmaDecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -370,8 +370,6 @@ class DiffusionGemmaModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
self.norm = RMSNorm(text_config.hidden_size, eps=text_config.rms_norm_eps)
|
||||
|
||||
@@ -40,7 +40,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
)
|
||||
from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import BumpAllocator, add_prefix, make_layers
|
||||
from sglang.srt.utils import BumpAllocator, add_prefix, make_pp_layers
|
||||
|
||||
_GATED_NORM_LOW_RANK = 16
|
||||
|
||||
@@ -525,7 +525,7 @@ class GigaChat35Model(nn.Module):
|
||||
|
||||
self.alt_stream = torch.cuda.Stream() if torch.cuda.is_available() else None
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: GigaChat35DecoderLayer(
|
||||
config=config,
|
||||
@@ -534,8 +534,6 @@ class GigaChat35Model(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
kv_cache_scales_loader,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
Glm4Config = None
|
||||
@@ -310,7 +310,7 @@ class Glm4Model(nn.Module):
|
||||
|
||||
# Use the provided decoder layer type or default to Glm4DecoderLayer
|
||||
decoder_layer_type = decoder_layer_type or Glm4DecoderLayer
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -319,8 +319,6 @@ class Glm4Model(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -97,7 +97,7 @@ from sglang.srt.utils import (
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -1039,7 +1039,7 @@ class Glm4MoeModel(nn.Module):
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Glm4MoeDecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -1049,8 +1049,6 @@ class Glm4MoeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -82,7 +82,7 @@ from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -740,7 +740,7 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.alt_stream = get_stream("alt") if _is_cuda else None
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Glm4MoeLiteDecoderLayer(
|
||||
config=config,
|
||||
@@ -749,8 +749,6 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -122,7 +122,7 @@ from sglang.srt.utils.common import (
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
set_weight_attrs,
|
||||
)
|
||||
|
||||
@@ -1008,7 +1008,7 @@ class Glm5NextModel(nn.Module):
|
||||
else None
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Glm5NextDecoderLayer(
|
||||
config=config,
|
||||
@@ -1017,8 +1017,6 @@ class Glm5NextModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -90,7 +90,7 @@ from sglang.srt.utils import (
|
||||
is_flashinfer_available,
|
||||
is_hip,
|
||||
is_npu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
@@ -667,7 +667,7 @@ class GptOssModel(nn.Module):
|
||||
|
||||
# Use the provided decoder layer type or default to GptOssDecoderLayer
|
||||
decoder_layer_type = decoder_layer_type or GptOssDecoderLayer
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -675,8 +675,6 @@ class GptOssModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -26,7 +26,7 @@ from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.transformers import maybe_prefix
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import make_layers
|
||||
from sglang.srt.utils import make_pp_layers
|
||||
|
||||
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
|
||||
|
||||
@@ -347,11 +347,9 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -124,7 +124,7 @@ from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_platform,
|
||||
)
|
||||
from sglang.srt.utils import is_hip, is_npu, make_layers
|
||||
from sglang.srt.utils import is_hip, is_npu, make_pp_layers
|
||||
from sglang.srt.utils.common import (
|
||||
BumpAllocator,
|
||||
add_prefix,
|
||||
@@ -2867,7 +2867,7 @@ class KimiK3LinearModel(nn.Module):
|
||||
# Disable on HIP code path.
|
||||
self.alt_streams = None if _is_hip else [torch.cuda.Stream() for _ in range(3)]
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: KimiK3DecoderLayer(
|
||||
layer_idx=idx,
|
||||
@@ -2876,8 +2876,6 @@ class KimiK3LinearModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_streams=self.alt_streams,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA as KimiMLAAtten
|
||||
from sglang.srt.models.llama import LlamaMLP as KimiMLP
|
||||
from sglang.srt.models.transformers import maybe_prefix
|
||||
from sglang.srt.runtime_context import get_parallel, get_stream
|
||||
from sglang.srt.utils import is_xpu, make_layers
|
||||
from sglang.srt.utils import is_xpu, make_pp_layers
|
||||
from sglang.srt.utils.common import BumpAllocator, add_prefix, set_weight_attrs
|
||||
|
||||
|
||||
@@ -686,7 +686,7 @@ class KimiLinearModel(nn.Module):
|
||||
|
||||
self.alt_stream = None if is_xpu() else get_stream("alt")
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: KimiDecoderLayer(
|
||||
layer_idx=idx,
|
||||
@@ -695,8 +695,6 @@ class KimiLinearModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.utils import apply_qk_norm
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import LazyValue, add_prefix, make_layers
|
||||
from sglang.srt.utils import LazyValue, add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -527,7 +527,7 @@ class LagunaModel(nn.Module):
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
decoder_layer_type = decoder_layer_type or LagunaDecoderLayer
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -535,8 +535,6 @@ class LagunaModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -80,7 +80,7 @@ from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -709,7 +709,7 @@ class LLaDA2MoeModel(nn.Module):
|
||||
|
||||
self.embedding_dropout = torch.nn.Dropout(config.embedding_dropout)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: LLaDA2MoeBlock(
|
||||
layer_id=idx,
|
||||
@@ -718,8 +718,6 @@ class LLaDA2MoeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -53,7 +53,13 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_npu, is_xpu, make_layers
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_cuda,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.utils import get_exception_traceback
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -394,7 +400,7 @@ class LlamaModel(nn.Module):
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: LlamaDecoderLayer(
|
||||
config=config,
|
||||
@@ -403,8 +409,6 @@ class LlamaModel(nn.Module):
|
||||
start_layer=pp_start_layer,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
ceil_align,
|
||||
is_non_idle_and_non_empty,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
MiMoV2Config = None
|
||||
@@ -994,7 +994,7 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
# Use the provided decoder layer type or default to MiMoV2DecoderLayer
|
||||
decoder_layer_type = decoder_layer_type or MiMoV2DecoderLayer
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
layer_fn=lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -1002,8 +1002,6 @@ class MiMoV2Model(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -100,7 +100,7 @@ from sglang.srt.utils import (
|
||||
is_non_idle_and_non_empty,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
@@ -1110,11 +1110,9 @@ class MiniMaxM2Model(nn.Module):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
layer_fn,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -99,7 +99,7 @@ from sglang.srt.utils import (
|
||||
is_hip,
|
||||
is_npu,
|
||||
log_info_on_rank0,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
@@ -1443,11 +1443,9 @@ class MiniMaxM3Model(nn.Module):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
layer_fn,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -11,7 +11,7 @@ from sglang.srt.models.llama import (
|
||||
LlamaForCausalLM,
|
||||
LlamaModel,
|
||||
)
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
|
||||
def _get_llama_4_attn_scale(
|
||||
@@ -141,7 +141,7 @@ class Ministral3Model(LlamaModel):
|
||||
# Override layer creation to use Ministral3Attention
|
||||
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Ministral3DecoderLayer(
|
||||
config=config,
|
||||
@@ -150,8 +150,6 @@ class Ministral3Model(LlamaModel):
|
||||
start_layer=self.start_layer,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -294,13 +294,11 @@ class MixtralModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: MixtralDecoderLayer(
|
||||
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="layers",
|
||||
return_tuple=True,
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -296,7 +296,7 @@ class NanbeigeModel(nn.Module):
|
||||
|
||||
# Use the provided decoder layer type or default to NanbeigeDecoderLayer
|
||||
decoder_layer_type = decoder_layer_type or NanbeigeDecoderLayer
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -305,8 +305,6 @@ class NanbeigeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -84,7 +84,7 @@ from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
get_current_device_stream_fast,
|
||||
is_cuda,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
)
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
from sglang.utils import logger
|
||||
@@ -701,11 +701,9 @@ class NemotronHModel(nn.Module):
|
||||
layer_class = ALL_DECODER_LAYER_TYPES[config.hybrid_override_pattern[idx]]
|
||||
return layer_class(config, idx, quant_config=quant_config, prefix=prefix)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
len(config.hybrid_override_pattern),
|
||||
get_layer,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -40,7 +40,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.models.llama import LlamaAttention, LlamaMLP
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.utils import logger
|
||||
|
||||
|
||||
@@ -197,11 +197,9 @@ class DeciModel(nn.Module):
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
pp_rank=get_parallel().pp_group.rank_in_group,
|
||||
pp_size=get_parallel().pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if get_parallel().pp_group.is_last_rank:
|
||||
|
||||
@@ -43,7 +43,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
kv_cache_scales_loader,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.utils import get_exception_traceback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -273,13 +273,11 @@ class OPTDecoder(nn.Module):
|
||||
else:
|
||||
self.final_layer_norm = None
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: OPTDecoderLayer(
|
||||
config=config, layer_id=idx, quant_config=quant_config, prefix=prefix
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
|
||||
@@ -231,13 +231,11 @@ class OrionModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: OrionDecoderLayer(
|
||||
config, layer_id=idx, quant_config=quant_config, prefix=prefix
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
|
||||
class PersimmonMLP(nn.Module):
|
||||
@@ -209,14 +209,12 @@ class PersimmonModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: PersimmonDecoderLayer(
|
||||
config, quant_config=quant_config, prefix=prefix, idx=idx
|
||||
),
|
||||
prefix="model.layers",
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
)
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -25,7 +25,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
|
||||
@torch.jit.script
|
||||
@@ -308,7 +308,7 @@ class Phi3SmallModel(nn.Module):
|
||||
prefix=add_prefix("embed_tokens", prefix),
|
||||
)
|
||||
self.mup_embedding_multiplier = config.mup_embedding_multiplier
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Phi3SmallDecoderLayer(
|
||||
config,
|
||||
@@ -316,8 +316,6 @@ class Phi3SmallModel(nn.Module):
|
||||
quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
Qwen2Config = None
|
||||
@@ -353,7 +353,7 @@ class Qwen2Model(nn.Module):
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -363,8 +363,6 @@ class Qwen2Model(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -107,7 +107,7 @@ from sglang.srt.utils import (
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_npu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
use_intel_amx_backend,
|
||||
)
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
@@ -1130,7 +1130,7 @@ class Qwen2MoeModel(nn.Module):
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: decoder_layer_type(
|
||||
layer_id=idx,
|
||||
@@ -1140,8 +1140,6 @@ class Qwen2MoeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -134,7 +134,7 @@ from sglang.srt.utils import (
|
||||
is_hip,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
set_weight_attrs,
|
||||
use_intel_amx_backend,
|
||||
)
|
||||
@@ -1691,11 +1691,9 @@ class Qwen3_5ForCausalLM(nn.Module):
|
||||
is_nextn=is_nextn,
|
||||
)
|
||||
|
||||
self.layers, self._start_layer, self._end_layer = make_layers(
|
||||
self.layers, self._start_layer, self._end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
get_layer,
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@ from sglang.srt.utils import (
|
||||
bind_or_assign,
|
||||
is_cuda,
|
||||
is_nvidia_cublas_version_ge_12_9,
|
||||
make_layers,
|
||||
make_pp_layers,
|
||||
next_power_of_2,
|
||||
)
|
||||
|
||||
@@ -1111,7 +1111,7 @@ class SarvamMLAModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = nn.Identity()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SarvamMoEMLADecoderLayer(
|
||||
config=config,
|
||||
@@ -1120,8 +1120,6 @@ class SarvamMLAModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=self.alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix="model.layers",
|
||||
)
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ from sglang.srt.models.utils import (
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, make_layers
|
||||
from sglang.srt.utils import add_prefix, is_cuda, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_is_cuda = is_cuda()
|
||||
@@ -357,7 +357,7 @@ class SDARModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SDARBlock(
|
||||
layer_id=idx,
|
||||
@@ -366,8 +366,6 @@ class SDARModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -59,7 +59,7 @@ from sglang.srt.models.utils import (
|
||||
enable_fused_set_kv_buffer,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers
|
||||
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_pp_layers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
_is_cuda = is_cuda()
|
||||
@@ -422,7 +422,7 @@ class SDARMoeModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SDARMoeBlock(
|
||||
config=config,
|
||||
@@ -431,8 +431,6 @@ class SDARMoeModel(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
# limitations under the License.
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/solar.py
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||
|
||||
|
||||
@@ -302,7 +302,7 @@ class SolarModel(nn.Module):
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: SolarDecoderLayer(
|
||||
config=config,
|
||||
@@ -310,8 +310,6 @@ class SolarModel(nn.Module):
|
||||
layer_id=idx,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
_check_skips_stay_in_stage(config, self.start_layer, self.end_layer)
|
||||
|
||||
@@ -34,7 +34,7 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
Spark2_5Config = None
|
||||
|
||||
@@ -322,7 +322,7 @@ class Spark2_5Model(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: Spark2_5DecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -330,8 +330,6 @@ class Spark2_5Model(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -49,7 +49,12 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
|
||||
from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers
|
||||
from sglang.srt.utils import (
|
||||
add_prefix,
|
||||
is_cuda,
|
||||
is_non_idle_and_non_empty,
|
||||
make_pp_layers,
|
||||
)
|
||||
|
||||
Step3p5Config = None
|
||||
|
||||
@@ -663,7 +668,7 @@ class Step3p5Model(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
# 1,
|
||||
lambda idx, prefix: Step3p5DecoderLayer(
|
||||
@@ -673,8 +678,6 @@ class Step3p5Model(nn.Module):
|
||||
prefix=prefix,
|
||||
alt_stream=alt_stream,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -84,7 +84,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers
|
||||
|
||||
_XLLM_SOURCE_ROUTER_PARTITIONS_CONFIG_KEY = "xllm_source_router_gemm_partitions"
|
||||
_XLLM_SOURCE_ROUTER_PARTITIONS_MISSING = object()
|
||||
@@ -1592,7 +1592,7 @@ class XllmModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: XllmDecoderLayer(
|
||||
layer_id=idx,
|
||||
@@ -1600,8 +1600,6 @@ class XllmModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
if self.pp_group.is_last_rank:
|
||||
|
||||
@@ -76,7 +76,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
|
||||
from sglang.srt.model_executor.forward_context import get_attn_backend
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix, make_layers, set_weight_attrs
|
||||
from sglang.srt.utils import add_prefix, make_pp_layers, set_weight_attrs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1395,7 +1395,7 @@ class ZayaModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
self.layers, self.start_layer, self.end_layer = make_layers(
|
||||
self.layers, self.start_layer, self.end_layer = make_pp_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda idx, prefix: _build_layer(
|
||||
layer_id=idx,
|
||||
@@ -1403,8 +1403,6 @@ class ZayaModel(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
pp_rank=self.pp_group.rank_in_group,
|
||||
pp_size=self.pp_group.world_size,
|
||||
prefix=add_prefix("layers", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -1544,6 +1544,29 @@ def make_layers(
|
||||
return modules, start_layer, end_layer
|
||||
|
||||
|
||||
def make_pp_layers(
|
||||
num_hidden_layers: int,
|
||||
layer_fn: LayerFn,
|
||||
prefix: str = "",
|
||||
return_tuple: bool = False,
|
||||
offloader_kwargs: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[torch.nn.Module, int, int]:
|
||||
"""Make this pipeline stage's layers, and return them with the stage's range.
|
||||
|
||||
Layers outside ``[start_layer, end_layer)`` are ``PPMissingLayer`` stand-ins.
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return make_layers(
|
||||
num_hidden_layers,
|
||||
layer_fn,
|
||||
pp_rank=parallel.pp_rank,
|
||||
pp_size=parallel.pp_size,
|
||||
prefix=prefix,
|
||||
return_tuple=return_tuple,
|
||||
offloader_kwargs=offloader_kwargs,
|
||||
)
|
||||
|
||||
|
||||
def set_random_seed(seed: int) -> None:
|
||||
"""Set the random seed for all libraries."""
|
||||
random.seed(seed)
|
||||
|
||||
Reference in New Issue
Block a user