[Refactor] Add make_pp_layers so models stop handing their PP position down (#41816)

This commit is contained in:
Cheng Wan
2026-09-30 13:28:34 -07:00
committed by GitHub
parent 00659ac099
commit e7f6a99313
50 changed files with 183 additions and 192 deletions
+2 -4
View File
@@ -57,7 +57,7 @@ from sglang.srt.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -322,13 +322,11 @@ class ApertusModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: ApertusDecoderLayer(
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+2 -4
View File
@@ -51,7 +51,7 @@ from sglang.srt.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -296,13 +296,11 @@ class ArceeModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: ArceeDecoderLayer(
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+7 -4
View File
@@ -83,7 +83,12 @@ from sglang.srt.models.utils import (
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers
from sglang.srt.utils import (
add_prefix,
is_cuda,
is_non_idle_and_non_empty,
make_pp_layers,
)
LoraConfig = None
logger = logging.getLogger(__name__)
@@ -780,7 +785,7 @@ class BailingMoEModel(nn.Module):
self.embedding_dropout = torch.nn.Dropout(config.embedding_dropout)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: BailingMoEBlock(
layer_id=idx,
@@ -789,8 +794,6 @@ class BailingMoEModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
@@ -78,7 +78,7 @@ from sglang.srt.utils import (
is_gfx95_supported,
is_hip,
is_npu,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.common import rank0_log
@@ -951,11 +951,9 @@ class BailingMoELinearModel(nn.Module):
alt_stream=self.alt_stream,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
self.num_layers,
layer_fn,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -99,7 +99,7 @@ from sglang.srt.utils import (
is_cuda,
is_flashinfer_available,
log_info_on_rank0,
make_layers,
make_pp_layers,
)
_is_fp8_fnuz = is_fp8_fnuz()
@@ -1297,11 +1297,9 @@ class BailingMoELinearModel(nn.Module):
alt_stream=self.alt_stream,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
self.num_layers,
layer_fn,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -209,7 +209,7 @@ from sglang.srt.utils import (
add_prefix,
is_non_idle_and_non_empty,
is_sm90_supported,
make_layers,
make_pp_layers,
use_intel_amx_backend,
)
from sglang.srt.utils.custom_op import register_custom_op
@@ -2923,7 +2923,7 @@ class DeepseekV2Model(nn.Module):
else None
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: DeepseekV2DecoderLayer(
config=config,
@@ -2933,8 +2933,6 @@ class DeepseekV2Model(nn.Module):
alt_stream=self.alt_stream,
skip_rope=config.qk_rope_head_dim == 0,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
offloader_kwargs=dict(
submodule_accessor=lambda layer: (
+2 -4
View File
@@ -196,7 +196,7 @@ from sglang.srt.utils import (
is_gfx942_supported,
is_gfx1250_supported,
log_info_on_rank0,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -4337,7 +4337,7 @@ class DeepseekV4Model(nn.Module):
else None
)
self.engram_layout = EngramLayout.from_config(config)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: DeepseekV4DecoderLayer(
config=config,
@@ -4349,8 +4349,6 @@ class DeepseekV4Model(nn.Module):
hc_stats_stream=self.hc_stats_stream,
moe_routed_quant_stream=self.moe_routed_quant_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
@@ -131,7 +131,7 @@ from sglang.srt.utils import (
is_cuda,
is_non_idle_and_non_empty,
log_info_on_rank0,
make_layers,
make_pp_layers,
)
_is_cuda = is_cuda()
@@ -1718,7 +1718,7 @@ class Dots3Model(nn.Module):
self.embed_tokens = PPMissingLayer()
self.alt_stream = torch.cuda.Stream() if _is_cuda else None
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Dots3DecoderLayer(
config=config,
@@ -1727,8 +1727,6 @@ class Dots3Model(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -43,7 +43,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.models.deepseek_v2 import DeepseekV2MLP as Ernie4_5_VLMoeMLP
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils.hf_transformers.common import get_rope_config
logger = logging.getLogger(__name__)
@@ -485,7 +485,7 @@ class Ernie4_5_VLMoeModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Ernie4_5_VLMoeDecoderLayer(
layer_id=idx,
@@ -493,8 +493,6 @@ class Ernie4_5_VLMoeModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -38,7 +38,7 @@ from sglang.srt.model_loader.weight_utils import (
maybe_remap_kv_scale_name,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.utils import get_exception_traceback, logger
@@ -338,7 +338,7 @@ class Exaone4Model(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Exaone4DecoderLayer(
config=config,
@@ -346,8 +346,6 @@ class Exaone4Model(nn.Module):
layer_id=idx,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -59,7 +59,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_executor.runner import get_is_capture_mode
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_pp_layers
logger = logging.getLogger(__name__)
@@ -537,7 +537,7 @@ class ExaoneMoEModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: ExaoneMoEDecoderLayer(
layer_id=idx,
@@ -546,8 +546,6 @@ class ExaoneMoEModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -58,7 +58,7 @@ from sglang.srt.models.utils import (
create_fused_set_kv_buffer_arg,
)
from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -862,7 +862,7 @@ class Gemma4TextModel(PreTrainedModel):
self.per_layer_input_scale = None
self.per_layer_projection_scale = None
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Gemma4DecoderLayer(
layer_id=idx,
@@ -870,8 +870,6 @@ class Gemma4TextModel(PreTrainedModel):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -57,7 +57,7 @@ from sglang.srt.models.gemma4_mm import (
)
from sglang.srt.models.gemma4_vision import Gemma4VisionEncoder
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -362,7 +362,7 @@ class DiffusionGemmaModel(nn.Module):
prefix=add_prefix("self_conditioning", prefix),
)
self.pp_group = get_parallel().pp_group
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
text_config.num_hidden_layers,
lambda idx, prefix: DiffusionGemmaDecoderLayer(
layer_id=idx,
@@ -370,8 +370,6 @@ class DiffusionGemmaModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
self.norm = RMSNorm(text_config.hidden_size, eps=text_config.rms_norm_eps)
+2 -4
View File
@@ -40,7 +40,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
)
from sglang.srt.models.qwen3_next import Qwen3GatedDeltaNet
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import BumpAllocator, add_prefix, make_layers
from sglang.srt.utils import BumpAllocator, add_prefix, make_pp_layers
_GATED_NORM_LOW_RANK = 16
@@ -525,7 +525,7 @@ class GigaChat35Model(nn.Module):
self.alt_stream = torch.cuda.Stream() if torch.cuda.is_available() else None
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: GigaChat35DecoderLayer(
config=config,
@@ -534,8 +534,6 @@ class GigaChat35Model(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -47,7 +47,7 @@ from sglang.srt.model_loader.weight_utils import (
kv_cache_scales_loader,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils.hf_transformers_utils import get_rope_config
Glm4Config = None
@@ -310,7 +310,7 @@ class Glm4Model(nn.Module):
# Use the provided decoder layer type or default to Glm4DecoderLayer
decoder_layer_type = decoder_layer_type or Glm4DecoderLayer
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -319,8 +319,6 @@ class Glm4Model(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -97,7 +97,7 @@ from sglang.srt.utils import (
is_non_idle_and_non_empty,
is_npu,
log_info_on_rank0,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -1039,7 +1039,7 @@ class Glm4MoeModel(nn.Module):
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Glm4MoeDecoderLayer(
layer_id=idx,
@@ -1049,8 +1049,6 @@ class Glm4MoeModel(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -82,7 +82,7 @@ from sglang.srt.utils import (
add_prefix,
is_non_idle_and_non_empty,
is_npu,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -740,7 +740,7 @@ class Glm4MoeLiteModel(nn.Module):
self.embed_tokens = PPMissingLayer()
self.alt_stream = get_stream("alt") if _is_cuda else None
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Glm4MoeLiteDecoderLayer(
config=config,
@@ -749,8 +749,6 @@ class Glm4MoeLiteModel(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -122,7 +122,7 @@ from sglang.srt.utils.common import (
LazyValue,
add_prefix,
log_info_on_rank0,
make_layers,
make_pp_layers,
set_weight_attrs,
)
@@ -1008,7 +1008,7 @@ class Glm5NextModel(nn.Module):
else None
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Glm5NextDecoderLayer(
config=config,
@@ -1017,8 +1017,6 @@ class Glm5NextModel(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -90,7 +90,7 @@ from sglang.srt.utils import (
is_flashinfer_available,
is_hip,
is_npu,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.custom_op import register_custom_op
@@ -667,7 +667,7 @@ class GptOssModel(nn.Module):
# Use the provided decoder layer type or default to GptOssDecoderLayer
decoder_layer_type = decoder_layer_type or GptOssDecoderLayer
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -675,8 +675,6 @@ class GptOssModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -26,7 +26,7 @@ from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.transformers import maybe_prefix
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import make_layers
from sglang.srt.utils import make_pp_layers
from .granitemoe import GraniteMoeMoE, GraniteMoeSharedMLP
@@ -347,11 +347,9 @@ class GraniteMoeHybridModel(nn.Module):
prefix=prefix,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
get_layer,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -124,7 +124,7 @@ from sglang.srt.runtime_context import (
get_parallel,
get_platform,
)
from sglang.srt.utils import is_hip, is_npu, make_layers
from sglang.srt.utils import is_hip, is_npu, make_pp_layers
from sglang.srt.utils.common import (
BumpAllocator,
add_prefix,
@@ -2867,7 +2867,7 @@ class KimiK3LinearModel(nn.Module):
# Disable on HIP code path.
self.alt_streams = None if _is_hip else [torch.cuda.Stream() for _ in range(3)]
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: KimiK3DecoderLayer(
layer_idx=idx,
@@ -2876,8 +2876,6 @@ class KimiK3LinearModel(nn.Module):
prefix=prefix,
alt_streams=self.alt_streams,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -54,7 +54,7 @@ from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA as KimiMLAAtten
from sglang.srt.models.llama import LlamaMLP as KimiMLP
from sglang.srt.models.transformers import maybe_prefix
from sglang.srt.runtime_context import get_parallel, get_stream
from sglang.srt.utils import is_xpu, make_layers
from sglang.srt.utils import is_xpu, make_pp_layers
from sglang.srt.utils.common import BumpAllocator, add_prefix, set_weight_attrs
@@ -686,7 +686,7 @@ class KimiLinearModel(nn.Module):
self.alt_stream = None if is_xpu() else get_stream("alt")
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: KimiDecoderLayer(
layer_idx=idx,
@@ -695,8 +695,6 @@ class KimiLinearModel(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -55,7 +55,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.models.utils import apply_qk_norm
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import LazyValue, add_prefix, make_layers
from sglang.srt.utils import LazyValue, add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -527,7 +527,7 @@ class LagunaModel(nn.Module):
self.embed_tokens = PPMissingLayer()
decoder_layer_type = decoder_layer_type or LagunaDecoderLayer
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -535,8 +535,6 @@ class LagunaModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -80,7 +80,7 @@ from sglang.srt.utils import (
is_cuda,
is_non_idle_and_non_empty,
is_npu,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -709,7 +709,7 @@ class LLaDA2MoeModel(nn.Module):
self.embedding_dropout = torch.nn.Dropout(config.embedding_dropout)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: LLaDA2MoeBlock(
layer_id=idx,
@@ -718,8 +718,6 @@ class LLaDA2MoeModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+8 -4
View File
@@ -53,7 +53,13 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, is_cuda, is_npu, is_xpu, make_layers
from sglang.srt.utils import (
add_prefix,
is_cuda,
is_npu,
is_xpu,
make_pp_layers,
)
from sglang.utils import get_exception_traceback
_is_cuda = is_cuda()
@@ -394,7 +400,7 @@ class LlamaModel(nn.Module):
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: LlamaDecoderLayer(
config=config,
@@ -403,8 +409,6 @@ class LlamaModel(nn.Module):
start_layer=pp_start_layer,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+2 -4
View File
@@ -86,7 +86,7 @@ from sglang.srt.utils import (
add_prefix,
ceil_align,
is_non_idle_and_non_empty,
make_layers,
make_pp_layers,
)
MiMoV2Config = None
@@ -994,7 +994,7 @@ class MiMoV2Model(nn.Module):
# Use the provided decoder layer type or default to MiMoV2DecoderLayer
decoder_layer_type = decoder_layer_type or MiMoV2DecoderLayer
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
layer_fn=lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -1002,8 +1002,6 @@ class MiMoV2Model(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -100,7 +100,7 @@ from sglang.srt.utils import (
is_non_idle_and_non_empty,
is_npu,
is_xpu,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.custom_op import register_custom_op
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -1110,11 +1110,9 @@ class MiniMaxM2Model(nn.Module):
prefix=prefix,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
layer_fn,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -99,7 +99,7 @@ from sglang.srt.utils import (
is_hip,
is_npu,
log_info_on_rank0,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -1443,11 +1443,9 @@ class MiniMaxM3Model(nn.Module):
prefix=prefix,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
layer_fn,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -11,7 +11,7 @@ from sglang.srt.models.llama import (
LlamaForCausalLM,
LlamaModel,
)
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
def _get_llama_4_attn_scale(
@@ -141,7 +141,7 @@ class Ministral3Model(LlamaModel):
# Override layer creation to use Ministral3Attention
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Ministral3DecoderLayer(
config=config,
@@ -150,8 +150,6 @@ class Ministral3Model(LlamaModel):
start_layer=self.start_layer,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+2 -4
View File
@@ -54,7 +54,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -294,13 +294,11 @@ class MixtralModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: MixtralDecoderLayer(
config=config, quant_config=quant_config, layer_id=idx, prefix=prefix
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="layers",
return_tuple=True,
)
+2 -4
View File
@@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
logger = logging.getLogger(__name__)
@@ -296,7 +296,7 @@ class NanbeigeModel(nn.Module):
# Use the provided decoder layer type or default to NanbeigeDecoderLayer
decoder_layer_type = decoder_layer_type or NanbeigeDecoderLayer
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -305,8 +305,6 @@ class NanbeigeModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -84,7 +84,7 @@ from sglang.srt.utils import (
add_prefix,
get_current_device_stream_fast,
is_cuda,
make_layers,
make_pp_layers,
)
from sglang.srt.utils.custom_op import register_custom_op
from sglang.utils import logger
@@ -701,11 +701,9 @@ class NemotronHModel(nn.Module):
layer_class = ALL_DECODER_LAYER_TYPES[config.hybrid_override_pattern[idx]]
return layer_class(config, idx, quant_config=quant_config, prefix=prefix)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
len(config.hybrid_override_pattern),
get_layer,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -40,7 +40,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.llama import LlamaAttention, LlamaMLP
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.utils import logger
@@ -197,11 +197,9 @@ class DeciModel(nn.Module):
prefix=prefix,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
get_layer,
pp_rank=get_parallel().pp_group.rank_in_group,
pp_size=get_parallel().pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if get_parallel().pp_group.is_last_rank:
+2 -4
View File
@@ -43,7 +43,7 @@ from sglang.srt.model_loader.weight_utils import (
kv_cache_scales_loader,
)
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.utils import get_exception_traceback
logger = logging.getLogger(__name__)
@@ -273,13 +273,11 @@ class OPTDecoder(nn.Module):
else:
self.final_layer_norm = None
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: OPTDecoderLayer(
config=config, layer_id=idx, quant_config=quant_config, prefix=prefix
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+2 -4
View File
@@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -231,13 +231,11 @@ class OrionModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: OrionDecoderLayer(
config, layer_id=idx, quant_config=quant_config, prefix=prefix
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -23,7 +23,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
class PersimmonMLP(nn.Module):
@@ -209,14 +209,12 @@ class PersimmonModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: PersimmonDecoderLayer(
config, quant_config=quant_config, prefix=prefix, idx=idx
),
prefix="model.layers",
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -25,7 +25,7 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
@torch.jit.script
@@ -308,7 +308,7 @@ class Phi3SmallModel(nn.Module):
prefix=add_prefix("embed_tokens", prefix),
)
self.mup_embedding_multiplier = config.mup_embedding_multiplier
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Phi3SmallDecoderLayer(
config,
@@ -316,8 +316,6 @@ class Phi3SmallModel(nn.Module):
quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -55,7 +55,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils.hf_transformers_utils import get_rope_config
Qwen2Config = None
@@ -353,7 +353,7 @@ class Qwen2Model(nn.Module):
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -363,8 +363,6 @@ class Qwen2Model(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -107,7 +107,7 @@ from sglang.srt.utils import (
is_cuda,
is_hip,
is_npu,
make_layers,
make_pp_layers,
use_intel_amx_backend,
)
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -1130,7 +1130,7 @@ class Qwen2MoeModel(nn.Module):
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: decoder_layer_type(
layer_id=idx,
@@ -1140,8 +1140,6 @@ class Qwen2MoeModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -134,7 +134,7 @@ from sglang.srt.utils import (
is_hip,
is_npu,
is_xpu,
make_layers,
make_pp_layers,
set_weight_attrs,
use_intel_amx_backend,
)
@@ -1691,11 +1691,9 @@ class Qwen3_5ForCausalLM(nn.Module):
is_nextn=is_nextn,
)
self.layers, self._start_layer, self._end_layer = make_layers(
self.layers, self._start_layer, self._end_layer = make_pp_layers(
config.num_hidden_layers,
get_layer,
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
+2 -4
View File
@@ -72,7 +72,7 @@ from sglang.srt.utils import (
bind_or_assign,
is_cuda,
is_nvidia_cublas_version_ge_12_9,
make_layers,
make_pp_layers,
next_power_of_2,
)
@@ -1111,7 +1111,7 @@ class SarvamMLAModel(nn.Module):
else:
self.embed_tokens = nn.Identity()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: SarvamMoEMLADecoderLayer(
config=config,
@@ -1120,8 +1120,6 @@ class SarvamMLAModel(nn.Module):
prefix=prefix,
alt_stream=self.alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix="model.layers",
)
+2 -4
View File
@@ -46,7 +46,7 @@ from sglang.srt.models.utils import (
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, is_cuda, make_layers
from sglang.srt.utils import add_prefix, is_cuda, make_pp_layers
logger = logging.getLogger(__name__)
_is_cuda = is_cuda()
@@ -357,7 +357,7 @@ class SDARModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: SDARBlock(
layer_id=idx,
@@ -366,8 +366,6 @@ class SDARModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -59,7 +59,7 @@ from sglang.srt.models.utils import (
enable_fused_set_kv_buffer,
)
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers
from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_pp_layers
logger = logging.getLogger(__name__)
_is_cuda = is_cuda()
@@ -422,7 +422,7 @@ class SDARMoeModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: SDARMoeBlock(
config=config,
@@ -431,8 +431,6 @@ class SDARMoeModel(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+2 -4
View File
@@ -54,7 +54,7 @@ from sglang.srt.model_loader.weight_utils import (
# limitations under the License.
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/solar.py
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
from sglang.srt.utils.hf_transformers_utils import get_rope_config
@@ -302,7 +302,7 @@ class SolarModel(nn.Module):
)
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: SolarDecoderLayer(
config=config,
@@ -310,8 +310,6 @@ class SolarModel(nn.Module):
layer_id=idx,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=f"{prefix}.layers",
)
_check_skips_stay_in_stage(config, self.start_layer, self.end_layer)
+2 -4
View File
@@ -34,7 +34,7 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
Spark2_5Config = None
@@ -322,7 +322,7 @@ class Spark2_5Model(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: Spark2_5DecoderLayer(
layer_id=idx,
@@ -330,8 +330,6 @@ class Spark2_5Model(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+7 -4
View File
@@ -49,7 +49,12 @@ from sglang.srt.layers.vocab_parallel_embedding import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_exec, get_parallel, get_stream
from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers
from sglang.srt.utils import (
add_prefix,
is_cuda,
is_non_idle_and_non_empty,
make_pp_layers,
)
Step3p5Config = None
@@ -663,7 +668,7 @@ class Step3p5Model(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
# 1,
lambda idx, prefix: Step3p5DecoderLayer(
@@ -673,8 +678,6 @@ class Step3p5Model(nn.Module):
prefix=prefix,
alt_stream=alt_stream,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -84,7 +84,7 @@ from sglang.srt.model_executor.cuda_graph_config import (
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.utils import add_prefix, make_layers
from sglang.srt.utils import add_prefix, make_pp_layers
_XLLM_SOURCE_ROUTER_PARTITIONS_CONFIG_KEY = "xllm_source_router_gemm_partitions"
_XLLM_SOURCE_ROUTER_PARTITIONS_MISSING = object()
@@ -1592,7 +1592,7 @@ class XllmModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: XllmDecoderLayer(
layer_id=idx,
@@ -1600,8 +1600,6 @@ class XllmModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
if self.pp_group.is_last_rank:
+2 -4
View File
@@ -76,7 +76,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe
from sglang.srt.model_executor.forward_context import get_attn_backend
from sglang.srt.model_loader.weight_utils import default_weight_loader
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix, make_layers, set_weight_attrs
from sglang.srt.utils import add_prefix, make_pp_layers, set_weight_attrs
logger = logging.getLogger(__name__)
@@ -1395,7 +1395,7 @@ class ZayaModel(nn.Module):
else:
self.embed_tokens = PPMissingLayer()
self.layers, self.start_layer, self.end_layer = make_layers(
self.layers, self.start_layer, self.end_layer = make_pp_layers(
config.num_hidden_layers,
lambda idx, prefix: _build_layer(
layer_id=idx,
@@ -1403,8 +1403,6 @@ class ZayaModel(nn.Module):
quant_config=quant_config,
prefix=prefix,
),
pp_rank=self.pp_group.rank_in_group,
pp_size=self.pp_group.world_size,
prefix=add_prefix("layers", prefix),
)
+23
View File
@@ -1544,6 +1544,29 @@ def make_layers(
return modules, start_layer, end_layer
def make_pp_layers(
num_hidden_layers: int,
layer_fn: LayerFn,
prefix: str = "",
return_tuple: bool = False,
offloader_kwargs: Optional[Dict[str, Any]] = None,
) -> Tuple[torch.nn.Module, int, int]:
"""Make this pipeline stage's layers, and return them with the stage's range.
Layers outside ``[start_layer, end_layer)`` are ``PPMissingLayer`` stand-ins.
"""
parallel = get_parallel()
return make_layers(
num_hidden_layers,
layer_fn,
pp_rank=parallel.pp_rank,
pp_size=parallel.pp_size,
prefix=prefix,
return_tuple=return_tuple,
offloader_kwargs=offloader_kwargs,
)
def set_random_seed(seed: int) -> None:
"""Set the random seed for all libraries."""
random.seed(seed)