[Refactor] Let linear layers select their parallel group (#42426)
This commit is contained in:
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -55,6 +55,40 @@ _disable_hip_linear_quant = _is_hip and get_bool_env_var(
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
|
||||
|
||||
|
||||
def _resolve_linear_partition(
|
||||
parallel_group: Optional[LinearParallelGroup],
|
||||
tp_rank: Optional[int],
|
||||
tp_size: Optional[int],
|
||||
) -> Tuple[int, int]:
|
||||
"""Resolve a layer's partition in its construction scope.
|
||||
|
||||
Explicit integers remain available for layouts that do not correspond to a
|
||||
process group. A group selection and explicit integers cannot be combined.
|
||||
The returned values belong to the layer, including during later reloads.
|
||||
"""
|
||||
if parallel_group is not None:
|
||||
if tp_rank is not None or tp_size is not None:
|
||||
raise ValueError(
|
||||
"parallel_group cannot be combined with tp_rank or tp_size"
|
||||
)
|
||||
if parallel_group == "replicated":
|
||||
return 0, 1
|
||||
if parallel_group not in ("tp", "attn_tp"):
|
||||
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
|
||||
parallel = get_parallel()
|
||||
if parallel_group == "attn_tp":
|
||||
return parallel.attn_tp_rank, parallel.attn_tp_size
|
||||
return parallel.tp_rank, parallel.tp_size
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
return tp_rank, tp_size
|
||||
|
||||
|
||||
WEIGHT_LOADER_V2_SUPPORTED = [
|
||||
"CompressedTensorsLinearMethod",
|
||||
"AWQLinearMethod",
|
||||
@@ -326,6 +360,8 @@ class ColumnParallelLinear(LinearBase):
|
||||
quant_config: Quantization configure.
|
||||
output_sizes: list of output sizes packed into one output, like for QKV
|
||||
the list would be size 3.
|
||||
parallel_group: Select TP, attention TP, or an unsharded partition.
|
||||
Mutually exclusive with explicit tp_rank and tp_size.
|
||||
prefix: The name of the layer in the state dict, including all parents
|
||||
(e.g. model.layers.0.qkv_proj)
|
||||
"""
|
||||
@@ -345,6 +381,8 @@ class ColumnParallelLinear(LinearBase):
|
||||
tp_size: Optional[int] = None,
|
||||
use_presharded_weights: bool = False,
|
||||
skip_block_quant_check: bool = False,
|
||||
*,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
super().__init__(
|
||||
input_size, output_size, skip_bias_add, params_dtype, quant_config, prefix
|
||||
@@ -355,10 +393,8 @@ class ColumnParallelLinear(LinearBase):
|
||||
self.use_presharded_weights = use_presharded_weights
|
||||
|
||||
# Divide the weight matrix along the last dimension.
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
|
||||
self.parallel_group = parallel_group
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
assert self.quant_method is not None
|
||||
self.output_size_per_partition = divide(self.output_size, tp_size)
|
||||
@@ -504,7 +540,14 @@ class ColumnParallelLinear(LinearBase):
|
||||
output_parallel = self.quant_method.apply(self, input_, bias)
|
||||
if self.gather_output:
|
||||
# All-gather across the partitions.
|
||||
output = tensor_model_parallel_all_gather(output_parallel)
|
||||
if self.parallel_group == "replicated":
|
||||
output = output_parallel
|
||||
elif self.parallel_group == "attn_tp":
|
||||
output = get_parallel().attn_tp_group.all_gather(output_parallel, -1)
|
||||
elif self.parallel_group == "tp":
|
||||
output = get_parallel().tp_group.all_gather(output_parallel, -1)
|
||||
else:
|
||||
output = tensor_model_parallel_all_gather(output_parallel)
|
||||
else:
|
||||
output = output_parallel
|
||||
output_bias = self.bias if self.skip_bias_add else None
|
||||
@@ -555,13 +598,12 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
use_presharded_weights: bool = False,
|
||||
*,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
self.with_bias = bias
|
||||
self.output_sizes = output_sizes
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
assert all(output_size % tp_size == 0 for output_size in output_sizes)
|
||||
self.use_presharded_weights = use_presharded_weights
|
||||
@@ -578,6 +620,7 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
|
||||
tp_size=tp_size,
|
||||
use_presharded_weights=use_presharded_weights,
|
||||
)
|
||||
self.parallel_group = parallel_group
|
||||
self.prefix = prefix
|
||||
|
||||
def weight_loader(
|
||||
@@ -988,6 +1031,8 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
skip_block_quant_check: bool = False,
|
||||
kv_tp_rank: Optional[int] = None,
|
||||
kv_tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
self.with_bias = bias
|
||||
self.hidden_size = hidden_size
|
||||
@@ -998,10 +1043,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
total_num_kv_heads = total_num_heads
|
||||
self.total_num_kv_heads = total_num_kv_heads
|
||||
# Divide the weight matrix along the last dimension.
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
if kv_tp_rank is None:
|
||||
kv_tp_rank = tp_rank
|
||||
@@ -1046,6 +1088,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
use_presharded_weights=self.use_presharded_weights,
|
||||
skip_block_quant_check=skip_block_quant_check,
|
||||
)
|
||||
self.parallel_group = parallel_group
|
||||
|
||||
def _get_shard_offset_mapping(self, loaded_shard_id: str):
|
||||
shard_offset_mapping = {
|
||||
@@ -1439,6 +1482,9 @@ class RowParallelLinear(LinearBase):
|
||||
We skip adding bias but instead return it.
|
||||
params_dtype: Data type for the parameters.
|
||||
quant_config: Quantization configure.
|
||||
parallel_group: Select TP, attention TP, or an unsharded weight partition.
|
||||
Mutually exclusive with explicit tp_rank and tp_size.
|
||||
Reduction and allocation policies remain independent.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1456,6 +1502,8 @@ class RowParallelLinear(LinearBase):
|
||||
tp_size: Optional[int] = None,
|
||||
use_presharded_weights: bool = False,
|
||||
use_dp_attention_reduce: bool = False,
|
||||
*,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
quant_config = None if _disable_hip_linear_quant else quant_config
|
||||
super().__init__(
|
||||
@@ -1465,13 +1513,11 @@ class RowParallelLinear(LinearBase):
|
||||
self.with_bias = bias
|
||||
self.input_is_parallel = input_is_parallel
|
||||
self.reduce_results = reduce_results
|
||||
self.parallel_group = parallel_group
|
||||
self.use_dp_attention_reduce = use_dp_attention_reduce
|
||||
|
||||
# Divide the weight matrix along the last dimension.
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
self.input_size_per_partition = divide(input_size, self.tp_size)
|
||||
assert self.quant_method is not None
|
||||
@@ -1682,6 +1728,8 @@ class RowParallelLinear(LinearBase):
|
||||
)
|
||||
if quantize_communications:
|
||||
output = tensor_model_parallel_quant_all_reduce(output_parallel)
|
||||
elif self.parallel_group == "tp":
|
||||
output = get_parallel().tp_group.all_reduce(output_parallel)
|
||||
else:
|
||||
output = tensor_model_parallel_all_reduce(output_parallel)
|
||||
else:
|
||||
|
||||
@@ -509,7 +509,6 @@ class BailingMoEAttention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_kv_heads = config.num_key_value_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -542,8 +541,7 @@ class BailingMoEAttention(nn.Module):
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("query_key_value", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
@@ -557,8 +555,7 @@ class BailingMoEAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("dense", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
if hasattr(config, "partial_rotary_factor"):
|
||||
|
||||
@@ -459,8 +459,7 @@ class BailingMoELinearAttention(nn.Module):
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
@@ -473,8 +472,7 @@ class BailingMoELinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
self.hidden_inner_size,
|
||||
@@ -482,8 +480,7 @@ class BailingMoELinearAttention(nn.Module):
|
||||
bias=config.use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.out_proj",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
)
|
||||
self.attn = RadixAttention(
|
||||
|
||||
@@ -198,8 +198,7 @@ class CLIPAttention(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=parallel.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
# group; reduce over the attention-TP group so attention DP and attention
|
||||
@@ -216,8 +215,7 @@ class CLIPAttention(nn.Module):
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("proj", prefix),
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=parallel.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -77,8 +78,8 @@ class ExaoneMoEMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
gateup_quant_config = quant_config
|
||||
@@ -95,8 +96,7 @@ class ExaoneMoEMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=gateup_quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -105,8 +105,7 @@ class ExaoneMoEMLP(nn.Module):
|
||||
quant_config=down_quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -194,7 +193,7 @@ class ExaoneMoESparseMoEBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
dict(parallel_group="replicated")
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
else {}
|
||||
),
|
||||
@@ -310,7 +309,6 @@ class ExaoneMoEAttention(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -351,8 +349,7 @@ class ExaoneMoEAttention(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=qkv_quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
@@ -361,8 +358,7 @@ class ExaoneMoEAttention(nn.Module):
|
||||
quant_config=o_quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps)
|
||||
|
||||
@@ -157,8 +157,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -167,8 +166,7 @@ class FalconH1HybridAttentionDecoderLayer(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.attn = RadixAttention(
|
||||
|
||||
@@ -233,7 +233,6 @@ class Glm4MoeAttention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
self.start_layer = start_layer
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -264,8 +263,7 @@ class Glm4MoeAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -274,8 +272,7 @@ class Glm4MoeAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -382,7 +382,6 @@ class GptOssAttention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
self.sliding_window_size = sliding_window_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -413,8 +412,7 @@ class GptOssAttention(nn.Module):
|
||||
bias=attention_bias,
|
||||
params_dtype=params_dtype,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -431,8 +429,7 @@ class GptOssAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
params_dtype=params_dtype,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
|
||||
@@ -353,10 +353,9 @@ class InklingAttention(nn.Module):
|
||||
output_size=self.hidden_size,
|
||||
bias=o_bias,
|
||||
prefix=add_prefix("wo_ud", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
reduce_results=False,
|
||||
parallel_group="attn_tp",
|
||||
use_dp_attention_reduce=True,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
# --enable-scattered-sconv: the output reduction becomes a hidden-dim
|
||||
|
||||
@@ -579,8 +579,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -589,8 +588,7 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
self.attn = RadixAttention(
|
||||
|
||||
@@ -310,8 +310,7 @@ class IQuestQ1Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# The decoder boundary reduces partial attention outputs. Only a
|
||||
@@ -327,8 +326,7 @@ class IQuestQ1Attention(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -68,8 +69,8 @@ class LagunaMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if hidden_act != "silu":
|
||||
@@ -82,8 +83,7 @@ class LagunaMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -92,8 +92,7 @@ class LagunaMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
@@ -188,7 +187,7 @@ class LagunaMoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_expert", prefix),
|
||||
**(dict(tp_rank=0, tp_size=1) if self._shared_expert_tp1 else {}),
|
||||
**(dict(parallel_group="replicated") if self._shared_expert_tp1 else {}),
|
||||
)
|
||||
|
||||
def get_moe_weights(self):
|
||||
@@ -256,7 +255,6 @@ class LagunaAttention(nn.Module):
|
||||
self.gating = gating != "disabled"
|
||||
self.gate_per_head = gating == "per-head"
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -279,8 +277,7 @@ class LagunaAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -288,8 +285,7 @@ class LagunaAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
@@ -306,8 +302,7 @@ class LagunaAttention(nn.Module):
|
||||
bias=False,
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("g_proj", prefix),
|
||||
)
|
||||
else:
|
||||
|
||||
@@ -427,7 +427,6 @@ class LLaDA2MoeAttention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.total_kv_heads = config.num_key_value_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -460,8 +459,7 @@ class LLaDA2MoeAttention(nn.Module):
|
||||
bias=(config.use_bias or config.use_qkv_bias),
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("query_key_value", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
if self.use_qk_norm:
|
||||
@@ -475,8 +473,7 @@ class LLaDA2MoeAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("dense", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
if hasattr(config, "partial_rotary_factor"):
|
||||
|
||||
@@ -212,7 +212,6 @@ class Llama4Attention(nn.Module):
|
||||
self.use_rope = (layer_id + 1) % 4 != 0
|
||||
self.use_qk_norm = config.use_qk_norm and self.use_rope
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -264,8 +263,7 @@ class Llama4Attention(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=qkv_quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -274,8 +272,7 @@ class Llama4Attention(nn.Module):
|
||||
bias=bias_o_proj,
|
||||
quant_config=o_quant_config,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
)
|
||||
is_neox_style = True
|
||||
|
||||
@@ -164,7 +164,6 @@ class MellumAttention(Qwen3MoeAttention):
|
||||
self.hidden_size = hidden_size
|
||||
self.start_layer = start_layer
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.config = config
|
||||
@@ -194,8 +193,7 @@ class MellumAttention(Qwen3MoeAttention):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -204,8 +202,7 @@ class MellumAttention(Qwen3MoeAttention):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -650,7 +650,6 @@ class MiniMaxM2Attention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
# Use attention TP rank/size for dp-attention support
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
# Get dimensions from config
|
||||
@@ -696,8 +695,7 @@ class MiniMaxM2Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -707,8 +705,7 @@ class MiniMaxM2Attention(nn.Module):
|
||||
bias=False,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -898,7 +898,6 @@ class MossVLSelfAttention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -928,8 +927,7 @@ class MossVLSelfAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -937,8 +935,7 @@ class MossVLSelfAttention(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=config.attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -38,6 +38,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
@@ -105,9 +106,9 @@ class NemotronHMLP(nn.Module):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
reduce_results: bool = True,
|
||||
tp_rank: int | None = None,
|
||||
tp_size: int | None = None,
|
||||
prefix: str = "",
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -116,8 +117,7 @@ class NemotronHMLP(nn.Module):
|
||||
output_size=intermediate_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
prefix=f"{prefix}.up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
@@ -126,8 +126,7 @@ class NemotronHMLP(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = ReLU2()
|
||||
@@ -232,7 +231,7 @@ class NemotronHMoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
dict(parallel_group="replicated")
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_flashinfer()
|
||||
or get_moe_a2a_backend().is_flashinfer_megamoe()
|
||||
@@ -541,7 +540,6 @@ class NemotronHAttention(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
@@ -571,8 +569,7 @@ class NemotronHAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -580,8 +577,7 @@ class NemotronHAttention(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
use_dp_attention_reduce=is_dp_attention_enabled(),
|
||||
prefix=f"{prefix}.o_proj",
|
||||
|
||||
@@ -246,7 +246,6 @@ class PhiMoEAttention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -279,8 +278,7 @@ class PhiMoEAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
# TODO: this layer shards over attention TP but reduces over the full TP
|
||||
@@ -294,8 +292,7 @@ class PhiMoEAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
|
||||
@@ -56,6 +56,7 @@ from sglang.srt.layers.layer_boundary import (
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -176,8 +177,8 @@ class Qwen2MoeMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
@@ -186,8 +187,7 @@ class Qwen2MoeMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -196,8 +196,7 @@ class Qwen2MoeMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
@@ -365,7 +364,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("shared_expert", prefix),
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
dict(parallel_group="replicated")
|
||||
if (
|
||||
get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mori()
|
||||
@@ -911,7 +910,6 @@ class Qwen2MoeAttention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -941,8 +939,7 @@ class Qwen2MoeAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=qkv_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -951,8 +948,7 @@ class Qwen2MoeAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -86,7 +86,6 @@ class Qwen3Attention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
self.start_layer = start_layer
|
||||
self.total_num_heads = num_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -126,8 +125,7 @@ class Qwen3Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -135,8 +133,7 @@ class Qwen3Attention(nn.Module):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -512,7 +512,6 @@ class Qwen3MoeAttention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
self.start_layer = start_layer
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.config = config
|
||||
@@ -543,8 +542,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -553,8 +551,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -641,8 +641,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
and quant_config.get_name() != "modelopt_fp4"
|
||||
else None
|
||||
),
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -652,8 +651,7 @@ class Qwen3HybridAttentionDecoderLayer(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -99,7 +99,6 @@ class SDARAttention(nn.Module):
|
||||
self.layer_id = layer_id
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -129,8 +128,7 @@ class SDARAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -139,8 +137,7 @@ class SDARAttention(nn.Module):
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -105,7 +105,6 @@ class Spark2_5Attention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.total_num_heads = num_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -139,8 +138,7 @@ class Spark2_5Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("q_k_v_proj", prefix),
|
||||
)
|
||||
if self.headwise_attn_output_gate:
|
||||
@@ -149,8 +147,7 @@ class Spark2_5Attention(nn.Module):
|
||||
self.total_num_heads,
|
||||
bias=False,
|
||||
quant_config=None, # g_proj keeps bf16.
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("g_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -159,8 +156,7 @@ class Spark2_5Attention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("out_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
@@ -67,6 +68,8 @@ class Step3p5MLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -76,6 +79,7 @@ class Step3p5MLP(nn.Module):
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
parallel_group=parallel_group,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
@@ -83,6 +87,7 @@ class Step3p5MLP(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
parallel_group=parallel_group,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
@@ -287,7 +292,6 @@ class Step3p5Attention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.total_num_heads = num_heads
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -318,8 +322,7 @@ class Step3p5Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -327,8 +330,7 @@ class Step3p5Attention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
@@ -339,8 +341,7 @@ class Step3p5Attention(nn.Module):
|
||||
hidden_size,
|
||||
self.total_num_heads,
|
||||
bias=False,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("g_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -490,7 +491,7 @@ class Step3p5DecoderLayer(nn.Module):
|
||||
prefix=add_prefix("share_expert", prefix),
|
||||
reduce_results=False,
|
||||
**(
|
||||
dict(tp_rank=0, tp_size=1)
|
||||
dict(parallel_group="replicated")
|
||||
if get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_nixl()
|
||||
|
||||
@@ -1081,7 +1081,6 @@ class XllmAttention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -1107,8 +1106,7 @@ class XllmAttention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=qkv_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -1117,8 +1115,7 @@ class XllmAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=qkv_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user