[Refactor] Select parallel groups for remaining model projections (#42587)
This commit is contained in:
@@ -231,9 +231,11 @@ class MambaMixer2(torch.nn.Module):
|
||||
# - NOTE: currently for the world size DOES NOT divide groups
|
||||
# case, we only support the case when n_groups == 1
|
||||
if is_dp_attention_enabled():
|
||||
parallel_group = "attn_tp"
|
||||
self.tp_size = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
else:
|
||||
parallel_group = "tp"
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
|
||||
@@ -284,8 +286,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
bias=use_conv_bias,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.in_proj = MergedColumnParallelLinear(
|
||||
@@ -300,8 +301,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
bias=use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
else:
|
||||
# This is the n_groups == 1 case,
|
||||
@@ -313,8 +313,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
bias=use_conv_bias,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.conv1d",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.in_proj = ColumnParallelLinear(
|
||||
@@ -323,8 +322,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
bias=use_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.in_proj",
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
# - because in_proj is a concatenation of 3 weights, we
|
||||
@@ -431,8 +429,7 @@ class MambaMixer2(torch.nn.Module):
|
||||
bias=use_bias,
|
||||
input_is_parallel=True,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
reduce_results=reduce_results,
|
||||
use_dp_attention_reduce=reduce_results and is_dp_attention_enabled(),
|
||||
prefix=f"{prefix}.out_proj",
|
||||
|
||||
@@ -8,6 +8,7 @@ import itertools
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.parameter import Parameter, UninitializedParameter
|
||||
@@ -55,15 +56,44 @@ _disable_hip_linear_quant = _is_hip and get_bool_env_var(
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
LinearParallelGroup = Literal["tp", "attn_tp", "replicated"]
|
||||
|
||||
class ReplicatedParallelGroup(msgspec.Struct, frozen=True):
|
||||
"""Share weight partitions across consecutive ranks; collectives stay separate."""
|
||||
|
||||
group: Literal["tp", "attn_tp"]
|
||||
replica_size: int
|
||||
|
||||
def __post_init__(self):
|
||||
if self.group not in ("tp", "attn_tp"):
|
||||
raise ValueError(f"Unknown replicated base group: {self.group!r}")
|
||||
if type(self.replica_size) is not int or self.replica_size < 1:
|
||||
raise ValueError("replica_size must be a positive integer")
|
||||
|
||||
|
||||
LinearParallelGroup = (
|
||||
Literal["tp", "attn_tp", "replicated", "shared_experts_tp"]
|
||||
| ReplicatedParallelGroup
|
||||
)
|
||||
|
||||
|
||||
def resolve_linear_parallel_group(
|
||||
parallel_group: LinearParallelGroup,
|
||||
) -> Tuple[int, int]:
|
||||
"""Freeze a group's weight partition in the current construction scope."""
|
||||
if isinstance(parallel_group, ReplicatedParallelGroup):
|
||||
rank, size = resolve_linear_parallel_group(parallel_group.group)
|
||||
return rank // parallel_group.replica_size, divide(
|
||||
size, parallel_group.replica_size
|
||||
)
|
||||
if parallel_group == "replicated":
|
||||
return 0, 1
|
||||
if parallel_group == "shared_experts_tp":
|
||||
group = get_parallel().shared_experts_tp_group
|
||||
if group is None:
|
||||
raise ValueError(
|
||||
"The shared-expert TP group must exist before construction"
|
||||
)
|
||||
return group.rank_in_group, group.world_size
|
||||
if parallel_group not in ("tp", "attn_tp"):
|
||||
raise ValueError(f"Unknown linear parallel_group: {parallel_group!r}")
|
||||
parallel = get_parallel()
|
||||
@@ -1005,6 +1035,9 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
be replicated while the query heads are partitioned.
|
||||
|
||||
Args:
|
||||
parallel_group: Group selecting the query weight partition.
|
||||
kv_parallel_group: Optional independent key/value weight partition;
|
||||
defaults to the query partition, including replicated head layouts.
|
||||
hidden_size: input hidden state size of the transformer.
|
||||
head_size: size of each attention head.
|
||||
total_num_heads: total number of attention query heads.
|
||||
@@ -1040,6 +1073,7 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
kv_tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
kv_parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
self.with_bias = bias
|
||||
self.hidden_size = hidden_size
|
||||
@@ -1052,10 +1086,15 @@ class QKVParallelLinear(ColumnParallelLinear):
|
||||
# Divide the weight matrix along the last dimension.
|
||||
tp_rank, tp_size = _resolve_linear_partition(parallel_group, tp_rank, tp_size)
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
if kv_tp_rank is None:
|
||||
kv_tp_rank = tp_rank
|
||||
if kv_tp_size is None:
|
||||
kv_tp_size = tp_size
|
||||
if kv_parallel_group is not None:
|
||||
kv_tp_rank, kv_tp_size = _resolve_linear_partition(
|
||||
kv_parallel_group, kv_tp_rank, kv_tp_size
|
||||
)
|
||||
else:
|
||||
if kv_tp_rank is None:
|
||||
kv_tp_rank = tp_rank
|
||||
if kv_tp_size is None:
|
||||
kv_tp_size = tp_size
|
||||
self.kv_tp_rank, self.kv_tp_size = kv_tp_rank, kv_tp_size
|
||||
self.num_heads = divide(self.total_num_heads, tp_size)
|
||||
if kv_tp_size >= self.total_num_kv_heads:
|
||||
@@ -1766,10 +1805,8 @@ class MergedColumnParallelRepeatedLinear(LinearBase):
|
||||
skip_bias_add: If true, skip adding bias but instead return it.
|
||||
params_dtype: Data type for the parameters.
|
||||
quant_config: Quantization configure.
|
||||
tp_rank: Rank to shard the column-parallel part on. Defaults to the
|
||||
global TP rank; pass the attention-TP rank to shard on attn-TP
|
||||
instead (see KimiDeltaAttention's shard_on_attn_tp).
|
||||
tp_size: World size matching ``tp_rank``. Defaults to global TP size.
|
||||
parallel_group: Select TP, attention TP, or an unsharded partition for
|
||||
the column-parallel part. Repeated weights remain unsharded.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1783,6 +1820,7 @@ class MergedColumnParallelRepeatedLinear(LinearBase):
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
output_size = sum(column_output_sizes) + sum(repeated_output_sizes)
|
||||
super().__init__(
|
||||
@@ -1794,11 +1832,9 @@ class MergedColumnParallelRepeatedLinear(LinearBase):
|
||||
prefix=prefix,
|
||||
)
|
||||
self.num_column_parallel = len(column_output_sizes)
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
self.tp_rank, self.tp_size = _resolve_linear_partition(
|
||||
parallel_group, tp_rank, tp_size
|
||||
)
|
||||
|
||||
self.output_partition_sizes = [
|
||||
divide(x, self.tp_size) for x in column_output_sizes
|
||||
@@ -1843,10 +1879,8 @@ class ColumnParallelBatchedLinear(nn.Module):
|
||||
input_size: input dimension of the linear layer.
|
||||
output_size: output dimension of the linear layer.
|
||||
dtype: Data type for the parameters.
|
||||
tp_rank: Rank to shard the output dimension on. Defaults to the global
|
||||
TP rank; pass the attention-TP rank to shard on attn-TP instead
|
||||
(see KimiDeltaAttention's shard_on_attn_tp).
|
||||
tp_size: World size matching ``tp_rank``. Defaults to global TP size.
|
||||
parallel_group: Select TP, attention TP, or an unsharded partition for
|
||||
the output dimension. The batch dimension remains unsharded.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1857,13 +1891,12 @@ class ColumnParallelBatchedLinear(nn.Module):
|
||||
dtype: torch.dtype,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
):
|
||||
super().__init__()
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
self.tp_rank, self.tp_size = tp_rank, tp_size
|
||||
self.tp_rank, self.tp_size = _resolve_linear_partition(
|
||||
parallel_group, tp_rank, tp_size
|
||||
)
|
||||
self.weight = nn.Parameter(
|
||||
torch.empty(batch, output_size // self.tp_size, input_size, dtype=dtype),
|
||||
requires_grad=False,
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -23,6 +23,9 @@ from torch import nn
|
||||
from sglang.srt.utils import set_weight_attrs
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.linear import LinearParallelGroup
|
||||
|
||||
_ROUTED_LINEAR_CHUNK_SIZE = 64 * 1024
|
||||
|
||||
|
||||
@@ -245,10 +248,12 @@ class RoutedValueExperts(nn.Module):
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
*,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
parallel_group: LinearParallelGroup = "attn_tp",
|
||||
) -> None:
|
||||
from sglang.srt.layers.linear import resolve_linear_parallel_group
|
||||
|
||||
super().__init__()
|
||||
tp_rank, tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
if output_size % tp_size:
|
||||
raise ValueError(
|
||||
f"MoVA value width {output_size} is not divisible by TP={tp_size}"
|
||||
|
||||
@@ -162,8 +162,6 @@ class DsV3MLA(DeepseekV2AttentionMLA):
|
||||
alt_stream,
|
||||
skip_rope,
|
||||
)
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.gated_attention_proj_granularity_type = getattr(
|
||||
config, "gated_attention_proj_granularity_type", None
|
||||
)
|
||||
@@ -175,8 +173,7 @@ class DsV3MLA(DeepseekV2AttentionMLA):
|
||||
bias=False,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
quant_config=None,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
elif self.gated_attention_proj_granularity_type == "element_wise":
|
||||
self.g_proj = ColumnParallelLinear(
|
||||
@@ -184,8 +181,7 @@ class DsV3MLA(DeepseekV2AttentionMLA):
|
||||
self.num_heads * self.v_head_dim,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.output_gate",
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
else:
|
||||
self.g_proj = None
|
||||
|
||||
@@ -112,9 +112,6 @@ class Exaone4Attention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_parallel().tp_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
@@ -144,8 +141,7 @@ class Exaone4Attention(nn.Module):
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -155,8 +151,7 @@ class Exaone4Attention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
is_neox_style = True
|
||||
|
||||
@@ -296,8 +296,7 @@ class GigaChat35AttentionMLA(deepseek_v2.DeepseekV2AttentionMLA):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("attn_gate", prefix),
|
||||
tp_rank=get_parallel().attn_tp_rank,
|
||||
tp_size=get_parallel().attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.o_proj.register_forward_pre_hook(self._o_proj_gate_hook)
|
||||
|
||||
|
||||
@@ -396,7 +396,6 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
head_shard_size = get_parallel().attn_tp_size
|
||||
head_shard_rank = get_parallel().attn_tp_rank
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.config = config
|
||||
@@ -435,8 +434,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.fg_sizes,
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.fused_qkvbfg_a_proj",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.split_sizes = [
|
||||
3 * projection_size // head_shard_size,
|
||||
@@ -448,8 +446,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
dtype=self.fused_qkvbfg_a_proj.params_dtype,
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
else:
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
@@ -459,8 +456,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.num_k_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
|
||||
@@ -471,8 +467,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
[self.head_dim, self.head_dim],
|
||||
quant_config=None,
|
||||
prefix=f"{prefix}.fused_bfg_a_proj",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
self.bfg_split_sizes = [self.local_num_heads, 2 * self.head_dim]
|
||||
self.fused_fg_b_proj = ColumnParallelBatchedLinear(
|
||||
@@ -480,8 +475,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
dtype=self.fused_bfg_a_proj.params_dtype,
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
else:
|
||||
self.f_a_proj = ReplicatedLinear(
|
||||
@@ -498,8 +492,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
@@ -508,8 +501,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
@@ -525,8 +517,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.dt_bias = nn.Parameter(
|
||||
@@ -544,8 +535,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.qkv_conv1d",
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
# ColumnParallelLinear's loader cannot reshape conv1d weights, so add the
|
||||
# singleton dimension after construction.
|
||||
@@ -569,8 +559,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=head_shard_rank,
|
||||
tp_size=head_shard_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
conv_weights = self.qkv_conv1d.weight.squeeze(1)
|
||||
|
||||
@@ -357,14 +357,12 @@ class HYV4Attention(DeepseekV2AttentionMLA):
|
||||
alt_stream=alt_stream,
|
||||
is_nextn=is_nextn,
|
||||
)
|
||||
parallel = get_parallel()
|
||||
self.linear_gate = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
config.num_attention_heads * config.v_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=parallel.attn_tp_rank,
|
||||
tp_size=parallel.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.linear_gate",
|
||||
)
|
||||
self.local_gate_width = self.num_local_heads * config.v_head_dim
|
||||
|
||||
@@ -199,8 +199,7 @@ class InklingDecoderLayer(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("mlp", prefix),
|
||||
fused=True,
|
||||
tp_rank=get_parallel().attn_tp_rank,
|
||||
tp_size=get_parallel().attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
tp_group=get_parallel().attn_tp_group,
|
||||
use_dp_attention_reduce=True,
|
||||
)
|
||||
|
||||
@@ -269,7 +269,6 @@ class InklingQKVRLinear(MergedColumnParallelLinear):
|
||||
inkling_head_dim: int,
|
||||
inkling_num_heads: int,
|
||||
inkling_d_rel: int,
|
||||
inkling_tp_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -277,7 +276,7 @@ class InklingQKVRLinear(MergedColumnParallelLinear):
|
||||
self.inkling_head_dim = inkling_head_dim
|
||||
self.inkling_num_heads = inkling_num_heads
|
||||
self.inkling_d_rel = inkling_d_rel
|
||||
self.inkling_tp_size = inkling_tp_size
|
||||
self.inkling_tp_size = self.tp_size
|
||||
|
||||
|
||||
class InklingAttention(nn.Module):
|
||||
@@ -305,7 +304,6 @@ class InklingAttention(nn.Module):
|
||||
self.hidden_size = hidden_size
|
||||
self.alt_stream = alt_stream
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.tp_size = attn_tp_size
|
||||
@@ -339,14 +337,12 @@ class InklingAttention(nn.Module):
|
||||
output_sizes=output_sizes,
|
||||
bias=q_bias,
|
||||
prefix=add_prefix("qkvr", prefix),
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
quant_config=quant_config,
|
||||
inkling_num_kv_heads=self.num_total_kv_heads,
|
||||
inkling_head_dim=self.head_dim,
|
||||
inkling_num_heads=self.num_total_heads,
|
||||
inkling_d_rel=self.d_rel,
|
||||
inkling_tp_size=attn_tp_size,
|
||||
)
|
||||
self.wo_ud = RowParallelLinear(
|
||||
input_size=self.head_dim * self.num_total_heads,
|
||||
|
||||
@@ -5,6 +5,7 @@ import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
from sglang.srt.layers.linear import LinearParallelGroup, resolve_linear_parallel_group
|
||||
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod
|
||||
@@ -91,13 +92,10 @@ class InklingDenseMLP(LlamaMLP):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
fused: bool = False,
|
||||
tp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
parallel_group: LinearParallelGroup = "replicated",
|
||||
tp_group: torch.distributed.ProcessGroup | None = None,
|
||||
use_dp_attention_reduce: bool = False,
|
||||
) -> None:
|
||||
self.tp_rank = tp_rank
|
||||
self.tp_size = tp_size
|
||||
self.tp_group = tp_group
|
||||
|
||||
super().__init__(
|
||||
@@ -107,8 +105,7 @@ class InklingDenseMLP(LlamaMLP):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
reduce_results=False,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=use_dp_attention_reduce,
|
||||
)
|
||||
|
||||
@@ -160,8 +157,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin):
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
inference_moe_w13_interleaved: bool = True,
|
||||
tp_rank: int = 0,
|
||||
tp_size: int = 1,
|
||||
parallel_group: LinearParallelGroup = "replicated",
|
||||
tp_group: torch.distributed.ProcessGroup | None = None,
|
||||
linearized_bf16: bool = False,
|
||||
):
|
||||
@@ -186,8 +182,9 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin):
|
||||
self.hidden_size = d_model
|
||||
self.layer_id = layer_id
|
||||
|
||||
self.moe_tp_rank = tp_rank
|
||||
self.moe_tp_size = tp_size
|
||||
self.moe_tp_rank, self.moe_tp_size = resolve_linear_parallel_group(
|
||||
parallel_group
|
||||
)
|
||||
self.tp_group = tp_group
|
||||
|
||||
local_intermediate_size = shared_d_mlp // self.moe_tp_size
|
||||
|
||||
@@ -718,7 +718,6 @@ def _build_inkling_shared_experts(
|
||||
if n_shared_experts <= 0:
|
||||
return None
|
||||
# The shared dense MLP is reconstructed by the full-TP all-reduce in forward.
|
||||
parallel = get_parallel()
|
||||
if shared_expert_sink:
|
||||
shared_prefix = add_prefix("shared_experts", prefix)
|
||||
shared_sink_serves_fp4 = InklingBatchDenseMLP._resolve_fp4_strategy(
|
||||
@@ -746,8 +745,7 @@ def _build_inkling_shared_experts(
|
||||
prefix=shared_prefix,
|
||||
quant_config=quant_config,
|
||||
inference_moe_w13_interleaved=inference_moe_w13_interleaved,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
return InklingBatchDenseMLP(
|
||||
@@ -765,8 +763,7 @@ def _build_inkling_shared_experts(
|
||||
layer_id=layer_id,
|
||||
prefix=add_prefix("shared_experts", prefix),
|
||||
quant_config=quant_config,
|
||||
tp_rank=parallel.tp_rank,
|
||||
tp_size=parallel.tp_size,
|
||||
parallel_group="tp",
|
||||
tp_group=get_parallel().tp_group,
|
||||
)
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelBatchedLinear,
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
MergedColumnParallelRepeatedLinear,
|
||||
QKVParallelLinear,
|
||||
@@ -305,8 +306,7 @@ class KimiK3MLP(nn.Module):
|
||||
prefix: str = "",
|
||||
activation_situ_beta: float | None = None,
|
||||
activation_situ_linear_beta: float | None = None,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
parallel_group: Optional[LinearParallelGroup] = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# The Ascend path shards the dense MLP inside each attention-TP
|
||||
@@ -317,22 +317,17 @@ class KimiK3MLP(nn.Module):
|
||||
self._dense_attn_tp = (
|
||||
get_parallel().enable_dense_mlp_attn_tp
|
||||
and is_dp_attention_enabled()
|
||||
and tp_rank is None
|
||||
and tp_size is None
|
||||
)
|
||||
if self._dense_attn_tp:
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
_tp_kwargs = (
|
||||
dict(tp_rank=tp_rank, tp_size=tp_size) if tp_size is not None else {}
|
||||
and parallel_group is None
|
||||
)
|
||||
if parallel_group is None:
|
||||
parallel_group = "attn_tp" if self._dense_attn_tp else "tp"
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
**_tp_kwargs,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -342,7 +337,7 @@ class KimiK3MLP(nn.Module):
|
||||
reduce_results=reduce_results,
|
||||
use_dp_attention_reduce=self._dense_attn_tp,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
**_tp_kwargs,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
if hidden_act == "silu":
|
||||
self.act_fn = SiluAndMul()
|
||||
@@ -586,9 +581,9 @@ class KimiK3MoE(nn.Module):
|
||||
self._ep_a2a and shared_tp is not None and shared_tp > 1
|
||||
)
|
||||
self._shared_experts_tp_group = None
|
||||
shared_experts_tp_kwargs = {}
|
||||
shared_experts_parallel_group = None
|
||||
if self._shared_experts_tp1:
|
||||
shared_experts_tp_kwargs = dict(tp_rank=0, tp_size=1)
|
||||
shared_experts_parallel_group = "replicated"
|
||||
elif self._shared_experts_tp_comm:
|
||||
group = (
|
||||
parallel.shared_experts_tp_group
|
||||
@@ -597,8 +592,8 @@ class KimiK3MoE(nn.Module):
|
||||
)
|
||||
assert group.world_size == shared_tp
|
||||
self._shared_experts_tp_group = group
|
||||
shared_experts_tp_kwargs = dict(
|
||||
tp_rank=group.rank_in_group, tp_size=group.world_size
|
||||
shared_experts_parallel_group = (
|
||||
"shared_experts_tp" if requested_shared_tp is not None else "attn_tp"
|
||||
)
|
||||
if self.num_shared_experts is not None and self.num_shared_experts > 0:
|
||||
shared_intermediate_size = moe_intermediate_size * self.num_shared_experts
|
||||
@@ -616,7 +611,7 @@ class KimiK3MoE(nn.Module):
|
||||
prefix=f"{prefix}.shared_experts",
|
||||
activation_situ_beta=config.activation_situ_beta,
|
||||
activation_situ_linear_beta=config.activation_situ_linear_beta,
|
||||
**shared_experts_tp_kwargs,
|
||||
parallel_group=shared_experts_parallel_group,
|
||||
)
|
||||
else:
|
||||
self.shared_experts = None
|
||||
@@ -1556,7 +1551,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
)
|
||||
|
||||
# The full-rank [q, k, v, g] merged projection is explicitly sharded
|
||||
# with attn_tp_rank/attn_tp_size, so it also supports DP attention.
|
||||
# using attention-TP placement, so it also supports DP attention.
|
||||
# The low-rank fused path still uses full-TP-only projection helpers.
|
||||
# For the full-rank gate (K3) the checkpoint quantizes only the MoE
|
||||
# experts; attention linears resolve to UnquantizedLinearMethod, so a
|
||||
@@ -1579,8 +1574,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.fused_qkvg_proj",
|
||||
)
|
||||
self.split_sizes = [
|
||||
@@ -1592,8 +1586,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.b_proj",
|
||||
# TP8 shards K3's 96 beta rows below the 128-row FP8 block.
|
||||
skip_block_quant_check=self._bfa_uses_block_fp8,
|
||||
@@ -1610,8 +1603,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
# Merged [f_a | b] weight, built after weight loading by
|
||||
@@ -1657,7 +1649,6 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
2, self.head_dim, projection_size, dtype=_dtype
|
||||
)
|
||||
else:
|
||||
attn_tp_rank = self.attn_tp_rank
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
@@ -1665,8 +1656,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
self.num_k_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
v_head_size=self.head_v_dim,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
@@ -1683,8 +1673,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
)
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
@@ -1692,8 +1681,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.b_proj",
|
||||
)
|
||||
|
||||
@@ -1703,8 +1691,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
else:
|
||||
@@ -1720,8 +1707,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
)
|
||||
|
||||
@@ -1735,8 +1721,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
output_sizes=[projection_size, projection_size, projection_size],
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.qkv_conv1d",
|
||||
)
|
||||
self.qkv_conv1d.weight.data = self.qkv_conv1d.weight.data.unsqueeze(1)
|
||||
@@ -1780,8 +1765,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
# comm lives there).
|
||||
reduce_results=not self.all_reduce_fusion,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
# Reduce within the attn-TP group: the default full-TP collective
|
||||
# is the wrong group at attn_tp>1 (sums across DP groups) and
|
||||
# deadlocks idle DP ranks. Off under all_reduce_fusion: the fused
|
||||
@@ -2290,8 +2274,7 @@ class KimiK3MLAAttention(DeepseekV2AttentionMLA):
|
||||
projection_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=get_parallel().attn_tp_rank,
|
||||
tp_size=get_parallel().attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=f"{prefix}.g_proj",
|
||||
)
|
||||
# Output gate multiplies the TP-local attention output right
|
||||
|
||||
@@ -225,12 +225,11 @@ class KimiDeltaAttention(nn.Module):
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
# Group the weights are sharded over. Defaults to the global TP group,
|
||||
# which is what plain Kimi-Linear has always used.
|
||||
parallel_group = "attn_tp" if shard_on_attn_tp else "tp"
|
||||
if shard_on_attn_tp:
|
||||
self.shard_tp_size = self.attn_tp_size
|
||||
self.shard_tp_rank = get_parallel().attn_tp_rank
|
||||
else:
|
||||
self.shard_tp_size = self.tp_size
|
||||
self.shard_tp_rank = get_parallel().tp_rank
|
||||
self.hidden_size = hidden_size
|
||||
self.config = config
|
||||
self.head_dim = config.linear_attn_config["head_dim"]
|
||||
@@ -277,8 +276,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkvbfg_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.split_sizes = [3 * projection_size // self.shard_tp_size]
|
||||
if self.fuse_no_lora_beta:
|
||||
@@ -295,8 +293,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
self.num_heads,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
elif self.do_fuse_qkvbfg:
|
||||
# Fuse: q, k, v, beta (column parallel) + f_a, g_a (replicated)
|
||||
@@ -314,8 +311,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
self.fg_sizes,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fused_qkvbfg_a_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.split_sizes = [
|
||||
3 * projection_size // self.shard_tp_size,
|
||||
@@ -327,12 +323,10 @@ class KimiDeltaAttention(nn.Module):
|
||||
self.head_dim,
|
||||
projection_size,
|
||||
dtype=config.dtype,
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
else:
|
||||
# Unfused path: separate QKVParallelLinear
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_dim,
|
||||
@@ -340,8 +334,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
self.num_k_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
v_head_size=self.head_v_dim,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
@@ -360,8 +353,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.f_b_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.b_proj = ColumnParallelLinear(
|
||||
@@ -370,8 +362,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.b_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.g_a_proj = ReplicatedLinear(
|
||||
@@ -387,8 +378,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.g_b_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
self.dt_bias = nn.Parameter(
|
||||
@@ -405,8 +395,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
params_dtype=torch.float32,
|
||||
prefix=f"{prefix}.qkv_conv1d",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
# unsqueeze to fit conv1d weights shape into the linear weights shape.
|
||||
# Can't do this in `weight_loader` since it already exists in
|
||||
@@ -428,8 +417,7 @@ class KimiDeltaAttention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
tp_rank=self.shard_tp_rank,
|
||||
tp_size=self.shard_tp_size,
|
||||
parallel_group=parallel_group,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layer_boundary.residual.add_norm import NORM_QUANT_READOUT
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -89,8 +90,7 @@ class LlamaMLP(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
reduce_results: bool = True,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
use_dp_attention_reduce: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -100,8 +100,7 @@ class LlamaMLP(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("gate_up_proj", prefix),
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
@@ -110,8 +109,7 @@ class LlamaMLP(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("down_proj", prefix),
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
use_dp_attention_reduce=use_dp_attention_reduce,
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
|
||||
@@ -606,7 +606,6 @@ class MiMoV2Attention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -641,8 +640,7 @@ class MiMoV2Attention(nn.Module):
|
||||
v_head_size=self.v_head_dim,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
skip_block_quant_check=True,
|
||||
)
|
||||
@@ -652,8 +650,7 @@ class MiMoV2Attention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -52,6 +52,7 @@ from sglang.srt.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
ReplicatedParallelGroup,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
@@ -601,6 +602,9 @@ class MiniMaxM3Attention(nn.Module):
|
||||
self.idx_replica_size = attn_tp_size // self.idx_head_tp_size
|
||||
self.idx_head_rank = attn_tp_rank // self.idx_replica_size
|
||||
self.num_idx_heads = self.total_idx_heads // self.idx_head_tp_size
|
||||
index_parallel_group = ReplicatedParallelGroup(
|
||||
"attn_tp", self.idx_replica_size
|
||||
)
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
@@ -609,8 +613,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -620,8 +623,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
bias=False,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -642,8 +644,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
v_head_size=(0 if self.disable_index_value else self.idx_head_dim),
|
||||
tp_rank=self.idx_head_rank,
|
||||
tp_size=self.idx_head_tp_size,
|
||||
parallel_group=index_parallel_group,
|
||||
prefix=add_prefix("index_qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -658,8 +659,7 @@ class MiniMaxM3Attention(nn.Module):
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("index_o_proj", prefix),
|
||||
tp_rank=self.idx_head_rank,
|
||||
tp_size=self.idx_head_tp_size,
|
||||
parallel_group=index_parallel_group,
|
||||
)
|
||||
self.index_rotary_emb = self.rotary_emb
|
||||
|
||||
|
||||
@@ -70,8 +70,7 @@ class NemotronHMTPAttentionDecoderLayer(NemotronHAttentionDecoderLayer):
|
||||
output_size=config.hidden_size,
|
||||
bias=False,
|
||||
gather_output=not _dp_attn,
|
||||
tp_rank=get_parallel().attn_tp_rank if _dp_attn else None,
|
||||
tp_size=get_parallel().attn_tp_size if _dp_attn else None,
|
||||
parallel_group="attn_tp" if _dp_attn else "tp",
|
||||
params_dtype=(
|
||||
config.dtype if hasattr(config, "dtype") else torch.bfloat16
|
||||
),
|
||||
@@ -152,8 +151,7 @@ class NemotronHMTPMoEDecoderLayer(NemotronHMoEDecoderLayer):
|
||||
output_size=config.hidden_size,
|
||||
bias=False,
|
||||
gather_output=not _dp_attn,
|
||||
tp_rank=get_parallel().attn_tp_rank if _dp_attn else None,
|
||||
tp_size=get_parallel().attn_tp_size if _dp_attn else None,
|
||||
parallel_group="attn_tp" if _dp_attn else "tp",
|
||||
params_dtype=(
|
||||
config.dtype if hasattr(config, "dtype") else torch.bfloat16
|
||||
),
|
||||
|
||||
@@ -72,9 +72,12 @@ from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
# Layers - Linear
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedParallelGroup,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
@@ -341,8 +344,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
output_size=self.conv_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("conv1d", prefix),
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
@@ -354,8 +356,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
value_dim=self.value_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("in_proj_qkvz", prefix),
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.in_proj_ba = self.create_ba_proj(
|
||||
@@ -363,8 +364,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
num_v_heads=self.num_v_heads,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("in_proj_ba", prefix),
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
# Override weight loaders for packed checkpoint format.
|
||||
@@ -462,8 +462,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
input_is_parallel=True,
|
||||
reduce_results=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("out_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -578,8 +577,8 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
value_dim: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> MergedColumnParallelLinear:
|
||||
return MergedColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
@@ -587,8 +586,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
def create_ba_proj(
|
||||
@@ -597,8 +595,8 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
num_v_heads: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> MergedColumnParallelLinear:
|
||||
# Qwen3.5 has separate in_proj_b and in_proj_a weights in the
|
||||
# checkpoint, which are loaded into the fused in_proj_ba parameter
|
||||
@@ -609,8 +607,7 @@ class Qwen3_5GatedDeltaNet(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
def fix_query_key_value_ordering(
|
||||
@@ -1187,8 +1184,10 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
# A Qwen3.5 draft is rewritten to the MTP arch (model_config._config_draft_model),
|
||||
# so is_nextn marks it. Drafts are TP-sharded and do not replicate KV under DCP.
|
||||
dcp_size = 1 if is_nextn else get_parallel().attn_dcp_size
|
||||
self.kv_tp_size = self.attn_tp_size // dcp_size
|
||||
self.kv_tp_rank = self.attn_tp_rank // dcp_size
|
||||
kv_parallel_group = ReplicatedParallelGroup("attn_tp", dcp_size)
|
||||
self.kv_tp_rank, self.kv_tp_size = resolve_linear_parallel_group(
|
||||
kv_parallel_group
|
||||
)
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
assert self.total_num_heads % self.attn_tp_size == 0
|
||||
self.num_heads = self.total_num_heads // self.attn_tp_size
|
||||
@@ -1236,10 +1235,8 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
kv_tp_rank=self.kv_tp_rank,
|
||||
kv_tp_size=self.kv_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
kv_parallel_group=kv_parallel_group,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -1249,8 +1246,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=False,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -132,8 +133,7 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
output_size=self.conv_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("conv1d", prefix),
|
||||
)
|
||||
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
|
||||
@@ -145,8 +145,7 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
value_dim=self.value_dim,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("in_proj_qkvz", prefix),
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
self.in_proj_ba = MergedColumnParallelLinear(
|
||||
@@ -155,8 +154,7 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("in_proj_ba", prefix),
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
# Override weight_loader for packed checkpoint format.
|
||||
@@ -231,8 +229,7 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
quant_config=quant_config,
|
||||
input_is_parallel=True,
|
||||
reduce_results=False,
|
||||
tp_rank=self.attn_tp_rank,
|
||||
tp_size=self.attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("out_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -319,8 +316,8 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
value_dim: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
*,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> MergedColumnParallelLinear:
|
||||
return MergedColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
@@ -328,8 +325,7 @@ class Qwen3GatedDeltaNet(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
|
||||
def fix_query_key_value_ordering(
|
||||
|
||||
@@ -191,7 +191,6 @@ class SDARMoeAttention(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
@@ -226,8 +225,7 @@ class SDARMoeAttention(nn.Module):
|
||||
bias=getattr(config, "attention_bias", False),
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
|
||||
@@ -189,7 +189,6 @@ class Step3TextAttention(nn.Module):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self.total_num_heads = num_heads
|
||||
@@ -219,8 +218,7 @@ class Step3TextAttention(nn.Module):
|
||||
[self.q_size, self.kv_size, self.kv_size],
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=0, # In fact, we need a MergedReplicatedLinear
|
||||
tp_size=1,
|
||||
parallel_group="replicated",
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -229,8 +227,7 @@ class Step3TextAttention(nn.Module):
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
@@ -242,8 +239,7 @@ class Step3TextAttention(nn.Module):
|
||||
self.head_dim * self.total_num_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("wq", prefix),
|
||||
)
|
||||
self.rotary_emb = get_rope(
|
||||
|
||||
@@ -1250,8 +1250,7 @@ class _XllmMoVAAttentionBase(nn.Module):
|
||||
self.total_num_heads * self.head_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("q_proj", prefix),
|
||||
)
|
||||
self.k_proj = ColumnParallelLinear(
|
||||
@@ -1259,8 +1258,7 @@ class _XllmMoVAAttentionBase(nn.Module):
|
||||
self.total_num_kv_heads * self.head_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("k_proj", prefix),
|
||||
)
|
||||
self.gate_proj = ColumnParallelLinear(
|
||||
@@ -1268,8 +1266,7 @@ class _XllmMoVAAttentionBase(nn.Module):
|
||||
self.total_num_heads * self.head_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("gate_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
@@ -1277,8 +1274,7 @@ class _XllmMoVAAttentionBase(nn.Module):
|
||||
config.hidden_size,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
reduce_results=False,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
@@ -1346,8 +1342,7 @@ class XllmGatedAttention(_XllmMoVAAttentionBase):
|
||||
self.total_num_kv_heads * self.head_dim,
|
||||
bias=False,
|
||||
quant_config=None,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
prefix=add_prefix("v_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -1394,8 +1389,7 @@ class XllmMoVAAttention(_XllmMoVAAttentionBase):
|
||||
self.num_values,
|
||||
config.hidden_size,
|
||||
self.total_num_kv_heads * self.head_dim,
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group="attn_tp",
|
||||
)
|
||||
|
||||
def _project_value(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -62,8 +62,10 @@ from sglang.srt.layers.layer_boundary.residual.add_norm import PLAIN_ADD
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearParallelGroup,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
resolve_linear_parallel_group,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
|
||||
@@ -432,8 +434,7 @@ class CCA(nn.Module):
|
||||
layer_id: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
tp_rank: Optional[int] = None,
|
||||
tp_size: Optional[int] = None,
|
||||
parallel_group: LinearParallelGroup = "tp",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -446,12 +447,7 @@ class CCA(nn.Module):
|
||||
self.padding1 = self.cca_time1 - 1
|
||||
self.total_padding = self.padding0 + self.padding1
|
||||
|
||||
if tp_rank is None:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
if tp_size is None:
|
||||
tp_size = get_parallel().tp_size
|
||||
self.tp_rank = int(tp_rank)
|
||||
self.tp_size = int(tp_size)
|
||||
self.tp_rank, self.tp_size = resolve_linear_parallel_group(parallel_group)
|
||||
|
||||
# Full (global) head counts retained for weight loading and shape asserts.
|
||||
self.num_q_heads_full = int(cca_num_q_heads)
|
||||
@@ -506,8 +502,7 @@ class CCA(nn.Module):
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_q", prefix),
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
self.linear_k = ColumnParallelLinear(
|
||||
self.hidden_size,
|
||||
@@ -516,8 +511,7 @@ class CCA(nn.Module):
|
||||
gather_output=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("linear_k", prefix),
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
parallel_group=parallel_group,
|
||||
)
|
||||
else:
|
||||
self.linear_q = ReplicatedLinear(
|
||||
|
||||
Reference in New Issue
Block a user