[Refactor] Stop passing models' layers the placement they already read (#41815)
This commit is contained in:
@@ -779,8 +779,6 @@ class MqaAttentionBase(nn.Module):
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
prefix: str,
|
||||
*,
|
||||
attn_tp_rank: Optional[int] = None,
|
||||
attn_tp_size: Optional[int] = None,
|
||||
compress_ratio: Optional[int] = None,
|
||||
fuse_wqa_wkv: Optional[bool] = None,
|
||||
wo_a_fp8: Optional[bool] = None,
|
||||
@@ -791,11 +789,8 @@ class MqaAttentionBase(nn.Module):
|
||||
super().__init__()
|
||||
self.is_dsv41 = getattr(config, "model_type", None) == "deepseek_v41"
|
||||
self.dsa_enable_prefill_cp = is_dsa_enable_prefill_cp()
|
||||
if attn_tp_rank is None or attn_tp_size is None:
|
||||
attn_tp_rank = get_parallel().attn_tp_rank
|
||||
attn_tp_size = get_parallel().attn_tp_size
|
||||
self.attn_tp_rank: int = attn_tp_rank
|
||||
self.attn_tp_size: int = attn_tp_size
|
||||
self.attn_tp_rank: int = get_parallel().attn_tp_rank
|
||||
self.attn_tp_size: int = get_parallel().attn_tp_size
|
||||
|
||||
self.layer_id = layer_id
|
||||
self.dim = config.hidden_size
|
||||
|
||||
@@ -109,8 +109,6 @@ class DSparkAttention(MqaAttentionBase):
|
||||
layer_id,
|
||||
quant_config,
|
||||
prefix,
|
||||
attn_tp_rank=get_parallel().attn_tp_rank,
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
compress_ratio=0,
|
||||
fuse_wqa_wkv=False,
|
||||
wo_a_fp8=False,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import logging
|
||||
from array import array
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -393,7 +392,6 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
super().__init__()
|
||||
head_shard_size = get_parallel().attn_tp_size
|
||||
head_shard_rank = get_parallel().attn_tp_rank
|
||||
_head_shard_rank_getter = partial(getattr, get_parallel(), "attn_tp_rank")
|
||||
|
||||
self.hidden_size = hidden_size
|
||||
self.config = config
|
||||
@@ -532,7 +530,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
|
||||
set_weight_attrs(
|
||||
self.dt_bias,
|
||||
{"weight_loader": sharded_weight_loader(0, _head_shard_rank_getter)},
|
||||
{"weight_loader": sharded_weight_loader(0)},
|
||||
)
|
||||
|
||||
self.qkv_conv1d = MergedColumnParallelLinear(
|
||||
@@ -553,7 +551,7 @@ class Glm5NextLinearAttention(nn.Module):
|
||||
)
|
||||
set_weight_attrs(
|
||||
self.A_log,
|
||||
{"weight_loader": sharded_weight_loader(2, _head_shard_rank_getter)},
|
||||
{"weight_loader": sharded_weight_loader(2)},
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(
|
||||
|
||||
@@ -331,21 +331,20 @@ class Grok1Attention(nn.Module):
|
||||
self.config = config
|
||||
self.layer_id = layer_id
|
||||
self.hidden_size = hidden_size
|
||||
attn_tp_rank = get_parallel().tp_rank
|
||||
attn_tp_size = get_parallel().tp_size
|
||||
tp_size = get_parallel().tp_size
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % attn_tp_size == 0
|
||||
self.num_heads = self.total_num_heads // attn_tp_size
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= attn_tp_size:
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert self.total_num_kv_heads % attn_tp_size == 0
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert attn_tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // attn_tp_size)
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.head_dim = getattr(config, "head_dim", 128)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
@@ -362,8 +361,6 @@ class Grok1Attention(nn.Module):
|
||||
self.total_num_kv_heads,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
load_presharded_attn=self.load_presharded_attn,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
@@ -373,8 +370,6 @@ class Grok1Attention(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
use_presharded_weights=self.load_presharded_attn,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
)
|
||||
|
||||
@@ -294,8 +294,6 @@ class IQuestQ1Attention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
tp_rank=attn_tp_rank,
|
||||
tp_size=attn_tp_size,
|
||||
kv_tp_rank=attn_tp_rank,
|
||||
kv_tp_size=attn_tp_size,
|
||||
prefix=add_prefix("qkv_proj", prefix),
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
|
||||
@@ -829,7 +829,6 @@ class ZayaAttention(nn.Module):
|
||||
# divisible by tp_size; the KV-replicated GQA-TP variant (tp_size >
|
||||
# num_k_heads) is intentionally rejected with a clear error message
|
||||
# because both per-K-head paths assume each rank holds whole K heads.
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.tp_size = get_parallel().tp_size
|
||||
# The head split, the ``o_proj`` RowParallel all-reduce, and the
|
||||
# RadixAttention KV cache are all organized on the *global* TP group,
|
||||
@@ -873,8 +872,6 @@ class ZayaAttention(nn.Module):
|
||||
layer_id=layer_id,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("qkv", prefix),
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
# RowParallel o_proj: per-rank input is the rank's q heads, full
|
||||
@@ -887,8 +884,6 @@ class ZayaAttention(nn.Module):
|
||||
reduce_results=True,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("o_proj", prefix),
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
)
|
||||
|
||||
rope_theta = float(getattr(config, "rope_theta", 1_000_000.0))
|
||||
|
||||
Reference in New Issue
Block a user