[Refactor] Load auxiliary and expert checkpoints using owner partitions (#42589)
This commit is contained in:
@@ -431,6 +431,7 @@ class ScaledActivation(nn.Module):
|
||||
self.act = act_module
|
||||
self.input_is_parallel = input_is_parallel
|
||||
if input_is_parallel:
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
tp_size = get_parallel().tp_size
|
||||
intermediate_size_per_partition = divide(intermediate_size, tp_size)
|
||||
else:
|
||||
@@ -448,9 +449,8 @@ class ScaledActivation(nn.Module):
|
||||
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
|
||||
param_data = param.data
|
||||
if self.input_is_parallel:
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_size = param_data.shape[0]
|
||||
start_idx = tp_rank * shard_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
+1
-2
@@ -26,7 +26,6 @@ from sglang.srt.layers.quantization.utils import (
|
||||
per_tensor_dequantize,
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_bool_env_var, is_hip, set_weight_attrs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -98,7 +97,7 @@ class CompressedTensorsW8A8Fp8MoE(CompressedTensorsMoEScheme):
|
||||
if self.block_quant:
|
||||
assert self.weight_block_size is not None
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
tp_size = get_parallel().tp_size
|
||||
tp_size = layer.moe_tp_size
|
||||
block_n, block_k = (
|
||||
self.weight_block_size[0],
|
||||
self.weight_block_size[1],
|
||||
|
||||
@@ -5,7 +5,6 @@ import re
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_cuda
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -61,33 +60,12 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
|
||||
loaded_params: set[str] = set()
|
||||
mxfp4_block = 32
|
||||
|
||||
moe_tp_rank = get_parallel().moe_tp_rank
|
||||
moe_tp_size = get_parallel().moe_tp_size
|
||||
moe_ep_rank = get_parallel().moe_ep_rank
|
||||
moe_ep_size = get_parallel().moe_ep_size
|
||||
|
||||
intermediate_size = model.config.intermediate_size
|
||||
assert intermediate_size % mxfp4_block == 0, (
|
||||
f"{intermediate_size=} must be divisible by {mxfp4_block=}"
|
||||
)
|
||||
intermediate_size_block = intermediate_size // mxfp4_block
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(intermediate_size_block / moe_tp_size)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert model.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
for name, weight in weights:
|
||||
# Quark stores experts separately as
|
||||
# `experts.{N}.{gate_up_proj|down_proj}.{suffix}`; pull the
|
||||
@@ -96,6 +74,30 @@ def _load_gptoss_quark_expert_weights(model, weights, quark_expert_pat):
|
||||
if m is None:
|
||||
continue
|
||||
prefix, expert_str, proj, suffix = m.groups()
|
||||
experts = model.get_submodule(prefix)
|
||||
moe_tp_rank = experts.moe_tp_rank
|
||||
moe_tp_size = experts.moe_tp_size
|
||||
moe_ep_rank = experts.moe_ep_rank
|
||||
moe_ep_size = experts.moe_ep_size
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert model.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_local_experts = model.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
global_expert_id = int(expert_str)
|
||||
if global_expert_id < moe_ep_rank_start or global_expert_id >= moe_ep_rank_end:
|
||||
continue
|
||||
|
||||
@@ -386,17 +386,17 @@ class BailingGroupRMSNormGate(RMSNormGated):
|
||||
dtype=dtype,
|
||||
activation="sigmoid",
|
||||
)
|
||||
self.tp_size = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.weight.weight_loader = self.weight_loader
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
) -> None:
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard_size = loaded_weight.shape[0] // self.tp_size
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].contiguous())
|
||||
return
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ class LayerNorm(nn.Module):
|
||||
def __init__(self, param_shape=None, eps=1e-5):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(param_shape))
|
||||
self.tp_rank = get_parallel().tp_rank if self.weight.dim() != 1 else 0
|
||||
self.variance_epsilon = eps
|
||||
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
|
||||
|
||||
@@ -94,12 +95,11 @@ class LayerNorm(nn.Module):
|
||||
return hidden_states, residuals
|
||||
|
||||
def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
|
||||
tp_rank = get_parallel().tp_rank
|
||||
shard_dim = 0 if param.dim() != 1 else None
|
||||
param_data = param.data
|
||||
if shard_dim is not None:
|
||||
shard_size = param_data.shape[shard_dim]
|
||||
start_idx = tp_rank * shard_size
|
||||
start_idx = self.tp_rank * shard_size
|
||||
loaded_weight = loaded_weight.narrow(shard_dim, start_idx, shard_size)
|
||||
assert param_data.shape == loaded_weight.shape
|
||||
param_data.copy_(loaded_weight)
|
||||
|
||||
@@ -103,6 +103,7 @@ class DbrxExperts(nn.Module):
|
||||
):
|
||||
super().__init__()
|
||||
self.tp_size = get_parallel().tp_size
|
||||
self.tp_rank = get_parallel().tp_rank
|
||||
self.num_total_experts = config.ffn_config.moe_num_experts
|
||||
self.top_k = config.ffn_config.moe_top_k
|
||||
self.d_model = config.d_model
|
||||
@@ -155,10 +156,9 @@ class DbrxExperts(nn.Module):
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
|
||||
):
|
||||
tp_rank = get_parallel().tp_rank
|
||||
param_data = param.data
|
||||
shard_size = self.intermediate_size
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
# DBRX uses GLU for each experts.
|
||||
# GLU has 3 linear layers: w1, v1 and w2.
|
||||
if weight_name.endswith("w1"):
|
||||
|
||||
@@ -942,11 +942,6 @@ class GptOssForCausalLM(nn.Module):
|
||||
loaded_params: set[str] = set()
|
||||
mxfp4_block = 32
|
||||
|
||||
moe_tp_rank = get_parallel().moe_tp_rank
|
||||
moe_tp_size = get_parallel().moe_tp_size
|
||||
moe_ep_rank = get_parallel().moe_ep_rank
|
||||
moe_ep_size = get_parallel().moe_ep_size
|
||||
|
||||
intermediate_size = self.config.intermediate_size
|
||||
original_intermediate_size = getattr(
|
||||
self.config, "original_intermediate_size", intermediate_size
|
||||
@@ -956,30 +951,49 @@ class GptOssForCausalLM(nn.Module):
|
||||
)
|
||||
intermediate_size_block = intermediate_size // mxfp4_block
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert self.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_global_experts = self.config.num_local_experts
|
||||
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size, original_intermediate_size
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
weight_device = next(iter(params_dict.values())).device
|
||||
|
||||
for name, weight in weights:
|
||||
weight = weight.to(weight_device)
|
||||
|
||||
if not any(
|
||||
suffix in name
|
||||
for suffix in (
|
||||
"gate_up_proj_blocks",
|
||||
"down_proj_blocks",
|
||||
"gate_up_proj_scales",
|
||||
"down_proj_scales",
|
||||
"gate_up_proj_bias",
|
||||
"down_proj_bias",
|
||||
)
|
||||
):
|
||||
continue
|
||||
experts = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
moe_tp_rank = experts.moe_tp_rank
|
||||
moe_tp_size = experts.moe_tp_size
|
||||
moe_ep_rank = experts.moe_ep_rank
|
||||
moe_ep_size = experts.moe_ep_size
|
||||
|
||||
per_rank_intermediate_size_block = math.ceil(
|
||||
intermediate_size_block / moe_tp_size
|
||||
)
|
||||
|
||||
per_rank_intermediate_size = per_rank_intermediate_size_block * mxfp4_block
|
||||
|
||||
# Calculate common slicing bounds for current rank
|
||||
assert self.config.num_local_experts % moe_ep_size == 0
|
||||
moe_num_global_experts = self.config.num_local_experts
|
||||
moe_num_local_experts = self.config.num_local_experts // moe_ep_size
|
||||
|
||||
moe_tp_rank_start = moe_tp_rank * per_rank_intermediate_size
|
||||
moe_tp_rank_end = min(
|
||||
(moe_tp_rank + 1) * per_rank_intermediate_size,
|
||||
original_intermediate_size,
|
||||
)
|
||||
|
||||
moe_ep_rank_start = moe_ep_rank * moe_num_local_experts
|
||||
moe_ep_rank_end = (moe_ep_rank + 1) * moe_num_local_experts
|
||||
|
||||
if "gate_up_proj_blocks" in name:
|
||||
# Handle MLP gate and up projection weights
|
||||
new_name = name.replace("gate_up_proj_blocks", "w13_weight")
|
||||
@@ -1240,8 +1254,10 @@ class GptOssForCausalLM(nn.Module):
|
||||
weight_loader = param.weight_loader
|
||||
if "bias" not in name:
|
||||
loaded_weight = loaded_weight.transpose(-2, -1)
|
||||
if "w2_weight_bias" in name and get_parallel().moe_tp_rank != 0:
|
||||
loaded_weight = loaded_weight.zero_()
|
||||
if "w2_weight_bias" in name:
|
||||
experts = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
if experts.moe_tp_rank != 0:
|
||||
loaded_weight = loaded_weight.zero_()
|
||||
|
||||
weight_loader(
|
||||
param,
|
||||
|
||||
@@ -100,29 +100,20 @@ STACKED_DENSE_PARAMS_MAPPING = [
|
||||
("down_proj", "down_proj", None),
|
||||
]
|
||||
|
||||
# Online RL weight-sync streams routed experts one at a time as FULL (unsharded)
|
||||
# per-expert tensors named `...mlp.experts.{j}.gate_proj/up_proj/down_proj.weight`.
|
||||
# Disk checkpoints only ever carry the fused w13_weight/w2_weight, so this pattern
|
||||
# never fires on the ordinary loading path.
|
||||
# Online RL updates use full per-expert gate/up/down tensors; disk checkpoints
|
||||
# contain fused w13/w2 weights and do not match this pattern.
|
||||
_PER_EXPERT_WEIGHT_RE = re.compile(
|
||||
r"^(?P<pfx>.+\.mlp\.experts)\.(?P<eid>\d+)\.(?P<proj>gate_proj|up_proj|down_proj)\.weight$"
|
||||
)
|
||||
|
||||
|
||||
def _shard_full_to_local(
|
||||
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int
|
||||
loaded_weight: torch.Tensor, dst: torch.Tensor, dim: int, *, tp_rank: int
|
||||
) -> torch.Tensor:
|
||||
"""Slice a FULL (unsharded) per-expert weight to this MoE-TP rank's shard along `dim`.
|
||||
|
||||
The online weight-sync ships full per-expert tensors (parallelism-agnostic HF
|
||||
layout); sglang owns its own MoE-TP sharding, so narrow here per
|
||||
get_parallel().moe_tp_rank. With TP1 the dims already match and this is the
|
||||
identity, so the ordinary path is byte-for-byte unchanged.
|
||||
"""
|
||||
"""Shard full HF per-expert updates with the owning module's MoE TP rank."""
|
||||
if loaded_weight.shape[dim] == dst.shape[dim]:
|
||||
return loaded_weight
|
||||
rank = get_parallel().moe_tp_rank
|
||||
return loaded_weight.narrow(dim, rank * dst.shape[dim], dst.shape[dim])
|
||||
return loaded_weight.narrow(dim, tp_rank * dst.shape[dim], dst.shape[dim])
|
||||
|
||||
|
||||
KV_REPLICATED_SUFFIXES = (
|
||||
@@ -1238,12 +1229,10 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
return False
|
||||
param = params_dict[name]
|
||||
if loaded_weight.shape != param.shape:
|
||||
# shared experts shard over the full tp group; routed over moe_tp
|
||||
tp_rank = (
|
||||
get_parallel().tp_rank
|
||||
if ".shared_experts" in name
|
||||
else get_parallel().moe_tp_rank
|
||||
)
|
||||
# Both routed and shared modules own their constructed TP layout.
|
||||
module = self.get_submodule(name.rsplit(".", 1)[0])
|
||||
module = getattr(module, "base_layer", module)
|
||||
tp_rank = module.moe_tp_rank
|
||||
for dim in range(loaded_weight.ndim):
|
||||
if loaded_weight.shape[dim] == param.shape[dim]:
|
||||
continue
|
||||
@@ -1297,10 +1286,8 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
hold only its contiguous slice (the fused loader shards the intermediate dim only).
|
||||
No-op when EP is off or for replicated shared-expert tensors.
|
||||
"""
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
if (
|
||||
ep_size <= 1
|
||||
or ".experts." not in name
|
||||
".experts." not in name
|
||||
# per-expert RL sync tensors do their own EP remap in _load_per_expert_param;
|
||||
# a full per-expert tensor whose dim 0 happens to equal n_routed_experts must
|
||||
# not be pre-narrowed here.
|
||||
@@ -1309,8 +1296,17 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
or loaded_weight.shape[0] != self.text_config.n_routed_experts
|
||||
):
|
||||
return loaded_weight
|
||||
local = self.text_config.n_routed_experts // ep_size
|
||||
start = get_parallel().moe_ep_rank * local
|
||||
prefix = name.split(".experts.", 1)[0] + ".experts"
|
||||
try:
|
||||
module = self.get_submodule(prefix)
|
||||
except AttributeError:
|
||||
# Checkpoints may contain a layer absent from this model.
|
||||
return loaded_weight
|
||||
module = getattr(module, "base_layer", module)
|
||||
if module.moe_ep_size <= 1:
|
||||
return loaded_weight
|
||||
local = self.text_config.n_routed_experts // module.moe_ep_size
|
||||
start = module.moe_ep_rank * local
|
||||
return loaded_weight.narrow(0, start, local).contiguous()
|
||||
|
||||
def _load_fused_moe_param(
|
||||
@@ -1403,10 +1399,10 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
f"per-expert RL weight-sync does not support the trtllm MoE layout ({target}); "
|
||||
"serve RL rollouts with the triton MoE runner"
|
||||
)
|
||||
ep_size = get_parallel().moe_ep_size
|
||||
ep_size = moe.moe_ep_size
|
||||
if ep_size > 1:
|
||||
local = self.text_config.n_routed_experts // ep_size
|
||||
first = get_parallel().moe_ep_rank * local
|
||||
first = moe.moe_ep_rank * local
|
||||
if not (first <= eid < first + local):
|
||||
loaded_params.add(target) # another rank owns this expert
|
||||
return True
|
||||
@@ -1415,7 +1411,9 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
dst = params_dict[target].data[
|
||||
eid
|
||||
] # [H, I_local]; shard intermediate (dim 1)
|
||||
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=1))
|
||||
dst.copy_(
|
||||
_shard_full_to_local(loaded_weight, dst, dim=1, tp_rank=moe.moe_tp_rank)
|
||||
)
|
||||
else:
|
||||
w13 = params_dict[target].data[
|
||||
eid
|
||||
@@ -1429,7 +1427,9 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
dst = w13[idx * half : (idx + 1) * half] # contiguous [gate || up]
|
||||
else:
|
||||
dst = w13[idx::2] # Inkling-interleaved rows
|
||||
dst.copy_(_shard_full_to_local(loaded_weight, dst, dim=0))
|
||||
dst.copy_(
|
||||
_shard_full_to_local(loaded_weight, dst, dim=0, tp_rank=moe.moe_tp_rank)
|
||||
)
|
||||
loaded_params.add(target)
|
||||
return True
|
||||
|
||||
@@ -1590,18 +1590,21 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
continue
|
||||
|
||||
if ".experts.w13_weight" in name:
|
||||
experts = modules_dict.get(name.rsplit(".", 1)[0])
|
||||
experts = getattr(experts, "base_layer", experts)
|
||||
# bf16 routed layers run the stock FusedMoE forward (not moe_tp_forward)
|
||||
# under --enable-lora, or natively on trtllm_routed for UNQUANTIZED
|
||||
# checkpoints: de-interleave per moe_tp block for the stock weight prep.
|
||||
if (
|
||||
loaded_weight.dtype != torch.uint8
|
||||
experts is not None
|
||||
and loaded_weight.dtype != torch.uint8
|
||||
and self.text_config.inference_moe_w13_interleaved
|
||||
and (
|
||||
lora_compatible_layout_enabled()
|
||||
or bf16_routed_uses_stock_fused_moe(self.quant_config)
|
||||
)
|
||||
):
|
||||
tp = get_parallel().moe_tp_size
|
||||
tp = experts.moe_tp_size
|
||||
n_e, two_f, hid = loaded_weight.shape
|
||||
loaded_weight = deinterleave_gate_up(
|
||||
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
|
||||
@@ -1625,16 +1628,19 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
):
|
||||
continue
|
||||
if ".shared_experts.shared_w13_weight" in name:
|
||||
experts = modules_dict.get(name.rsplit(".", 1)[0])
|
||||
experts = getattr(experts, "base_layer", experts)
|
||||
# InklingSharedFusedMoE's bf16 path needs contiguous [gate||up] w13 for the
|
||||
# SRT runner's silu_and_mul, unlike the interleaved bmm/moe_tp_forward paths.
|
||||
# Per-rank blocks are sized by the FULL tp group (InklingSharedFusedMoE always
|
||||
# shards over it at EP=1), NOT moe_tp (= tp/ep, wrong under --ep-size > 1).
|
||||
if (
|
||||
loaded_weight.dtype != torch.uint8
|
||||
experts is not None
|
||||
and loaded_weight.dtype != torch.uint8
|
||||
and self.text_config.inference_moe_w13_interleaved
|
||||
and use_inkling_shared_fused_moe()
|
||||
):
|
||||
tp = get_parallel().tp_size
|
||||
tp = experts.moe_tp_size
|
||||
n_e, two_f, hid = loaded_weight.shape
|
||||
loaded_weight = deinterleave_gate_up(
|
||||
loaded_weight.view(n_e, tp, two_f // tp, hid), dim=2
|
||||
|
||||
@@ -1739,7 +1739,7 @@ class KimiK3DeltaAttention(nn.Module):
|
||||
def _a_log_weight_loader(
|
||||
param: torch.Tensor, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
tp_rank = self.attn_tp_rank
|
||||
shard_size = param.data.shape[2] # local_num_heads
|
||||
start_idx = tp_rank * shard_size
|
||||
|
||||
|
||||
@@ -173,6 +173,7 @@ class MultiHeadRMSNorm(nn.Module):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.tp_world = get_parallel().attn_tp_size
|
||||
self.tp_rank = get_parallel().attn_tp_rank
|
||||
self.num_heads = num_heads
|
||||
self.num_heads_per_tp = num_heads // self.tp_world
|
||||
self.head_dim = head_dim
|
||||
@@ -183,16 +184,13 @@ class MultiHeadRMSNorm(nn.Module):
|
||||
self.apply_layernorm_1p = apply_layernorm_1p
|
||||
self.variance_epsilon = eps
|
||||
|
||||
@staticmethod
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
) -> None:
|
||||
tp_world = get_parallel().attn_tp_size
|
||||
tp_rank = get_parallel().attn_tp_rank
|
||||
|
||||
shard_size = loaded_weight.shape[0] // tp_world
|
||||
shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
|
||||
shard_size = loaded_weight.shape[0] // self.tp_world
|
||||
shard = slice(self.tp_rank * shard_size, (self.tp_rank + 1) * shard_size)
|
||||
param.data.copy_(loaded_weight[shard].reshape_as(param))
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user