[Bugfix] Fix PP tied embeddings weight loading for qwen3.5 4B dense model (#21347)
This commit is contained in:
@@ -1384,6 +1384,17 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
|||||||
name = name.replace(r"model.language_model.", r"model.")
|
name = name.replace(r"model.language_model.", r"model.")
|
||||||
if ".self_attn." in name:
|
if ".self_attn." in name:
|
||||||
name = name.replace(".self_attn", "")
|
name = name.replace(".self_attn", "")
|
||||||
|
if (
|
||||||
|
self.config.tie_word_embeddings
|
||||||
|
and self.pp_group.is_last_rank
|
||||||
|
and "model.embed_tokens.weight" in name
|
||||||
|
):
|
||||||
|
if "lm_head.weight" in params_dict:
|
||||||
|
lm_head_param = params_dict["lm_head.weight"]
|
||||||
|
weight_loader = getattr(
|
||||||
|
lm_head_param, "weight_loader", default_weight_loader
|
||||||
|
)
|
||||||
|
weight_loader(lm_head_param, loaded_weight)
|
||||||
layer_id = get_layer_id(name)
|
layer_id = get_layer_id(name)
|
||||||
if (
|
if (
|
||||||
layer_id is not None
|
layer_id is not None
|
||||||
@@ -1549,6 +1560,17 @@ class Qwen3_5MoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
|
|||||||
name = name.replace(r"model.language_model.", r"model.")
|
name = name.replace(r"model.language_model.", r"model.")
|
||||||
if ".self_attn." in name:
|
if ".self_attn." in name:
|
||||||
name = name.replace(".self_attn", "")
|
name = name.replace(".self_attn", "")
|
||||||
|
if (
|
||||||
|
self.config.tie_word_embeddings
|
||||||
|
and self.pp_group.is_last_rank
|
||||||
|
and "model.embed_tokens.weight" in name
|
||||||
|
):
|
||||||
|
if "lm_head.weight" in params_dict:
|
||||||
|
lm_head_param = params_dict["lm_head.weight"]
|
||||||
|
weight_loader = getattr(
|
||||||
|
lm_head_param, "weight_loader", default_weight_loader
|
||||||
|
)
|
||||||
|
weight_loader(lm_head_param, loaded_weight)
|
||||||
|
|
||||||
layer_id = get_layer_id(name)
|
layer_id = get_layer_id(name)
|
||||||
if (
|
if (
|
||||||
|
|||||||
Reference in New Issue
Block a user