[NPU] [Bugfix] Fix rope sin cos pre-calculation for Qwen3.5 during Pipeline Parallelism (#29753)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Артем Савкин <58187114+OrangeRedeng@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Артем Савкин
parent
01ac2cf36d
commit
fa66e7d816
@@ -559,6 +559,11 @@ class InternS2MobiusAttentionDecoderLayer(
|
||||
self.rope_theta, rope_scaling = get_rope_config(config)
|
||||
self.partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
|
||||
self.layer_id = layer_id
|
||||
# Inherited forward_prepare_npu reads this. Mobius is never PP-split and its
|
||||
# schedule is layer_types, not full_attention_interval.
|
||||
self.refreshes_rope_cos_sin = layer_id == config.layer_types.index(
|
||||
"full_attention"
|
||||
)
|
||||
if rope_scaling and not ("rope_type" in rope_scaling or "type" in rope_scaling):
|
||||
rope_scaling = None
|
||||
self.attn_output_gate = getattr(config, "attn_output_gate", True)
|
||||
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.configs.qwen3_5 import (
|
||||
)
|
||||
|
||||
# Distributed
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
@@ -1227,6 +1228,31 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
|
||||
return ffn_exit.finish(hidden_states)
|
||||
|
||||
|
||||
def _refreshes_rope_cos_sin(
|
||||
config: Qwen3_5TextConfig, layer_id: int, is_nextn: bool
|
||||
) -> bool:
|
||||
"""rotary_emb is shared by all layers, so its per-forward cos/sin is refreshed
|
||||
once, at the first full-attention layer this PP stage owns.
|
||||
|
||||
layer_id must be a full-attention layer; linear-attention layers near the
|
||||
stage start would also pass the check below. Only Qwen3_5AttentionDecoderLayer
|
||||
calls this, and it is built only for "attention" entries of layers_block_type.
|
||||
"""
|
||||
# Drafts keep the unsplit numbering.
|
||||
stage_start = 0
|
||||
if not is_nextn:
|
||||
stage_start, _ = get_pp_indices(
|
||||
config.num_hidden_layers,
|
||||
get_parallel().pp_rank,
|
||||
get_parallel().pp_size,
|
||||
)
|
||||
owned_by_stage = layer_id >= stage_start
|
||||
# Full-attention layers are interval apart, so only the stage's first one
|
||||
# lies within interval of its start.
|
||||
first_in_stage = layer_id < stage_start + config.full_attention_interval
|
||||
return owned_by_stage and first_in_stage
|
||||
|
||||
|
||||
class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
"""Qwen3.5 Decoder Layer with Full Attention."""
|
||||
|
||||
@@ -1247,6 +1273,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
self.hidden_size = config.hidden_size
|
||||
self.attn_tp_rank = get_parallel().attn_tp_rank
|
||||
self.attn_tp_size = get_parallel().attn_tp_size
|
||||
self.refreshes_rope_cos_sin = _refreshes_rope_cos_sin(
|
||||
config, layer_id, is_nextn
|
||||
)
|
||||
# A Qwen3.5 draft is rewritten to the MTP arch (model_config._config_draft_model),
|
||||
# so is_nextn marks it. Drafts are TP-sharded and do not replicate KV under DCP.
|
||||
dcp_size = 1 if is_nextn else get_parallel().attn_dcp_size
|
||||
@@ -1511,8 +1540,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
|
||||
|
||||
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
# Calculate first full attention layer ID based on config
|
||||
if self.attn.layer_id == (self.config.full_attention_interval - 1):
|
||||
if self.refreshes_rope_cos_sin:
|
||||
self.rotary_emb.get_cos_sin_with_position(positions)
|
||||
|
||||
q, k, v, gate = split_qkvgate_gemma_rmsnorm_rope(
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig
|
||||
from sglang.srt.distributed import get_pp_indices
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.models.qwen3_5 import Qwen3_5MoeForConditionalGeneration
|
||||
from sglang.srt.models.qwen3_5 import (
|
||||
Qwen3_5MoeForConditionalGeneration,
|
||||
_refreshes_rope_cos_sin,
|
||||
)
|
||||
from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
@@ -99,6 +106,62 @@ class TestQwen3_5PipelineParallel(CustomTestCase):
|
||||
self.assertEqual(loaded, {"model.embed_tokens.weight"})
|
||||
torch.testing.assert_close(model.model.embed_tokens.weight, expected)
|
||||
|
||||
def test_rope_refresh_at_first_attention_layer_of_each_pp_stage(self):
|
||||
"""Each stage must refresh the shared rope cos/sin exactly once per forward.
|
||||
Refreshing only at the model's first attention layer left later stages
|
||||
without one."""
|
||||
# (layers, interval, pp_size, SGLANG_PP_LAYER_PARTITION)
|
||||
cases = [
|
||||
(48, 4, 1, None),
|
||||
(48, 4, 2, None),
|
||||
(48, 4, 4, None),
|
||||
(40, 4, 3, None), # uneven split: 13/13/14
|
||||
(48, 4, 3, "7,16,25"), # stages start on attention layers
|
||||
]
|
||||
for num_layers, interval, pp_size, partition in cases:
|
||||
config = Qwen3_5TextConfig(
|
||||
num_hidden_layers=num_layers, full_attention_interval=interval
|
||||
)
|
||||
env = {} if partition is None else {"SGLANG_PP_LAYER_PARTITION": partition}
|
||||
for pp_rank in range(pp_size):
|
||||
with (
|
||||
self.subTest(
|
||||
num_layers=num_layers,
|
||||
pp_size=pp_size,
|
||||
pp_rank=pp_rank,
|
||||
partition=partition,
|
||||
),
|
||||
patch.dict(os.environ, env),
|
||||
patch(
|
||||
"sglang.srt.models.qwen3_5.get_parallel",
|
||||
return_value=SimpleNamespace(pp_rank=pp_rank, pp_size=pp_size),
|
||||
),
|
||||
):
|
||||
start, end = get_pp_indices(num_layers, pp_rank, pp_size)
|
||||
attn_layers = [
|
||||
i
|
||||
for i in range(start, end)
|
||||
if config.layers_block_type[i] == "attention"
|
||||
]
|
||||
refreshing = [
|
||||
i
|
||||
for i in attn_layers
|
||||
if _refreshes_rope_cos_sin(config, i, is_nextn=False)
|
||||
]
|
||||
self.assertEqual(refreshing, attn_layers[:1])
|
||||
|
||||
def test_rope_refresh_for_draft_ignores_target_pp_split(self):
|
||||
# The MTP draft's single layer is numbered 0 whatever stage it runs on.
|
||||
config = Qwen3_5TextConfig(num_hidden_layers=1, full_attention_interval=1)
|
||||
with (
|
||||
patch.dict(os.environ, {"SGLANG_PP_LAYER_PARTITION": "24,24"}),
|
||||
patch(
|
||||
"sglang.srt.models.qwen3_5.get_parallel",
|
||||
return_value=SimpleNamespace(pp_rank=1, pp_size=2),
|
||||
),
|
||||
):
|
||||
self.assertTrue(_refreshes_rope_cos_sin(config, 0, is_nextn=True))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user