[NPU] [Bugfix] Fix rope sin cos pre-calculation for Qwen3.5 during Pipeline Parallelism (#29753)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Артем Савкин <58187114+OrangeRedeng@users.noreply.github.com>
This commit is contained in:
Tamir Baydasov
2026-10-15 09:58:47 +03:00
committed by GitHub
co-authored by Claude Opus 5.5 Артем Савкин
parent 01ac2cf36d
commit fa66e7d816
3 changed files with 99 additions and 3 deletions
@@ -559,6 +559,11 @@ class InternS2MobiusAttentionDecoderLayer(
self.rope_theta, rope_scaling = get_rope_config(config)
self.partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0)
self.layer_id = layer_id
# Inherited forward_prepare_npu reads this. Mobius is never PP-split and its
# schedule is layer_types, not full_attention_interval.
self.refreshes_rope_cos_sin = layer_id == config.layer_types.index(
"full_attention"
)
if rope_scaling and not ("rope_type" in rope_scaling or "type" in rope_scaling):
rope_scaling = None
self.attn_output_gate = getattr(config, "attn_output_gate", True)
+30 -2
View File
@@ -45,6 +45,7 @@ from sglang.srt.configs.qwen3_5 import (
)
# Distributed
from sglang.srt.distributed import get_pp_indices
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
@@ -1227,6 +1228,31 @@ class Qwen3_5LinearDecoderLayer(nn.Module):
return ffn_exit.finish(hidden_states)
def _refreshes_rope_cos_sin(
config: Qwen3_5TextConfig, layer_id: int, is_nextn: bool
) -> bool:
"""rotary_emb is shared by all layers, so its per-forward cos/sin is refreshed
once, at the first full-attention layer this PP stage owns.
layer_id must be a full-attention layer; linear-attention layers near the
stage start would also pass the check below. Only Qwen3_5AttentionDecoderLayer
calls this, and it is built only for "attention" entries of layers_block_type.
"""
# Drafts keep the unsplit numbering.
stage_start = 0
if not is_nextn:
stage_start, _ = get_pp_indices(
config.num_hidden_layers,
get_parallel().pp_rank,
get_parallel().pp_size,
)
owned_by_stage = layer_id >= stage_start
# Full-attention layers are interval apart, so only the stage's first one
# lies within interval of its start.
first_in_stage = layer_id < stage_start + config.full_attention_interval
return owned_by_stage and first_in_stage
class Qwen3_5AttentionDecoderLayer(nn.Module):
"""Qwen3.5 Decoder Layer with Full Attention."""
@@ -1247,6 +1273,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
self.hidden_size = config.hidden_size
self.attn_tp_rank = get_parallel().attn_tp_rank
self.attn_tp_size = get_parallel().attn_tp_size
self.refreshes_rope_cos_sin = _refreshes_rope_cos_sin(
config, layer_id, is_nextn
)
# A Qwen3.5 draft is rewritten to the MTP arch (model_config._config_draft_model),
# so is_nextn marks it. Drafts are TP-sharded and do not replicate KV under DCP.
dcp_size = 1 if is_nextn else get_parallel().attn_dcp_size
@@ -1511,8 +1540,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module):
def forward_prepare_npu(self, positions, hidden_states, forward_batch):
qkv, _ = self.qkv_proj(hidden_states)
# Calculate first full attention layer ID based on config
if self.attn.layer_id == (self.config.full_attention_interval - 1):
if self.refreshes_rope_cos_sin:
self.rotary_emb.get_cos_sin_with_position(positions)
q, k, v, gate = split_qkvgate_gemma_rmsnorm_rope(
@@ -1,10 +1,17 @@
import os
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig
from sglang.srt.distributed import get_pp_indices
from sglang.srt.layers.utils import PPMissingLayer
from sglang.srt.models.qwen3_5 import Qwen3_5MoeForConditionalGeneration
from sglang.srt.models.qwen3_5 import (
Qwen3_5MoeForConditionalGeneration,
_refreshes_rope_cos_sin,
)
from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -99,6 +106,62 @@ class TestQwen3_5PipelineParallel(CustomTestCase):
self.assertEqual(loaded, {"model.embed_tokens.weight"})
torch.testing.assert_close(model.model.embed_tokens.weight, expected)
def test_rope_refresh_at_first_attention_layer_of_each_pp_stage(self):
"""Each stage must refresh the shared rope cos/sin exactly once per forward.
Refreshing only at the model's first attention layer left later stages
without one."""
# (layers, interval, pp_size, SGLANG_PP_LAYER_PARTITION)
cases = [
(48, 4, 1, None),
(48, 4, 2, None),
(48, 4, 4, None),
(40, 4, 3, None), # uneven split: 13/13/14
(48, 4, 3, "7,16,25"), # stages start on attention layers
]
for num_layers, interval, pp_size, partition in cases:
config = Qwen3_5TextConfig(
num_hidden_layers=num_layers, full_attention_interval=interval
)
env = {} if partition is None else {"SGLANG_PP_LAYER_PARTITION": partition}
for pp_rank in range(pp_size):
with (
self.subTest(
num_layers=num_layers,
pp_size=pp_size,
pp_rank=pp_rank,
partition=partition,
),
patch.dict(os.environ, env),
patch(
"sglang.srt.models.qwen3_5.get_parallel",
return_value=SimpleNamespace(pp_rank=pp_rank, pp_size=pp_size),
),
):
start, end = get_pp_indices(num_layers, pp_rank, pp_size)
attn_layers = [
i
for i in range(start, end)
if config.layers_block_type[i] == "attention"
]
refreshing = [
i
for i in attn_layers
if _refreshes_rope_cos_sin(config, i, is_nextn=False)
]
self.assertEqual(refreshing, attn_layers[:1])
def test_rope_refresh_for_draft_ignores_target_pp_split(self):
# The MTP draft's single layer is numbered 0 whatever stage it runs on.
config = Qwen3_5TextConfig(num_hidden_layers=1, full_attention_interval=1)
with (
patch.dict(os.environ, {"SGLANG_PP_LAYER_PARTITION": "24,24"}),
patch(
"sglang.srt.models.qwen3_5.get_parallel",
return_value=SimpleNamespace(pp_rank=1, pp_size=2),
),
):
self.assertTrue(_refreshes_rope_cos_sin(config, 0, is_nextn=True))
if __name__ == "__main__":
unittest.main()