From fa66e7d816d77bdff44fec3fa2aa333a2934d703 Mon Sep 17 00:00:00 2001 From: Tamir Baydasov <41994229+TamirBaydasov@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:58:47 +0300 Subject: [PATCH] [NPU] [Bugfix] Fix rope sin cos pre-calculation for Qwen3.5 during Pipeline Parallelism (#29753) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Claude Opus 5.5 Co-authored-by: Артем Савкин <58187114+OrangeRedeng@users.noreply.github.com> --- python/sglang/srt/models/interns2_mobius.py | 5 ++ python/sglang/srt/models/qwen3_5.py | 32 ++++++++- .../models/test_qwen3_5_pipeline_parallel.py | 65 ++++++++++++++++++- 3 files changed, 99 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index fadb815cb4..bd0781cefe 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -559,6 +559,11 @@ class InternS2MobiusAttentionDecoderLayer( self.rope_theta, rope_scaling = get_rope_config(config) self.partial_rotary_factor = getattr(config, "partial_rotary_factor", 1.0) self.layer_id = layer_id + # Inherited forward_prepare_npu reads this. Mobius is never PP-split and its + # schedule is layer_types, not full_attention_interval. + self.refreshes_rope_cos_sin = layer_id == config.layer_types.index( + "full_attention" + ) if rope_scaling and not ("rope_type" in rope_scaling or "type" in rope_scaling): rope_scaling = None self.attn_output_gate = getattr(config, "attn_output_gate", True) diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index def6a402d5..d04b5ae2ac 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -45,6 +45,7 @@ from sglang.srt.configs.qwen3_5 import ( ) # Distributed +from sglang.srt.distributed import get_pp_indices from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation @@ -1227,6 +1228,31 @@ class Qwen3_5LinearDecoderLayer(nn.Module): return ffn_exit.finish(hidden_states) +def _refreshes_rope_cos_sin( + config: Qwen3_5TextConfig, layer_id: int, is_nextn: bool +) -> bool: + """rotary_emb is shared by all layers, so its per-forward cos/sin is refreshed + once, at the first full-attention layer this PP stage owns. + + layer_id must be a full-attention layer; linear-attention layers near the + stage start would also pass the check below. Only Qwen3_5AttentionDecoderLayer + calls this, and it is built only for "attention" entries of layers_block_type. + """ + # Drafts keep the unsplit numbering. + stage_start = 0 + if not is_nextn: + stage_start, _ = get_pp_indices( + config.num_hidden_layers, + get_parallel().pp_rank, + get_parallel().pp_size, + ) + owned_by_stage = layer_id >= stage_start + # Full-attention layers are interval apart, so only the stage's first one + # lies within interval of its start. + first_in_stage = layer_id < stage_start + config.full_attention_interval + return owned_by_stage and first_in_stage + + class Qwen3_5AttentionDecoderLayer(nn.Module): """Qwen3.5 Decoder Layer with Full Attention.""" @@ -1247,6 +1273,9 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): self.hidden_size = config.hidden_size self.attn_tp_rank = get_parallel().attn_tp_rank self.attn_tp_size = get_parallel().attn_tp_size + self.refreshes_rope_cos_sin = _refreshes_rope_cos_sin( + config, layer_id, is_nextn + ) # A Qwen3.5 draft is rewritten to the MTP arch (model_config._config_draft_model), # so is_nextn marks it. Drafts are TP-sharded and do not replicate KV under DCP. dcp_size = 1 if is_nextn else get_parallel().attn_dcp_size @@ -1511,8 +1540,7 @@ class Qwen3_5AttentionDecoderLayer(nn.Module): def forward_prepare_npu(self, positions, hidden_states, forward_batch): qkv, _ = self.qkv_proj(hidden_states) - # Calculate first full attention layer ID based on config - if self.attn.layer_id == (self.config.full_attention_interval - 1): + if self.refreshes_rope_cos_sin: self.rotary_emb.get_cos_sin_with_position(positions) q, k, v, gate = split_qkvgate_gemma_rmsnorm_rope( diff --git a/test/registered/unit/models/test_qwen3_5_pipeline_parallel.py b/test/registered/unit/models/test_qwen3_5_pipeline_parallel.py index 402df38e78..9fe8079ba5 100644 --- a/test/registered/unit/models/test_qwen3_5_pipeline_parallel.py +++ b/test/registered/unit/models/test_qwen3_5_pipeline_parallel.py @@ -1,10 +1,17 @@ +import os import unittest from types import SimpleNamespace +from unittest.mock import patch import torch +from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig +from sglang.srt.distributed import get_pp_indices from sglang.srt.layers.utils import PPMissingLayer -from sglang.srt.models.qwen3_5 import Qwen3_5MoeForConditionalGeneration +from sglang.srt.models.qwen3_5 import ( + Qwen3_5MoeForConditionalGeneration, + _refreshes_rope_cos_sin, +) from sglang.srt.models.qwen3_5_mtp import Qwen3_5ForCausalLMMTP from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -99,6 +106,62 @@ class TestQwen3_5PipelineParallel(CustomTestCase): self.assertEqual(loaded, {"model.embed_tokens.weight"}) torch.testing.assert_close(model.model.embed_tokens.weight, expected) + def test_rope_refresh_at_first_attention_layer_of_each_pp_stage(self): + """Each stage must refresh the shared rope cos/sin exactly once per forward. + Refreshing only at the model's first attention layer left later stages + without one.""" + # (layers, interval, pp_size, SGLANG_PP_LAYER_PARTITION) + cases = [ + (48, 4, 1, None), + (48, 4, 2, None), + (48, 4, 4, None), + (40, 4, 3, None), # uneven split: 13/13/14 + (48, 4, 3, "7,16,25"), # stages start on attention layers + ] + for num_layers, interval, pp_size, partition in cases: + config = Qwen3_5TextConfig( + num_hidden_layers=num_layers, full_attention_interval=interval + ) + env = {} if partition is None else {"SGLANG_PP_LAYER_PARTITION": partition} + for pp_rank in range(pp_size): + with ( + self.subTest( + num_layers=num_layers, + pp_size=pp_size, + pp_rank=pp_rank, + partition=partition, + ), + patch.dict(os.environ, env), + patch( + "sglang.srt.models.qwen3_5.get_parallel", + return_value=SimpleNamespace(pp_rank=pp_rank, pp_size=pp_size), + ), + ): + start, end = get_pp_indices(num_layers, pp_rank, pp_size) + attn_layers = [ + i + for i in range(start, end) + if config.layers_block_type[i] == "attention" + ] + refreshing = [ + i + for i in attn_layers + if _refreshes_rope_cos_sin(config, i, is_nextn=False) + ] + self.assertEqual(refreshing, attn_layers[:1]) + + def test_rope_refresh_for_draft_ignores_target_pp_split(self): + # The MTP draft's single layer is numbered 0 whatever stage it runs on. + config = Qwen3_5TextConfig(num_hidden_layers=1, full_attention_interval=1) + with ( + patch.dict(os.environ, {"SGLANG_PP_LAYER_PARTITION": "24,24"}), + patch( + "sglang.srt.models.qwen3_5.get_parallel", + return_value=SimpleNamespace(pp_rank=1, pp_size=2), + ), + ): + self.assertTrue(_refreshes_rope_cos_sin(config, 0, is_nextn=True)) + if __name__ == "__main__": unittest.main()