From cc6e8b0f258ea80f8e88affc77e7d6ae8725ce8b Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 23 Sep 2026 19:52:55 -0700 Subject: [PATCH] [Fix] Derive per-runner hybrid SWA layer ids on ModelLayerInfo instead of mutating ModelConfig (#40983) --- python/sglang/srt/configs/model_config.py | 3 + .../srt/mem_cache/kv_cache_configurator.py | 61 +++------- .../sglang/srt/model_executor/model_runner.py | 8 +- .../model_runner_components/layer_setup.py | 70 +++++++----- .../srt/model_executor/pool_configurator.py | 6 +- python/sglang/srt/models/inkling.py | 1 + .../test_layer_setup.py | 104 +++++++++++++++++- .../model_executor/test_pool_configurator.py | 8 +- 8 files changed, 172 insertions(+), 89 deletions(-) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index f52ea2f808..937e2d702e 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1033,6 +1033,9 @@ class ModelConfig: is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config) and not self.disable_hybrid_swa_memory ) + # Whole-model split, read-only; per-runner slices live on ModelLayerInfo. + self.swa_attention_layer_ids: Optional[List[int]] = None + self.full_attention_layer_ids: Optional[List[int]] = None if self.is_hybrid_swa: logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}") diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 23fffa68d7..c0e89ebcc3 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -294,15 +294,9 @@ class KVCacheConfigurator: self.mambaish_config = mambaish_config(self.model_config) self.hybrid_gdn_config = hybrid_gdn_config(self.model_config) self.hybrid_kda_config = hybrid_kda_config(self.model_config) - self.is_hybrid_swa_mtp_draft = ( - self.is_draft_worker - and self.draft_model_idx is not None - and self.is_hybrid_swa - and getattr(self.model_config.hf_text_config, "mtp_local_layer_ids", None) - is not None - ) - self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and ( - self.draft_model_idx in self.model_config.swa_attention_layer_ids + self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft + self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool( + self.layer_info.swa_attention_layer_ids ) def hybrid_swa_token_capacity( @@ -784,19 +778,9 @@ class KVCacheConfigurator: swa_head_dim = head_dim swa_v_head_dim = head_dim - # From the sglang ModelConfig WRAPPER, never the HF config's - # full_attention_layer_ids: that property feeds the conv/attention - # pairing, not the KV-lifetime split, and returns ALL layers. - swa_attention_layer_ids = [ - i - for i in self.model_config.swa_attention_layer_ids - if self.layer_info.start_layer <= i < self.layer_info.end_layer - ] - full_attention_layer_ids = [ - i - for i in self.model_config.full_attention_layer_ids - if self.layer_info.start_layer <= i < self.layer_info.end_layer - ] + # Not the HF config's full_attention_layer_ids: that one returns ALL layers. + swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids + full_attention_layer_ids = self.layer_info.full_attention_layer_ids n_local_layers = self.layer_info.end_layer - self.layer_info.start_layer assert ( len(full_attention_layer_ids) + len(swa_attention_layer_ids) @@ -903,17 +887,8 @@ class KVCacheConfigurator: swa_head_dim = head_dim swa_v_head_dim = head_dim - # Filter layer ids to this worker's [start_layer, end_layer) range. - swa_attention_layer_ids = [ - i - for i in self.model_config.swa_attention_layer_ids - if self.layer_info.start_layer <= i < self.layer_info.end_layer - ] - full_attention_layer_ids = [ - i - for i in self.model_config.full_attention_layer_ids - if self.layer_info.start_layer <= i < self.layer_info.end_layer - ] + swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids + full_attention_layer_ids = self.layer_info.full_attention_layer_ids total_bytes = unified_memory_pool_bytes # An uncapped, draft-free pool owns the profiled budget, including bytes @@ -1521,8 +1496,8 @@ class KVCacheConfigurator: get_parallel().attn_tp_size, get_parallel().attn_dcp_size ), head_dim=self.model_config.head_dim, - swa_attention_layer_ids=self.model_config.swa_attention_layer_ids, - full_attention_layer_ids=self.model_config.full_attention_layer_ids, + swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids, + full_attention_layer_ids=self.layer_info.full_attention_layer_ids, device=self.device, token_to_kv_pool_class=NPUMHATokenToKVPool, **kwargs, @@ -1732,8 +1707,8 @@ class KVCacheConfigurator: dtype=self.kv_cache_dtype, head_num=0, head_dim=0, - swa_attention_layer_ids=self.model_config.swa_attention_layer_ids, - full_attention_layer_ids=self.model_config.full_attention_layer_ids, + swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids, + full_attention_layer_ids=self.layer_info.full_attention_layer_ids, device=self.device, full_kv_pool_class=full_pool_class, swa_kv_pool_class=MLATokenToKVPool, @@ -1799,16 +1774,8 @@ class KVCacheConfigurator: if self.kv_cache_dtype_str == "mxfp8" else mha_pool_class ) - swa_attention_layer_ids = self.model_config.swa_attention_layer_ids - full_attention_layer_ids = self.model_config.full_attention_layer_ids - if self.is_hybrid_swa_mtp_draft: - if self.draft_swa_full_capacity: - # Route local MTP depths through the SWA ring pool. - swa_attention_layer_ids = [self.draft_model_idx] - full_attention_layer_ids = [] - else: - swa_attention_layer_ids = [] - full_attention_layer_ids = [self.draft_model_idx] + swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids + full_attention_layer_ids = self.layer_info.full_attention_layer_ids # The draft SWA ring must cover the target allocator's full token capacity. size_swa = ( full_max_total_num_tokens diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 5a30720783..c3085ee01a 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -132,7 +132,6 @@ from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import ( from sglang.srt.model_executor.model_runner_components.layer_setup import ( AttentionAndMoeLayers, ModelLayerInfo, - adjust_hybrid_swa_layer_ids, compute_attention_and_moe_layers, resolve_layer_indices, ) @@ -666,12 +665,7 @@ class ModelRunner: model=self.model, model_config=self.model_config, is_draft_worker=self.is_draft_worker, - ) - adjust_hybrid_swa_layer_ids( - model_config=self.model_config, - start_layer=self.layer_info.start_layer, - end_layer=self.layer_info.end_layer, - is_hybrid_swa=self.is_hybrid_swa, + draft_model_idx=self.draft_model_idx, ) self.maybe_apply_post_load_model_transforms() self.maybe_init_lora_manager() diff --git a/python/sglang/srt/model_executor/model_runner_components/layer_setup.py b/python/sglang/srt/model_executor/model_runner_components/layer_setup.py index dae8990fe6..01d8453287 100644 --- a/python/sglang/srt/model_executor/model_runner_components/layer_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/layer_setup.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, NamedTuple +from typing import TYPE_CHECKING, Any, NamedTuple, Optional import msgspec from torch import nn @@ -130,6 +130,11 @@ class ModelLayerInfo(msgspec.Struct, frozen=True, kw_only=True): start_layer: int end_layer: int num_effective_layers: int + # Global ids of the layers this runner owns; None when the model has no split. + swa_attention_layer_ids: Optional[list[int]] = None + full_attention_layer_ids: Optional[list[int]] = None + # Owns the single block at layer_id == draft_model_idx, not a [start, end) slice. + is_hybrid_swa_mtp_draft: bool = False def resolve_layer_indices( @@ -137,6 +142,7 @@ def resolve_layer_indices( model: Any, model_config: ModelConfig, is_draft_worker: bool, + draft_model_idx: Optional[int] = None, ) -> ModelLayerInfo: # For MTP models like DeepSeek-V3 or GLM-4.5, the MTP layer(s) are used separately as draft # models for speculative decoding. In those cases, `num_nextn_predict_layers` is used to @@ -152,10 +158,43 @@ def resolve_layer_indices( if loop_num > 1: num_effective_layers = num_effective_layers * loop_num + is_hybrid_swa_mtp_draft = ( + is_draft_worker + and draft_model_idx is not None + and model_config.is_hybrid_swa + and getattr(model, "mtp_layer_id_is_depth", False) + ) + owned_layers = ( + range(draft_model_idx, draft_model_idx + 1) + if is_hybrid_swa_mtp_draft + else range(pp_range.start_layer, pp_range.end_layer) + ) + swa_attention_layer_ids, full_attention_layer_ids = ( + _resolve_local_hybrid_swa_layer_ids( + model_config=model_config, owned_layers=owned_layers + ) + ) + return ModelLayerInfo( start_layer=pp_range.start_layer, end_layer=pp_range.end_layer, num_effective_layers=num_effective_layers, + swa_attention_layer_ids=swa_attention_layer_ids, + full_attention_layer_ids=full_attention_layer_ids, + is_hybrid_swa_mtp_draft=is_hybrid_swa_mtp_draft, + ) + + +def _resolve_local_hybrid_swa_layer_ids( + *, + model_config: ModelConfig, + owned_layers: range, +) -> tuple[Optional[list[int]], Optional[list[int]]]: + if model_config.swa_attention_layer_ids is None: + return None, None + return ( + [i for i in model_config.swa_attention_layer_ids if i in owned_layers], + [i for i in model_config.full_attention_layer_ids if i in owned_layers], ) @@ -196,32 +235,3 @@ def _resolve_pp_layer_range(*, model: Any, model_num_layers: int) -> _PPLayerRan start_layer=getattr(model, "start_layer", 0), end_layer=getattr(model, "end_layer", model_num_layers), ) - - -def adjust_hybrid_swa_layer_ids( - *, - model_config: ModelConfig, - start_layer: int, - end_layer: int, - is_hybrid_swa: bool, -) -> None: - if not is_hybrid_swa: - return - - if model_config.is_deepseek_v4_arch: - return - - full_attention_layer_ids = [ - layer_idx - for layer_idx in range(start_layer, end_layer + 1) - if hasattr(model_config, "full_attention_layer_ids") - and layer_idx in model_config.full_attention_layer_ids - ] - swa_attention_layer_ids = [ - layer_idx - for layer_idx in range(start_layer, end_layer + 1) - if hasattr(model_config, "swa_attention_layer_ids") - and layer_idx in model_config.swa_attention_layer_ids - ] - model_config.swa_attention_layer_ids = swa_attention_layer_ids - model_config.full_attention_layer_ids = full_attention_layer_ids diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index bbb41d5635..fdf01e606b 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -605,8 +605,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): kv_size = torch._utils._element_size(kv_cache_dtype) tp_size = get_parallel().attn_tp_size - self._full_layers_num = len(model_config.full_attention_layer_ids) - self._swa_layers_num = len(model_config.swa_attention_layer_ids) + self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids) + self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids) assert self._swa_layers_num > 0, ( "Hybrid SWA model must have at least one SWA layer" ) @@ -915,7 +915,7 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator): return False if kvc.sliding_window_size is None: return False - return len(kvc.model_config.full_attention_layer_ids) > 0 + return len(kvc.layer_info.full_attention_layer_ids) > 0 def calculate_pool_sizes( self, available_bytes: int, page_size: int diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 3193c903fb..e9eeb9e616 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -1826,6 +1826,7 @@ class InklingForConditionalGenerationMTP(nn.Module): """ fall_back_to_pt_during_load = False + mtp_layer_id_is_depth = True def __init__( self, diff --git a/test/registered/unit/model_executor/model_runner_components/test_layer_setup.py b/test/registered/unit/model_executor/model_runner_components/test_layer_setup.py index 94faa44d6d..bf9a5b16ef 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_layer_setup.py +++ b/test/registered/unit/model_executor/model_runner_components/test_layer_setup.py @@ -1,17 +1,22 @@ """Unit tests for model-runner layer discovery.""" +import os import unittest from types import SimpleNamespace +from unittest.mock import patch +from sglang.srt.distributed.utils import get_pp_indices from sglang.srt.model_executor.model_runner_components.layer_setup import ( compute_attention_and_moe_layers, + resolve_layer_indices, ) from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase register_cpu_ci(est_time=6, suite="base-a-test-cpu") -class TestComputeAttentionAndMoeLayers(unittest.TestCase): +class TestComputeAttentionAndMoeLayers(CustomTestCase): def test_deepseek_mla_registers_mha_companion(self): attn_mqa = SimpleNamespace() attn_mha = SimpleNamespace() @@ -46,5 +51,102 @@ class TestComputeAttentionAndMoeLayers(unittest.TestCase): self.assertEqual(mha_companion_layers, [None, None, None]) +NUM_LAYERS = 36 +GLOBAL_FULL_IDS = list(range(1, NUM_LAYERS, 2)) +GLOBAL_SWA_IDS = list(range(0, NUM_LAYERS, 2)) + + +def _hybrid_swa_model_config(*, architecture="Hybrid"): + return SimpleNamespace( + hf_config=SimpleNamespace(architectures=[architecture]), + num_hidden_layers=NUM_LAYERS, + num_attention_layers=NUM_LAYERS, + num_nextn_predict_layers=None, + is_hybrid_swa=True, + full_attention_layer_ids=list(GLOBAL_FULL_IDS), + swa_attention_layer_ids=list(GLOBAL_SWA_IDS), + ) + + +class TestResolveHybridSWALayerIds(CustomTestCase): + def test_pipeline_stages_own_each_hybrid_layer_once(self): + """Every hybrid layer belongs to exactly one pipeline stage.""" + for pp_size in (1, 8): + with self.subTest(pp_size=pp_size), patch.dict(os.environ): + os.environ.pop("SGLANG_PP_LAYER_PARTITION", None) + full_ids = [] + swa_ids = [] + for rank in range(pp_size): + start, end = get_pp_indices(NUM_LAYERS, rank, pp_size) + model = SimpleNamespace(start_layer=start, end_layer=end) + layer_info = resolve_layer_indices( + model=model, + model_config=_hybrid_swa_model_config(), + is_draft_worker=False, + ) + self.assertEqual( + sorted( + layer_info.full_attention_layer_ids + + layer_info.swa_attention_layer_ids + ), + list(range(start, end)), + ) + full_ids.extend(layer_info.full_attention_layer_ids) + swa_ids.extend(layer_info.swa_attention_layer_ids) + + self.assertEqual(full_ids, GLOBAL_FULL_IDS) + self.assertEqual(swa_ids, GLOBAL_SWA_IDS) + + def test_model_config_is_shared_and_never_modified(self): + """Resolving one runner's view must not change the shared ModelConfig.""" + model_config = _hybrid_swa_model_config() + for start, end in ((0, 4), (4, 9), (0, 1)): + resolve_layer_indices( + model=SimpleNamespace(start_layer=start, end_layer=end), + model_config=model_config, + is_draft_worker=False, + ) + self.assertEqual(model_config.full_attention_layer_ids, GLOBAL_FULL_IDS) + self.assertEqual(model_config.swa_attention_layer_ids, GLOBAL_SWA_IDS) + + def test_multi_layer_mtp_draft_owns_its_depth(self): + """Each MTP draft runner owns the depth it serves, not layer 0.""" + model_config = _hybrid_swa_model_config( + architecture="InklingForConditionalGenerationMTP" + ) + model_config.swa_attention_layer_ids = [0, 2] + model_config.full_attention_layer_ids = [1] + model_config.num_nextn_predict_layers = 3 + + for depth, swa, full in ((0, [0], []), (1, [], [1]), (2, [2], [])): + with self.subTest(depth=depth): + layer_info = resolve_layer_indices( + model=SimpleNamespace(mtp_layer_id_is_depth=True), + model_config=model_config, + is_draft_worker=True, + draft_model_idx=depth, + ) + self.assertTrue(layer_info.is_hybrid_swa_mtp_draft) + self.assertEqual(layer_info.swa_attention_layer_ids, swa) + self.assertEqual(layer_info.full_attention_layer_ids, full) + + def test_single_block_mtp_draft_keeps_layer_range_view(self): + """A draft whose block is always layer 0 is an ordinary [0, 1) slice.""" + model_config = _hybrid_swa_model_config(architecture="MiMoV2MTP") + model_config.swa_attention_layer_ids = [0] + model_config.full_attention_layer_ids = [] + + layer_info = resolve_layer_indices( + model=SimpleNamespace(), + model_config=model_config, + is_draft_worker=True, + draft_model_idx=1, + ) + + self.assertFalse(layer_info.is_hybrid_swa_mtp_draft) + self.assertEqual(layer_info.swa_attention_layer_ids, [0]) + self.assertEqual(layer_info.full_attention_layer_ids, []) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 26fc85b22a..91f0192a3d 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -175,8 +175,14 @@ def _make_model_runner( spec.is_none.return_value = True mr.spec_algorithm = spec + # Single stage: the runner's local hybrid-SWA view is the whole model's split. mr.layer_info = SimpleNamespace( - start_layer=0, end_layer=num_layers, num_effective_layers=num_layers + start_layer=0, + end_layer=num_layers, + num_effective_layers=num_layers, + swa_attention_layer_ids=list(mc.swa_attention_layer_ids), + full_attention_layer_ids=list(mc.full_attention_layer_ids), + is_hybrid_swa_mtp_draft=False, ) mr.attn_dp_size = 1 mr.pp_size = 1