[Fix] Derive per-runner hybrid SWA layer ids on ModelLayerInfo instead of mutating ModelConfig (#40983)

This commit is contained in:
Liangsheng Yin
2026-09-23 19:52:55 -07:00
committed by GitHub
parent e98b2f7538
commit cc6e8b0f25
8 changed files with 172 additions and 89 deletions
@@ -1033,6 +1033,9 @@ class ModelConfig:
is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config)
and not self.disable_hybrid_swa_memory
)
# Whole-model split, read-only; per-runner slices live on ModelLayerInfo.
self.swa_attention_layer_ids: Optional[List[int]] = None
self.full_attention_layer_ids: Optional[List[int]] = None
if self.is_hybrid_swa:
logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}")
@@ -294,15 +294,9 @@ class KVCacheConfigurator:
self.mambaish_config = mambaish_config(self.model_config)
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
self.is_hybrid_swa_mtp_draft = (
self.is_draft_worker
and self.draft_model_idx is not None
and self.is_hybrid_swa
and getattr(self.model_config.hf_text_config, "mtp_local_layer_ids", None)
is not None
)
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and (
self.draft_model_idx in self.model_config.swa_attention_layer_ids
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
self.layer_info.swa_attention_layer_ids
)
def hybrid_swa_token_capacity(
@@ -784,19 +778,9 @@ class KVCacheConfigurator:
swa_head_dim = head_dim
swa_v_head_dim = head_dim
# From the sglang ModelConfig WRAPPER, never the HF config's
# full_attention_layer_ids: that property feeds the conv/attention
# pairing, not the KV-lifetime split, and returns ALL layers.
swa_attention_layer_ids = [
i
for i in self.model_config.swa_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
full_attention_layer_ids = [
i
for i in self.model_config.full_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
n_local_layers = self.layer_info.end_layer - self.layer_info.start_layer
assert (
len(full_attention_layer_ids) + len(swa_attention_layer_ids)
@@ -903,17 +887,8 @@ class KVCacheConfigurator:
swa_head_dim = head_dim
swa_v_head_dim = head_dim
# Filter layer ids to this worker's [start_layer, end_layer) range.
swa_attention_layer_ids = [
i
for i in self.model_config.swa_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
full_attention_layer_ids = [
i
for i in self.model_config.full_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
total_bytes = unified_memory_pool_bytes
# An uncapped, draft-free pool owns the profiled budget, including bytes
@@ -1521,8 +1496,8 @@ class KVCacheConfigurator:
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
device=self.device,
token_to_kv_pool_class=NPUMHATokenToKVPool,
**kwargs,
@@ -1732,8 +1707,8 @@ class KVCacheConfigurator:
dtype=self.kv_cache_dtype,
head_num=0,
head_dim=0,
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
device=self.device,
full_kv_pool_class=full_pool_class,
swa_kv_pool_class=MLATokenToKVPool,
@@ -1799,16 +1774,8 @@ class KVCacheConfigurator:
if self.kv_cache_dtype_str == "mxfp8"
else mha_pool_class
)
swa_attention_layer_ids = self.model_config.swa_attention_layer_ids
full_attention_layer_ids = self.model_config.full_attention_layer_ids
if self.is_hybrid_swa_mtp_draft:
if self.draft_swa_full_capacity:
# Route local MTP depths through the SWA ring pool.
swa_attention_layer_ids = [self.draft_model_idx]
full_attention_layer_ids = []
else:
swa_attention_layer_ids = []
full_attention_layer_ids = [self.draft_model_idx]
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
# The draft SWA ring must cover the target allocator's full token capacity.
size_swa = (
full_max_total_num_tokens
@@ -132,7 +132,6 @@ from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
from sglang.srt.model_executor.model_runner_components.layer_setup import (
AttentionAndMoeLayers,
ModelLayerInfo,
adjust_hybrid_swa_layer_ids,
compute_attention_and_moe_layers,
resolve_layer_indices,
)
@@ -666,12 +665,7 @@ class ModelRunner:
model=self.model,
model_config=self.model_config,
is_draft_worker=self.is_draft_worker,
)
adjust_hybrid_swa_layer_ids(
model_config=self.model_config,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
is_hybrid_swa=self.is_hybrid_swa,
draft_model_idx=self.draft_model_idx,
)
self.maybe_apply_post_load_model_transforms()
self.maybe_init_lora_manager()
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
import msgspec
from torch import nn
@@ -130,6 +130,11 @@ class ModelLayerInfo(msgspec.Struct, frozen=True, kw_only=True):
start_layer: int
end_layer: int
num_effective_layers: int
# Global ids of the layers this runner owns; None when the model has no split.
swa_attention_layer_ids: Optional[list[int]] = None
full_attention_layer_ids: Optional[list[int]] = None
# Owns the single block at layer_id == draft_model_idx, not a [start, end) slice.
is_hybrid_swa_mtp_draft: bool = False
def resolve_layer_indices(
@@ -137,6 +142,7 @@ def resolve_layer_indices(
model: Any,
model_config: ModelConfig,
is_draft_worker: bool,
draft_model_idx: Optional[int] = None,
) -> ModelLayerInfo:
# For MTP models like DeepSeek-V3 or GLM-4.5, the MTP layer(s) are used separately as draft
# models for speculative decoding. In those cases, `num_nextn_predict_layers` is used to
@@ -152,10 +158,43 @@ def resolve_layer_indices(
if loop_num > 1:
num_effective_layers = num_effective_layers * loop_num
is_hybrid_swa_mtp_draft = (
is_draft_worker
and draft_model_idx is not None
and model_config.is_hybrid_swa
and getattr(model, "mtp_layer_id_is_depth", False)
)
owned_layers = (
range(draft_model_idx, draft_model_idx + 1)
if is_hybrid_swa_mtp_draft
else range(pp_range.start_layer, pp_range.end_layer)
)
swa_attention_layer_ids, full_attention_layer_ids = (
_resolve_local_hybrid_swa_layer_ids(
model_config=model_config, owned_layers=owned_layers
)
)
return ModelLayerInfo(
start_layer=pp_range.start_layer,
end_layer=pp_range.end_layer,
num_effective_layers=num_effective_layers,
swa_attention_layer_ids=swa_attention_layer_ids,
full_attention_layer_ids=full_attention_layer_ids,
is_hybrid_swa_mtp_draft=is_hybrid_swa_mtp_draft,
)
def _resolve_local_hybrid_swa_layer_ids(
*,
model_config: ModelConfig,
owned_layers: range,
) -> tuple[Optional[list[int]], Optional[list[int]]]:
if model_config.swa_attention_layer_ids is None:
return None, None
return (
[i for i in model_config.swa_attention_layer_ids if i in owned_layers],
[i for i in model_config.full_attention_layer_ids if i in owned_layers],
)
@@ -196,32 +235,3 @@ def _resolve_pp_layer_range(*, model: Any, model_num_layers: int) -> _PPLayerRan
start_layer=getattr(model, "start_layer", 0),
end_layer=getattr(model, "end_layer", model_num_layers),
)
def adjust_hybrid_swa_layer_ids(
*,
model_config: ModelConfig,
start_layer: int,
end_layer: int,
is_hybrid_swa: bool,
) -> None:
if not is_hybrid_swa:
return
if model_config.is_deepseek_v4_arch:
return
full_attention_layer_ids = [
layer_idx
for layer_idx in range(start_layer, end_layer + 1)
if hasattr(model_config, "full_attention_layer_ids")
and layer_idx in model_config.full_attention_layer_ids
]
swa_attention_layer_ids = [
layer_idx
for layer_idx in range(start_layer, end_layer + 1)
if hasattr(model_config, "swa_attention_layer_ids")
and layer_idx in model_config.swa_attention_layer_ids
]
model_config.swa_attention_layer_ids = swa_attention_layer_ids
model_config.full_attention_layer_ids = full_attention_layer_ids
@@ -605,8 +605,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
kv_size = torch._utils._element_size(kv_cache_dtype)
tp_size = get_parallel().attn_tp_size
self._full_layers_num = len(model_config.full_attention_layer_ids)
self._swa_layers_num = len(model_config.swa_attention_layer_ids)
self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids)
self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids)
assert self._swa_layers_num > 0, (
"Hybrid SWA model must have at least one SWA layer"
)
@@ -915,7 +915,7 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
return False
if kvc.sliding_window_size is None:
return False
return len(kvc.model_config.full_attention_layer_ids) > 0
return len(kvc.layer_info.full_attention_layer_ids) > 0
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
+1
View File
@@ -1826,6 +1826,7 @@ class InklingForConditionalGenerationMTP(nn.Module):
"""
fall_back_to_pt_during_load = False
mtp_layer_id_is_depth = True
def __init__(
self,
@@ -1,17 +1,22 @@
"""Unit tests for model-runner layer discovery."""
import os
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.distributed.utils import get_pp_indices
from sglang.srt.model_executor.model_runner_components.layer_setup import (
compute_attention_and_moe_layers,
resolve_layer_indices,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
class TestComputeAttentionAndMoeLayers(unittest.TestCase):
class TestComputeAttentionAndMoeLayers(CustomTestCase):
def test_deepseek_mla_registers_mha_companion(self):
attn_mqa = SimpleNamespace()
attn_mha = SimpleNamespace()
@@ -46,5 +51,102 @@ class TestComputeAttentionAndMoeLayers(unittest.TestCase):
self.assertEqual(mha_companion_layers, [None, None, None])
NUM_LAYERS = 36
GLOBAL_FULL_IDS = list(range(1, NUM_LAYERS, 2))
GLOBAL_SWA_IDS = list(range(0, NUM_LAYERS, 2))
def _hybrid_swa_model_config(*, architecture="Hybrid"):
return SimpleNamespace(
hf_config=SimpleNamespace(architectures=[architecture]),
num_hidden_layers=NUM_LAYERS,
num_attention_layers=NUM_LAYERS,
num_nextn_predict_layers=None,
is_hybrid_swa=True,
full_attention_layer_ids=list(GLOBAL_FULL_IDS),
swa_attention_layer_ids=list(GLOBAL_SWA_IDS),
)
class TestResolveHybridSWALayerIds(CustomTestCase):
def test_pipeline_stages_own_each_hybrid_layer_once(self):
"""Every hybrid layer belongs to exactly one pipeline stage."""
for pp_size in (1, 8):
with self.subTest(pp_size=pp_size), patch.dict(os.environ):
os.environ.pop("SGLANG_PP_LAYER_PARTITION", None)
full_ids = []
swa_ids = []
for rank in range(pp_size):
start, end = get_pp_indices(NUM_LAYERS, rank, pp_size)
model = SimpleNamespace(start_layer=start, end_layer=end)
layer_info = resolve_layer_indices(
model=model,
model_config=_hybrid_swa_model_config(),
is_draft_worker=False,
)
self.assertEqual(
sorted(
layer_info.full_attention_layer_ids
+ layer_info.swa_attention_layer_ids
),
list(range(start, end)),
)
full_ids.extend(layer_info.full_attention_layer_ids)
swa_ids.extend(layer_info.swa_attention_layer_ids)
self.assertEqual(full_ids, GLOBAL_FULL_IDS)
self.assertEqual(swa_ids, GLOBAL_SWA_IDS)
def test_model_config_is_shared_and_never_modified(self):
"""Resolving one runner's view must not change the shared ModelConfig."""
model_config = _hybrid_swa_model_config()
for start, end in ((0, 4), (4, 9), (0, 1)):
resolve_layer_indices(
model=SimpleNamespace(start_layer=start, end_layer=end),
model_config=model_config,
is_draft_worker=False,
)
self.assertEqual(model_config.full_attention_layer_ids, GLOBAL_FULL_IDS)
self.assertEqual(model_config.swa_attention_layer_ids, GLOBAL_SWA_IDS)
def test_multi_layer_mtp_draft_owns_its_depth(self):
"""Each MTP draft runner owns the depth it serves, not layer 0."""
model_config = _hybrid_swa_model_config(
architecture="InklingForConditionalGenerationMTP"
)
model_config.swa_attention_layer_ids = [0, 2]
model_config.full_attention_layer_ids = [1]
model_config.num_nextn_predict_layers = 3
for depth, swa, full in ((0, [0], []), (1, [], [1]), (2, [2], [])):
with self.subTest(depth=depth):
layer_info = resolve_layer_indices(
model=SimpleNamespace(mtp_layer_id_is_depth=True),
model_config=model_config,
is_draft_worker=True,
draft_model_idx=depth,
)
self.assertTrue(layer_info.is_hybrid_swa_mtp_draft)
self.assertEqual(layer_info.swa_attention_layer_ids, swa)
self.assertEqual(layer_info.full_attention_layer_ids, full)
def test_single_block_mtp_draft_keeps_layer_range_view(self):
"""A draft whose block is always layer 0 is an ordinary [0, 1) slice."""
model_config = _hybrid_swa_model_config(architecture="MiMoV2MTP")
model_config.swa_attention_layer_ids = [0]
model_config.full_attention_layer_ids = []
layer_info = resolve_layer_indices(
model=SimpleNamespace(),
model_config=model_config,
is_draft_worker=True,
draft_model_idx=1,
)
self.assertFalse(layer_info.is_hybrid_swa_mtp_draft)
self.assertEqual(layer_info.swa_attention_layer_ids, [0])
self.assertEqual(layer_info.full_attention_layer_ids, [])
if __name__ == "__main__":
unittest.main()
@@ -175,8 +175,14 @@ def _make_model_runner(
spec.is_none.return_value = True
mr.spec_algorithm = spec
# Single stage: the runner's local hybrid-SWA view is the whole model's split.
mr.layer_info = SimpleNamespace(
start_layer=0, end_layer=num_layers, num_effective_layers=num_layers
start_layer=0,
end_layer=num_layers,
num_effective_layers=num_layers,
swa_attention_layer_ids=list(mc.swa_attention_layer_ids),
full_attention_layer_ids=list(mc.full_attention_layer_ids),
is_hybrid_swa_mtp_draft=False,
)
mr.attn_dp_size = 1
mr.pp_size = 1