[Fix] Derive per-runner hybrid SWA layer ids on ModelLayerInfo instead of mutating ModelConfig (#40983)
This commit is contained in:
@@ -1033,6 +1033,9 @@ class ModelConfig:
|
||||
is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config)
|
||||
and not self.disable_hybrid_swa_memory
|
||||
)
|
||||
# Whole-model split, read-only; per-runner slices live on ModelLayerInfo.
|
||||
self.swa_attention_layer_ids: Optional[List[int]] = None
|
||||
self.full_attention_layer_ids: Optional[List[int]] = None
|
||||
|
||||
if self.is_hybrid_swa:
|
||||
logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}")
|
||||
|
||||
@@ -294,15 +294,9 @@ class KVCacheConfigurator:
|
||||
self.mambaish_config = mambaish_config(self.model_config)
|
||||
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
|
||||
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
|
||||
self.is_hybrid_swa_mtp_draft = (
|
||||
self.is_draft_worker
|
||||
and self.draft_model_idx is not None
|
||||
and self.is_hybrid_swa
|
||||
and getattr(self.model_config.hf_text_config, "mtp_local_layer_ids", None)
|
||||
is not None
|
||||
)
|
||||
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and (
|
||||
self.draft_model_idx in self.model_config.swa_attention_layer_ids
|
||||
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
|
||||
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
|
||||
self.layer_info.swa_attention_layer_ids
|
||||
)
|
||||
|
||||
def hybrid_swa_token_capacity(
|
||||
@@ -784,19 +778,9 @@ class KVCacheConfigurator:
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
|
||||
# From the sglang ModelConfig WRAPPER, never the HF config's
|
||||
# full_attention_layer_ids: that property feeds the conv/attention
|
||||
# pairing, not the KV-lifetime split, and returns ALL layers.
|
||||
swa_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.swa_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.full_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
n_local_layers = self.layer_info.end_layer - self.layer_info.start_layer
|
||||
assert (
|
||||
len(full_attention_layer_ids) + len(swa_attention_layer_ids)
|
||||
@@ -903,17 +887,8 @@ class KVCacheConfigurator:
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
|
||||
# Filter layer ids to this worker's [start_layer, end_layer) range.
|
||||
swa_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.swa_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.full_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
|
||||
total_bytes = unified_memory_pool_bytes
|
||||
# An uncapped, draft-free pool owns the profiled budget, including bytes
|
||||
@@ -1521,8 +1496,8 @@ class KVCacheConfigurator:
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
|
||||
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
|
||||
device=self.device,
|
||||
token_to_kv_pool_class=NPUMHATokenToKVPool,
|
||||
**kwargs,
|
||||
@@ -1732,8 +1707,8 @@ class KVCacheConfigurator:
|
||||
dtype=self.kv_cache_dtype,
|
||||
head_num=0,
|
||||
head_dim=0,
|
||||
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
|
||||
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
|
||||
device=self.device,
|
||||
full_kv_pool_class=full_pool_class,
|
||||
swa_kv_pool_class=MLATokenToKVPool,
|
||||
@@ -1799,16 +1774,8 @@ class KVCacheConfigurator:
|
||||
if self.kv_cache_dtype_str == "mxfp8"
|
||||
else mha_pool_class
|
||||
)
|
||||
swa_attention_layer_ids = self.model_config.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.model_config.full_attention_layer_ids
|
||||
if self.is_hybrid_swa_mtp_draft:
|
||||
if self.draft_swa_full_capacity:
|
||||
# Route local MTP depths through the SWA ring pool.
|
||||
swa_attention_layer_ids = [self.draft_model_idx]
|
||||
full_attention_layer_ids = []
|
||||
else:
|
||||
swa_attention_layer_ids = []
|
||||
full_attention_layer_ids = [self.draft_model_idx]
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
# The draft SWA ring must cover the target allocator's full token capacity.
|
||||
size_swa = (
|
||||
full_max_total_num_tokens
|
||||
|
||||
@@ -132,7 +132,6 @@ from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
AttentionAndMoeLayers,
|
||||
ModelLayerInfo,
|
||||
adjust_hybrid_swa_layer_ids,
|
||||
compute_attention_and_moe_layers,
|
||||
resolve_layer_indices,
|
||||
)
|
||||
@@ -666,12 +665,7 @@ class ModelRunner:
|
||||
model=self.model,
|
||||
model_config=self.model_config,
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
)
|
||||
adjust_hybrid_swa_layer_ids(
|
||||
model_config=self.model_config,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
is_hybrid_swa=self.is_hybrid_swa,
|
||||
draft_model_idx=self.draft_model_idx,
|
||||
)
|
||||
self.maybe_apply_post_load_model_transforms()
|
||||
self.maybe_init_lora_manager()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
|
||||
|
||||
import msgspec
|
||||
from torch import nn
|
||||
@@ -130,6 +130,11 @@ class ModelLayerInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
start_layer: int
|
||||
end_layer: int
|
||||
num_effective_layers: int
|
||||
# Global ids of the layers this runner owns; None when the model has no split.
|
||||
swa_attention_layer_ids: Optional[list[int]] = None
|
||||
full_attention_layer_ids: Optional[list[int]] = None
|
||||
# Owns the single block at layer_id == draft_model_idx, not a [start, end) slice.
|
||||
is_hybrid_swa_mtp_draft: bool = False
|
||||
|
||||
|
||||
def resolve_layer_indices(
|
||||
@@ -137,6 +142,7 @@ def resolve_layer_indices(
|
||||
model: Any,
|
||||
model_config: ModelConfig,
|
||||
is_draft_worker: bool,
|
||||
draft_model_idx: Optional[int] = None,
|
||||
) -> ModelLayerInfo:
|
||||
# For MTP models like DeepSeek-V3 or GLM-4.5, the MTP layer(s) are used separately as draft
|
||||
# models for speculative decoding. In those cases, `num_nextn_predict_layers` is used to
|
||||
@@ -152,10 +158,43 @@ def resolve_layer_indices(
|
||||
if loop_num > 1:
|
||||
num_effective_layers = num_effective_layers * loop_num
|
||||
|
||||
is_hybrid_swa_mtp_draft = (
|
||||
is_draft_worker
|
||||
and draft_model_idx is not None
|
||||
and model_config.is_hybrid_swa
|
||||
and getattr(model, "mtp_layer_id_is_depth", False)
|
||||
)
|
||||
owned_layers = (
|
||||
range(draft_model_idx, draft_model_idx + 1)
|
||||
if is_hybrid_swa_mtp_draft
|
||||
else range(pp_range.start_layer, pp_range.end_layer)
|
||||
)
|
||||
swa_attention_layer_ids, full_attention_layer_ids = (
|
||||
_resolve_local_hybrid_swa_layer_ids(
|
||||
model_config=model_config, owned_layers=owned_layers
|
||||
)
|
||||
)
|
||||
|
||||
return ModelLayerInfo(
|
||||
start_layer=pp_range.start_layer,
|
||||
end_layer=pp_range.end_layer,
|
||||
num_effective_layers=num_effective_layers,
|
||||
swa_attention_layer_ids=swa_attention_layer_ids,
|
||||
full_attention_layer_ids=full_attention_layer_ids,
|
||||
is_hybrid_swa_mtp_draft=is_hybrid_swa_mtp_draft,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_local_hybrid_swa_layer_ids(
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
owned_layers: range,
|
||||
) -> tuple[Optional[list[int]], Optional[list[int]]]:
|
||||
if model_config.swa_attention_layer_ids is None:
|
||||
return None, None
|
||||
return (
|
||||
[i for i in model_config.swa_attention_layer_ids if i in owned_layers],
|
||||
[i for i in model_config.full_attention_layer_ids if i in owned_layers],
|
||||
)
|
||||
|
||||
|
||||
@@ -196,32 +235,3 @@ def _resolve_pp_layer_range(*, model: Any, model_num_layers: int) -> _PPLayerRan
|
||||
start_layer=getattr(model, "start_layer", 0),
|
||||
end_layer=getattr(model, "end_layer", model_num_layers),
|
||||
)
|
||||
|
||||
|
||||
def adjust_hybrid_swa_layer_ids(
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
is_hybrid_swa: bool,
|
||||
) -> None:
|
||||
if not is_hybrid_swa:
|
||||
return
|
||||
|
||||
if model_config.is_deepseek_v4_arch:
|
||||
return
|
||||
|
||||
full_attention_layer_ids = [
|
||||
layer_idx
|
||||
for layer_idx in range(start_layer, end_layer + 1)
|
||||
if hasattr(model_config, "full_attention_layer_ids")
|
||||
and layer_idx in model_config.full_attention_layer_ids
|
||||
]
|
||||
swa_attention_layer_ids = [
|
||||
layer_idx
|
||||
for layer_idx in range(start_layer, end_layer + 1)
|
||||
if hasattr(model_config, "swa_attention_layer_ids")
|
||||
and layer_idx in model_config.swa_attention_layer_ids
|
||||
]
|
||||
model_config.swa_attention_layer_ids = swa_attention_layer_ids
|
||||
model_config.full_attention_layer_ids = full_attention_layer_ids
|
||||
|
||||
@@ -605,8 +605,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
kv_size = torch._utils._element_size(kv_cache_dtype)
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self._full_layers_num = len(model_config.full_attention_layer_ids)
|
||||
self._swa_layers_num = len(model_config.swa_attention_layer_ids)
|
||||
self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids)
|
||||
self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids)
|
||||
assert self._swa_layers_num > 0, (
|
||||
"Hybrid SWA model must have at least one SWA layer"
|
||||
)
|
||||
@@ -915,7 +915,7 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
return False
|
||||
if kvc.sliding_window_size is None:
|
||||
return False
|
||||
return len(kvc.model_config.full_attention_layer_ids) > 0
|
||||
return len(kvc.layer_info.full_attention_layer_ids) > 0
|
||||
|
||||
def calculate_pool_sizes(
|
||||
self, available_bytes: int, page_size: int
|
||||
|
||||
@@ -1826,6 +1826,7 @@ class InklingForConditionalGenerationMTP(nn.Module):
|
||||
"""
|
||||
|
||||
fall_back_to_pt_during_load = False
|
||||
mtp_layer_id_is_depth = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
"""Unit tests for model-runner layer discovery."""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from sglang.srt.distributed.utils import get_pp_indices
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
compute_attention_and_moe_layers,
|
||||
resolve_layer_indices,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
|
||||
|
||||
|
||||
class TestComputeAttentionAndMoeLayers(unittest.TestCase):
|
||||
class TestComputeAttentionAndMoeLayers(CustomTestCase):
|
||||
def test_deepseek_mla_registers_mha_companion(self):
|
||||
attn_mqa = SimpleNamespace()
|
||||
attn_mha = SimpleNamespace()
|
||||
@@ -46,5 +51,102 @@ class TestComputeAttentionAndMoeLayers(unittest.TestCase):
|
||||
self.assertEqual(mha_companion_layers, [None, None, None])
|
||||
|
||||
|
||||
NUM_LAYERS = 36
|
||||
GLOBAL_FULL_IDS = list(range(1, NUM_LAYERS, 2))
|
||||
GLOBAL_SWA_IDS = list(range(0, NUM_LAYERS, 2))
|
||||
|
||||
|
||||
def _hybrid_swa_model_config(*, architecture="Hybrid"):
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(architectures=[architecture]),
|
||||
num_hidden_layers=NUM_LAYERS,
|
||||
num_attention_layers=NUM_LAYERS,
|
||||
num_nextn_predict_layers=None,
|
||||
is_hybrid_swa=True,
|
||||
full_attention_layer_ids=list(GLOBAL_FULL_IDS),
|
||||
swa_attention_layer_ids=list(GLOBAL_SWA_IDS),
|
||||
)
|
||||
|
||||
|
||||
class TestResolveHybridSWALayerIds(CustomTestCase):
|
||||
def test_pipeline_stages_own_each_hybrid_layer_once(self):
|
||||
"""Every hybrid layer belongs to exactly one pipeline stage."""
|
||||
for pp_size in (1, 8):
|
||||
with self.subTest(pp_size=pp_size), patch.dict(os.environ):
|
||||
os.environ.pop("SGLANG_PP_LAYER_PARTITION", None)
|
||||
full_ids = []
|
||||
swa_ids = []
|
||||
for rank in range(pp_size):
|
||||
start, end = get_pp_indices(NUM_LAYERS, rank, pp_size)
|
||||
model = SimpleNamespace(start_layer=start, end_layer=end)
|
||||
layer_info = resolve_layer_indices(
|
||||
model=model,
|
||||
model_config=_hybrid_swa_model_config(),
|
||||
is_draft_worker=False,
|
||||
)
|
||||
self.assertEqual(
|
||||
sorted(
|
||||
layer_info.full_attention_layer_ids
|
||||
+ layer_info.swa_attention_layer_ids
|
||||
),
|
||||
list(range(start, end)),
|
||||
)
|
||||
full_ids.extend(layer_info.full_attention_layer_ids)
|
||||
swa_ids.extend(layer_info.swa_attention_layer_ids)
|
||||
|
||||
self.assertEqual(full_ids, GLOBAL_FULL_IDS)
|
||||
self.assertEqual(swa_ids, GLOBAL_SWA_IDS)
|
||||
|
||||
def test_model_config_is_shared_and_never_modified(self):
|
||||
"""Resolving one runner's view must not change the shared ModelConfig."""
|
||||
model_config = _hybrid_swa_model_config()
|
||||
for start, end in ((0, 4), (4, 9), (0, 1)):
|
||||
resolve_layer_indices(
|
||||
model=SimpleNamespace(start_layer=start, end_layer=end),
|
||||
model_config=model_config,
|
||||
is_draft_worker=False,
|
||||
)
|
||||
self.assertEqual(model_config.full_attention_layer_ids, GLOBAL_FULL_IDS)
|
||||
self.assertEqual(model_config.swa_attention_layer_ids, GLOBAL_SWA_IDS)
|
||||
|
||||
def test_multi_layer_mtp_draft_owns_its_depth(self):
|
||||
"""Each MTP draft runner owns the depth it serves, not layer 0."""
|
||||
model_config = _hybrid_swa_model_config(
|
||||
architecture="InklingForConditionalGenerationMTP"
|
||||
)
|
||||
model_config.swa_attention_layer_ids = [0, 2]
|
||||
model_config.full_attention_layer_ids = [1]
|
||||
model_config.num_nextn_predict_layers = 3
|
||||
|
||||
for depth, swa, full in ((0, [0], []), (1, [], [1]), (2, [2], [])):
|
||||
with self.subTest(depth=depth):
|
||||
layer_info = resolve_layer_indices(
|
||||
model=SimpleNamespace(mtp_layer_id_is_depth=True),
|
||||
model_config=model_config,
|
||||
is_draft_worker=True,
|
||||
draft_model_idx=depth,
|
||||
)
|
||||
self.assertTrue(layer_info.is_hybrid_swa_mtp_draft)
|
||||
self.assertEqual(layer_info.swa_attention_layer_ids, swa)
|
||||
self.assertEqual(layer_info.full_attention_layer_ids, full)
|
||||
|
||||
def test_single_block_mtp_draft_keeps_layer_range_view(self):
|
||||
"""A draft whose block is always layer 0 is an ordinary [0, 1) slice."""
|
||||
model_config = _hybrid_swa_model_config(architecture="MiMoV2MTP")
|
||||
model_config.swa_attention_layer_ids = [0]
|
||||
model_config.full_attention_layer_ids = []
|
||||
|
||||
layer_info = resolve_layer_indices(
|
||||
model=SimpleNamespace(),
|
||||
model_config=model_config,
|
||||
is_draft_worker=True,
|
||||
draft_model_idx=1,
|
||||
)
|
||||
|
||||
self.assertFalse(layer_info.is_hybrid_swa_mtp_draft)
|
||||
self.assertEqual(layer_info.swa_attention_layer_ids, [0])
|
||||
self.assertEqual(layer_info.full_attention_layer_ids, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -175,8 +175,14 @@ def _make_model_runner(
|
||||
spec.is_none.return_value = True
|
||||
mr.spec_algorithm = spec
|
||||
|
||||
# Single stage: the runner's local hybrid-SWA view is the whole model's split.
|
||||
mr.layer_info = SimpleNamespace(
|
||||
start_layer=0, end_layer=num_layers, num_effective_layers=num_layers
|
||||
start_layer=0,
|
||||
end_layer=num_layers,
|
||||
num_effective_layers=num_layers,
|
||||
swa_attention_layer_ids=list(mc.swa_attention_layer_ids),
|
||||
full_attention_layer_ids=list(mc.full_attention_layer_ids),
|
||||
is_hybrid_swa_mtp_draft=False,
|
||||
)
|
||||
mr.attn_dp_size = 1
|
||||
mr.pp_size = 1
|
||||
|
||||
Reference in New Issue
Block a user