[Fix] Derive per-runner hybrid SWA layer ids on ModelLayerInfo instead of mutating ModelConfig (#40983)

This commit is contained in:
Liangsheng Yin
2026-09-23 19:52:55 -07:00
committed by GitHub
parent e98b2f7538
commit cc6e8b0f25
8 changed files with 172 additions and 89 deletions
@@ -1033,6 +1033,9 @@ class ModelConfig:
is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config)
and not self.disable_hybrid_swa_memory
)
# Whole-model split, read-only; per-runner slices live on ModelLayerInfo.
self.swa_attention_layer_ids: Optional[List[int]] = None
self.full_attention_layer_ids: Optional[List[int]] = None
if self.is_hybrid_swa:
logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}")
@@ -294,15 +294,9 @@ class KVCacheConfigurator:
self.mambaish_config = mambaish_config(self.model_config)
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
self.is_hybrid_swa_mtp_draft = (
self.is_draft_worker
and self.draft_model_idx is not None
and self.is_hybrid_swa
and getattr(self.model_config.hf_text_config, "mtp_local_layer_ids", None)
is not None
)
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and (
self.draft_model_idx in self.model_config.swa_attention_layer_ids
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
self.layer_info.swa_attention_layer_ids
)
def hybrid_swa_token_capacity(
@@ -784,19 +778,9 @@ class KVCacheConfigurator:
swa_head_dim = head_dim
swa_v_head_dim = head_dim
# From the sglang ModelConfig WRAPPER, never the HF config's
# full_attention_layer_ids: that property feeds the conv/attention
# pairing, not the KV-lifetime split, and returns ALL layers.
swa_attention_layer_ids = [
i
for i in self.model_config.swa_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
full_attention_layer_ids = [
i
for i in self.model_config.full_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
n_local_layers = self.layer_info.end_layer - self.layer_info.start_layer
assert (
len(full_attention_layer_ids) + len(swa_attention_layer_ids)
@@ -903,17 +887,8 @@ class KVCacheConfigurator:
swa_head_dim = head_dim
swa_v_head_dim = head_dim
# Filter layer ids to this worker's [start_layer, end_layer) range.
swa_attention_layer_ids = [
i
for i in self.model_config.swa_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
full_attention_layer_ids = [
i
for i in self.model_config.full_attention_layer_ids
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
total_bytes = unified_memory_pool_bytes
# An uncapped, draft-free pool owns the profiled budget, including bytes
@@ -1521,8 +1496,8 @@ class KVCacheConfigurator:
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
device=self.device,
token_to_kv_pool_class=NPUMHATokenToKVPool,
**kwargs,
@@ -1732,8 +1707,8 @@ class KVCacheConfigurator:
dtype=self.kv_cache_dtype,
head_num=0,
head_dim=0,
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
device=self.device,
full_kv_pool_class=full_pool_class,
swa_kv_pool_class=MLATokenToKVPool,
@@ -1799,16 +1774,8 @@ class KVCacheConfigurator:
if self.kv_cache_dtype_str == "mxfp8"
else mha_pool_class
)
swa_attention_layer_ids = self.model_config.swa_attention_layer_ids
full_attention_layer_ids = self.model_config.full_attention_layer_ids
if self.is_hybrid_swa_mtp_draft:
if self.draft_swa_full_capacity:
# Route local MTP depths through the SWA ring pool.
swa_attention_layer_ids = [self.draft_model_idx]
full_attention_layer_ids = []
else:
swa_attention_layer_ids = []
full_attention_layer_ids = [self.draft_model_idx]
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
# The draft SWA ring must cover the target allocator's full token capacity.
size_swa = (
full_max_total_num_tokens
@@ -132,7 +132,6 @@ from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
from sglang.srt.model_executor.model_runner_components.layer_setup import (
AttentionAndMoeLayers,
ModelLayerInfo,
adjust_hybrid_swa_layer_ids,
compute_attention_and_moe_layers,
resolve_layer_indices,
)
@@ -666,12 +665,7 @@ class ModelRunner:
model=self.model,
model_config=self.model_config,
is_draft_worker=self.is_draft_worker,
)
adjust_hybrid_swa_layer_ids(
model_config=self.model_config,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
is_hybrid_swa=self.is_hybrid_swa,
draft_model_idx=self.draft_model_idx,
)
self.maybe_apply_post_load_model_transforms()
self.maybe_init_lora_manager()
@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, NamedTuple
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
import msgspec
from torch import nn
@@ -130,6 +130,11 @@ class ModelLayerInfo(msgspec.Struct, frozen=True, kw_only=True):
start_layer: int
end_layer: int
num_effective_layers: int
# Global ids of the layers this runner owns; None when the model has no split.
swa_attention_layer_ids: Optional[list[int]] = None
full_attention_layer_ids: Optional[list[int]] = None
# Owns the single block at layer_id == draft_model_idx, not a [start, end) slice.
is_hybrid_swa_mtp_draft: bool = False
def resolve_layer_indices(
@@ -137,6 +142,7 @@ def resolve_layer_indices(
model: Any,
model_config: ModelConfig,
is_draft_worker: bool,
draft_model_idx: Optional[int] = None,
) -> ModelLayerInfo:
# For MTP models like DeepSeek-V3 or GLM-4.5, the MTP layer(s) are used separately as draft
# models for speculative decoding. In those cases, `num_nextn_predict_layers` is used to
@@ -152,10 +158,43 @@ def resolve_layer_indices(
if loop_num > 1:
num_effective_layers = num_effective_layers * loop_num
is_hybrid_swa_mtp_draft = (
is_draft_worker
and draft_model_idx is not None
and model_config.is_hybrid_swa
and getattr(model, "mtp_layer_id_is_depth", False)
)
owned_layers = (
range(draft_model_idx, draft_model_idx + 1)
if is_hybrid_swa_mtp_draft
else range(pp_range.start_layer, pp_range.end_layer)
)
swa_attention_layer_ids, full_attention_layer_ids = (
_resolve_local_hybrid_swa_layer_ids(
model_config=model_config, owned_layers=owned_layers
)
)
return ModelLayerInfo(
start_layer=pp_range.start_layer,
end_layer=pp_range.end_layer,
num_effective_layers=num_effective_layers,
swa_attention_layer_ids=swa_attention_layer_ids,
full_attention_layer_ids=full_attention_layer_ids,
is_hybrid_swa_mtp_draft=is_hybrid_swa_mtp_draft,
)
def _resolve_local_hybrid_swa_layer_ids(
*,
model_config: ModelConfig,
owned_layers: range,
) -> tuple[Optional[list[int]], Optional[list[int]]]:
if model_config.swa_attention_layer_ids is None:
return None, None
return (
[i for i in model_config.swa_attention_layer_ids if i in owned_layers],
[i for i in model_config.full_attention_layer_ids if i in owned_layers],
)
@@ -196,32 +235,3 @@ def _resolve_pp_layer_range(*, model: Any, model_num_layers: int) -> _PPLayerRan
start_layer=getattr(model, "start_layer", 0),
end_layer=getattr(model, "end_layer", model_num_layers),
)
def adjust_hybrid_swa_layer_ids(
*,
model_config: ModelConfig,
start_layer: int,
end_layer: int,
is_hybrid_swa: bool,
) -> None:
if not is_hybrid_swa:
return
if model_config.is_deepseek_v4_arch:
return
full_attention_layer_ids = [
layer_idx
for layer_idx in range(start_layer, end_layer + 1)
if hasattr(model_config, "full_attention_layer_ids")
and layer_idx in model_config.full_attention_layer_ids
]
swa_attention_layer_ids = [
layer_idx
for layer_idx in range(start_layer, end_layer + 1)
if hasattr(model_config, "swa_attention_layer_ids")
and layer_idx in model_config.swa_attention_layer_ids
]
model_config.swa_attention_layer_ids = swa_attention_layer_ids
model_config.full_attention_layer_ids = full_attention_layer_ids
@@ -605,8 +605,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
kv_size = torch._utils._element_size(kv_cache_dtype)
tp_size = get_parallel().attn_tp_size
self._full_layers_num = len(model_config.full_attention_layer_ids)
self._swa_layers_num = len(model_config.swa_attention_layer_ids)
self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids)
self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids)
assert self._swa_layers_num > 0, (
"Hybrid SWA model must have at least one SWA layer"
)
@@ -915,7 +915,7 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
return False
if kvc.sliding_window_size is None:
return False
return len(kvc.model_config.full_attention_layer_ids) > 0
return len(kvc.layer_info.full_attention_layer_ids) > 0
def calculate_pool_sizes(
self, available_bytes: int, page_size: int
+1
View File
@@ -1826,6 +1826,7 @@ class InklingForConditionalGenerationMTP(nn.Module):
"""
fall_back_to_pt_during_load = False
mtp_layer_id_is_depth = True
def __init__(
self,