[Fix] Derive per-runner hybrid SWA layer ids on ModelLayerInfo instead of mutating ModelConfig (#40983)
This commit is contained in:
@@ -1033,6 +1033,9 @@ class ModelConfig:
|
||||
is_hybrid_swa_model(self.hf_config.architectures, self.hf_text_config)
|
||||
and not self.disable_hybrid_swa_memory
|
||||
)
|
||||
# Whole-model split, read-only; per-runner slices live on ModelLayerInfo.
|
||||
self.swa_attention_layer_ids: Optional[List[int]] = None
|
||||
self.full_attention_layer_ids: Optional[List[int]] = None
|
||||
|
||||
if self.is_hybrid_swa:
|
||||
logger.debug(f"Hybrid swa model: {self.hf_config.architectures=}")
|
||||
|
||||
@@ -294,15 +294,9 @@ class KVCacheConfigurator:
|
||||
self.mambaish_config = mambaish_config(self.model_config)
|
||||
self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
|
||||
self.hybrid_kda_config = hybrid_kda_config(self.model_config)
|
||||
self.is_hybrid_swa_mtp_draft = (
|
||||
self.is_draft_worker
|
||||
and self.draft_model_idx is not None
|
||||
and self.is_hybrid_swa
|
||||
and getattr(self.model_config.hf_text_config, "mtp_local_layer_ids", None)
|
||||
is not None
|
||||
)
|
||||
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and (
|
||||
self.draft_model_idx in self.model_config.swa_attention_layer_ids
|
||||
self.is_hybrid_swa_mtp_draft = self.layer_info.is_hybrid_swa_mtp_draft
|
||||
self.draft_swa_full_capacity = self.is_hybrid_swa_mtp_draft and bool(
|
||||
self.layer_info.swa_attention_layer_ids
|
||||
)
|
||||
|
||||
def hybrid_swa_token_capacity(
|
||||
@@ -784,19 +778,9 @@ class KVCacheConfigurator:
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
|
||||
# From the sglang ModelConfig WRAPPER, never the HF config's
|
||||
# full_attention_layer_ids: that property feeds the conv/attention
|
||||
# pairing, not the KV-lifetime split, and returns ALL layers.
|
||||
swa_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.swa_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.full_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
n_local_layers = self.layer_info.end_layer - self.layer_info.start_layer
|
||||
assert (
|
||||
len(full_attention_layer_ids) + len(swa_attention_layer_ids)
|
||||
@@ -903,17 +887,8 @@ class KVCacheConfigurator:
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
|
||||
# Filter layer ids to this worker's [start_layer, end_layer) range.
|
||||
swa_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.swa_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
full_attention_layer_ids = [
|
||||
i
|
||||
for i in self.model_config.full_attention_layer_ids
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
|
||||
total_bytes = unified_memory_pool_bytes
|
||||
# An uncapped, draft-free pool owns the profiled budget, including bytes
|
||||
@@ -1521,8 +1496,8 @@ class KVCacheConfigurator:
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
|
||||
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
|
||||
device=self.device,
|
||||
token_to_kv_pool_class=NPUMHATokenToKVPool,
|
||||
**kwargs,
|
||||
@@ -1732,8 +1707,8 @@ class KVCacheConfigurator:
|
||||
dtype=self.kv_cache_dtype,
|
||||
head_num=0,
|
||||
head_dim=0,
|
||||
swa_attention_layer_ids=self.model_config.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.model_config.full_attention_layer_ids,
|
||||
swa_attention_layer_ids=self.layer_info.swa_attention_layer_ids,
|
||||
full_attention_layer_ids=self.layer_info.full_attention_layer_ids,
|
||||
device=self.device,
|
||||
full_kv_pool_class=full_pool_class,
|
||||
swa_kv_pool_class=MLATokenToKVPool,
|
||||
@@ -1799,16 +1774,8 @@ class KVCacheConfigurator:
|
||||
if self.kv_cache_dtype_str == "mxfp8"
|
||||
else mha_pool_class
|
||||
)
|
||||
swa_attention_layer_ids = self.model_config.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.model_config.full_attention_layer_ids
|
||||
if self.is_hybrid_swa_mtp_draft:
|
||||
if self.draft_swa_full_capacity:
|
||||
# Route local MTP depths through the SWA ring pool.
|
||||
swa_attention_layer_ids = [self.draft_model_idx]
|
||||
full_attention_layer_ids = []
|
||||
else:
|
||||
swa_attention_layer_ids = []
|
||||
full_attention_layer_ids = [self.draft_model_idx]
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
# The draft SWA ring must cover the target allocator's full token capacity.
|
||||
size_swa = (
|
||||
full_max_total_num_tokens
|
||||
|
||||
@@ -132,7 +132,6 @@ from sglang.srt.model_executor.model_runner_components.kv_pool_runtime import (
|
||||
from sglang.srt.model_executor.model_runner_components.layer_setup import (
|
||||
AttentionAndMoeLayers,
|
||||
ModelLayerInfo,
|
||||
adjust_hybrid_swa_layer_ids,
|
||||
compute_attention_and_moe_layers,
|
||||
resolve_layer_indices,
|
||||
)
|
||||
@@ -666,12 +665,7 @@ class ModelRunner:
|
||||
model=self.model,
|
||||
model_config=self.model_config,
|
||||
is_draft_worker=self.is_draft_worker,
|
||||
)
|
||||
adjust_hybrid_swa_layer_ids(
|
||||
model_config=self.model_config,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
is_hybrid_swa=self.is_hybrid_swa,
|
||||
draft_model_idx=self.draft_model_idx,
|
||||
)
|
||||
self.maybe_apply_post_load_model_transforms()
|
||||
self.maybe_init_lora_manager()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional
|
||||
|
||||
import msgspec
|
||||
from torch import nn
|
||||
@@ -130,6 +130,11 @@ class ModelLayerInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
start_layer: int
|
||||
end_layer: int
|
||||
num_effective_layers: int
|
||||
# Global ids of the layers this runner owns; None when the model has no split.
|
||||
swa_attention_layer_ids: Optional[list[int]] = None
|
||||
full_attention_layer_ids: Optional[list[int]] = None
|
||||
# Owns the single block at layer_id == draft_model_idx, not a [start, end) slice.
|
||||
is_hybrid_swa_mtp_draft: bool = False
|
||||
|
||||
|
||||
def resolve_layer_indices(
|
||||
@@ -137,6 +142,7 @@ def resolve_layer_indices(
|
||||
model: Any,
|
||||
model_config: ModelConfig,
|
||||
is_draft_worker: bool,
|
||||
draft_model_idx: Optional[int] = None,
|
||||
) -> ModelLayerInfo:
|
||||
# For MTP models like DeepSeek-V3 or GLM-4.5, the MTP layer(s) are used separately as draft
|
||||
# models for speculative decoding. In those cases, `num_nextn_predict_layers` is used to
|
||||
@@ -152,10 +158,43 @@ def resolve_layer_indices(
|
||||
if loop_num > 1:
|
||||
num_effective_layers = num_effective_layers * loop_num
|
||||
|
||||
is_hybrid_swa_mtp_draft = (
|
||||
is_draft_worker
|
||||
and draft_model_idx is not None
|
||||
and model_config.is_hybrid_swa
|
||||
and getattr(model, "mtp_layer_id_is_depth", False)
|
||||
)
|
||||
owned_layers = (
|
||||
range(draft_model_idx, draft_model_idx + 1)
|
||||
if is_hybrid_swa_mtp_draft
|
||||
else range(pp_range.start_layer, pp_range.end_layer)
|
||||
)
|
||||
swa_attention_layer_ids, full_attention_layer_ids = (
|
||||
_resolve_local_hybrid_swa_layer_ids(
|
||||
model_config=model_config, owned_layers=owned_layers
|
||||
)
|
||||
)
|
||||
|
||||
return ModelLayerInfo(
|
||||
start_layer=pp_range.start_layer,
|
||||
end_layer=pp_range.end_layer,
|
||||
num_effective_layers=num_effective_layers,
|
||||
swa_attention_layer_ids=swa_attention_layer_ids,
|
||||
full_attention_layer_ids=full_attention_layer_ids,
|
||||
is_hybrid_swa_mtp_draft=is_hybrid_swa_mtp_draft,
|
||||
)
|
||||
|
||||
|
||||
def _resolve_local_hybrid_swa_layer_ids(
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
owned_layers: range,
|
||||
) -> tuple[Optional[list[int]], Optional[list[int]]]:
|
||||
if model_config.swa_attention_layer_ids is None:
|
||||
return None, None
|
||||
return (
|
||||
[i for i in model_config.swa_attention_layer_ids if i in owned_layers],
|
||||
[i for i in model_config.full_attention_layer_ids if i in owned_layers],
|
||||
)
|
||||
|
||||
|
||||
@@ -196,32 +235,3 @@ def _resolve_pp_layer_range(*, model: Any, model_num_layers: int) -> _PPLayerRan
|
||||
start_layer=getattr(model, "start_layer", 0),
|
||||
end_layer=getattr(model, "end_layer", model_num_layers),
|
||||
)
|
||||
|
||||
|
||||
def adjust_hybrid_swa_layer_ids(
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
is_hybrid_swa: bool,
|
||||
) -> None:
|
||||
if not is_hybrid_swa:
|
||||
return
|
||||
|
||||
if model_config.is_deepseek_v4_arch:
|
||||
return
|
||||
|
||||
full_attention_layer_ids = [
|
||||
layer_idx
|
||||
for layer_idx in range(start_layer, end_layer + 1)
|
||||
if hasattr(model_config, "full_attention_layer_ids")
|
||||
and layer_idx in model_config.full_attention_layer_ids
|
||||
]
|
||||
swa_attention_layer_ids = [
|
||||
layer_idx
|
||||
for layer_idx in range(start_layer, end_layer + 1)
|
||||
if hasattr(model_config, "swa_attention_layer_ids")
|
||||
and layer_idx in model_config.swa_attention_layer_ids
|
||||
]
|
||||
model_config.swa_attention_layer_ids = swa_attention_layer_ids
|
||||
model_config.full_attention_layer_ids = full_attention_layer_ids
|
||||
|
||||
@@ -605,8 +605,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
kv_size = torch._utils._element_size(kv_cache_dtype)
|
||||
tp_size = get_parallel().attn_tp_size
|
||||
|
||||
self._full_layers_num = len(model_config.full_attention_layer_ids)
|
||||
self._swa_layers_num = len(model_config.swa_attention_layer_ids)
|
||||
self._full_layers_num = len(kvc.layer_info.full_attention_layer_ids)
|
||||
self._swa_layers_num = len(kvc.layer_info.swa_attention_layer_ids)
|
||||
assert self._swa_layers_num > 0, (
|
||||
"Hybrid SWA model must have at least one SWA layer"
|
||||
)
|
||||
@@ -915,7 +915,7 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
return False
|
||||
if kvc.sliding_window_size is None:
|
||||
return False
|
||||
return len(kvc.model_config.full_attention_layer_ids) > 0
|
||||
return len(kvc.layer_info.full_attention_layer_ids) > 0
|
||||
|
||||
def calculate_pool_sizes(
|
||||
self, available_bytes: int, page_size: int
|
||||
|
||||
@@ -1826,6 +1826,7 @@ class InklingForConditionalGenerationMTP(nn.Module):
|
||||
"""
|
||||
|
||||
fall_back_to_pt_during_load = False
|
||||
mtp_layer_id_is_depth = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user