[Feat] unified-memory: Support asymmetric KV rows (#43325)

This commit is contained in:
caihuali95
2026-10-18 05:37:50 -07:00
committed by GitHub
parent 1d17066bad
commit 19cb24f1e1
7 changed files with 274 additions and 109 deletions
+12 -19
View File
@@ -754,6 +754,12 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
)
# Unified-memory backends that carry v_head_dim through to the kernel, so a
# V row narrower than K (MiMoV2: 192 / 128) reads correctly. FlashInfer's
# paged_kv_t compiles ONE head_dim for both; trtllm_mha refuses unequal dims.
ASYMMETRIC_KV_BACKENDS = frozenset({"triton", "fa3", "fa4"})
def handle_page_major_kv_layout(server_args: Any):
# --enable-unified-memory implies --enable-page-major-kv-layout, so the
# unified pool goes through this one gate (declared before the guard).
@@ -774,23 +780,7 @@ def handle_page_major_kv_layout(server_args: Any):
"reimplementation. Run with --enable-unified-memory, or drop "
"--enable-page-major-kv-layout."
)
from sglang.srt.mem_cache.unified_memory_pool import (
unified_memory_supported_for_model,
)
model_config = model_config_of(server_args)
assert unified_memory_supported_for_model(
model_config, use_mla_backend=use_mla_backend(server_args)
), (
"--enable-unified-memory does not yet admit asymmetric K/V rows "
"(head_dim != v_head_dim); this model has "
f"head_dim={model_config.head_dim}, "
f"v_head_dim={model_config.v_head_dim}, "
f"swa_head_dim={model_config.swa_head_dim}, "
f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major "
"views can hold them, but the backends' write and read paths are "
"not audited for it; run this model without --enable-unified-memory."
)
# Allow-list. Every backend below reads through the translator, so what
# gates one is only whether its kernels address the per-layer views by
# their strides (the slot stride is the whole entry, not one row):
@@ -798,6 +788,8 @@ def handle_page_major_kv_layout(server_args: Any):
# snap).
# * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside
# Triton. fa4 is the fa3 class.
# * MHA/SWA models with asymmetric K/V rows: only the backends that
# carry v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS).
# Names are the RESOLVED ids from attention_backends_of.
if use_mla_backend(server_args):
allowed_full = {
@@ -809,6 +801,8 @@ def handle_page_major_kv_layout(server_args: Any):
"tokenspeed_mla",
"flashmla",
}
elif model_config.has_asymmetric_kv:
allowed_full = set(ASYMMETRIC_KV_BACKENDS)
else:
allowed_full = {
"triton",
@@ -822,9 +816,8 @@ def handle_page_major_kv_layout(server_args: Any):
assert backends <= allowed_full, (
"--enable-page-major-kv-layout: the resolved attention backends "
f"{sorted(backends)} are not in the allowed set "
f"{sorted(allowed_full)} for this configuration (unified memory "
"allows the stride-aware per-layer-view families). Pass a "
"compatible --attention-backend."
f"{sorted(allowed_full)} for this configuration. Pass a compatible "
"--attention-backend."
)
# The Mamba/KDA state is stored in envelope-strided views; only
# stride-audited kernels may read it (Stage 4 audit, per slot):
@@ -281,6 +281,21 @@ class _InitializedPools(msgspec.Struct, frozen=True, kw_only=True):
unified_memory_pool: Optional[UnifiedKVPool] = None
class _UnifiedSWAHeadGeometry(msgspec.Struct, frozen=True, kw_only=True):
"""Per-GPU head geometry of a hybrid-SWA host's full and SWA sub-pools.
The byte solve prices `_full_per_token` / `_swa_per_token` from the same
ModelConfig fields, so the priced and the allocated entry agree.
"""
head_num: int
head_dim: int
v_head_dim: int
swa_head_num: int
swa_head_dim: int
swa_v_head_dim: int
class _PoolSizes(msgspec.Struct, frozen=True, kw_only=True):
max_total_num_tokens: int
max_running_requests: int
@@ -859,6 +874,7 @@ class KVCacheConfigurator:
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
v_head_dim=self.model_config.v_head_dim,
fused_draft=self._fused_draft_for_mamba_factory(),
page_size=self.page_size,
start_layer=self.layer_info.start_layer,
@@ -929,26 +945,7 @@ class KVCacheConfigurator:
if get_spec().speculative_num_draft_tokens is not None:
extra_max_context_len += get_spec().speculative_num_draft_tokens
head_num = self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
)
head_dim = self.model_config.head_dim
if self.is_hybrid_swa_compress:
# Asymmetric full/SWA head geometry (Inkling): SWA dims from the
# hf text config, same as the 2-pool SWA wrapper.
v_head_dim = self.model_config.hf_text_config.v_head_dim
swa_head_num = max(
1,
self.model_config.hf_text_config.swa_num_key_value_heads
// get_parallel().attn_tp_size,
)
swa_head_dim = self.model_config.hf_text_config.swa_head_dim
swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim
else:
v_head_dim = head_dim
swa_head_num = head_num
swa_head_dim = head_dim
swa_v_head_dim = head_dim
geometry = self._unified_swa_head_geometry()
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
@@ -973,12 +970,12 @@ class KVCacheConfigurator:
return init_unified_mamba_swa_pools(
device=self.device,
kv_cache_dtype=self.kv_cache_dtype,
head_num=head_num,
head_dim=head_dim,
v_head_dim=v_head_dim,
swa_head_num=swa_head_num,
swa_head_dim=swa_head_dim,
swa_v_head_dim=swa_v_head_dim,
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
swa_head_num=geometry.swa_head_num,
swa_head_dim=geometry.swa_head_dim,
swa_v_head_dim=geometry.swa_v_head_dim,
page_size=self.page_size,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
@@ -1015,12 +1012,40 @@ class KVCacheConfigurator:
fused_draft=self._fused_draft_for_mamba_factory(),
)
def _unified_swa_head_geometry(self) -> _UnifiedSWAHeadGeometry:
"""Head geometry of a hybrid-SWA host's full and SWA sub-pools."""
mc = self.model_config
attn_tp = get_parallel().attn_tp_size
head_num = mc.get_num_kv_heads(attn_tp, get_parallel().attn_dcp_size)
if self.is_hybrid_swa_compress:
# NPU compress path: the SWA dims live only on the hf text config.
return _UnifiedSWAHeadGeometry(
head_num=head_num,
head_dim=mc.head_dim,
v_head_dim=mc.hf_text_config.v_head_dim,
swa_head_num=max(
1, mc.hf_text_config.swa_num_key_value_heads // attn_tp
),
swa_head_dim=mc.hf_text_config.swa_head_dim,
swa_v_head_dim=mc.hf_text_config.swa_v_head_dim,
)
return _UnifiedSWAHeadGeometry(
head_num=head_num,
head_dim=mc.head_dim,
v_head_dim=mc.v_head_dim,
swa_head_num=mc.get_swa_num_kv_heads(attn_tp),
swa_head_dim=mc.swa_head_dim,
swa_v_head_dim=mc.swa_v_head_dim,
)
def _fused_draft_decision(self):
"""Whether, and where, the draft's layers fuse into the target's
sub-pools. An empty decision means fusion does not apply (unified
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
loaded at target boot); a declined one says why the draft cannot fuse.
The pool factories decide whether a decline refuses the boot."""
The pool factories decide whether a decline refuses the boot.
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
to the kernel."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
draft_kv_profile,
@@ -1141,10 +1166,22 @@ class KVCacheConfigurator:
return place_fused_draft(
profile=profile,
num_runners=num_runners,
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
store_dtype=_store_dtype_for(self.kv_cache_dtype),
kv_dtype=self.kv_cache_dtype,
)
def _draft_backends_carry_v_head_dim(self) -> bool:
"""Whether every resolved attention backend, the draft's included,
carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The
draft worker inherits the target's backend unless one is set."""
from sglang.srt.arg_groups.kv_cache_hook import ASYMMETRIC_KV_BACKENDS
backends = set(attention_backends())
backends.add(get_spec().speculative_draft_attention_backend)
backends.discard(None)
return backends <= ASYMMETRIC_KV_BACKENDS
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
pad), built from the same spec as the pool factory's so the priced and
@@ -1183,6 +1220,18 @@ class KVCacheConfigurator:
grow_direction="down",
draft_region=region,
)
if self.is_hybrid_swa:
geometry = self._unified_swa_head_geometry()
return MHASubPoolSpec(
name="full",
layer_num=len(full_attention_layer_ids),
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
grow_direction="down",
draft_region=region,
)
return MHASubPoolSpec(
name="full",
layer_num=len(full_attention_layer_ids),
@@ -1190,6 +1239,7 @@ class KVCacheConfigurator:
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
v_head_dim=self.model_config.v_head_dim,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
grow_direction="down",
draft_region=region,
@@ -1294,26 +1344,7 @@ class KVCacheConfigurator:
)
req_to_token_pool = self._build_req_to_token_pool(max_num_reqs=max_num_reqs)
head_num = self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
)
head_dim = self.model_config.head_dim
if self.is_hybrid_swa_compress:
# Asymmetric head dims between full and SWA (NPU compress path):
# pull SWA-specific dims from the hf text config.
v_head_dim = self.model_config.hf_text_config.v_head_dim
swa_head_num = max(
1,
self.model_config.hf_text_config.swa_num_key_value_heads
// get_parallel().attn_tp_size,
)
swa_head_dim = self.model_config.hf_text_config.swa_head_dim
swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim
else:
v_head_dim = head_dim
swa_head_num = head_num
swa_head_dim = head_dim
swa_v_head_dim = head_dim
geometry = self._unified_swa_head_geometry()
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
@@ -1327,12 +1358,12 @@ class KVCacheConfigurator:
bundle = init_unified_swa_pools(
device=self.device,
kv_cache_dtype=self.kv_cache_dtype,
head_num=head_num,
head_dim=head_dim,
v_head_dim=v_head_dim,
swa_head_num=swa_head_num,
swa_head_dim=swa_head_dim,
swa_v_head_dim=swa_v_head_dim,
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
swa_head_num=geometry.swa_head_num,
swa_head_dim=geometry.swa_head_dim,
swa_v_head_dim=geometry.swa_v_head_dim,
page_size=self.page_size,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
@@ -226,12 +226,14 @@ def place_fused_draft(
profile: DraftKVProfile,
num_runners: int,
store_dtype: torch.dtype,
asymmetric_rows_ok: bool,
kv_dtype: Optional[torch.dtype] = None,
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host sub-pool serves declines the whole
draft."""
``"full"``. A layer kind no host sub-pool serves declines the whole draft,
as do asymmetric K/V rows unless the caller vouches that every attention
backend carries v_head_dim through to the kernel."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
@@ -251,22 +253,24 @@ def place_fused_draft(
)
)
geometry = profile.full
if geometry.head_dim != geometry.v_head_dim:
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
return FusedDraftDecision(
declined=(
"the draft's K/V rows are asymmetric "
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), "
"which is not admitted yet"
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
"and a resolved attention backend does not carry v_head_dim "
"through to the kernel"
)
)
row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return FusedDraftDecision(
declined=(
f"the draft's K/V rows are {row_bytes} B, not a multiple of the "
f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
for dim in (geometry.head_dim, geometry.v_head_dim):
row_bytes = geometry.head_num * dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return FusedDraftDecision(
declined=(
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
)
)
)
full_counts = tuple(full for full, _ in counts)
region = DenseDraftRegion(
lane_num=sum(full_counts),
@@ -342,11 +342,6 @@ class MambaSubPoolSpec(SubPoolSpec):
# ---------------------------------------------------------------------------
def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) -> bool:
"""Whether this model's KV geometry can back the unified memory pool."""
return use_mla_backend or not model_config.has_asymmetric_kv
def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
"""Physical ids must fit the int32 read-index buffers."""
assert n_rows < 2**31, (
@@ -1507,6 +1502,7 @@ def init_unified_mamba_pools(
end_layer: int,
is_draft_worker: bool,
use_mla_backend: bool,
v_head_dim: Optional[int] = None,
kv_lora_rank: Optional[int] = None,
qk_rope_head_dim: Optional[int] = None,
fused_draft: Optional[FusedDraftPlacement] = None,
@@ -1564,6 +1560,7 @@ def init_unified_mamba_pools(
layer_num=len(full_attention_layer_ids),
head_num=head_num,
head_dim=head_dim,
v_head_dim=v_head_dim,
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",