[Feat] unified-memory: Support asymmetric KV rows (#43325)
This commit is contained in:
@@ -754,6 +754,12 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
|
||||
)
|
||||
|
||||
|
||||
# Unified-memory backends that carry v_head_dim through to the kernel, so a
|
||||
# V row narrower than K (MiMoV2: 192 / 128) reads correctly. FlashInfer's
|
||||
# paged_kv_t compiles ONE head_dim for both; trtllm_mha refuses unequal dims.
|
||||
ASYMMETRIC_KV_BACKENDS = frozenset({"triton", "fa3", "fa4"})
|
||||
|
||||
|
||||
def handle_page_major_kv_layout(server_args: Any):
|
||||
# --enable-unified-memory implies --enable-page-major-kv-layout, so the
|
||||
# unified pool goes through this one gate (declared before the guard).
|
||||
@@ -774,23 +780,7 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
"reimplementation. Run with --enable-unified-memory, or drop "
|
||||
"--enable-page-major-kv-layout."
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
unified_memory_supported_for_model,
|
||||
)
|
||||
|
||||
model_config = model_config_of(server_args)
|
||||
assert unified_memory_supported_for_model(
|
||||
model_config, use_mla_backend=use_mla_backend(server_args)
|
||||
), (
|
||||
"--enable-unified-memory does not yet admit asymmetric K/V rows "
|
||||
"(head_dim != v_head_dim); this model has "
|
||||
f"head_dim={model_config.head_dim}, "
|
||||
f"v_head_dim={model_config.v_head_dim}, "
|
||||
f"swa_head_dim={model_config.swa_head_dim}, "
|
||||
f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major "
|
||||
"views can hold them, but the backends' write and read paths are "
|
||||
"not audited for it; run this model without --enable-unified-memory."
|
||||
)
|
||||
# Allow-list. Every backend below reads through the translator, so what
|
||||
# gates one is only whether its kernels address the per-layer views by
|
||||
# their strides (the slot stride is the whole entry, not one row):
|
||||
@@ -798,6 +788,8 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
# snap).
|
||||
# * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside
|
||||
# Triton. fa4 is the fa3 class.
|
||||
# * MHA/SWA models with asymmetric K/V rows: only the backends that
|
||||
# carry v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS).
|
||||
# Names are the RESOLVED ids from attention_backends_of.
|
||||
if use_mla_backend(server_args):
|
||||
allowed_full = {
|
||||
@@ -809,6 +801,8 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
}
|
||||
elif model_config.has_asymmetric_kv:
|
||||
allowed_full = set(ASYMMETRIC_KV_BACKENDS)
|
||||
else:
|
||||
allowed_full = {
|
||||
"triton",
|
||||
@@ -822,9 +816,8 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
assert backends <= allowed_full, (
|
||||
"--enable-page-major-kv-layout: the resolved attention backends "
|
||||
f"{sorted(backends)} are not in the allowed set "
|
||||
f"{sorted(allowed_full)} for this configuration (unified memory "
|
||||
"allows the stride-aware per-layer-view families). Pass a "
|
||||
"compatible --attention-backend."
|
||||
f"{sorted(allowed_full)} for this configuration. Pass a compatible "
|
||||
"--attention-backend."
|
||||
)
|
||||
# The Mamba/KDA state is stored in envelope-strided views; only
|
||||
# stride-audited kernels may read it (Stage 4 audit, per slot):
|
||||
|
||||
@@ -281,6 +281,21 @@ class _InitializedPools(msgspec.Struct, frozen=True, kw_only=True):
|
||||
unified_memory_pool: Optional[UnifiedKVPool] = None
|
||||
|
||||
|
||||
class _UnifiedSWAHeadGeometry(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-GPU head geometry of a hybrid-SWA host's full and SWA sub-pools.
|
||||
|
||||
The byte solve prices `_full_per_token` / `_swa_per_token` from the same
|
||||
ModelConfig fields, so the priced and the allocated entry agree.
|
||||
"""
|
||||
|
||||
head_num: int
|
||||
head_dim: int
|
||||
v_head_dim: int
|
||||
swa_head_num: int
|
||||
swa_head_dim: int
|
||||
swa_v_head_dim: int
|
||||
|
||||
|
||||
class _PoolSizes(msgspec.Struct, frozen=True, kw_only=True):
|
||||
max_total_num_tokens: int
|
||||
max_running_requests: int
|
||||
@@ -859,6 +874,7 @@ class KVCacheConfigurator:
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
v_head_dim=self.model_config.v_head_dim,
|
||||
fused_draft=self._fused_draft_for_mamba_factory(),
|
||||
page_size=self.page_size,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
@@ -929,26 +945,7 @@ class KVCacheConfigurator:
|
||||
if get_spec().speculative_num_draft_tokens is not None:
|
||||
extra_max_context_len += get_spec().speculative_num_draft_tokens
|
||||
|
||||
head_num = self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
)
|
||||
head_dim = self.model_config.head_dim
|
||||
if self.is_hybrid_swa_compress:
|
||||
# Asymmetric full/SWA head geometry (Inkling): SWA dims from the
|
||||
# hf text config, same as the 2-pool SWA wrapper.
|
||||
v_head_dim = self.model_config.hf_text_config.v_head_dim
|
||||
swa_head_num = max(
|
||||
1,
|
||||
self.model_config.hf_text_config.swa_num_key_value_heads
|
||||
// get_parallel().attn_tp_size,
|
||||
)
|
||||
swa_head_dim = self.model_config.hf_text_config.swa_head_dim
|
||||
swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim
|
||||
else:
|
||||
v_head_dim = head_dim
|
||||
swa_head_num = head_num
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
geometry = self._unified_swa_head_geometry()
|
||||
|
||||
# Not the HF config's full_attention_layer_ids: that one returns ALL layers.
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
@@ -973,12 +970,12 @@ class KVCacheConfigurator:
|
||||
return init_unified_mamba_swa_pools(
|
||||
device=self.device,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
swa_head_num=swa_head_num,
|
||||
swa_head_dim=swa_head_dim,
|
||||
swa_v_head_dim=swa_v_head_dim,
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
swa_head_num=geometry.swa_head_num,
|
||||
swa_head_dim=geometry.swa_head_dim,
|
||||
swa_v_head_dim=geometry.swa_v_head_dim,
|
||||
page_size=self.page_size,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
@@ -1015,12 +1012,40 @@ class KVCacheConfigurator:
|
||||
fused_draft=self._fused_draft_for_mamba_factory(),
|
||||
)
|
||||
|
||||
def _unified_swa_head_geometry(self) -> _UnifiedSWAHeadGeometry:
|
||||
"""Head geometry of a hybrid-SWA host's full and SWA sub-pools."""
|
||||
mc = self.model_config
|
||||
attn_tp = get_parallel().attn_tp_size
|
||||
head_num = mc.get_num_kv_heads(attn_tp, get_parallel().attn_dcp_size)
|
||||
if self.is_hybrid_swa_compress:
|
||||
# NPU compress path: the SWA dims live only on the hf text config.
|
||||
return _UnifiedSWAHeadGeometry(
|
||||
head_num=head_num,
|
||||
head_dim=mc.head_dim,
|
||||
v_head_dim=mc.hf_text_config.v_head_dim,
|
||||
swa_head_num=max(
|
||||
1, mc.hf_text_config.swa_num_key_value_heads // attn_tp
|
||||
),
|
||||
swa_head_dim=mc.hf_text_config.swa_head_dim,
|
||||
swa_v_head_dim=mc.hf_text_config.swa_v_head_dim,
|
||||
)
|
||||
return _UnifiedSWAHeadGeometry(
|
||||
head_num=head_num,
|
||||
head_dim=mc.head_dim,
|
||||
v_head_dim=mc.v_head_dim,
|
||||
swa_head_num=mc.get_swa_num_kv_heads(attn_tp),
|
||||
swa_head_dim=mc.swa_head_dim,
|
||||
swa_v_head_dim=mc.swa_v_head_dim,
|
||||
)
|
||||
|
||||
def _fused_draft_decision(self):
|
||||
"""Whether, and where, the draft's layers fuse into the target's
|
||||
sub-pools. An empty decision means fusion does not apply (unified
|
||||
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
|
||||
loaded at target boot); a declined one says why the draft cannot fuse.
|
||||
The pool factories decide whether a decline refuses the boot."""
|
||||
The pool factories decide whether a decline refuses the boot.
|
||||
Asymmetric K/V rows fuse only on backends that carry v_head_dim through
|
||||
to the kernel."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
draft_kv_profile,
|
||||
@@ -1141,10 +1166,22 @@ class KVCacheConfigurator:
|
||||
return place_fused_draft(
|
||||
profile=profile,
|
||||
num_runners=num_runners,
|
||||
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
kv_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def _draft_backends_carry_v_head_dim(self) -> bool:
|
||||
"""Whether every resolved attention backend, the draft's included,
|
||||
carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The
|
||||
draft worker inherits the target's backend unless one is set."""
|
||||
from sglang.srt.arg_groups.kv_cache_hook import ASYMMETRIC_KV_BACKENDS
|
||||
|
||||
backends = set(attention_backends())
|
||||
backends.add(get_spec().speculative_draft_attention_backend)
|
||||
backends.discard(None)
|
||||
return backends <= ASYMMETRIC_KV_BACKENDS
|
||||
|
||||
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
|
||||
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
|
||||
pad), built from the same spec as the pool factory's so the priced and
|
||||
@@ -1183,6 +1220,18 @@ class KVCacheConfigurator:
|
||||
grow_direction="down",
|
||||
draft_region=region,
|
||||
)
|
||||
if self.is_hybrid_swa:
|
||||
geometry = self._unified_swa_head_geometry()
|
||||
return MHASubPoolSpec(
|
||||
name="full",
|
||||
layer_num=len(full_attention_layer_ids),
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
grow_direction="down",
|
||||
draft_region=region,
|
||||
)
|
||||
return MHASubPoolSpec(
|
||||
name="full",
|
||||
layer_num=len(full_attention_layer_ids),
|
||||
@@ -1190,6 +1239,7 @@ class KVCacheConfigurator:
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
v_head_dim=self.model_config.v_head_dim,
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
grow_direction="down",
|
||||
draft_region=region,
|
||||
@@ -1294,26 +1344,7 @@ class KVCacheConfigurator:
|
||||
)
|
||||
req_to_token_pool = self._build_req_to_token_pool(max_num_reqs=max_num_reqs)
|
||||
|
||||
head_num = self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
)
|
||||
head_dim = self.model_config.head_dim
|
||||
if self.is_hybrid_swa_compress:
|
||||
# Asymmetric head dims between full and SWA (NPU compress path):
|
||||
# pull SWA-specific dims from the hf text config.
|
||||
v_head_dim = self.model_config.hf_text_config.v_head_dim
|
||||
swa_head_num = max(
|
||||
1,
|
||||
self.model_config.hf_text_config.swa_num_key_value_heads
|
||||
// get_parallel().attn_tp_size,
|
||||
)
|
||||
swa_head_dim = self.model_config.hf_text_config.swa_head_dim
|
||||
swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim
|
||||
else:
|
||||
v_head_dim = head_dim
|
||||
swa_head_num = head_num
|
||||
swa_head_dim = head_dim
|
||||
swa_v_head_dim = head_dim
|
||||
geometry = self._unified_swa_head_geometry()
|
||||
|
||||
swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids
|
||||
full_attention_layer_ids = self.layer_info.full_attention_layer_ids
|
||||
@@ -1327,12 +1358,12 @@ class KVCacheConfigurator:
|
||||
bundle = init_unified_swa_pools(
|
||||
device=self.device,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
swa_head_num=swa_head_num,
|
||||
swa_head_dim=swa_head_dim,
|
||||
swa_v_head_dim=swa_v_head_dim,
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
swa_head_num=geometry.swa_head_num,
|
||||
swa_head_dim=geometry.swa_head_dim,
|
||||
swa_v_head_dim=geometry.swa_v_head_dim,
|
||||
page_size=self.page_size,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
|
||||
@@ -226,12 +226,14 @@ def place_fused_draft(
|
||||
profile: DraftKVProfile,
|
||||
num_runners: int,
|
||||
store_dtype: torch.dtype,
|
||||
asymmetric_rows_ok: bool,
|
||||
kv_dtype: Optional[torch.dtype] = None,
|
||||
) -> FusedDraftDecision:
|
||||
"""Assign every draft layer of every runner to the host sub-pool whose
|
||||
lifetime covers what the layer reads: a full-attention layer rides in
|
||||
``"full"``. A layer kind no host sub-pool serves declines the whole
|
||||
draft."""
|
||||
``"full"``. A layer kind no host sub-pool serves declines the whole draft,
|
||||
as do asymmetric K/V rows unless the caller vouches that every attention
|
||||
backend carries v_head_dim through to the kernel."""
|
||||
counts, reason = _runner_layer_counts(profile, num_runners)
|
||||
if counts is None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
@@ -251,22 +253,24 @@ def place_fused_draft(
|
||||
)
|
||||
)
|
||||
geometry = profile.full
|
||||
if geometry.head_dim != geometry.v_head_dim:
|
||||
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
"the draft's K/V rows are asymmetric "
|
||||
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), "
|
||||
"which is not admitted yet"
|
||||
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
|
||||
"and a resolved attention backend does not carry v_head_dim "
|
||||
"through to the kernel"
|
||||
)
|
||||
)
|
||||
row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize
|
||||
if row_bytes % ROW_ALIGN_BYTES:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft's K/V rows are {row_bytes} B, not a multiple of the "
|
||||
f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
|
||||
for dim in (geometry.head_dim, geometry.v_head_dim):
|
||||
row_bytes = geometry.head_num * dim * store_dtype.itemsize
|
||||
if row_bytes % ROW_ALIGN_BYTES:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
|
||||
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
|
||||
)
|
||||
)
|
||||
)
|
||||
full_counts = tuple(full for full, _ in counts)
|
||||
region = DenseDraftRegion(
|
||||
lane_num=sum(full_counts),
|
||||
|
||||
@@ -342,11 +342,6 @@ class MambaSubPoolSpec(SubPoolSpec):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) -> bool:
|
||||
"""Whether this model's KV geometry can back the unified memory pool."""
|
||||
return use_mla_backend or not model_config.has_asymmetric_kv
|
||||
|
||||
|
||||
def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
|
||||
"""Physical ids must fit the int32 read-index buffers."""
|
||||
assert n_rows < 2**31, (
|
||||
@@ -1507,6 +1502,7 @@ def init_unified_mamba_pools(
|
||||
end_layer: int,
|
||||
is_draft_worker: bool,
|
||||
use_mla_backend: bool,
|
||||
v_head_dim: Optional[int] = None,
|
||||
kv_lora_rank: Optional[int] = None,
|
||||
qk_rope_head_dim: Optional[int] = None,
|
||||
fused_draft: Optional[FusedDraftPlacement] = None,
|
||||
@@ -1564,6 +1560,7 @@ def init_unified_mamba_pools(
|
||||
layer_num=len(full_attention_layer_ids),
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="down",
|
||||
|
||||
Reference in New Issue
Block a user