From 19cb24f1e1c39edbdba7de4d46719150e6e55dec Mon Sep 17 00:00:00 2001 From: caihuali95 <42954765+caihuali95@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:37:50 -0700 Subject: [PATCH] [Feat] unified-memory: Support asymmetric KV rows (#43325) --- python/sglang/srt/arg_groups/kv_cache_hook.py | 31 ++-- .../srt/mem_cache/kv_cache_configurator.py | 137 +++++++++++------- .../srt/mem_cache/layout/fused_draft.py | 28 ++-- .../srt/mem_cache/unified_memory_pool.py | 7 +- .../mem_cache/test_fused_draft_placement.py | 99 ++++++++++++- .../unit/mem_cache/test_unified_tri_pool.py | 31 ++++ .../test_page_major_backend_allowlist.py | 50 ++++--- 7 files changed, 274 insertions(+), 109 deletions(-) diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 65c942dfe8..6c5ec13d92 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -754,6 +754,12 @@ def _validate_unified_memory_dcp(server_args: Any) -> None: ) +# Unified-memory backends that carry v_head_dim through to the kernel, so a +# V row narrower than K (MiMoV2: 192 / 128) reads correctly. FlashInfer's +# paged_kv_t compiles ONE head_dim for both; trtllm_mha refuses unequal dims. +ASYMMETRIC_KV_BACKENDS = frozenset({"triton", "fa3", "fa4"}) + + def handle_page_major_kv_layout(server_args: Any): # --enable-unified-memory implies --enable-page-major-kv-layout, so the # unified pool goes through this one gate (declared before the guard). @@ -774,23 +780,7 @@ def handle_page_major_kv_layout(server_args: Any): "reimplementation. Run with --enable-unified-memory, or drop " "--enable-page-major-kv-layout." ) - from sglang.srt.mem_cache.unified_memory_pool import ( - unified_memory_supported_for_model, - ) - model_config = model_config_of(server_args) - assert unified_memory_supported_for_model( - model_config, use_mla_backend=use_mla_backend(server_args) - ), ( - "--enable-unified-memory does not yet admit asymmetric K/V rows " - "(head_dim != v_head_dim); this model has " - f"head_dim={model_config.head_dim}, " - f"v_head_dim={model_config.v_head_dim}, " - f"swa_head_dim={model_config.swa_head_dim}, " - f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major " - "views can hold them, but the backends' write and read paths are " - "not audited for it; run this model without --enable-unified-memory." - ) # Allow-list. Every backend below reads through the translator, so what # gates one is only whether its kernels address the per-layer views by # their strides (the slot stride is the whole entry, not one row): @@ -798,6 +788,8 @@ def handle_page_major_kv_layout(server_args: Any): # snap). # * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside # Triton. fa4 is the fa3 class. + # * MHA/SWA models with asymmetric K/V rows: only the backends that + # carry v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). # Names are the RESOLVED ids from attention_backends_of. if use_mla_backend(server_args): allowed_full = { @@ -809,6 +801,8 @@ def handle_page_major_kv_layout(server_args: Any): "tokenspeed_mla", "flashmla", } + elif model_config.has_asymmetric_kv: + allowed_full = set(ASYMMETRIC_KV_BACKENDS) else: allowed_full = { "triton", @@ -822,9 +816,8 @@ def handle_page_major_kv_layout(server_args: Any): assert backends <= allowed_full, ( "--enable-page-major-kv-layout: the resolved attention backends " f"{sorted(backends)} are not in the allowed set " - f"{sorted(allowed_full)} for this configuration (unified memory " - "allows the stride-aware per-layer-view families). Pass a " - "compatible --attention-backend." + f"{sorted(allowed_full)} for this configuration. Pass a compatible " + "--attention-backend." ) # The Mamba/KDA state is stored in envelope-strided views; only # stride-audited kernels may read it (Stage 4 audit, per slot): diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 8859cbbbdb..0a9eda5be7 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -281,6 +281,21 @@ class _InitializedPools(msgspec.Struct, frozen=True, kw_only=True): unified_memory_pool: Optional[UnifiedKVPool] = None +class _UnifiedSWAHeadGeometry(msgspec.Struct, frozen=True, kw_only=True): + """Per-GPU head geometry of a hybrid-SWA host's full and SWA sub-pools. + + The byte solve prices `_full_per_token` / `_swa_per_token` from the same + ModelConfig fields, so the priced and the allocated entry agree. + """ + + head_num: int + head_dim: int + v_head_dim: int + swa_head_num: int + swa_head_dim: int + swa_v_head_dim: int + + class _PoolSizes(msgspec.Struct, frozen=True, kw_only=True): max_total_num_tokens: int max_running_requests: int @@ -859,6 +874,7 @@ class KVCacheConfigurator: get_parallel().attn_tp_size, get_parallel().attn_dcp_size ), head_dim=self.model_config.head_dim, + v_head_dim=self.model_config.v_head_dim, fused_draft=self._fused_draft_for_mamba_factory(), page_size=self.page_size, start_layer=self.layer_info.start_layer, @@ -929,26 +945,7 @@ class KVCacheConfigurator: if get_spec().speculative_num_draft_tokens is not None: extra_max_context_len += get_spec().speculative_num_draft_tokens - head_num = self.model_config.get_num_kv_heads( - get_parallel().attn_tp_size, get_parallel().attn_dcp_size - ) - head_dim = self.model_config.head_dim - if self.is_hybrid_swa_compress: - # Asymmetric full/SWA head geometry (Inkling): SWA dims from the - # hf text config, same as the 2-pool SWA wrapper. - v_head_dim = self.model_config.hf_text_config.v_head_dim - swa_head_num = max( - 1, - self.model_config.hf_text_config.swa_num_key_value_heads - // get_parallel().attn_tp_size, - ) - swa_head_dim = self.model_config.hf_text_config.swa_head_dim - swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim - else: - v_head_dim = head_dim - swa_head_num = head_num - swa_head_dim = head_dim - swa_v_head_dim = head_dim + geometry = self._unified_swa_head_geometry() # Not the HF config's full_attention_layer_ids: that one returns ALL layers. swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids @@ -973,12 +970,12 @@ class KVCacheConfigurator: return init_unified_mamba_swa_pools( device=self.device, kv_cache_dtype=self.kv_cache_dtype, - head_num=head_num, - head_dim=head_dim, - v_head_dim=v_head_dim, - swa_head_num=swa_head_num, - swa_head_dim=swa_head_dim, - swa_v_head_dim=swa_v_head_dim, + head_num=geometry.head_num, + head_dim=geometry.head_dim, + v_head_dim=geometry.v_head_dim, + swa_head_num=geometry.swa_head_num, + swa_head_dim=geometry.swa_head_dim, + swa_v_head_dim=geometry.swa_v_head_dim, page_size=self.page_size, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, @@ -1015,12 +1012,40 @@ class KVCacheConfigurator: fused_draft=self._fused_draft_for_mamba_factory(), ) + def _unified_swa_head_geometry(self) -> _UnifiedSWAHeadGeometry: + """Head geometry of a hybrid-SWA host's full and SWA sub-pools.""" + mc = self.model_config + attn_tp = get_parallel().attn_tp_size + head_num = mc.get_num_kv_heads(attn_tp, get_parallel().attn_dcp_size) + if self.is_hybrid_swa_compress: + # NPU compress path: the SWA dims live only on the hf text config. + return _UnifiedSWAHeadGeometry( + head_num=head_num, + head_dim=mc.head_dim, + v_head_dim=mc.hf_text_config.v_head_dim, + swa_head_num=max( + 1, mc.hf_text_config.swa_num_key_value_heads // attn_tp + ), + swa_head_dim=mc.hf_text_config.swa_head_dim, + swa_v_head_dim=mc.hf_text_config.swa_v_head_dim, + ) + return _UnifiedSWAHeadGeometry( + head_num=head_num, + head_dim=mc.head_dim, + v_head_dim=mc.v_head_dim, + swa_head_num=mc.get_swa_num_kv_heads(attn_tp), + swa_head_dim=mc.swa_head_dim, + swa_v_head_dim=mc.swa_v_head_dim, + ) + def _fused_draft_decision(self): """Whether, and where, the draft's layers fuse into the target's sub-pools. An empty decision means fusion does not apply (unified memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config loaded at target boot); a declined one says why the draft cannot fuse. - The pool factories decide whether a decline refuses the boot.""" + The pool factories decide whether a decline refuses the boot. + Asymmetric K/V rows fuse only on backends that carry v_head_dim through + to the kernel.""" from sglang.srt.mem_cache.layout.fused_draft import ( FusedDraftDecision, draft_kv_profile, @@ -1141,10 +1166,22 @@ class KVCacheConfigurator: return place_fused_draft( profile=profile, num_runners=num_runners, + asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(), store_dtype=_store_dtype_for(self.kv_cache_dtype), kv_dtype=self.kv_cache_dtype, ) + def _draft_backends_carry_v_head_dim(self) -> bool: + """Whether every resolved attention backend, the draft's included, + carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The + draft worker inherits the target's backend unless one is set.""" + from sglang.srt.arg_groups.kv_cache_hook import ASYMMETRIC_KV_BACKENDS + + backends = set(attention_backends()) + backends.add(get_spec().speculative_draft_attention_backend) + backends.discard(None) + return backends <= ASYMMETRIC_KV_BACKENDS + def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]: """Per-token bytes of ``sub_pool_name``'s fused entry (host + draft + pad), built from the same spec as the pool factory's so the priced and @@ -1183,6 +1220,18 @@ class KVCacheConfigurator: grow_direction="down", draft_region=region, ) + if self.is_hybrid_swa: + geometry = self._unified_swa_head_geometry() + return MHASubPoolSpec( + name="full", + layer_num=len(full_attention_layer_ids), + head_num=geometry.head_num, + head_dim=geometry.head_dim, + v_head_dim=geometry.v_head_dim, + store_dtype=_store_dtype_for(self.kv_cache_dtype), + grow_direction="down", + draft_region=region, + ) return MHASubPoolSpec( name="full", layer_num=len(full_attention_layer_ids), @@ -1190,6 +1239,7 @@ class KVCacheConfigurator: get_parallel().attn_tp_size, get_parallel().attn_dcp_size ), head_dim=self.model_config.head_dim, + v_head_dim=self.model_config.v_head_dim, store_dtype=_store_dtype_for(self.kv_cache_dtype), grow_direction="down", draft_region=region, @@ -1294,26 +1344,7 @@ class KVCacheConfigurator: ) req_to_token_pool = self._build_req_to_token_pool(max_num_reqs=max_num_reqs) - head_num = self.model_config.get_num_kv_heads( - get_parallel().attn_tp_size, get_parallel().attn_dcp_size - ) - head_dim = self.model_config.head_dim - if self.is_hybrid_swa_compress: - # Asymmetric head dims between full and SWA (NPU compress path): - # pull SWA-specific dims from the hf text config. - v_head_dim = self.model_config.hf_text_config.v_head_dim - swa_head_num = max( - 1, - self.model_config.hf_text_config.swa_num_key_value_heads - // get_parallel().attn_tp_size, - ) - swa_head_dim = self.model_config.hf_text_config.swa_head_dim - swa_v_head_dim = self.model_config.hf_text_config.swa_v_head_dim - else: - v_head_dim = head_dim - swa_head_num = head_num - swa_head_dim = head_dim - swa_v_head_dim = head_dim + geometry = self._unified_swa_head_geometry() swa_attention_layer_ids = self.layer_info.swa_attention_layer_ids full_attention_layer_ids = self.layer_info.full_attention_layer_ids @@ -1327,12 +1358,12 @@ class KVCacheConfigurator: bundle = init_unified_swa_pools( device=self.device, kv_cache_dtype=self.kv_cache_dtype, - head_num=head_num, - head_dim=head_dim, - v_head_dim=v_head_dim, - swa_head_num=swa_head_num, - swa_head_dim=swa_head_dim, - swa_v_head_dim=swa_v_head_dim, + head_num=geometry.head_num, + head_dim=geometry.head_dim, + v_head_dim=geometry.v_head_dim, + swa_head_num=geometry.swa_head_num, + swa_head_dim=geometry.swa_head_dim, + swa_v_head_dim=geometry.swa_v_head_dim, page_size=self.page_size, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, diff --git a/python/sglang/srt/mem_cache/layout/fused_draft.py b/python/sglang/srt/mem_cache/layout/fused_draft.py index c404b4e100..ba2c830a72 100644 --- a/python/sglang/srt/mem_cache/layout/fused_draft.py +++ b/python/sglang/srt/mem_cache/layout/fused_draft.py @@ -226,12 +226,14 @@ def place_fused_draft( profile: DraftKVProfile, num_runners: int, store_dtype: torch.dtype, + asymmetric_rows_ok: bool, kv_dtype: Optional[torch.dtype] = None, ) -> FusedDraftDecision: """Assign every draft layer of every runner to the host sub-pool whose lifetime covers what the layer reads: a full-attention layer rides in - ``"full"``. A layer kind no host sub-pool serves declines the whole - draft.""" + ``"full"``. A layer kind no host sub-pool serves declines the whole draft, + as do asymmetric K/V rows unless the caller vouches that every attention + backend carries v_head_dim through to the kernel.""" counts, reason = _runner_layer_counts(profile, num_runners) if counts is None: return FusedDraftDecision(declined=reason) @@ -251,22 +253,24 @@ def place_fused_draft( ) ) geometry = profile.full - if geometry.head_dim != geometry.v_head_dim: + if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok: return FusedDraftDecision( declined=( "the draft's K/V rows are asymmetric " - f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), " - "which is not admitted yet" + f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) " + "and a resolved attention backend does not carry v_head_dim " + "through to the kernel" ) ) - row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize - if row_bytes % ROW_ALIGN_BYTES: - return FusedDraftDecision( - declined=( - f"the draft's K/V rows are {row_bytes} B, not a multiple of the " - f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs" + for dim in (geometry.head_dim, geometry.v_head_dim): + row_bytes = geometry.head_num * dim * store_dtype.itemsize + if row_bytes % ROW_ALIGN_BYTES: + return FusedDraftDecision( + declined=( + f"the draft's K/V rows are {row_bytes} B, not a multiple of " + f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs" + ) ) - ) full_counts = tuple(full for full, _ in counts) region = DenseDraftRegion( lane_num=sum(full_counts), diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 661b15d6f8..cdebc8b996 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -342,11 +342,6 @@ class MambaSubPoolSpec(SubPoolSpec): # --------------------------------------------------------------------------- -def unified_memory_supported_for_model(model_config, *, use_mla_backend: bool) -> bool: - """Whether this model's KV geometry can back the unified memory pool.""" - return use_mla_backend or not model_config.has_asymmetric_kv - - def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None: """Physical ids must fit the int32 read-index buffers.""" assert n_rows < 2**31, ( @@ -1507,6 +1502,7 @@ def init_unified_mamba_pools( end_layer: int, is_draft_worker: bool, use_mla_backend: bool, + v_head_dim: Optional[int] = None, kv_lora_rank: Optional[int] = None, qk_rope_head_dim: Optional[int] = None, fused_draft: Optional[FusedDraftPlacement] = None, @@ -1564,6 +1560,7 @@ def init_unified_mamba_pools( layer_num=len(full_attention_layer_ids), head_num=head_num, head_dim=head_dim, + v_head_dim=v_head_dim, store_dtype=store_dtype, kv_cache_dtype=kv_cache_dtype, grow_direction="down", diff --git a/test/registered/unit/mem_cache/test_fused_draft_placement.py b/test/registered/unit/mem_cache/test_fused_draft_placement.py index cc1abf68a0..3075fcba60 100644 --- a/test/registered/unit/mem_cache/test_fused_draft_placement.py +++ b/test/registered/unit/mem_cache/test_fused_draft_placement.py @@ -21,8 +21,9 @@ Pinned: (one shared region would let the runners clobber each other's KV); - a per-depth head serves one depth per runner and needs one runner per depth; - - a draft with SWA or recurrent-state layers of its own, or asymmetric K/V - rows, declines; + - a draft with SWA or recurrent-state layers of its own declines, and so do + asymmetric K/V rows unless every resolved backend, the draft's included, + carries v_head_dim through to the kernel; - so does a draft whose attention backend is off the translated MHA rails (it would read the fused rows with virtual ids), a draft under --dcp-size > 1 (each rank's rows hold only its share of the tokens), an @@ -39,6 +40,8 @@ from unittest.mock import patch import torch +from sglang.srt.arg_groups.kv_cache_hook import ASYMMETRIC_KV_BACKENDS +from sglang.srt.mem_cache import kv_cache_configurator as kcc from sglang.srt.mem_cache.layout.fused_draft import ( DenseDraftRegion, DraftKVGeometry, @@ -72,11 +75,12 @@ def _profile( ) -def _place(profile, num_runners=1): +def _place(profile, num_runners=1, asymmetric_rows_ok=False): return place_fused_draft( profile=profile, num_runners=num_runners, store_dtype=_DTYPE, + asymmetric_rows_ok=asymmetric_rows_ok, ) @@ -112,6 +116,23 @@ class TestPlaceFusedDraft(CustomTestCase): self.assertIsNone(decision.placement) self.assertIsNotNone(decision.declined) + def test_asymmetric_rows_fuse_with_their_v_width_when_backends_allow(self): + placement = _place( + _profile(head_dim=64, v_head_dim=32), asymmetric_rows_ok=True + ).placement + self.assertIsNotNone(placement) + self.assertEqual(placement.region.resolved_v_head_dim(), 32) + self.assertEqual(placement.region.entry_bytes(), 4 * (64 + 32) * 2) + # Symmetric rows never consult the rule. + self.assertIsNotNone(_place(_profile(), asymmetric_rows_ok=False).placement) + + def test_a_misaligned_v_row_declines_even_when_backends_allow(self): + # K rows are 4 x 64 x 2 B = 512 B; V rows 4 x 5 x 2 B = 40 B are not a + # 16-B-aligned entry part. + decision = _place(_profile(head_dim=64, v_head_dim=5), asymmetric_rows_ok=True) + self.assertIsNone(decision.placement) + self.assertIn("40 B", decision.declined) + def test_region_carries_the_profile_geometry(self): placement = _place(_profile()).placement self.assertEqual( @@ -122,6 +143,78 @@ class TestPlaceFusedDraft(CustomTestCase): ) +class TestAsymmetricBackendRule(CustomTestCase): + """The rule reads the TARGET's resolved backends and the draft's explicit + one (unset inherits the target's); one shared-head_dim backend anywhere + disqualifies asymmetric rows.""" + + def _carries(self, *, backends, draft_backend=None): + cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + with ( + patch.object(kcc, "attention_backends", return_value=tuple(backends)), + patch.object( + kcc, + "get_spec", + return_value=SimpleNamespace( + speculative_draft_attention_backend=draft_backend + ), + ), + ): + return cfg._draft_backends_carry_v_head_dim() + + def test_every_split_stride_backend_qualifies(self): + for backend in sorted(ASYMMETRIC_KV_BACKENDS): + self.assertTrue(self._carries(backends=(backend,)), backend) + + def test_a_shared_head_dim_backend_anywhere_disqualifies(self): + self.assertNotIn("flashinfer", ASYMMETRIC_KV_BACKENDS) + self.assertFalse(self._carries(backends=("flashinfer",))) + self.assertFalse( + self._carries(backends=("triton",), draft_backend="flashinfer") + ) + + +class TestUnifiedSWAHeadGeometry(CustomTestCase): + """BUG REGRESSION. The hybrid-SWA factories hard-coded a symmetric shape + on GPU hosts, so an asymmetric model (MiMo-V2-Flash, 192/128) allocated + 192-wide V rows in both sub-pools while the boot solve priced 128.""" + + def _configurator(self): + cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator) + cfg.is_hybrid_swa = True + cfg.is_hybrid_swa_compress = False + cfg.use_mla_backend = False + cfg.kv_cache_dtype = _DTYPE + cfg.layer_info = SimpleNamespace(full_attention_layer_ids=[0, 3]) + cfg.model_config = SimpleNamespace( + get_num_kv_heads=lambda tp, dcp: 4, + head_dim=192, + v_head_dim=128, + get_swa_num_kv_heads=lambda tp: 8, + swa_head_dim=192, + swa_v_head_dim=128, + ) + return cfg + + def test_both_sub_pools_take_the_model_config_geometry(self): + with get_parallel().override(attn_tp_size=1, attn_dcp_size=1): + g = self._configurator()._unified_swa_head_geometry() + self.assertEqual((g.head_num, g.head_dim, g.v_head_dim), (4, 192, 128)) + self.assertEqual( + (g.swa_head_num, g.swa_head_dim, g.swa_v_head_dim), (8, 192, 128) + ) + + def test_the_fused_price_reads_the_same_geometry(self): + region = DenseDraftRegion( + lane_num=1, head_num=4, head_dim=64, store_dtype=_DTYPE + ) + with get_parallel().override(attn_tp_size=1, attn_dcp_size=1): + spec = self._configurator()._full_host_spec(region) + self.assertEqual( + (spec.layer_num, spec.head_dim, spec.v_head_dim), (2, 192, 128) + ) + + class TestFusedDraftDecision(CustomTestCase): """The target's boot decision over a host whose full sub-pool fuses.""" diff --git a/test/registered/unit/mem_cache/test_unified_tri_pool.py b/test/registered/unit/mem_cache/test_unified_tri_pool.py index 66a79051f0..cee8087ae3 100644 --- a/test/registered/unit/mem_cache/test_unified_tri_pool.py +++ b/test/registered/unit/mem_cache/test_unified_tri_pool.py @@ -35,6 +35,10 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import ( MultiEndedAllocator, ) from sglang.srt.mem_cache.base_prefix_cache import EvictParams +from sglang.srt.mem_cache.layout.fused_draft import ( + DenseDraftRegion, + FusedDraftPlacement, +) from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_memory_pool import ( UnifiedKVPool, @@ -1048,7 +1052,11 @@ class TestTriFactorySizing(unittest.TestCase): kw = self._factory_kwargs() model = SimpleNamespace( get_num_kv_heads=lambda tp, dcp: 2, + get_swa_num_kv_heads=lambda tp: 2, head_dim=4, + v_head_dim=4, + swa_head_dim=4, + swa_v_head_dim=4, context_len=16, full_attention_layer_ids=[0], swa_attention_layer_ids=[1], @@ -1076,6 +1084,11 @@ class TestTriFactorySizing(unittest.TestCase): forward_stream=None, # Spec off: no draft region to fuse. _fused_draft_for_mamba_factory=lambda: None, + _unified_swa_head_geometry=lambda: ( + cfg.KVCacheConfigurator._unified_swa_head_geometry( + configurator + ) + ), ) # Run the production configurator AND factory. Reverting # either top-level flag forwarding must break cleanup. @@ -1209,6 +1222,24 @@ class TestTriFactorySizing(unittest.TestCase): ) self.assertEqual(pool.total_bytes, want) + def test_full_side_carries_the_fused_draft_region(self): + """The boot solve prices the fused entry whenever a placement resolves, + so a factory that dropped the kwarg would allocate UNFUSED under that + price and under-budget the private draft pool.""" + region = DenseDraftRegion( + lane_num=1, head_num=2, head_dim=4, store_dtype=torch.float16 + ) + placement = FusedDraftPlacement(region=region, runner_lane_counts=(1,)) + bundle = init_unified_mamba_swa_pools( + **self._factory_kwargs(fused_draft=placement) + ) + pool = bundle.unified_memory_pool + self.assertIs(pool.fused_draft, placement) + self.assertIs(pool.spec("full").draft_region, region) + self.assertIs(pool.require_draft_host_spec("full"), pool.spec("full")) + for other in ("swa", "mamba"): + self.assertIsNone(pool.spec(other).draft_region, other) + def test_bs1_floor_fails_loud_before_construction(self): """A budget far below one worst-case request must raise BEFORE any pool construction -- under-sizing is a retract LIVELOCK at runtime.""" diff --git a/test/registered/unit/server_args/test_page_major_backend_allowlist.py b/test/registered/unit/server_args/test_page_major_backend_allowlist.py index d80fa7c9db..e35d1c3835 100644 --- a/test/registered/unit/server_args/test_page_major_backend_allowlist.py +++ b/test/registered/unit/server_args/test_page_major_backend_allowlist.py @@ -19,13 +19,15 @@ MHA/SWA arm, and no page-major arm at all without the unified pool. `fa3` is the resolved default on pre-Blackwell hosts, so its absence from an arm makes `--enable-unified-memory` fail to boot under its own default configuration. -The same handler screens the pool itself: asymmetric K/V rows are not admitted -yet (the token-major views can hold them, but the backends' read and write -paths are not audited for them), so an asymmetric-K/V model (MiMoV2: head_dim -192 != v_head_dim 128) is rejected on EVERY backend, Triton included. MLA -models are exempt -- their sub-pool keeps one latent row per layer, and real -MLA configs (Kimi-Linear: head_dim 72, v_head_dim 128) report asymmetric dims -while running the unified pool today. +The same handler narrows the MHA arm for asymmetric K/V rows (MiMoV2: +head_dim 192 != v_head_dim 128) to `ASYMMETRIC_KV_BACKENDS`, the backends +that carry v_head_dim through to the kernel; flashinfer's `paged_kv_t` +compiles one head_dim for both K and V, and trtllm_mha refuses unequal QK/V +dims outright. `fa4` is the resolved default for such a model on SM100, so +that arm keeps it for the same reason. MLA models are exempt -- their +sub-pool keeps one latent row per layer, and real MLA configs (Kimi-Linear: +head_dim 72, v_head_dim 128) report asymmetric dims while running the +unified pool today. """ import unittest @@ -33,7 +35,10 @@ from types import SimpleNamespace import msgspec -from sglang.srt.arg_groups.kv_cache_hook import handle_page_major_kv_layout +from sglang.srt.arg_groups.kv_cache_hook import ( + ASYMMETRIC_KV_BACKENDS, + handle_page_major_kv_layout, +) from sglang.srt.configs.model_config import AttentionArch from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci @@ -113,9 +118,7 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): the MLA nor the MHA arm can narrow away.""" for use_mla in (True, False): self.assertTrue(_accepts("triton", use_mla=use_mla)) - # The asymmetric-K/V screen is a property of the model, not of the - # backend, so it rejects even Triton. - self.assertFalse(_accepts("triton", use_mla=False, has_asymmetric_kv=True)) + self.assertTrue(_accepts("triton", use_mla=False, has_asymmetric_kv=True)) def test_per_layer_view_mla_backends_allowed_under_unified_mla(self): for backend in self.PER_LAYER_VIEW_MLA_BACKENDS: @@ -138,14 +141,27 @@ class TestPageMajorBackendAllowlist(unittest.TestCase): f"{backend} is an MLA kernel and must stay out of the MHA arm", ) - def test_asymmetric_kv_mha_model_cannot_use_unified_memory(self): - """The rejection is the POOL's, not a backend's, so it must fire on - every backend -- Triton included.""" - for backend in ("triton",) + self.PER_LAYER_VIEW_MHA_BACKENDS: + def test_asymmetric_kv_mha_model_narrows_to_v_head_dim_backends(self): + """head_dim != v_head_dim (MiMoV2): the token-major entry holds the + two row widths at different offsets, so admission is per backend -- + exactly the ones that carry v_head_dim through to the kernel. + + `fa4` must stay admitted: it is the resolved default for an + asymmetric-K/V model on SM100, so dropping it makes such a model + unbootable under its own default configuration.""" + for backend in ASYMMETRIC_KV_BACKENDS: + self.assertTrue( + _accepts(backend, use_mla=False, has_asymmetric_kv=True), + f"{backend} carries v_head_dim through and must admit an " + "asymmetric-K/V model", + ) + one_head_dim = set(self.PER_LAYER_VIEW_MHA_BACKENDS) - ASYMMETRIC_KV_BACKENDS + self.assertEqual(one_head_dim, {"flashinfer", "trtllm_mha"}) + for backend in one_head_dim: self.assertFalse( _accepts(backend, use_mla=False, has_asymmetric_kv=True), - f"--enable-unified-memory + {backend} must be rejected for an " - "asymmetric-K/V model", + f"{backend} reads V at K's head_dim (or refuses unequal dims) " + "and must stay rejected for an asymmetric-K/V model", ) def test_asymmetric_dims_do_not_screen_out_mla(self):