From 79ab8f6d7c8f3d487558dde690bf386b3b2ce628 Mon Sep 17 00:00:00 2001 From: Shunkangz <182541032+Shunkangz@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:20:19 +0800 Subject: [PATCH] [Feat] KVShard: Support kv cache sharding for MTP (#40929) Co-authored-by: Zhangheng --- .../srt/mem_cache/kv_cache_configurator.py | 119 +++ .../sglang/srt/mem_cache/page_interleave.py | 74 +- .../srt/mem_cache/page_interleave_pool.py | 9 +- .../unit/managers/test_prefill_adder.py | 1 + .../test_page_interleave_gather_multi_gpu.py | 687 ++++++++++++++++++ .../mem_cache/test_page_interleave_shard.py | 655 ++++++++--------- 6 files changed, 1168 insertions(+), 377 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index af8e01ea89..fad48b1745 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -84,6 +84,7 @@ from sglang.srt.mem_cache.memory_pool import ( from sglang.srt.mem_cache.page_interleave import ( PageShardSpec, get_kv_shard_group, + get_shared_kv_shard_pool, make_page_shard_spec, ) from sglang.srt.mem_cache.page_interleave_pool import ( @@ -1763,6 +1764,15 @@ class KVCacheConfigurator: is_dsv4_model: bool, req_to_token_pool: ReqToTokenPool, ) -> KVCache: + # MTP drafts sharing the target's sharded allocator need the same shard + # layout with separate KV storage. Handle them before the normal pool + # dispatch; targets and drafts without that allocator fall through. + mtp_pool = self._build_mtp_kv_shard_pool( + sizes=sizes, is_dsa_model=is_dsa_model, is_dsv4_model=is_dsv4_model + ) + if mtp_pool is not None: + return mtp_pool + # Page-granularity envelope layout for the MHA-shaped (full / SWA) pools, # selected by swapping in the PageMajorMHATokenToKVPool subclass. The # default keeps upstream's per-layer layout. The Mamba state pool is routed @@ -1889,6 +1899,115 @@ class KVCacheConfigurator: ) return token_to_kv_pool + def _build_mtp_kv_shard_pool( + self, *, sizes: _PoolSizes, is_dsa_model: bool, is_dsv4_model: bool + ) -> Optional[KVCache]: + """Build a draft pool in its target's logical space, with its own KV. + + This is a pool-level opt-in through a shared sharded allocator. Target + pool creation and speculative serving remain separate integration work. + """ + if not self.is_draft_worker: + return None + + target_pool = get_shared_kv_shard_pool(self) + if target_pool is None: + return None + + if ( + self.spec_algorithm != SpeculativeAlgorithm.EAGLE + or (self.model_config.num_nextn_predict_layers or 0) <= 0 + or get_spec().enable_multi_layer_eagle + ): + raise ValueError( + "Shared KV sharding supports only an EAGLE MTP draft runner." + ) + if get_parallel().attn_dcp_size > 1: + raise ValueError("MTP KV sharding is incompatible with DCP.") + if ( + is_dsa_model + or is_dsv4_model + or self.is_hybrid_swa + or self.mambaish_config is not None + or self.sliding_window_size is not None + or is_minimax_sparse(self.model_config.hf_config) + or current_platform.is_out_of_tree() + or _is_npu + ): + raise ValueError("MTP KV sharding supports only dense MLA and MHA pools.") + if ( + self.post_capture_kv_active + or get_memory().enable_page_major_kv_layout + or self.kv_cache_dtype_str == "mxfp8" + or is_float4_e2m1fn_x2(self.kv_cache_dtype) + ): + raise ValueError("MTP KV sharding requires the plain per-layer KV layout.") + if ( + self.page_size != target_pool.page_size + or self.pool_page_size != target_pool.page_size + ): + raise ValueError( + "MTP and target KV pools must use the same physical page size." + ) + if sizes.max_total_num_tokens != target_pool.size: + raise ValueError( + "MTP and target sharded KV pools must have the same per-rank capacity." + ) + target_is_mla = isinstance(target_pool, PageInterleaveMLATokenToKVPool) + if self.use_mla_backend != target_is_mla or not isinstance( + target_pool, + (PageInterleaveMLATokenToKVPool, PageInterleaveMHATokenToKVPool), + ): + raise ValueError( + "MTP and target KV pools must use the same attention geometry." + ) + fields = ( + ("kv_lora_rank", "qk_rope_head_dim") + if self.use_mla_backend + else ("head_dim", "v_head_dim") + ) + if ( + self.kv_cache_dtype != target_pool.dtype + or any( + getattr(self.model_config, name) != getattr(target_pool, name) + for name in fields + ) + or ( + not self.use_mla_backend + and self.model_config.get_num_kv_heads(get_parallel().attn_tp_size) + != target_pool.head_num + ) + ): + raise ValueError( + "MTP and target KV pools must use the same KV geometry and dtype." + ) + + kwargs = dict( + size=target_pool.size, + page_size=target_pool.page_size, + dtype=self.kv_cache_dtype, + layer_num=self.layer_info.num_effective_layers, + device=self.device, + enable_memory_saver=get_exec().features.enable_memory_saver, + start_layer=self.layer_info.start_layer, + end_layer=self.layer_info.end_layer, + shard_spec=target_pool.shard_spec, + shard_group=target_pool.shard_group, + ) + if self.use_mla_backend: + return PageInterleaveMLATokenToKVPool( + **kwargs, + kv_lora_rank=self.model_config.kv_lora_rank, + qk_rope_head_dim=self.model_config.qk_rope_head_dim, + ) + return PageInterleaveMHATokenToKVPool( + **kwargs, + head_num=target_pool.head_num, + head_dim=self.model_config.head_dim, + v_head_dim=self.model_config.v_head_dim, + enable_alt_stream=not get_disagg().enable_pdmux, + ) + def _build_dsv4_kv_pool( self, *, diff --git a/python/sglang/srt/mem_cache/page_interleave.py b/python/sglang/srt/mem_cache/page_interleave.py index 3764cca215..5994676a92 100644 --- a/python/sglang/srt/mem_cache/page_interleave.py +++ b/python/sglang/srt/mem_cache/page_interleave.py @@ -37,12 +37,18 @@ from typing import TYPE_CHECKING, Optional, Tuple import msgspec import torch -from sglang.srt.runtime_context import get_parallel, get_schedule +from sglang.srt.mem_cache import page_interleave_pool +from sglang.srt.mem_cache.allocator.page_interleave import ( + PageInterleavePoolAllocator, +) +from sglang.srt.runtime_context import get_parallel, get_schedule, get_spec +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator + from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin logger = logging.getLogger(__name__) @@ -127,11 +133,42 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator: return cp_group +def get_shared_kv_shard_pool( + kvc: KVCacheConfigurator, +) -> Optional[PageInterleaveKVPoolMixin]: + """Find a draft's sharded target through their shared logical allocator.""" + if not kvc.is_draft_worker: + return None + + allocator = kvc.token_to_kv_pool_allocator + if not isinstance(allocator, PageInterleavePoolAllocator): + return None + + pool = allocator.get_kvcache() + if not isinstance(pool, page_interleave_pool.PageInterleaveKVPoolMixin): + raise ValueError("A sharded MTP draft requires the target's sharded KV pool.") + if ( + allocator.shard_spec != pool.shard_spec + or allocator.shard_size != pool.shard_spec.shard_size + or allocator.page_size != pool.page_size + or allocator.size != pool.size * allocator.shard_size + ): + raise ValueError( + "The shared allocator and target KV pool have different shard layouts." + ) + return pool + + def get_kv_shard_group_info( kvc: KVCacheConfigurator, ) -> Tuple[Optional[int], int]: """``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables.""" - if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding: + if kvc.is_draft_worker: + pool = get_shared_kv_shard_pool(kvc) + if pool is None: + return None, 1 + return pool.shard_spec.shard_rank, pool.shard_spec.shard_size + if not get_parallel().enable_kv_cache_sharding: return None, 1 group = get_kv_shard_group(kvc.use_mla_backend) if group.world_size <= 1: @@ -163,7 +200,14 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]: or chunk size. Both scratch slots are charged against HBM before sizing persistent KV; large model contexts therefore reduce persistent capacity. Admission still checks the actual batch's padded prefix and chunk spans. + + MTP drafts inherit the target's exact bounds: both pools serve the same + scheduled batch and logical locations, even if their model configs differ + in context length. """ + if kvc.is_draft_worker: + pool = get_shared_kv_shard_pool(kvc) + return pool.shard_spec if pool is not None else None shard_rank, shard_size = get_kv_shard_group_info(kvc) if shard_rank is None: return None @@ -181,11 +225,31 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]: ) -def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int: +def compute_page_shard_scratch_bytes( + kvc: KVCacheConfigurator, *, include_mtp: bool = False +) -> int: """Fixed HBM cost of the double-buffered assembly scratch, charged against the KV budget before pool sizing. Use the same spec as the pool builders: - two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.""" + two slots, each holding ONE layer's ``[prefix | chunk | trash page]``. + + ``include_mtp`` additionally budgets a target's single EAGLE MTP runner + at the same KV geometry, as required by the draft pool builder. Its two + scratch slots are reused across draft layers. Target sizing integration + can opt into this estimate without changing the per-pool default. + """ spec = make_page_shard_spec(kvc) if spec is None: return 0 - return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc) + scratch_bytes = 2 * spec.scratch_rows * _page_shard_row_bytes(kvc) + if include_mtp: + if ( + kvc.is_draft_worker + or kvc.spec_algorithm != SpeculativeAlgorithm.EAGLE + or (kvc.spec_aux_config.eagle_draft_num_layers or 0) <= 0 + or get_spec().enable_multi_layer_eagle + ): + raise ValueError( + "MTP scratch budgeting requires a target with one EAGLE MTP runner." + ) + scratch_bytes *= 2 + return scratch_bytes diff --git a/python/sglang/srt/mem_cache/page_interleave_pool.py b/python/sglang/srt/mem_cache/page_interleave_pool.py index 33a00f8cb6..8e09ce8ac4 100644 --- a/python/sglang/srt/mem_cache/page_interleave_pool.py +++ b/python/sglang/srt/mem_cache/page_interleave_pool.py @@ -61,6 +61,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import ( ) from sglang.kernels.ops.kvcache import dsa_index_k_cache from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.mem_cache import page_interleave from sglang.srt.mem_cache.index_key_cache import IndexKeyCache from sglang.srt.mem_cache.memory_pool import ( GPU_MEMORY_TYPE_KV_CACHE, @@ -70,15 +71,12 @@ from sglang.srt.mem_cache.memory_pool import ( RadixAttention, unwrap_write_loc, ) -from sglang.srt.mem_cache.page_interleave import ( - PageInterleavePlacement, - PageShardSpec, -) from sglang.srt.mem_cache.utils import get_mla_kv_buffer_triton from sglang.srt.utils import ceil_div, get_bool_env_var if TYPE_CHECKING: from sglang.srt.distributed.parallel_state import GroupCoordinator + from sglang.srt.mem_cache.page_interleave import PageShardSpec logger = logging.getLogger(__name__) @@ -123,7 +121,8 @@ class PageInterleaveKVPoolMixin: assert spec.chunk_tokens % spec.page_size == 0 self.shard_spec = spec - self.placement = PageInterleavePlacement(spec) + self.shard_group = shard_group + self.placement = page_interleave.PageInterleavePlacement(spec) self.shard_rank = spec.shard_rank self.shard_size = spec.shard_size self.device_module = torch.get_device_module(self.device) diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index f8f38c1e16..a7d41399aa 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -972,6 +972,7 @@ class TestPrefillAdder(CustomTestCase): extend_len = 1024 prefix_len = context_len - extend_len kvc = SimpleNamespace( + is_draft_worker=False, page_size=4, model_config=SimpleNamespace(context_len=context_len), ) diff --git a/test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py b/test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py new file mode 100644 index 0000000000..72a5489766 --- /dev/null +++ b/test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py @@ -0,0 +1,687 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Multi-GPU tests for logical-page KV cache sharding: the real NCCL +layer-ahead gather over a real 2-rank process group. + +``test_page_interleave_shard.py`` pins the plan arithmetic on CPU stubs. These +tests drive real pools, which is the only check that the plan the stubs +validate actually addresses the bytes the collective delivers: + +1. MLA pool sharded across the attention-TP group: replicated writes are + owner-filtered into disjoint pool stripes; a later batch's chunked-prefix + read (get_mla_kv_buffer) assembles the full prefix from all ranks via the + layer-ahead NCCL allgather and must return the canonical bytes. +2. MHA pool sharded across the attention-CP group: the post-allgather full + chunk is staged into the scratch chunk region and owner-persisted; a later + batch reads prefix+chunk through the translated page table (the scratch), + and the assembled rows must match the canonical bytes. +3. MTP draft pools sharing the target's sharded allocator, for MLA and MHA: + exact KV bytes, separate target/draft storage and scratch plans, fragmented + pages, partial tail pages, both MLA write APIs, and local/nonzero draft + layer IDs across batches. + +Registered as a base-b 2-gpu-large unit test; skips when fewer than 2 CUDA +devices are visible. Run directly with: + CUDA_VISIBLE_DEVICES=0,1 python3 \ + test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py +""" + +import os +import unittest +from types import SimpleNamespace + +import torch +import torch.multiprocessing as mp + +from sglang.srt.distributed import ( + init_distributed_environment, + initialize_model_parallel, +) +from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator +from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator +from sglang.srt.mem_cache.page_interleave import PageShardSpec, get_kv_shard_group +from sglang.srt.mem_cache.page_interleave_pool import ( + PageInterleaveMHATokenToKVPool, + PageInterleaveMLATokenToKVPool, +) +from sglang.srt.runtime_context import get_context, get_parallel, publish +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase, publish_build_topology + +register_cuda_ci(est_time=120, stage="base-b", runner_config="2-gpu-large") + +_GATHER_WORLD = 2 +_GATHER_LAYER_NUM = 4 +_GATHER_PAGE_SIZE = 16 +_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE +_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank +_GATHER_KV_LORA_RANK = 128 +_GATHER_QK_ROPE = 32 +_GATHER_HEAD_NUM = 2 +_GATHER_HEAD_DIM = 32 +_GATHER_DTYPE = torch.bfloat16 +_MTP_SIZE = 32 * _GATHER_PAGE_SIZE # Physical token capacity per rank. +_MTP_UNWRITTEN = -256 + + +def _mla_value(loc, dim): + """Deterministic canonical latent value for logical slot ``loc``.""" + loc = loc.to(torch.float32) + return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to( + _GATHER_DTYPE + ) + + +def _dist_init(rank, world, port, attn_cp_size): + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = str(port) + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world) + os.environ.setdefault("no_proxy", "127.0.0.1,localhost") + torch.cuda.set_device(rank) + + init_distributed_environment( + world_size=world, + rank=rank, + local_rank=rank, + distributed_init_method=f"tcp://127.0.0.1:{port}", + backend="nccl", + ) + # Publish the widths the groups below are about to be built at. The derived + # quotients (attn_tp_size, attn_dcp_size, ...) are projected from these + # leaves at publish; initialize_model_parallel no longer supplies them, and + # MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path. + publish( + ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size), + role="scheduler", + ) + publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank) + initialize_model_parallel() + + +def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4): + return PageShardSpec( + shard_rank=shard_rank, + shard_size=_GATHER_WORLD, + page_size=_GATHER_PAGE_SIZE, + max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE, + chunk_tokens=chunk_groups * _GATHER_GRANULE, + ) + + +def _fake_req_to_token(groups, seq_len, device): + """req_to_token row where sequence group j is allocator group groups[j].""" + row = torch.zeros( + (1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device + ) + for j, q in enumerate(groups): + row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange( + q * _GATHER_GRANULE, + (q + 1) * _GATHER_GRANULE, + dtype=torch.int32, + device=device, + ) + return row[:, :seq_len] if seq_len < row.shape[1] else row + + +def _check(rank, name, got, expect, atol=0.0): + ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0) + max_err = (got.float() - expect.float()).abs().max().item() + print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}") + assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})" + + +def _run_mla(rank, world, port): + _dist_init(rank, world, port, attn_cp_size=1) + + group = get_parallel().attn_tp_group + assert group.world_size == world + # Topology-first shard-group selection: no CP here, so MLA falls back to + # the attn-TP axis, while GQA has no replicated axis (world_size 1). + assert get_kv_shard_group(use_mla_backend=True) is group + assert get_kv_shard_group(use_mla_backend=False).world_size == 1 + spec = _gather_make_spec(shard_rank=group.rank_in_group) + + pool = PageInterleaveMLATokenToKVPool( + _GATHER_SIZE, + page_size=_GATHER_PAGE_SIZE, + dtype=_GATHER_DTYPE, + kv_lora_rank=_GATHER_KV_LORA_RANK, + qk_rope_head_dim=_GATHER_QK_ROPE, + layer_num=_GATHER_LAYER_NUM, + device=f"cuda:{rank}", + enable_memory_saver=False, + start_layer=0, + end_layer=_GATHER_LAYER_NUM - 1, + shard_spec=spec, + shard_group=group, + ) + device = pool.kv_buffer[0].device + + # ---- chunk 1: replicated write, owner-filtered persist ----------------- + # "Allocator" hands out fragmented groups (identical on every rank). + chunk1_groups = [5, 2, 9] + chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[ + 0 + ].long() + for layer_id in range(_GATHER_LAYER_NUM): + layer = SimpleNamespace(layer_id=layer_id) + vals = _mla_value( + chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE + ) + pool.set_mla_kv_buffer( + layer, + chunk1_locs, + vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1), + vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1), + ) + torch.cuda.synchronize() + torch.distributed.barrier() + + # Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps) + # on every rank, holding that rank's page of the group. + for q in chunk1_groups: + local_rows = torch.arange( + q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device + ) + owned_locs = ( + q * _GATHER_GRANULE + + group.rank_in_group * _GATHER_PAGE_SIZE + + torch.arange(_GATHER_PAGE_SIZE, device=device) + ) + got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE) + expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE) + _check(rank, f"mla owned stripe g{q}", got, expect) + + # ---- chunk 2: prefix gather + staged chunk, both read styles ----------- + seq_groups = chunk1_groups + [12] # one new chunk group + prefix_len = 3 * _GATHER_GRANULE + seq_len = prefix_len + _GATHER_GRANULE + req_to_token = _fake_req_to_token(seq_groups, seq_len, device) + chunk2_locs = req_to_token[0, prefix_len:seq_len].long() + pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len]) + + for layer_id in range(_GATHER_LAYER_NUM): + layer = SimpleNamespace(layer_id=layer_id) + # Write the current chunk (stages it into the slot + persists the + # owned stripe), like the extend forward does before attention. + chunk_vals = _mla_value( + chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE + ) + pool.set_mla_kv_buffer( + layer, + chunk2_locs, + chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1), + chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1), + ) + # Chunked-prefix MHA style: fetch an arbitrary sub-range of the + # prefix through get_mla_kv_buffer. + sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3] + k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE) + expect = _mla_value( + sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE + ) + _check( + rank, + f"mla prefix read l{layer_id}", + k_nope[:, 0, :], + expect[:, :_GATHER_KV_LORA_RANK], + ) + _check( + rank, + f"mla prefix rope l{layer_id}", + k_rope[:, 0, :], + expect[:, _GATHER_KV_LORA_RANK:], + ) + # Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk] + # from get_key_buffer through the translated page table. + all_locs = req_to_token[0, :seq_len].long() + rows = pool.translate_loc_to_scratch(all_locs) + kv_scratch = pool.get_key_buffer(layer_id) + _check( + rank, + f"mla absorbed read l{layer_id}", + kv_scratch[rows, 0, :], + _mla_value( + all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE + ), + ) + + torch.distributed.barrier() + if rank == 0: + print("PASS: MLA page-interleave shard (attn-TP axis)") + + +def _run_mha(rank, world, port): + _dist_init(rank, world, port, attn_cp_size=world) + + group = get_parallel().attn_cp_group + assert group.world_size == world + # Topology-first shard-group selection: with an active CP group, both + # GQA and MLA shard across CP (CP replicates KV for every attention + # type; the TP axis is only the no-CP MLA fallback). + assert get_kv_shard_group(use_mla_backend=False) is group + assert get_kv_shard_group(use_mla_backend=True) is group + spec = _gather_make_spec(shard_rank=group.rank_in_group) + + pool = PageInterleaveMHATokenToKVPool( + _GATHER_SIZE, + page_size=_GATHER_PAGE_SIZE, + dtype=_GATHER_DTYPE, + head_num=_GATHER_HEAD_NUM, + head_dim=_GATHER_HEAD_DIM, + layer_num=_GATHER_LAYER_NUM, + device=f"cuda:{rank}", + enable_memory_saver=False, + start_layer=0, + end_layer=_GATHER_LAYER_NUM - 1, + enable_alt_stream=False, + shard_spec=spec, + shard_group=group, + ) + device = pool.k_buffer[0].device + + def kv_value(locs, layer_id, is_v): + base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0) + return ( + base.view(-1, 1, 1) + + torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01 + + torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001 + ).to(_GATHER_DTYPE) + + # ---- chunk 1 (prefix-less batch): stage + owner-persist ---------------- + chunk1_groups = [7, 3] + chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[ + 0 + ].long() + req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device) + pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE]) + for layer_id in range(_GATHER_LAYER_NUM): + layer = SimpleNamespace(layer_id=layer_id) + pool.set_kv_buffer( + layer, + chunk1_locs, + kv_value(chunk1_locs, layer_id, False), + kv_value(chunk1_locs, layer_id, True), + ) + # The current chunk must be readable through the scratch right away. + k_scratch = pool.get_key_buffer(layer_id) + rows = pool.translate_loc_to_scratch(chunk1_locs) + _check( + rank, + f"mha chunk stage l{layer_id}", + k_scratch[rows], + kv_value(chunk1_locs, layer_id, False), + ) + torch.cuda.synchronize() + torch.distributed.barrier() + + # ---- chunk 2: prefix gathered from peers via translated page table ----- + seq_groups = chunk1_groups + [11] + prefix_len = 2 * _GATHER_GRANULE + seq_len = prefix_len + _GATHER_GRANULE + req_to_token = _fake_req_to_token(seq_groups, seq_len, device) + chunk2_locs = req_to_token[0, prefix_len:seq_len].long() + pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len]) + + for layer_id in range(_GATHER_LAYER_NUM): + layer = SimpleNamespace(layer_id=layer_id) + pool.set_kv_buffer( + layer, + chunk2_locs, + kv_value(chunk2_locs, layer_id, False), + kv_value(chunk2_locs, layer_id, True), + ) + all_locs = req_to_token[0, :seq_len].long() + rows = pool.translate_loc_to_scratch(all_locs) + k_scratch = pool.get_key_buffer(layer_id) + v_scratch = pool.get_value_buffer(layer_id) + _check( + rank, + f"mha seq read k l{layer_id}", + k_scratch[rows], + kv_value(all_locs, layer_id, False), + ) + _check( + rank, + f"mha seq read v l{layer_id}", + v_scratch[rows], + kv_value(all_locs, layer_id, True), + ) + + torch.distributed.barrier() + if rank == 0: + print("PASS: MHA page-interleave shard (attn-CP axis)") + + +def _mtp_make_pool(rank, group, kind, start_layer, layer_num): + kwargs = dict( + size=_MTP_SIZE, + page_size=_GATHER_PAGE_SIZE, + dtype=_GATHER_DTYPE, + layer_num=layer_num, + device=f"cuda:{rank}", + enable_memory_saver=False, + start_layer=start_layer, + end_layer=start_layer + layer_num - 1, + shard_spec=PageShardSpec( + shard_rank=group.rank_in_group, + shard_size=_GATHER_WORLD, + page_size=_GATHER_PAGE_SIZE, + max_prefix_tokens=16 * _GATHER_WORLD * _GATHER_PAGE_SIZE, + chunk_tokens=4 * _GATHER_PAGE_SIZE, + ), + shard_group=group, + ) + if kind == "mla": + pool = PageInterleaveMLATokenToKVPool( + **kwargs, + kv_lora_rank=_GATHER_KV_LORA_RANK, + qk_rope_head_dim=_GATHER_QK_ROPE, + ) + else: + pool = PageInterleaveMHATokenToKVPool( + **kwargs, + head_num=_GATHER_HEAD_NUM, + head_dim=_GATHER_HEAD_DIM, + enable_alt_stream=False, + ) + for local_layer in range(layer_num): + for buffer in _mtp_buffers(pool, kind, local_layer): + buffer.fill_(_MTP_UNWRITTEN) + return pool + + +def _mtp_buffers(pool, kind, local_layer): + if kind == "mla": + return (pool.kv_buffer[local_layer],) + return (pool.k_buffer[local_layer], pool.v_buffer[local_layer]) + + +def _mtp_make_draft_pool(target, allocator, kind, start_layer): + # Skip model loading while using the production pool dispatch and sharing + # the target allocator. A replicated draft pool must fail this test. + kvc = KVCacheConfigurator.__new__(KVCacheConfigurator) + kvc.device = target.device + kvc.is_draft_worker = True + kvc.page_size = _GATHER_PAGE_SIZE + kvc.use_mla_backend = kind == "mla" + kvc.kv_cache_dtype = _GATHER_DTYPE + kvc.kv_cache_dtype_str = "bfloat16" + kvc.post_capture_kv_active = False + kvc.is_hybrid_swa = False + kvc.sliding_window_size = None + kvc.mambaish_config = None + kvc.token_to_kv_pool_allocator = allocator + kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE + kvc.model_config = SimpleNamespace( + num_nextn_predict_layers=1, + hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]), + kv_lora_rank=_GATHER_KV_LORA_RANK, + qk_rope_head_dim=_GATHER_QK_ROPE, + head_dim=_GATHER_HEAD_DIM, + v_head_dim=_GATHER_HEAD_DIM, + get_num_kv_heads=lambda *_args: _GATHER_HEAD_NUM, + ) + kvc.layer_info = SimpleNamespace( + start_layer=start_layer, end_layer=start_layer + 1, num_effective_layers=1 + ) + with get_context().override_server_args( + page_size=_GATHER_PAGE_SIZE, + tp_size=_GATHER_WORLD, + attn_cp_size=_GATHER_WORLD if kind == "mha" else 1, + speculative_algorithm="EAGLE", + enable_multi_layer_eagle=False, + ): + pool = kvc._build_token_to_kv_pool( + sizes=SimpleNamespace(max_total_num_tokens=target.size), + is_dsa_model=False, + is_dsv4_model=False, + req_to_token_pool=None, + ) + assert pool.shard_spec is target.shard_spec + assert pool.shard_group is target.shard_group + for buffer in _mtp_buffers(pool, kind, 0): + buffer.fill_(_MTP_UNWRITTEN) + return pool + + +def _mtp_values(locs, width, tag, local_layer, is_v=False): + """Each location, feature, layer, and pool has distinguishable BF16 bytes. + + Small integers survive BF16 exactly; high-valued logical slot IDs plus tiny + feature offsets would round away the differences this test must detect. + The first two columns encode the full logical slot, avoiding hash aliases. + """ + features = torch.arange(width, device=locs.device) + values = ( + locs[:, None] * 37 + + features[None, :] * 17 + + tag * 73 + + local_layer * 11 + + int(is_v) * 97 + ) % 251 - 125 + values[:, 0] = locs % 128 + values[:, 1] = locs // 128 + values[:, 2] = tag + values[:, 3] = local_layer + values[:, 4] = int(is_v) + return values.to(_GATHER_DTYPE) + + +def _mtp_canonical(kind, locs, tag, local_layer): + if kind == "mla": + return ( + _mtp_values( + locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE, tag, local_layer + ).unsqueeze(1), + ) + return tuple( + _mtp_values( + locs, _GATHER_HEAD_NUM * _GATHER_HEAD_DIM, tag, local_layer, is_v + ).reshape(-1, _GATHER_HEAD_NUM, _GATHER_HEAD_DIM) + for is_v in (False, True) + ) + + +def _mtp_assert_bytes(got, expected, label): + assert got.shape == expected.shape, label + assert got.dtype == expected.dtype == _GATHER_DTYPE, label + assert torch.equal( + got.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8) + ), f"{label}: KV bytes differ" + + +def _mtp_logical_chain(device): + # Owners alternate from rank 1, while local pages run out of order. Some + # logical slots exceed the physical pool size, catching a replicated draft + # indexing them raw. + local_pages = torch.tensor([25, 3, 20, 6, 28, 4, 30, 8, 22], device=device) + owners = (1 + torch.arange(local_pages.numel(), device=device)) % _GATHER_WORLD + pages = local_pages * _GATHER_WORLD + owners + return ( + pages[:, None] * _GATHER_PAGE_SIZE + + torch.arange(_GATHER_PAGE_SIZE, device=device) + ).reshape(-1) + + +def _mtp_write(pool, kind, locs, tag, local_layer, epoch): + layer = SimpleNamespace(layer_id=pool.start_layer + local_layer) + values = _mtp_canonical(kind, locs, tag, local_layer) + if kind == "mha": + pool.set_kv_buffer(layer, locs, *values) + elif epoch % 2: + # Both MLA write entry points must owner-filter and stage MTP rows. + pool.set_mla_kv_buffer( + layer, + locs, + values[0][..., :_GATHER_KV_LORA_RANK], + values[0][..., _GATHER_KV_LORA_RANK:], + ) + else: + pool.set_kv_buffer( + layer, locs, values[0], values[0][..., :_GATHER_KV_LORA_RANK] + ) + + +def _mtp_check_persisted(pool, kind, locs, tag, rank): + owned_locs = locs[(locs // _GATHER_PAGE_SIZE) % _GATHER_WORLD == rank] + local_rows = ( + owned_locs // (_GATHER_WORLD * _GATHER_PAGE_SIZE) * _GATHER_PAGE_SIZE + + owned_locs % _GATHER_PAGE_SIZE + ) + for local_layer in range(pool.layer_num): + canonical = _mtp_canonical(kind, owned_locs, tag, local_layer) + for buffer, values in zip(_mtp_buffers(pool, kind, local_layer), canonical): + expected = torch.full_like(buffer, _MTP_UNWRITTEN) + expected[local_rows] = values + _mtp_assert_bytes( + buffer, expected, f"{kind} pool {tag} owned layer {local_layer}" + ) + + +def _mtp_check_scratch(pool, kind, locs, prefix_len, tag, local_layer): + layer_id = pool.start_layer + local_layer + rows = pool.translate_loc_to_scratch(locs) + expected = _mtp_canonical(kind, locs, tag, local_layer) + _mtp_assert_bytes( + pool.get_key_buffer(layer_id)[rows], expected[0], f"{kind} pool {tag} key" + ) + _mtp_assert_bytes( + pool.get_value_buffer(layer_id)[rows], + expected[0][..., :_GATHER_KV_LORA_RANK] if kind == "mla" else expected[1], + f"{kind} pool {tag} value", + ) + if kind == "mla" and prefix_len: + # Arbitrary prefix indices also exercise the chunked MLA consumer. + subset = locs[3:prefix_len:7].clone() + k_nope, k_rope = pool.get_mla_kv_buffer( + SimpleNamespace(layer_id=layer_id), subset, _GATHER_DTYPE + ) + reference = _mtp_canonical(kind, subset, tag, local_layer)[0] + _mtp_assert_bytes( + k_nope, reference[..., :_GATHER_KV_LORA_RANK], f"pool {tag} latent" + ) + _mtp_assert_bytes( + k_rope, reference[..., _GATHER_KV_LORA_RANK:], f"pool {tag} rope" + ) + + +def _run_mtp(rank, port, kind): + """Check exact KV bytes and independent target/draft state across batches.""" + _dist_init( + rank, _GATHER_WORLD, port, attn_cp_size=_GATHER_WORLD if kind == "mha" else 1 + ) + group = get_kv_shard_group(use_mla_backend=kind == "mla") + assert group.world_size == _GATHER_WORLD + target = _mtp_make_pool( + rank, group, kind, start_layer=0, layer_num=_GATHER_LAYER_NUM + ) + allocator = PageInterleavePoolAllocator( + size=target.size, + physical_page_size=_GATHER_PAGE_SIZE, + shard_size=_GATHER_WORLD, + dtype=_GATHER_DTYPE, + device=target.device, + kvcache=target, + need_sort=False, + shard_spec=target.shard_spec, + ) + # DeepSeek's NextN uses local layer 0. Also cover an absolute/offset layer ID + # so the one-layer gather terminates correctly on either scratch-slot parity. + drafts = [ + _mtp_make_draft_pool(target, allocator, kind, start_layer=start) + for start in (0, 61) + ] + pools = [target, *drafts] + for other in drafts: + assert other._page_pos.data_ptr() != target._page_pos.data_ptr() + assert other.kv_gather_comm is not target.kv_gather_comm + for slot, target_slot in zip(other._slots, target._slots): + for name in slot.tensors: + assert ( + slot.tensors[name].data_ptr() + != target_slot.tensors[name].data_ptr() + ) + + locs = _mtp_logical_chain(target.device) + req_to_token = locs.to(torch.int32).unsqueeze(0) + req_indices = torch.tensor([0], dtype=torch.int64, device=locs.device) + # Each cached prefix is page aligned. The third batch writes only five + # tokens of its tail page; the fourth recomputes and completes that page. + batches = [(0, 48), (48, 96), (96, 133), (128, 144)] + for epoch, (prefix_len, seq_len) in enumerate(batches, start=1): + active_locs = locs[:seq_len] + chunk_locs = locs[prefix_len:seq_len] + target_epoch = target._epoch + for tag, pool in enumerate(pools, start=1): + pool.begin_shard_extend(req_to_token, req_indices, [prefix_len], [seq_len]) + assert pool._epoch == epoch + for local_layer in range(pool.layer_num): + _mtp_write(pool, kind, chunk_locs, tag, local_layer, epoch) + _mtp_check_scratch( + pool, kind, active_locs, prefix_len, tag, local_layer + ) + _mtp_check_persisted(pool, kind, active_locs, tag, rank) + # MTP metadata/writes/reads must leave the target's last resident + # layer and gather plan usable, despite identical logical locations. + assert target._epoch == target_epoch + 1 + _mtp_check_scratch( + target, kind, active_locs, prefix_len, 1, _GATHER_LAYER_NUM - 1 + ) + if pool.layer_num == 1 and prefix_len: + resident = pool._slots[pool.start_layer % 2] + assert resident.resident_key == (pool.start_layer, epoch) + assert pool._slots[(pool.start_layer + 1) % 2].resident_key is None + if rank == 0: + print(f"PASS: {kind.upper()} target/MTP prefix={prefix_len}, seq={seq_len}") + + # Deactivating one draft's plan must not deactivate the other pools. + drafts[0].end_shard_extend() + assert not drafts[0]._shard_extend_active + assert target._shard_extend_active and drafts[1]._shard_extend_active + _mtp_check_scratch(drafts[1], kind, locs, 128, 3, 0) + _mtp_check_scratch(target, kind, locs, 128, 1, _GATHER_LAYER_NUM - 1) + torch.cuda.synchronize() + torch.distributed.barrier() + torch.distributed.destroy_process_group() + + +@unittest.skipIf( + torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices" +) +class TestPageInterleaveGatherMultiGpu(CustomTestCase): + """Real pools, real NCCL, 2 ranks — one mp.spawn per phase. + + Separate spawns (and separate ports) because each phase builds its own + process group with a different attention-CP width. + """ + + def test_mla_shard_over_attention_tp(self): + mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True) + + def test_mha_shard_over_attention_cp(self): + mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True) + + def test_mtp_mla_shard_over_attention_tp(self): + mp.spawn(_run_mtp, args=(29813, "mla"), nprocs=_GATHER_WORLD, join=True) + + def test_mtp_mha_shard_over_attention_cp(self): + mp.spawn(_run_mtp, args=(29814, "mha"), nprocs=_GATHER_WORLD, join=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 6dd882b414..2451c676ae 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -13,8 +13,7 @@ # ============================================================================== """Tests for logical-page KV cache sharding. -The CPU tests pin the pure arithmetic that rotated owner-classed -allocation hangs on: +Pins the pure arithmetic that rotated owner-classed allocation hangs on: 1. The placement bijection ``loc = Q*(N*ps) + r*ps + o`` — owner / local-row round-trip, disjoint equal partition across ranks. @@ -31,37 +30,31 @@ allocation hangs on: 5. ``begin_shard_extend`` plan capture (page positions, padded send rows, owner-congruence guard) with the gather stubbed out, following the SimpleNamespace binding pattern of ``test_dsa_layer_shard_utils.py``. -6. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's +6. MTP draft pool construction, shared allocator/layout validation, and scratch + sizing for separate target and draft pools. +7. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's skip of a non-final chunk this rank owns no page of. -7. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's +8. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's tokens, each token stored on exactly one rank. -The GPU tests exercise real MLA and MHA pools and NCCL gathers on 2 ranks -(``TestPageInterleaveGatherMultiGpu``). DSA indexer and attention coverage lives -in ``test_page_sharded_dsa_attention.py``. -GPU tests skip when too few CUDA devices are visible, so the same file runs -in both CPU and CUDA CI suites. +The real NCCL MLA/MHA/MTP gathers over 2 GPUs live in +``test_page_interleave_gather_multi_gpu.py``. DSA indexer and attention +coverage lives in ``test_page_sharded_dsa_attention.py``. """ -import os import shutil import unittest import unittest.mock from array import array -from contextlib import contextmanager +from contextlib import ExitStack, contextmanager from types import SimpleNamespace import numpy as np import torch -import torch.multiprocessing as mp from parameterized import parameterized_class from sglang.srt.disaggregation.common.conn import CommonKVSender from sglang.srt.disaggregation.utils import filter_kv_indices_for_shard_rank -from sglang.srt.distributed import ( - init_distributed_environment, - initialize_model_parallel, -) from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache import page_interleave @@ -77,6 +70,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.common import _evict_until_allocatable +from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator from sglang.srt.mem_cache.memory_pool import ( MHATokenToKVPool, MLATokenToKVPool, @@ -86,7 +80,8 @@ from sglang.srt.mem_cache.page_interleave import ( PageInterleavePlacement, PageShardSpec, compute_page_shard_scratch_bytes, - get_kv_shard_group, + get_kv_shard_group_info, + get_shared_kv_shard_pool, make_page_shard_spec, ) from sglang.srt.mem_cache.page_interleave_pool import ( @@ -97,19 +92,14 @@ from sglang.srt.mem_cache.page_interleave_pool import ( from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache -from sglang.srt.runtime_context import get_parallel, publish -from sglang.srt.server_args import ServerArgs +from sglang.srt.runtime_context import get_context, get_memory, get_parallel, get_spec +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import ceil_div -from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci +from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.mem_cache_utils import finish_req -from sglang.test.test_utils import ( - CustomTestCase, - enter_scope, - publish_build_topology, -) +from sglang.test.test_utils import CustomTestCase, enter_scope register_cpu_ci(est_time=30, suite="base-a-test-cpu") -register_cuda_ci(est_time=120, stage="base-c", runner_config="4-gpu-b200") N = 4 # shard size PS = 16 # physical page size @@ -150,6 +140,7 @@ class TestPageShardScratchSizing(CustomTestCase): ) kvc = SimpleNamespace( is_draft_worker=draft, + token_to_kv_pool_allocator=None, use_mla_backend=use_mla, page_size=16, kv_cache_dtype=torch.bfloat16, @@ -182,6 +173,28 @@ class TestPageShardScratchSizing(CustomTestCase): self.assertEqual(spec.chunk_tokens, 48) self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0) + def test_target_with_mtp_budgets_one_extra_scratch_pair(self): + with ( + self._fixture(context_len=100, chunk_tokens=64, shard_size=2) as kvc, + get_context().override_server_args(enable_multi_layer_eagle=False), + ): + kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE + # Target configs can omit next-N layers; use the resolved draft count. + kvc.model_config.num_nextn_predict_layers = None + kvc.spec_aux_config = SimpleNamespace(eagle_draft_num_layers=1) + expected = 2 * (8 * 128 + 64 + 16) * (16 + 8) * 2 + for layers in (1, 3): + with self.subTest(layers=layers): + kvc.spec_aux_config.eagle_draft_num_layers = layers + self.assertEqual(compute_page_shard_scratch_bytes(kvc), expected) + self.assertEqual( + compute_page_shard_scratch_bytes(kvc, include_mtp=True), + 2 * expected, + ) + kvc.spec_aux_config.eagle_draft_num_layers = None + with self.assertRaisesRegex(ValueError, "EAGLE MTP"): + compute_page_shard_scratch_bytes(kvc, include_mtp=True) + def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self): for use_mla in (True, False): with ( @@ -211,6 +224,255 @@ class TestPageShardScratchSizing(CustomTestCase): self.assertEqual(estimated, actual) +def _make_mtp_configurator(use_mla=True): + # Keep real pool/allocator types for validation; allocate only CPU metadata. + pool_class = ( + PageInterleaveMLATokenToKVPool if use_mla else PageInterleaveMHATokenToKVPool + ) + target = pool_class.__new__(pool_class) + target.shard_spec = PageShardSpec(1, 2, 16, 128, 64) + target.shard_group = SimpleNamespace(rank_in_group=1, world_size=2) + target.size, target.page_size, target.dtype = 128, 16, torch.bfloat16 + target.kv_lora_rank, target.qk_rope_head_dim = 8, 4 + target.head_num, target.head_dim, target.v_head_dim = 2, 4, 6 + kvc = KVCacheConfigurator.__new__(KVCacheConfigurator) + kvc.token_to_kv_pool_allocator = PageInterleavePoolAllocator( + size=target.size, + physical_page_size=target.page_size, + shard_size=target.shard_spec.shard_size, + dtype=target.dtype, + device="cpu", + kvcache=target, + need_sort=False, + shard_spec=target.shard_spec, + ) + kvc.device = "cpu" + kvc.is_draft_worker = True + kvc.page_size = target.page_size + kvc.use_mla_backend = use_mla + kvc.kv_cache_dtype = target.dtype + kvc.kv_cache_dtype_str = "bfloat16" + kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE + kvc.model_config = SimpleNamespace( + num_nextn_predict_layers=1, + context_len=9999, + kv_lora_rank=8, + qk_rope_head_dim=4, + head_dim=4, + v_head_dim=6, + get_num_kv_heads=lambda tp: 4 // tp, + hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]), + ) + kvc.layer_info = SimpleNamespace(start_layer=5, end_layer=6, num_effective_layers=1) + kvc.is_hybrid_swa = False + kvc.sliding_window_size = None + kvc.mambaish_config = None + kvc.post_capture_kv_active = False + return kvc, target + + +def _build_mtp_pool(kvc, *, capacity=128, is_dsa_model=False, is_dsv4_model=False): + return kvc._build_token_to_kv_pool( + sizes=SimpleNamespace(max_total_num_tokens=capacity), + is_dsa_model=is_dsa_model, + is_dsv4_model=is_dsv4_model, + req_to_token_pool=None, + ) + + +class TestMTPKVShardConfig(CustomTestCase): + def setUp(self): + super().setUp() + stack = ExitStack() + self.addCleanup(stack.close) + stack.enter_context( + get_context().override_server_args( + page_size=16, chunked_prefill_size=1024, enable_multi_layer_eagle=False + ) + ) + stack.enter_context( + get_parallel().override( + tp_size=2, attn_tp_size=2, moe_tp_size=2, attn_dcp_size=1 + ) + ) + + def test_draft_shares_target_layout_but_has_its_own_pool(self): + for use_mla in (True, False): + with self.subTest(use_mla=use_mla): + kvc, target = _make_mtp_configurator(use_mla) + pool_class = type(target) + # Mock allocation only; preserve real types and distinct instances. + with ( + unittest.mock.patch.object( + pool_class, "__init__", return_value=None + ) as init, + unittest.mock.patch.object( + page_interleave, + "get_kv_shard_group", + side_effect=AssertionError, + ), + ): + self.assertIs(get_shared_kv_shard_pool(kvc), target) + self.assertEqual(get_kv_shard_group_info(kvc), (1, 2)) + self.assertIs(make_page_shard_spec(kvc), target.shard_spec) + pool = _build_mtp_pool(kvc) + self.assertIsInstance(pool, pool_class) + self.assertIsNot(pool, target) + kwargs = init.call_args.kwargs + self.assertIs(kwargs["shard_spec"], target.shard_spec) + self.assertIs(kwargs["shard_group"], target.shard_group) + for name, expected in dict( + size=128, + page_size=16, + dtype=torch.bfloat16, + layer_num=1, + start_layer=5, + end_layer=6, + ).items(): + self.assertEqual(kwargs[name], expected, name) + allocator = kvc.token_to_kv_pool_allocator + self.assertIs( + kvc._build_token_to_kv_pool_allocator( + sizes=SimpleNamespace(max_total_num_tokens=128), + token_to_kv_pool=pool, + is_dsv4_model=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=allocator, + ), + allocator, + ) + # Scratch uses inherited bounds despite different draft context/chunk sizes. + row_bytes = (8 + 4) * 2 if use_mla else 2 * (4 + 6) * 2 + self.assertEqual( + compute_page_shard_scratch_bytes(kvc), + 2 * (128 + 64 + 16) * row_bytes, + ) + with self.assertRaisesRegex(ValueError, "target"): + compute_page_shard_scratch_bytes(kvc, include_mtp=True) + + def test_without_shared_allocator_keeps_existing_pool(self): + for draft, allocator in ((True, None), (True, object()), (False, None)): + with self.subTest(draft=draft, allocator=allocator): + kvc, _ = _make_mtp_configurator() + kvc.is_draft_worker = draft + kvc.token_to_kv_pool_allocator = allocator + self.assertIsNone(get_shared_kv_shard_pool(kvc)) + if draft: + self.assertEqual(get_kv_shard_group_info(kvc), (None, 1)) + self.assertIsNone(make_page_shard_spec(kvc)) + self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0) + with unittest.mock.patch.object( + KVCacheConfigurator, "_build_mla_kv_pool" + ) as build: + self.assertIs(_build_mtp_pool(kvc), build.return_value) + build.assert_called_once_with( + max_total_num_tokens=128, mla_pool_class=MLATokenToKVPool + ) + + def test_rejects_invalid_shared_allocator(self): + kvc, _ = _make_mtp_configurator() + for name, value, message in ( + ("_kvcache", object(), "target's sharded KV pool"), + ("shard_spec", PageShardSpec(0, 2, 16, 128, 64), "different shard layouts"), + ("shard_size", 4, "different shard layouts"), + ("page_size", 32, "different shard layouts"), + ("size", 128, "different shard layouts"), + ): + with ( + self.subTest(attribute=name), + unittest.mock.patch.object(kvc.token_to_kv_pool_allocator, name, value), + self.assertRaisesRegex(ValueError, message), + ): + get_shared_kv_shard_pool(kvc) + + def test_rejects_incompatible_pool_geometry(self): + for use_mla in (True, False): + kvc, _ = _make_mtp_configurator(use_mla) + geometry = ( + ("kv_lora_rank", "qk_rope_head_dim") + if use_mla + else ("head_dim", "v_head_dim") + ) + cases = [ + (kvc, "page_size", 32, "physical page size"), + (kvc, "kv_cache_dtype", torch.float16, "geometry and dtype"), + (kvc, "use_mla_backend", not use_mla, "attention geometry"), + *( + (kvc.model_config, field, 16, "geometry and dtype") + for field in geometry + ), + ] + if not use_mla: + cases.append( + ( + kvc.model_config, + "get_num_kv_heads", + lambda tp: 4, + "geometry and dtype", + ) + ) + for obj, name, value, message in cases: + with ( + self.subTest(use_mla=use_mla, attribute=name), + unittest.mock.patch.object(obj, name, value), + self.assertRaisesRegex(ValueError, message), + ): + _build_mtp_pool(kvc) + for capacity in (64, 256): + with ( + self.subTest(use_mla=use_mla, capacity=capacity), + self.assertRaisesRegex(ValueError, "same per-rank capacity"), + ): + _build_mtp_pool(kvc, capacity=capacity) + + def test_rejects_unsupported_draft_config(self): + kvc, _ = _make_mtp_configurator() + cases = [ + (kvc, "spec_algorithm", algorithm, "EAGLE MTP") + for algorithm in ( + SpeculativeAlgorithm.EAGLE3, + SpeculativeAlgorithm.DFLASH, + SpeculativeAlgorithm.FROZEN_KV_MTP, + ) + ] + [ + (kvc.model_config, "num_nextn_predict_layers", None, "EAGLE MTP"), + (kvc.model_config, "num_nextn_predict_layers", 0, "EAGLE MTP"), + (kvc, "is_hybrid_swa", True, "dense MLA and MHA"), + (kvc, "mambaish_config", object(), "dense MLA and MHA"), + (kvc, "sliding_window_size", 128, "dense MLA and MHA"), + (kvc, "post_capture_kv_active", True, "plain per-layer KV layout"), + (kvc, "kv_cache_dtype_str", "mxfp8", "plain per-layer KV layout"), + ] + for obj, name, value, message in cases: + with ( + self.subTest(attribute=name, value=value), + unittest.mock.patch.object(obj, name, value), + self.assertRaisesRegex(ValueError, message), + ): + _build_mtp_pool(kvc) + for options in ({"is_dsa_model": True}, {"is_dsv4_model": True}): + with ( + self.subTest(options=options), + self.assertRaisesRegex(ValueError, "dense MLA and MHA"), + ): + _build_mtp_pool(kvc, **options) + for context, options, message in ( + (get_spec(), {"enable_multi_layer_eagle": True}, "EAGLE MTP"), + (get_parallel(), {"attn_dcp_size": 2}, "incompatible with DCP"), + ( + get_memory(), + {"enable_page_major_kv_layout": True}, + "plain per-layer KV layout", + ), + ): + with ( + self.subTest(options=options), + context.override(**options), + self.assertRaisesRegex(ValueError, message), + ): + _build_mtp_pool(kvc) + + def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32): return PageShardSpec( shard_rank=shard_rank, @@ -1402,346 +1664,5 @@ class TestShardSendFilter(CustomTestCase): self.assertEqual(skips, expect_skip) -# ============================================================================= -# Multi-GPU: the real NCCL layer-ahead gather (2 GPUs). -# -# Everything above is pure arithmetic on a CPU stub. This section drives real -# pools over a real process group, which is the only check that the plan the -# stub validates actually addresses the bytes the collective delivers: -# -# 1. MLA pool sharded across the attention-TP group: replicated writes are -# owner-filtered into disjoint pool stripes; a later batch's chunked-prefix -# read (get_mla_kv_buffer) assembles the full prefix from all ranks via the -# layer-ahead NCCL allgather and must return the canonical bytes. -# 2. MHA pool sharded across the attention-CP group: the post-allgather full -# chunk is staged into the scratch chunk region and owner-persisted; a later -# batch reads prefix+chunk through the translated page table (the scratch), -# and the assembled rows must match the canonical bytes. -# -# Skipped unless 2 CUDA devices are visible, so it is inert on the CPU runner -# this file is registered to. Run it explicitly with: -# CUDA_VISIBLE_DEVICES=0,1 python3 test/registered/unit/mem_cache/\ -# test_page_interleave_shard.py TestPageInterleaveGatherMultiGpu -# ============================================================================= - -_GATHER_WORLD = 2 -_GATHER_LAYER_NUM = 4 -_GATHER_PAGE_SIZE = 16 -_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE -_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank -_GATHER_KV_LORA_RANK = 128 -_GATHER_QK_ROPE = 32 -_GATHER_HEAD_NUM = 2 -_GATHER_HEAD_DIM = 32 -_GATHER_DTYPE = torch.bfloat16 - - -def _mla_value(loc, dim): - """Deterministic canonical latent value for logical slot ``loc``.""" - loc = loc.to(torch.float32) - return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to( - _GATHER_DTYPE - ) - - -def _dist_init(rank, world, port, attn_cp_size): - os.environ["MASTER_ADDR"] = "127.0.0.1" - os.environ["MASTER_PORT"] = str(port) - os.environ["RANK"] = str(rank) - os.environ["WORLD_SIZE"] = str(world) - os.environ.setdefault("no_proxy", "127.0.0.1,localhost") - torch.cuda.set_device(rank) - - init_distributed_environment( - world_size=world, - rank=rank, - local_rank=rank, - distributed_init_method=f"tcp://127.0.0.1:{port}", - backend="nccl", - ) - # Publish the widths the groups below are about to be built at. The derived - # quotients (attn_tp_size, attn_dcp_size, ...) are projected from these - # leaves at publish; initialize_model_parallel no longer supplies them, and - # MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path. - publish( - ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size), - role="scheduler", - ) - publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank) - initialize_model_parallel() - - -def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4): - return PageShardSpec( - shard_rank=shard_rank, - shard_size=_GATHER_WORLD, - page_size=_GATHER_PAGE_SIZE, - max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE, - chunk_tokens=chunk_groups * _GATHER_GRANULE, - ) - - -def _fake_req_to_token(groups, seq_len, device): - """req_to_token row where sequence group j is allocator group groups[j].""" - row = torch.zeros( - (1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device - ) - for j, q in enumerate(groups): - row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange( - q * _GATHER_GRANULE, - (q + 1) * _GATHER_GRANULE, - dtype=torch.int32, - device=device, - ) - return row[:, :seq_len] if seq_len < row.shape[1] else row - - -def _check(rank, name, got, expect, atol=0.0): - ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0) - max_err = (got.float() - expect.float()).abs().max().item() - print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}") - assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})" - - -def _run_mla(rank, world, port): - _dist_init(rank, world, port, attn_cp_size=1) - - group = get_parallel().attn_tp_group - assert group.world_size == world - # Topology-first shard-group selection: no CP here, so MLA falls back to - # the attn-TP axis, while GQA has no replicated axis (world_size 1). - assert get_kv_shard_group(use_mla_backend=True) is group - assert get_kv_shard_group(use_mla_backend=False).world_size == 1 - spec = _gather_make_spec(shard_rank=group.rank_in_group) - - pool = PageInterleaveMLATokenToKVPool( - _GATHER_SIZE, - page_size=_GATHER_PAGE_SIZE, - dtype=_GATHER_DTYPE, - kv_lora_rank=_GATHER_KV_LORA_RANK, - qk_rope_head_dim=_GATHER_QK_ROPE, - layer_num=_GATHER_LAYER_NUM, - device=f"cuda:{rank}", - enable_memory_saver=False, - start_layer=0, - end_layer=_GATHER_LAYER_NUM - 1, - shard_spec=spec, - shard_group=group, - ) - device = pool.kv_buffer[0].device - - # ---- chunk 1: replicated write, owner-filtered persist ----------------- - # "Allocator" hands out fragmented groups (identical on every rank). - chunk1_groups = [5, 2, 9] - chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[ - 0 - ].long() - for layer_id in range(_GATHER_LAYER_NUM): - layer = SimpleNamespace(layer_id=layer_id) - vals = _mla_value( - chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE - ) - pool.set_mla_kv_buffer( - layer, - chunk1_locs, - vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1), - vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1), - ) - torch.cuda.synchronize() - torch.distributed.barrier() - - # Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps) - # on every rank, holding that rank's page of the group. - for q in chunk1_groups: - local_rows = torch.arange( - q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device - ) - owned_locs = ( - q * _GATHER_GRANULE - + group.rank_in_group * _GATHER_PAGE_SIZE - + torch.arange(_GATHER_PAGE_SIZE, device=device) - ) - got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE) - expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE) - _check(rank, f"mla owned stripe g{q}", got, expect) - - # ---- chunk 2: prefix gather + staged chunk, both read styles ----------- - seq_groups = chunk1_groups + [12] # one new chunk group - prefix_len = 3 * _GATHER_GRANULE - seq_len = prefix_len + _GATHER_GRANULE - req_to_token = _fake_req_to_token(seq_groups, seq_len, device) - chunk2_locs = req_to_token[0, prefix_len:seq_len].long() - pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len]) - - for layer_id in range(_GATHER_LAYER_NUM): - layer = SimpleNamespace(layer_id=layer_id) - # Write the current chunk (stages it into the slot + persists the - # owned stripe), like the extend forward does before attention. - chunk_vals = _mla_value( - chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE - ) - pool.set_mla_kv_buffer( - layer, - chunk2_locs, - chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1), - chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1), - ) - # Chunked-prefix MHA style: fetch an arbitrary sub-range of the - # prefix through get_mla_kv_buffer. - sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3] - k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE) - expect = _mla_value( - sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE - ) - _check( - rank, - f"mla prefix read l{layer_id}", - k_nope[:, 0, :], - expect[:, :_GATHER_KV_LORA_RANK], - ) - _check( - rank, - f"mla prefix rope l{layer_id}", - k_rope[:, 0, :], - expect[:, _GATHER_KV_LORA_RANK:], - ) - # Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk] - # from get_key_buffer through the translated page table. - all_locs = req_to_token[0, :seq_len].long() - rows = pool.translate_loc_to_scratch(all_locs) - kv_scratch = pool.get_key_buffer(layer_id) - _check( - rank, - f"mla absorbed read l{layer_id}", - kv_scratch[rows, 0, :], - _mla_value( - all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE - ), - ) - - torch.distributed.barrier() - if rank == 0: - print("PASS: MLA page-interleave shard (attn-TP axis)") - - -def _run_mha(rank, world, port): - _dist_init(rank, world, port, attn_cp_size=world) - - group = get_parallel().attn_cp_group - assert group.world_size == world - # Topology-first shard-group selection: with an active CP group, both - # GQA and MLA shard across CP (CP replicates KV for every attention - # type; the TP axis is only the no-CP MLA fallback). - assert get_kv_shard_group(use_mla_backend=False) is group - assert get_kv_shard_group(use_mla_backend=True) is group - spec = _gather_make_spec(shard_rank=group.rank_in_group) - - pool = PageInterleaveMHATokenToKVPool( - _GATHER_SIZE, - page_size=_GATHER_PAGE_SIZE, - dtype=_GATHER_DTYPE, - head_num=_GATHER_HEAD_NUM, - head_dim=_GATHER_HEAD_DIM, - layer_num=_GATHER_LAYER_NUM, - device=f"cuda:{rank}", - enable_memory_saver=False, - start_layer=0, - end_layer=_GATHER_LAYER_NUM - 1, - enable_alt_stream=False, - shard_spec=spec, - shard_group=group, - ) - device = pool.k_buffer[0].device - - def kv_value(locs, layer_id, is_v): - base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0) - return ( - base.view(-1, 1, 1) - + torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01 - + torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001 - ).to(_GATHER_DTYPE) - - # ---- chunk 1 (prefix-less batch): stage + owner-persist ---------------- - chunk1_groups = [7, 3] - chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[ - 0 - ].long() - req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device) - pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE]) - for layer_id in range(_GATHER_LAYER_NUM): - layer = SimpleNamespace(layer_id=layer_id) - pool.set_kv_buffer( - layer, - chunk1_locs, - kv_value(chunk1_locs, layer_id, False), - kv_value(chunk1_locs, layer_id, True), - ) - # The current chunk must be readable through the scratch right away. - k_scratch = pool.get_key_buffer(layer_id) - rows = pool.translate_loc_to_scratch(chunk1_locs) - _check( - rank, - f"mha chunk stage l{layer_id}", - k_scratch[rows], - kv_value(chunk1_locs, layer_id, False), - ) - torch.cuda.synchronize() - torch.distributed.barrier() - - # ---- chunk 2: prefix gathered from peers via translated page table ----- - seq_groups = chunk1_groups + [11] - prefix_len = 2 * _GATHER_GRANULE - seq_len = prefix_len + _GATHER_GRANULE - req_to_token = _fake_req_to_token(seq_groups, seq_len, device) - chunk2_locs = req_to_token[0, prefix_len:seq_len].long() - pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len]) - - for layer_id in range(_GATHER_LAYER_NUM): - layer = SimpleNamespace(layer_id=layer_id) - pool.set_kv_buffer( - layer, - chunk2_locs, - kv_value(chunk2_locs, layer_id, False), - kv_value(chunk2_locs, layer_id, True), - ) - all_locs = req_to_token[0, :seq_len].long() - rows = pool.translate_loc_to_scratch(all_locs) - k_scratch = pool.get_key_buffer(layer_id) - v_scratch = pool.get_value_buffer(layer_id) - _check( - rank, - f"mha seq read k l{layer_id}", - k_scratch[rows], - kv_value(all_locs, layer_id, False), - ) - _check( - rank, - f"mha seq read v l{layer_id}", - v_scratch[rows], - kv_value(all_locs, layer_id, True), - ) - - torch.distributed.barrier() - if rank == 0: - print("PASS: MHA page-interleave shard (attn-CP axis)") - - -@unittest.skipIf( - torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices" -) -class TestPageInterleaveGatherMultiGpu(CustomTestCase): - """Real pools, real NCCL, 2 ranks — one mp.spawn per phase. - - Separate spawns (and separate ports) because each phase builds its own - process group with a different attention-CP width. - """ - - def test_mla_shard_over_attention_tp(self): - mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True) - - def test_mha_shard_over_attention_cp(self): - mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True) - - if __name__ == "__main__": unittest.main()