[Feat] KVShard: Support kv cache sharding for MTP (#40929)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-10-23 15:20:19 +08:00
committed by GitHub
co-authored by Zhangheng
parent 6e667c4310
commit 79ab8f6d7c
6 changed files with 1168 additions and 377 deletions
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.memory_pool import (
from sglang.srt.mem_cache.page_interleave import (
PageShardSpec,
get_kv_shard_group,
get_shared_kv_shard_pool,
make_page_shard_spec,
)
from sglang.srt.mem_cache.page_interleave_pool import (
@@ -1763,6 +1764,15 @@ class KVCacheConfigurator:
is_dsv4_model: bool,
req_to_token_pool: ReqToTokenPool,
) -> KVCache:
# MTP drafts sharing the target's sharded allocator need the same shard
# layout with separate KV storage. Handle them before the normal pool
# dispatch; targets and drafts without that allocator fall through.
mtp_pool = self._build_mtp_kv_shard_pool(
sizes=sizes, is_dsa_model=is_dsa_model, is_dsv4_model=is_dsv4_model
)
if mtp_pool is not None:
return mtp_pool
# Page-granularity envelope layout for the MHA-shaped (full / SWA) pools,
# selected by swapping in the PageMajorMHATokenToKVPool subclass. The
# default keeps upstream's per-layer layout. The Mamba state pool is routed
@@ -1889,6 +1899,115 @@ class KVCacheConfigurator:
)
return token_to_kv_pool
def _build_mtp_kv_shard_pool(
self, *, sizes: _PoolSizes, is_dsa_model: bool, is_dsv4_model: bool
) -> Optional[KVCache]:
"""Build a draft pool in its target's logical space, with its own KV.
This is a pool-level opt-in through a shared sharded allocator. Target
pool creation and speculative serving remain separate integration work.
"""
if not self.is_draft_worker:
return None
target_pool = get_shared_kv_shard_pool(self)
if target_pool is None:
return None
if (
self.spec_algorithm != SpeculativeAlgorithm.EAGLE
or (self.model_config.num_nextn_predict_layers or 0) <= 0
or get_spec().enable_multi_layer_eagle
):
raise ValueError(
"Shared KV sharding supports only an EAGLE MTP draft runner."
)
if get_parallel().attn_dcp_size > 1:
raise ValueError("MTP KV sharding is incompatible with DCP.")
if (
is_dsa_model
or is_dsv4_model
or self.is_hybrid_swa
or self.mambaish_config is not None
or self.sliding_window_size is not None
or is_minimax_sparse(self.model_config.hf_config)
or current_platform.is_out_of_tree()
or _is_npu
):
raise ValueError("MTP KV sharding supports only dense MLA and MHA pools.")
if (
self.post_capture_kv_active
or get_memory().enable_page_major_kv_layout
or self.kv_cache_dtype_str == "mxfp8"
or is_float4_e2m1fn_x2(self.kv_cache_dtype)
):
raise ValueError("MTP KV sharding requires the plain per-layer KV layout.")
if (
self.page_size != target_pool.page_size
or self.pool_page_size != target_pool.page_size
):
raise ValueError(
"MTP and target KV pools must use the same physical page size."
)
if sizes.max_total_num_tokens != target_pool.size:
raise ValueError(
"MTP and target sharded KV pools must have the same per-rank capacity."
)
target_is_mla = isinstance(target_pool, PageInterleaveMLATokenToKVPool)
if self.use_mla_backend != target_is_mla or not isinstance(
target_pool,
(PageInterleaveMLATokenToKVPool, PageInterleaveMHATokenToKVPool),
):
raise ValueError(
"MTP and target KV pools must use the same attention geometry."
)
fields = (
("kv_lora_rank", "qk_rope_head_dim")
if self.use_mla_backend
else ("head_dim", "v_head_dim")
)
if (
self.kv_cache_dtype != target_pool.dtype
or any(
getattr(self.model_config, name) != getattr(target_pool, name)
for name in fields
)
or (
not self.use_mla_backend
and self.model_config.get_num_kv_heads(get_parallel().attn_tp_size)
!= target_pool.head_num
)
):
raise ValueError(
"MTP and target KV pools must use the same KV geometry and dtype."
)
kwargs = dict(
size=target_pool.size,
page_size=target_pool.page_size,
dtype=self.kv_cache_dtype,
layer_num=self.layer_info.num_effective_layers,
device=self.device,
enable_memory_saver=get_exec().features.enable_memory_saver,
start_layer=self.layer_info.start_layer,
end_layer=self.layer_info.end_layer,
shard_spec=target_pool.shard_spec,
shard_group=target_pool.shard_group,
)
if self.use_mla_backend:
return PageInterleaveMLATokenToKVPool(
**kwargs,
kv_lora_rank=self.model_config.kv_lora_rank,
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
)
return PageInterleaveMHATokenToKVPool(
**kwargs,
head_num=target_pool.head_num,
head_dim=self.model_config.head_dim,
v_head_dim=self.model_config.v_head_dim,
enable_alt_stream=not get_disagg().enable_pdmux,
)
def _build_dsv4_kv_pool(
self,
*,
+69 -5
View File
@@ -37,12 +37,18 @@ from typing import TYPE_CHECKING, Optional, Tuple
import msgspec
import torch
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.mem_cache import page_interleave_pool
from sglang.srt.mem_cache.allocator.page_interleave import (
PageInterleavePoolAllocator,
)
from sglang.srt.runtime_context import get_parallel, get_schedule, get_spec
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
logger = logging.getLogger(__name__)
@@ -127,11 +133,42 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
return cp_group
def get_shared_kv_shard_pool(
kvc: KVCacheConfigurator,
) -> Optional[PageInterleaveKVPoolMixin]:
"""Find a draft's sharded target through their shared logical allocator."""
if not kvc.is_draft_worker:
return None
allocator = kvc.token_to_kv_pool_allocator
if not isinstance(allocator, PageInterleavePoolAllocator):
return None
pool = allocator.get_kvcache()
if not isinstance(pool, page_interleave_pool.PageInterleaveKVPoolMixin):
raise ValueError("A sharded MTP draft requires the target's sharded KV pool.")
if (
allocator.shard_spec != pool.shard_spec
or allocator.shard_size != pool.shard_spec.shard_size
or allocator.page_size != pool.page_size
or allocator.size != pool.size * allocator.shard_size
):
raise ValueError(
"The shared allocator and target KV pool have different shard layouts."
)
return pool
def get_kv_shard_group_info(
kvc: KVCacheConfigurator,
) -> Tuple[Optional[int], int]:
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
if kvc.is_draft_worker:
pool = get_shared_kv_shard_pool(kvc)
if pool is None:
return None, 1
return pool.shard_spec.shard_rank, pool.shard_spec.shard_size
if not get_parallel().enable_kv_cache_sharding:
return None, 1
group = get_kv_shard_group(kvc.use_mla_backend)
if group.world_size <= 1:
@@ -163,7 +200,14 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
or chunk size. Both scratch slots are charged against HBM before sizing
persistent KV; large model contexts therefore reduce persistent capacity.
Admission still checks the actual batch's padded prefix and chunk spans.
MTP drafts inherit the target's exact bounds: both pools serve the same
scheduled batch and logical locations, even if their model configs differ
in context length.
"""
if kvc.is_draft_worker:
pool = get_shared_kv_shard_pool(kvc)
return pool.shard_spec if pool is not None else None
shard_rank, shard_size = get_kv_shard_group_info(kvc)
if shard_rank is None:
return None
@@ -181,11 +225,31 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
)
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
def compute_page_shard_scratch_bytes(
kvc: KVCacheConfigurator, *, include_mtp: bool = False
) -> int:
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
the KV budget before pool sizing. Use the same spec as the pool builders:
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.
``include_mtp`` additionally budgets a target's single EAGLE MTP runner
at the same KV geometry, as required by the draft pool builder. Its two
scratch slots are reused across draft layers. Target sizing integration
can opt into this estimate without changing the per-pool default.
"""
spec = make_page_shard_spec(kvc)
if spec is None:
return 0
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
scratch_bytes = 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
if include_mtp:
if (
kvc.is_draft_worker
or kvc.spec_algorithm != SpeculativeAlgorithm.EAGLE
or (kvc.spec_aux_config.eagle_draft_num_layers or 0) <= 0
or get_spec().enable_multi_layer_eagle
):
raise ValueError(
"MTP scratch budgeting requires a target with one EAGLE MTP runner."
)
scratch_bytes *= 2
return scratch_bytes
@@ -61,6 +61,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import (
)
from sglang.kernels.ops.kvcache import dsa_index_k_cache
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
from sglang.srt.mem_cache import page_interleave
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
from sglang.srt.mem_cache.memory_pool import (
GPU_MEMORY_TYPE_KV_CACHE,
@@ -70,15 +71,12 @@ from sglang.srt.mem_cache.memory_pool import (
RadixAttention,
unwrap_write_loc,
)
from sglang.srt.mem_cache.page_interleave import (
PageInterleavePlacement,
PageShardSpec,
)
from sglang.srt.mem_cache.utils import get_mla_kv_buffer_triton
from sglang.srt.utils import ceil_div, get_bool_env_var
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.page_interleave import PageShardSpec
logger = logging.getLogger(__name__)
@@ -123,7 +121,8 @@ class PageInterleaveKVPoolMixin:
assert spec.chunk_tokens % spec.page_size == 0
self.shard_spec = spec
self.placement = PageInterleavePlacement(spec)
self.shard_group = shard_group
self.placement = page_interleave.PageInterleavePlacement(spec)
self.shard_rank = spec.shard_rank
self.shard_size = spec.shard_size
self.device_module = torch.get_device_module(self.device)
@@ -972,6 +972,7 @@ class TestPrefillAdder(CustomTestCase):
extend_len = 1024
prefix_len = context_len - extend_len
kvc = SimpleNamespace(
is_draft_worker=False,
page_size=4,
model_config=SimpleNamespace(context_len=context_len),
)
@@ -0,0 +1,687 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Multi-GPU tests for logical-page KV cache sharding: the real NCCL
layer-ahead gather over a real 2-rank process group.
``test_page_interleave_shard.py`` pins the plan arithmetic on CPU stubs. These
tests drive real pools, which is the only check that the plan the stubs
validate actually addresses the bytes the collective delivers:
1. MLA pool sharded across the attention-TP group: replicated writes are
owner-filtered into disjoint pool stripes; a later batch's chunked-prefix
read (get_mla_kv_buffer) assembles the full prefix from all ranks via the
layer-ahead NCCL allgather and must return the canonical bytes.
2. MHA pool sharded across the attention-CP group: the post-allgather full
chunk is staged into the scratch chunk region and owner-persisted; a later
batch reads prefix+chunk through the translated page table (the scratch),
and the assembled rows must match the canonical bytes.
3. MTP draft pools sharing the target's sharded allocator, for MLA and MHA:
exact KV bytes, separate target/draft storage and scratch plans, fragmented
pages, partial tail pages, both MLA write APIs, and local/nonzero draft
layer IDs across batches.
Registered as a base-b 2-gpu-large unit test; skips when fewer than 2 CUDA
devices are visible. Run directly with:
CUDA_VISIBLE_DEVICES=0,1 python3 \
test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py
"""
import os
import unittest
from types import SimpleNamespace
import torch
import torch.multiprocessing as mp
from sglang.srt.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.mem_cache.page_interleave import PageShardSpec, get_kv_shard_group
from sglang.srt.mem_cache.page_interleave_pool import (
PageInterleaveMHATokenToKVPool,
PageInterleaveMLATokenToKVPool,
)
from sglang.srt.runtime_context import get_context, get_parallel, publish
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase, publish_build_topology
register_cuda_ci(est_time=120, stage="base-b", runner_config="2-gpu-large")
_GATHER_WORLD = 2
_GATHER_LAYER_NUM = 4
_GATHER_PAGE_SIZE = 16
_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE
_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank
_GATHER_KV_LORA_RANK = 128
_GATHER_QK_ROPE = 32
_GATHER_HEAD_NUM = 2
_GATHER_HEAD_DIM = 32
_GATHER_DTYPE = torch.bfloat16
_MTP_SIZE = 32 * _GATHER_PAGE_SIZE # Physical token capacity per rank.
_MTP_UNWRITTEN = -256
def _mla_value(loc, dim):
"""Deterministic canonical latent value for logical slot ``loc``."""
loc = loc.to(torch.float32)
return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to(
_GATHER_DTYPE
)
def _dist_init(rank, world, port, attn_cp_size):
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(port)
os.environ["RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world)
os.environ.setdefault("no_proxy", "127.0.0.1,localhost")
torch.cuda.set_device(rank)
init_distributed_environment(
world_size=world,
rank=rank,
local_rank=rank,
distributed_init_method=f"tcp://127.0.0.1:{port}",
backend="nccl",
)
# Publish the widths the groups below are about to be built at. The derived
# quotients (attn_tp_size, attn_dcp_size, ...) are projected from these
# leaves at publish; initialize_model_parallel no longer supplies them, and
# MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path.
publish(
ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size),
role="scheduler",
)
publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank)
initialize_model_parallel()
def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4):
return PageShardSpec(
shard_rank=shard_rank,
shard_size=_GATHER_WORLD,
page_size=_GATHER_PAGE_SIZE,
max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE,
chunk_tokens=chunk_groups * _GATHER_GRANULE,
)
def _fake_req_to_token(groups, seq_len, device):
"""req_to_token row where sequence group j is allocator group groups[j]."""
row = torch.zeros(
(1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device
)
for j, q in enumerate(groups):
row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange(
q * _GATHER_GRANULE,
(q + 1) * _GATHER_GRANULE,
dtype=torch.int32,
device=device,
)
return row[:, :seq_len] if seq_len < row.shape[1] else row
def _check(rank, name, got, expect, atol=0.0):
ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0)
max_err = (got.float() - expect.float()).abs().max().item()
print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}")
assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})"
def _run_mla(rank, world, port):
_dist_init(rank, world, port, attn_cp_size=1)
group = get_parallel().attn_tp_group
assert group.world_size == world
# Topology-first shard-group selection: no CP here, so MLA falls back to
# the attn-TP axis, while GQA has no replicated axis (world_size 1).
assert get_kv_shard_group(use_mla_backend=True) is group
assert get_kv_shard_group(use_mla_backend=False).world_size == 1
spec = _gather_make_spec(shard_rank=group.rank_in_group)
pool = PageInterleaveMLATokenToKVPool(
_GATHER_SIZE,
page_size=_GATHER_PAGE_SIZE,
dtype=_GATHER_DTYPE,
kv_lora_rank=_GATHER_KV_LORA_RANK,
qk_rope_head_dim=_GATHER_QK_ROPE,
layer_num=_GATHER_LAYER_NUM,
device=f"cuda:{rank}",
enable_memory_saver=False,
start_layer=0,
end_layer=_GATHER_LAYER_NUM - 1,
shard_spec=spec,
shard_group=group,
)
device = pool.kv_buffer[0].device
# ---- chunk 1: replicated write, owner-filtered persist -----------------
# "Allocator" hands out fragmented groups (identical on every rank).
chunk1_groups = [5, 2, 9]
chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[
0
].long()
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
vals = _mla_value(
chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
pool.set_mla_kv_buffer(
layer,
chunk1_locs,
vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
)
torch.cuda.synchronize()
torch.distributed.barrier()
# Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps)
# on every rank, holding that rank's page of the group.
for q in chunk1_groups:
local_rows = torch.arange(
q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device
)
owned_locs = (
q * _GATHER_GRANULE
+ group.rank_in_group * _GATHER_PAGE_SIZE
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
)
got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE)
expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE)
_check(rank, f"mla owned stripe g{q}", got, expect)
# ---- chunk 2: prefix gather + staged chunk, both read styles -----------
seq_groups = chunk1_groups + [12] # one new chunk group
prefix_len = 3 * _GATHER_GRANULE
seq_len = prefix_len + _GATHER_GRANULE
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
# Write the current chunk (stages it into the slot + persists the
# owned stripe), like the extend forward does before attention.
chunk_vals = _mla_value(
chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
pool.set_mla_kv_buffer(
layer,
chunk2_locs,
chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
)
# Chunked-prefix MHA style: fetch an arbitrary sub-range of the
# prefix through get_mla_kv_buffer.
sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3]
k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE)
expect = _mla_value(
sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
_check(
rank,
f"mla prefix read l{layer_id}",
k_nope[:, 0, :],
expect[:, :_GATHER_KV_LORA_RANK],
)
_check(
rank,
f"mla prefix rope l{layer_id}",
k_rope[:, 0, :],
expect[:, _GATHER_KV_LORA_RANK:],
)
# Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk]
# from get_key_buffer through the translated page table.
all_locs = req_to_token[0, :seq_len].long()
rows = pool.translate_loc_to_scratch(all_locs)
kv_scratch = pool.get_key_buffer(layer_id)
_check(
rank,
f"mla absorbed read l{layer_id}",
kv_scratch[rows, 0, :],
_mla_value(
all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
),
)
torch.distributed.barrier()
if rank == 0:
print("PASS: MLA page-interleave shard (attn-TP axis)")
def _run_mha(rank, world, port):
_dist_init(rank, world, port, attn_cp_size=world)
group = get_parallel().attn_cp_group
assert group.world_size == world
# Topology-first shard-group selection: with an active CP group, both
# GQA and MLA shard across CP (CP replicates KV for every attention
# type; the TP axis is only the no-CP MLA fallback).
assert get_kv_shard_group(use_mla_backend=False) is group
assert get_kv_shard_group(use_mla_backend=True) is group
spec = _gather_make_spec(shard_rank=group.rank_in_group)
pool = PageInterleaveMHATokenToKVPool(
_GATHER_SIZE,
page_size=_GATHER_PAGE_SIZE,
dtype=_GATHER_DTYPE,
head_num=_GATHER_HEAD_NUM,
head_dim=_GATHER_HEAD_DIM,
layer_num=_GATHER_LAYER_NUM,
device=f"cuda:{rank}",
enable_memory_saver=False,
start_layer=0,
end_layer=_GATHER_LAYER_NUM - 1,
enable_alt_stream=False,
shard_spec=spec,
shard_group=group,
)
device = pool.k_buffer[0].device
def kv_value(locs, layer_id, is_v):
base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0)
return (
base.view(-1, 1, 1)
+ torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01
+ torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001
).to(_GATHER_DTYPE)
# ---- chunk 1 (prefix-less batch): stage + owner-persist ----------------
chunk1_groups = [7, 3]
chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[
0
].long()
req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
pool.set_kv_buffer(
layer,
chunk1_locs,
kv_value(chunk1_locs, layer_id, False),
kv_value(chunk1_locs, layer_id, True),
)
# The current chunk must be readable through the scratch right away.
k_scratch = pool.get_key_buffer(layer_id)
rows = pool.translate_loc_to_scratch(chunk1_locs)
_check(
rank,
f"mha chunk stage l{layer_id}",
k_scratch[rows],
kv_value(chunk1_locs, layer_id, False),
)
torch.cuda.synchronize()
torch.distributed.barrier()
# ---- chunk 2: prefix gathered from peers via translated page table -----
seq_groups = chunk1_groups + [11]
prefix_len = 2 * _GATHER_GRANULE
seq_len = prefix_len + _GATHER_GRANULE
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
pool.set_kv_buffer(
layer,
chunk2_locs,
kv_value(chunk2_locs, layer_id, False),
kv_value(chunk2_locs, layer_id, True),
)
all_locs = req_to_token[0, :seq_len].long()
rows = pool.translate_loc_to_scratch(all_locs)
k_scratch = pool.get_key_buffer(layer_id)
v_scratch = pool.get_value_buffer(layer_id)
_check(
rank,
f"mha seq read k l{layer_id}",
k_scratch[rows],
kv_value(all_locs, layer_id, False),
)
_check(
rank,
f"mha seq read v l{layer_id}",
v_scratch[rows],
kv_value(all_locs, layer_id, True),
)
torch.distributed.barrier()
if rank == 0:
print("PASS: MHA page-interleave shard (attn-CP axis)")
def _mtp_make_pool(rank, group, kind, start_layer, layer_num):
kwargs = dict(
size=_MTP_SIZE,
page_size=_GATHER_PAGE_SIZE,
dtype=_GATHER_DTYPE,
layer_num=layer_num,
device=f"cuda:{rank}",
enable_memory_saver=False,
start_layer=start_layer,
end_layer=start_layer + layer_num - 1,
shard_spec=PageShardSpec(
shard_rank=group.rank_in_group,
shard_size=_GATHER_WORLD,
page_size=_GATHER_PAGE_SIZE,
max_prefix_tokens=16 * _GATHER_WORLD * _GATHER_PAGE_SIZE,
chunk_tokens=4 * _GATHER_PAGE_SIZE,
),
shard_group=group,
)
if kind == "mla":
pool = PageInterleaveMLATokenToKVPool(
**kwargs,
kv_lora_rank=_GATHER_KV_LORA_RANK,
qk_rope_head_dim=_GATHER_QK_ROPE,
)
else:
pool = PageInterleaveMHATokenToKVPool(
**kwargs,
head_num=_GATHER_HEAD_NUM,
head_dim=_GATHER_HEAD_DIM,
enable_alt_stream=False,
)
for local_layer in range(layer_num):
for buffer in _mtp_buffers(pool, kind, local_layer):
buffer.fill_(_MTP_UNWRITTEN)
return pool
def _mtp_buffers(pool, kind, local_layer):
if kind == "mla":
return (pool.kv_buffer[local_layer],)
return (pool.k_buffer[local_layer], pool.v_buffer[local_layer])
def _mtp_make_draft_pool(target, allocator, kind, start_layer):
# Skip model loading while using the production pool dispatch and sharing
# the target allocator. A replicated draft pool must fail this test.
kvc = KVCacheConfigurator.__new__(KVCacheConfigurator)
kvc.device = target.device
kvc.is_draft_worker = True
kvc.page_size = _GATHER_PAGE_SIZE
kvc.use_mla_backend = kind == "mla"
kvc.kv_cache_dtype = _GATHER_DTYPE
kvc.kv_cache_dtype_str = "bfloat16"
kvc.post_capture_kv_active = False
kvc.is_hybrid_swa = False
kvc.sliding_window_size = None
kvc.mambaish_config = None
kvc.token_to_kv_pool_allocator = allocator
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
kvc.model_config = SimpleNamespace(
num_nextn_predict_layers=1,
hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]),
kv_lora_rank=_GATHER_KV_LORA_RANK,
qk_rope_head_dim=_GATHER_QK_ROPE,
head_dim=_GATHER_HEAD_DIM,
v_head_dim=_GATHER_HEAD_DIM,
get_num_kv_heads=lambda *_args: _GATHER_HEAD_NUM,
)
kvc.layer_info = SimpleNamespace(
start_layer=start_layer, end_layer=start_layer + 1, num_effective_layers=1
)
with get_context().override_server_args(
page_size=_GATHER_PAGE_SIZE,
tp_size=_GATHER_WORLD,
attn_cp_size=_GATHER_WORLD if kind == "mha" else 1,
speculative_algorithm="EAGLE",
enable_multi_layer_eagle=False,
):
pool = kvc._build_token_to_kv_pool(
sizes=SimpleNamespace(max_total_num_tokens=target.size),
is_dsa_model=False,
is_dsv4_model=False,
req_to_token_pool=None,
)
assert pool.shard_spec is target.shard_spec
assert pool.shard_group is target.shard_group
for buffer in _mtp_buffers(pool, kind, 0):
buffer.fill_(_MTP_UNWRITTEN)
return pool
def _mtp_values(locs, width, tag, local_layer, is_v=False):
"""Each location, feature, layer, and pool has distinguishable BF16 bytes.
Small integers survive BF16 exactly; high-valued logical slot IDs plus tiny
feature offsets would round away the differences this test must detect.
The first two columns encode the full logical slot, avoiding hash aliases.
"""
features = torch.arange(width, device=locs.device)
values = (
locs[:, None] * 37
+ features[None, :] * 17
+ tag * 73
+ local_layer * 11
+ int(is_v) * 97
) % 251 - 125
values[:, 0] = locs % 128
values[:, 1] = locs // 128
values[:, 2] = tag
values[:, 3] = local_layer
values[:, 4] = int(is_v)
return values.to(_GATHER_DTYPE)
def _mtp_canonical(kind, locs, tag, local_layer):
if kind == "mla":
return (
_mtp_values(
locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE, tag, local_layer
).unsqueeze(1),
)
return tuple(
_mtp_values(
locs, _GATHER_HEAD_NUM * _GATHER_HEAD_DIM, tag, local_layer, is_v
).reshape(-1, _GATHER_HEAD_NUM, _GATHER_HEAD_DIM)
for is_v in (False, True)
)
def _mtp_assert_bytes(got, expected, label):
assert got.shape == expected.shape, label
assert got.dtype == expected.dtype == _GATHER_DTYPE, label
assert torch.equal(
got.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8)
), f"{label}: KV bytes differ"
def _mtp_logical_chain(device):
# Owners alternate from rank 1, while local pages run out of order. Some
# logical slots exceed the physical pool size, catching a replicated draft
# indexing them raw.
local_pages = torch.tensor([25, 3, 20, 6, 28, 4, 30, 8, 22], device=device)
owners = (1 + torch.arange(local_pages.numel(), device=device)) % _GATHER_WORLD
pages = local_pages * _GATHER_WORLD + owners
return (
pages[:, None] * _GATHER_PAGE_SIZE
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
).reshape(-1)
def _mtp_write(pool, kind, locs, tag, local_layer, epoch):
layer = SimpleNamespace(layer_id=pool.start_layer + local_layer)
values = _mtp_canonical(kind, locs, tag, local_layer)
if kind == "mha":
pool.set_kv_buffer(layer, locs, *values)
elif epoch % 2:
# Both MLA write entry points must owner-filter and stage MTP rows.
pool.set_mla_kv_buffer(
layer,
locs,
values[0][..., :_GATHER_KV_LORA_RANK],
values[0][..., _GATHER_KV_LORA_RANK:],
)
else:
pool.set_kv_buffer(
layer, locs, values[0], values[0][..., :_GATHER_KV_LORA_RANK]
)
def _mtp_check_persisted(pool, kind, locs, tag, rank):
owned_locs = locs[(locs // _GATHER_PAGE_SIZE) % _GATHER_WORLD == rank]
local_rows = (
owned_locs // (_GATHER_WORLD * _GATHER_PAGE_SIZE) * _GATHER_PAGE_SIZE
+ owned_locs % _GATHER_PAGE_SIZE
)
for local_layer in range(pool.layer_num):
canonical = _mtp_canonical(kind, owned_locs, tag, local_layer)
for buffer, values in zip(_mtp_buffers(pool, kind, local_layer), canonical):
expected = torch.full_like(buffer, _MTP_UNWRITTEN)
expected[local_rows] = values
_mtp_assert_bytes(
buffer, expected, f"{kind} pool {tag} owned layer {local_layer}"
)
def _mtp_check_scratch(pool, kind, locs, prefix_len, tag, local_layer):
layer_id = pool.start_layer + local_layer
rows = pool.translate_loc_to_scratch(locs)
expected = _mtp_canonical(kind, locs, tag, local_layer)
_mtp_assert_bytes(
pool.get_key_buffer(layer_id)[rows], expected[0], f"{kind} pool {tag} key"
)
_mtp_assert_bytes(
pool.get_value_buffer(layer_id)[rows],
expected[0][..., :_GATHER_KV_LORA_RANK] if kind == "mla" else expected[1],
f"{kind} pool {tag} value",
)
if kind == "mla" and prefix_len:
# Arbitrary prefix indices also exercise the chunked MLA consumer.
subset = locs[3:prefix_len:7].clone()
k_nope, k_rope = pool.get_mla_kv_buffer(
SimpleNamespace(layer_id=layer_id), subset, _GATHER_DTYPE
)
reference = _mtp_canonical(kind, subset, tag, local_layer)[0]
_mtp_assert_bytes(
k_nope, reference[..., :_GATHER_KV_LORA_RANK], f"pool {tag} latent"
)
_mtp_assert_bytes(
k_rope, reference[..., _GATHER_KV_LORA_RANK:], f"pool {tag} rope"
)
def _run_mtp(rank, port, kind):
"""Check exact KV bytes and independent target/draft state across batches."""
_dist_init(
rank, _GATHER_WORLD, port, attn_cp_size=_GATHER_WORLD if kind == "mha" else 1
)
group = get_kv_shard_group(use_mla_backend=kind == "mla")
assert group.world_size == _GATHER_WORLD
target = _mtp_make_pool(
rank, group, kind, start_layer=0, layer_num=_GATHER_LAYER_NUM
)
allocator = PageInterleavePoolAllocator(
size=target.size,
physical_page_size=_GATHER_PAGE_SIZE,
shard_size=_GATHER_WORLD,
dtype=_GATHER_DTYPE,
device=target.device,
kvcache=target,
need_sort=False,
shard_spec=target.shard_spec,
)
# DeepSeek's NextN uses local layer 0. Also cover an absolute/offset layer ID
# so the one-layer gather terminates correctly on either scratch-slot parity.
drafts = [
_mtp_make_draft_pool(target, allocator, kind, start_layer=start)
for start in (0, 61)
]
pools = [target, *drafts]
for other in drafts:
assert other._page_pos.data_ptr() != target._page_pos.data_ptr()
assert other.kv_gather_comm is not target.kv_gather_comm
for slot, target_slot in zip(other._slots, target._slots):
for name in slot.tensors:
assert (
slot.tensors[name].data_ptr()
!= target_slot.tensors[name].data_ptr()
)
locs = _mtp_logical_chain(target.device)
req_to_token = locs.to(torch.int32).unsqueeze(0)
req_indices = torch.tensor([0], dtype=torch.int64, device=locs.device)
# Each cached prefix is page aligned. The third batch writes only five
# tokens of its tail page; the fourth recomputes and completes that page.
batches = [(0, 48), (48, 96), (96, 133), (128, 144)]
for epoch, (prefix_len, seq_len) in enumerate(batches, start=1):
active_locs = locs[:seq_len]
chunk_locs = locs[prefix_len:seq_len]
target_epoch = target._epoch
for tag, pool in enumerate(pools, start=1):
pool.begin_shard_extend(req_to_token, req_indices, [prefix_len], [seq_len])
assert pool._epoch == epoch
for local_layer in range(pool.layer_num):
_mtp_write(pool, kind, chunk_locs, tag, local_layer, epoch)
_mtp_check_scratch(
pool, kind, active_locs, prefix_len, tag, local_layer
)
_mtp_check_persisted(pool, kind, active_locs, tag, rank)
# MTP metadata/writes/reads must leave the target's last resident
# layer and gather plan usable, despite identical logical locations.
assert target._epoch == target_epoch + 1
_mtp_check_scratch(
target, kind, active_locs, prefix_len, 1, _GATHER_LAYER_NUM - 1
)
if pool.layer_num == 1 and prefix_len:
resident = pool._slots[pool.start_layer % 2]
assert resident.resident_key == (pool.start_layer, epoch)
assert pool._slots[(pool.start_layer + 1) % 2].resident_key is None
if rank == 0:
print(f"PASS: {kind.upper()} target/MTP prefix={prefix_len}, seq={seq_len}")
# Deactivating one draft's plan must not deactivate the other pools.
drafts[0].end_shard_extend()
assert not drafts[0]._shard_extend_active
assert target._shard_extend_active and drafts[1]._shard_extend_active
_mtp_check_scratch(drafts[1], kind, locs, 128, 3, 0)
_mtp_check_scratch(target, kind, locs, 128, 1, _GATHER_LAYER_NUM - 1)
torch.cuda.synchronize()
torch.distributed.barrier()
torch.distributed.destroy_process_group()
@unittest.skipIf(
torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices"
)
class TestPageInterleaveGatherMultiGpu(CustomTestCase):
"""Real pools, real NCCL, 2 ranks — one mp.spawn per phase.
Separate spawns (and separate ports) because each phase builds its own
process group with a different attention-CP width.
"""
def test_mla_shard_over_attention_tp(self):
mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True)
def test_mha_shard_over_attention_cp(self):
mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True)
def test_mtp_mla_shard_over_attention_tp(self):
mp.spawn(_run_mtp, args=(29813, "mla"), nprocs=_GATHER_WORLD, join=True)
def test_mtp_mha_shard_over_attention_cp(self):
mp.spawn(_run_mtp, args=(29814, "mha"), nprocs=_GATHER_WORLD, join=True)
if __name__ == "__main__":
unittest.main()
@@ -13,8 +13,7 @@
# ==============================================================================
"""Tests for logical-page KV cache sharding.
The CPU tests pin the pure arithmetic that rotated owner-classed
allocation hangs on:
Pins the pure arithmetic that rotated owner-classed allocation hangs on:
1. The placement bijection ``loc = Q*(N*ps) + r*ps + o`` — owner / local-row
round-trip, disjoint equal partition across ranks.
@@ -31,37 +30,31 @@ allocation hangs on:
5. ``begin_shard_extend`` plan capture (page positions, padded send rows,
owner-congruence guard) with the gather stubbed out, following the
SimpleNamespace binding pattern of ``test_dsa_layer_shard_utils.py``.
6. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's
6. MTP draft pool construction, shared allocator/layout validation, and scratch
sizing for separate target and draft pools.
7. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's
skip of a non-final chunk this rank owns no page of.
7. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's
8. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's
tokens, each token stored on exactly one rank.
The GPU tests exercise real MLA and MHA pools and NCCL gathers on 2 ranks
(``TestPageInterleaveGatherMultiGpu``). DSA indexer and attention coverage lives
in ``test_page_sharded_dsa_attention.py``.
GPU tests skip when too few CUDA devices are visible, so the same file runs
in both CPU and CUDA CI suites.
The real NCCL MLA/MHA/MTP gathers over 2 GPUs live in
``test_page_interleave_gather_multi_gpu.py``. DSA indexer and attention
coverage lives in ``test_page_sharded_dsa_attention.py``.
"""
import os
import shutil
import unittest
import unittest.mock
from array import array
from contextlib import contextmanager
from contextlib import ExitStack, contextmanager
from types import SimpleNamespace
import numpy as np
import torch
import torch.multiprocessing as mp
from parameterized import parameterized_class
from sglang.srt.disaggregation.common.conn import CommonKVSender
from sglang.srt.disaggregation.utils import filter_kv_indices_for_shard_rank
from sglang.srt.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import ReqKvInfo
from sglang.srt.mem_cache import page_interleave
@@ -77,6 +70,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
)
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.common import _evict_until_allocatable
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
from sglang.srt.mem_cache.memory_pool import (
MHATokenToKVPool,
MLATokenToKVPool,
@@ -86,7 +80,8 @@ from sglang.srt.mem_cache.page_interleave import (
PageInterleavePlacement,
PageShardSpec,
compute_page_shard_scratch_bytes,
get_kv_shard_group,
get_kv_shard_group_info,
get_shared_kv_shard_pool,
make_page_shard_spec,
)
from sglang.srt.mem_cache.page_interleave_pool import (
@@ -97,19 +92,14 @@ from sglang.srt.mem_cache.page_interleave_pool import (
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.runtime_context import get_parallel, publish
from sglang.srt.server_args import ServerArgs
from sglang.srt.runtime_context import get_context, get_memory, get_parallel, get_spec
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils import ceil_div
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.mem_cache_utils import finish_req
from sglang.test.test_utils import (
CustomTestCase,
enter_scope,
publish_build_topology,
)
from sglang.test.test_utils import CustomTestCase, enter_scope
register_cpu_ci(est_time=30, suite="base-a-test-cpu")
register_cuda_ci(est_time=120, stage="base-c", runner_config="4-gpu-b200")
N = 4 # shard size
PS = 16 # physical page size
@@ -150,6 +140,7 @@ class TestPageShardScratchSizing(CustomTestCase):
)
kvc = SimpleNamespace(
is_draft_worker=draft,
token_to_kv_pool_allocator=None,
use_mla_backend=use_mla,
page_size=16,
kv_cache_dtype=torch.bfloat16,
@@ -182,6 +173,28 @@ class TestPageShardScratchSizing(CustomTestCase):
self.assertEqual(spec.chunk_tokens, 48)
self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0)
def test_target_with_mtp_budgets_one_extra_scratch_pair(self):
with (
self._fixture(context_len=100, chunk_tokens=64, shard_size=2) as kvc,
get_context().override_server_args(enable_multi_layer_eagle=False),
):
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
# Target configs can omit next-N layers; use the resolved draft count.
kvc.model_config.num_nextn_predict_layers = None
kvc.spec_aux_config = SimpleNamespace(eagle_draft_num_layers=1)
expected = 2 * (8 * 128 + 64 + 16) * (16 + 8) * 2
for layers in (1, 3):
with self.subTest(layers=layers):
kvc.spec_aux_config.eagle_draft_num_layers = layers
self.assertEqual(compute_page_shard_scratch_bytes(kvc), expected)
self.assertEqual(
compute_page_shard_scratch_bytes(kvc, include_mtp=True),
2 * expected,
)
kvc.spec_aux_config.eagle_draft_num_layers = None
with self.assertRaisesRegex(ValueError, "EAGLE MTP"):
compute_page_shard_scratch_bytes(kvc, include_mtp=True)
def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self):
for use_mla in (True, False):
with (
@@ -211,6 +224,255 @@ class TestPageShardScratchSizing(CustomTestCase):
self.assertEqual(estimated, actual)
def _make_mtp_configurator(use_mla=True):
# Keep real pool/allocator types for validation; allocate only CPU metadata.
pool_class = (
PageInterleaveMLATokenToKVPool if use_mla else PageInterleaveMHATokenToKVPool
)
target = pool_class.__new__(pool_class)
target.shard_spec = PageShardSpec(1, 2, 16, 128, 64)
target.shard_group = SimpleNamespace(rank_in_group=1, world_size=2)
target.size, target.page_size, target.dtype = 128, 16, torch.bfloat16
target.kv_lora_rank, target.qk_rope_head_dim = 8, 4
target.head_num, target.head_dim, target.v_head_dim = 2, 4, 6
kvc = KVCacheConfigurator.__new__(KVCacheConfigurator)
kvc.token_to_kv_pool_allocator = PageInterleavePoolAllocator(
size=target.size,
physical_page_size=target.page_size,
shard_size=target.shard_spec.shard_size,
dtype=target.dtype,
device="cpu",
kvcache=target,
need_sort=False,
shard_spec=target.shard_spec,
)
kvc.device = "cpu"
kvc.is_draft_worker = True
kvc.page_size = target.page_size
kvc.use_mla_backend = use_mla
kvc.kv_cache_dtype = target.dtype
kvc.kv_cache_dtype_str = "bfloat16"
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
kvc.model_config = SimpleNamespace(
num_nextn_predict_layers=1,
context_len=9999,
kv_lora_rank=8,
qk_rope_head_dim=4,
head_dim=4,
v_head_dim=6,
get_num_kv_heads=lambda tp: 4 // tp,
hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]),
)
kvc.layer_info = SimpleNamespace(start_layer=5, end_layer=6, num_effective_layers=1)
kvc.is_hybrid_swa = False
kvc.sliding_window_size = None
kvc.mambaish_config = None
kvc.post_capture_kv_active = False
return kvc, target
def _build_mtp_pool(kvc, *, capacity=128, is_dsa_model=False, is_dsv4_model=False):
return kvc._build_token_to_kv_pool(
sizes=SimpleNamespace(max_total_num_tokens=capacity),
is_dsa_model=is_dsa_model,
is_dsv4_model=is_dsv4_model,
req_to_token_pool=None,
)
class TestMTPKVShardConfig(CustomTestCase):
def setUp(self):
super().setUp()
stack = ExitStack()
self.addCleanup(stack.close)
stack.enter_context(
get_context().override_server_args(
page_size=16, chunked_prefill_size=1024, enable_multi_layer_eagle=False
)
)
stack.enter_context(
get_parallel().override(
tp_size=2, attn_tp_size=2, moe_tp_size=2, attn_dcp_size=1
)
)
def test_draft_shares_target_layout_but_has_its_own_pool(self):
for use_mla in (True, False):
with self.subTest(use_mla=use_mla):
kvc, target = _make_mtp_configurator(use_mla)
pool_class = type(target)
# Mock allocation only; preserve real types and distinct instances.
with (
unittest.mock.patch.object(
pool_class, "__init__", return_value=None
) as init,
unittest.mock.patch.object(
page_interleave,
"get_kv_shard_group",
side_effect=AssertionError,
),
):
self.assertIs(get_shared_kv_shard_pool(kvc), target)
self.assertEqual(get_kv_shard_group_info(kvc), (1, 2))
self.assertIs(make_page_shard_spec(kvc), target.shard_spec)
pool = _build_mtp_pool(kvc)
self.assertIsInstance(pool, pool_class)
self.assertIsNot(pool, target)
kwargs = init.call_args.kwargs
self.assertIs(kwargs["shard_spec"], target.shard_spec)
self.assertIs(kwargs["shard_group"], target.shard_group)
for name, expected in dict(
size=128,
page_size=16,
dtype=torch.bfloat16,
layer_num=1,
start_layer=5,
end_layer=6,
).items():
self.assertEqual(kwargs[name], expected, name)
allocator = kvc.token_to_kv_pool_allocator
self.assertIs(
kvc._build_token_to_kv_pool_allocator(
sizes=SimpleNamespace(max_total_num_tokens=128),
token_to_kv_pool=pool,
is_dsv4_model=False,
req_to_token_pool=None,
token_to_kv_pool_allocator=allocator,
),
allocator,
)
# Scratch uses inherited bounds despite different draft context/chunk sizes.
row_bytes = (8 + 4) * 2 if use_mla else 2 * (4 + 6) * 2
self.assertEqual(
compute_page_shard_scratch_bytes(kvc),
2 * (128 + 64 + 16) * row_bytes,
)
with self.assertRaisesRegex(ValueError, "target"):
compute_page_shard_scratch_bytes(kvc, include_mtp=True)
def test_without_shared_allocator_keeps_existing_pool(self):
for draft, allocator in ((True, None), (True, object()), (False, None)):
with self.subTest(draft=draft, allocator=allocator):
kvc, _ = _make_mtp_configurator()
kvc.is_draft_worker = draft
kvc.token_to_kv_pool_allocator = allocator
self.assertIsNone(get_shared_kv_shard_pool(kvc))
if draft:
self.assertEqual(get_kv_shard_group_info(kvc), (None, 1))
self.assertIsNone(make_page_shard_spec(kvc))
self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0)
with unittest.mock.patch.object(
KVCacheConfigurator, "_build_mla_kv_pool"
) as build:
self.assertIs(_build_mtp_pool(kvc), build.return_value)
build.assert_called_once_with(
max_total_num_tokens=128, mla_pool_class=MLATokenToKVPool
)
def test_rejects_invalid_shared_allocator(self):
kvc, _ = _make_mtp_configurator()
for name, value, message in (
("_kvcache", object(), "target's sharded KV pool"),
("shard_spec", PageShardSpec(0, 2, 16, 128, 64), "different shard layouts"),
("shard_size", 4, "different shard layouts"),
("page_size", 32, "different shard layouts"),
("size", 128, "different shard layouts"),
):
with (
self.subTest(attribute=name),
unittest.mock.patch.object(kvc.token_to_kv_pool_allocator, name, value),
self.assertRaisesRegex(ValueError, message),
):
get_shared_kv_shard_pool(kvc)
def test_rejects_incompatible_pool_geometry(self):
for use_mla in (True, False):
kvc, _ = _make_mtp_configurator(use_mla)
geometry = (
("kv_lora_rank", "qk_rope_head_dim")
if use_mla
else ("head_dim", "v_head_dim")
)
cases = [
(kvc, "page_size", 32, "physical page size"),
(kvc, "kv_cache_dtype", torch.float16, "geometry and dtype"),
(kvc, "use_mla_backend", not use_mla, "attention geometry"),
*(
(kvc.model_config, field, 16, "geometry and dtype")
for field in geometry
),
]
if not use_mla:
cases.append(
(
kvc.model_config,
"get_num_kv_heads",
lambda tp: 4,
"geometry and dtype",
)
)
for obj, name, value, message in cases:
with (
self.subTest(use_mla=use_mla, attribute=name),
unittest.mock.patch.object(obj, name, value),
self.assertRaisesRegex(ValueError, message),
):
_build_mtp_pool(kvc)
for capacity in (64, 256):
with (
self.subTest(use_mla=use_mla, capacity=capacity),
self.assertRaisesRegex(ValueError, "same per-rank capacity"),
):
_build_mtp_pool(kvc, capacity=capacity)
def test_rejects_unsupported_draft_config(self):
kvc, _ = _make_mtp_configurator()
cases = [
(kvc, "spec_algorithm", algorithm, "EAGLE MTP")
for algorithm in (
SpeculativeAlgorithm.EAGLE3,
SpeculativeAlgorithm.DFLASH,
SpeculativeAlgorithm.FROZEN_KV_MTP,
)
] + [
(kvc.model_config, "num_nextn_predict_layers", None, "EAGLE MTP"),
(kvc.model_config, "num_nextn_predict_layers", 0, "EAGLE MTP"),
(kvc, "is_hybrid_swa", True, "dense MLA and MHA"),
(kvc, "mambaish_config", object(), "dense MLA and MHA"),
(kvc, "sliding_window_size", 128, "dense MLA and MHA"),
(kvc, "post_capture_kv_active", True, "plain per-layer KV layout"),
(kvc, "kv_cache_dtype_str", "mxfp8", "plain per-layer KV layout"),
]
for obj, name, value, message in cases:
with (
self.subTest(attribute=name, value=value),
unittest.mock.patch.object(obj, name, value),
self.assertRaisesRegex(ValueError, message),
):
_build_mtp_pool(kvc)
for options in ({"is_dsa_model": True}, {"is_dsv4_model": True}):
with (
self.subTest(options=options),
self.assertRaisesRegex(ValueError, "dense MLA and MHA"),
):
_build_mtp_pool(kvc, **options)
for context, options, message in (
(get_spec(), {"enable_multi_layer_eagle": True}, "EAGLE MTP"),
(get_parallel(), {"attn_dcp_size": 2}, "incompatible with DCP"),
(
get_memory(),
{"enable_page_major_kv_layout": True},
"plain per-layer KV layout",
),
):
with (
self.subTest(options=options),
context.override(**options),
self.assertRaisesRegex(ValueError, message),
):
_build_mtp_pool(kvc)
def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32):
return PageShardSpec(
shard_rank=shard_rank,
@@ -1402,346 +1664,5 @@ class TestShardSendFilter(CustomTestCase):
self.assertEqual(skips, expect_skip)
# =============================================================================
# Multi-GPU: the real NCCL layer-ahead gather (2 GPUs).
#
# Everything above is pure arithmetic on a CPU stub. This section drives real
# pools over a real process group, which is the only check that the plan the
# stub validates actually addresses the bytes the collective delivers:
#
# 1. MLA pool sharded across the attention-TP group: replicated writes are
# owner-filtered into disjoint pool stripes; a later batch's chunked-prefix
# read (get_mla_kv_buffer) assembles the full prefix from all ranks via the
# layer-ahead NCCL allgather and must return the canonical bytes.
# 2. MHA pool sharded across the attention-CP group: the post-allgather full
# chunk is staged into the scratch chunk region and owner-persisted; a later
# batch reads prefix+chunk through the translated page table (the scratch),
# and the assembled rows must match the canonical bytes.
#
# Skipped unless 2 CUDA devices are visible, so it is inert on the CPU runner
# this file is registered to. Run it explicitly with:
# CUDA_VISIBLE_DEVICES=0,1 python3 test/registered/unit/mem_cache/\
# test_page_interleave_shard.py TestPageInterleaveGatherMultiGpu
# =============================================================================
_GATHER_WORLD = 2
_GATHER_LAYER_NUM = 4
_GATHER_PAGE_SIZE = 16
_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE
_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank
_GATHER_KV_LORA_RANK = 128
_GATHER_QK_ROPE = 32
_GATHER_HEAD_NUM = 2
_GATHER_HEAD_DIM = 32
_GATHER_DTYPE = torch.bfloat16
def _mla_value(loc, dim):
"""Deterministic canonical latent value for logical slot ``loc``."""
loc = loc.to(torch.float32)
return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to(
_GATHER_DTYPE
)
def _dist_init(rank, world, port, attn_cp_size):
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(port)
os.environ["RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world)
os.environ.setdefault("no_proxy", "127.0.0.1,localhost")
torch.cuda.set_device(rank)
init_distributed_environment(
world_size=world,
rank=rank,
local_rank=rank,
distributed_init_method=f"tcp://127.0.0.1:{port}",
backend="nccl",
)
# Publish the widths the groups below are about to be built at. The derived
# quotients (attn_tp_size, attn_dcp_size, ...) are projected from these
# leaves at publish; initialize_model_parallel no longer supplies them, and
# MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path.
publish(
ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size),
role="scheduler",
)
publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank)
initialize_model_parallel()
def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4):
return PageShardSpec(
shard_rank=shard_rank,
shard_size=_GATHER_WORLD,
page_size=_GATHER_PAGE_SIZE,
max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE,
chunk_tokens=chunk_groups * _GATHER_GRANULE,
)
def _fake_req_to_token(groups, seq_len, device):
"""req_to_token row where sequence group j is allocator group groups[j]."""
row = torch.zeros(
(1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device
)
for j, q in enumerate(groups):
row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange(
q * _GATHER_GRANULE,
(q + 1) * _GATHER_GRANULE,
dtype=torch.int32,
device=device,
)
return row[:, :seq_len] if seq_len < row.shape[1] else row
def _check(rank, name, got, expect, atol=0.0):
ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0)
max_err = (got.float() - expect.float()).abs().max().item()
print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}")
assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})"
def _run_mla(rank, world, port):
_dist_init(rank, world, port, attn_cp_size=1)
group = get_parallel().attn_tp_group
assert group.world_size == world
# Topology-first shard-group selection: no CP here, so MLA falls back to
# the attn-TP axis, while GQA has no replicated axis (world_size 1).
assert get_kv_shard_group(use_mla_backend=True) is group
assert get_kv_shard_group(use_mla_backend=False).world_size == 1
spec = _gather_make_spec(shard_rank=group.rank_in_group)
pool = PageInterleaveMLATokenToKVPool(
_GATHER_SIZE,
page_size=_GATHER_PAGE_SIZE,
dtype=_GATHER_DTYPE,
kv_lora_rank=_GATHER_KV_LORA_RANK,
qk_rope_head_dim=_GATHER_QK_ROPE,
layer_num=_GATHER_LAYER_NUM,
device=f"cuda:{rank}",
enable_memory_saver=False,
start_layer=0,
end_layer=_GATHER_LAYER_NUM - 1,
shard_spec=spec,
shard_group=group,
)
device = pool.kv_buffer[0].device
# ---- chunk 1: replicated write, owner-filtered persist -----------------
# "Allocator" hands out fragmented groups (identical on every rank).
chunk1_groups = [5, 2, 9]
chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[
0
].long()
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
vals = _mla_value(
chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
pool.set_mla_kv_buffer(
layer,
chunk1_locs,
vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
)
torch.cuda.synchronize()
torch.distributed.barrier()
# Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps)
# on every rank, holding that rank's page of the group.
for q in chunk1_groups:
local_rows = torch.arange(
q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device
)
owned_locs = (
q * _GATHER_GRANULE
+ group.rank_in_group * _GATHER_PAGE_SIZE
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
)
got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE)
expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE)
_check(rank, f"mla owned stripe g{q}", got, expect)
# ---- chunk 2: prefix gather + staged chunk, both read styles -----------
seq_groups = chunk1_groups + [12] # one new chunk group
prefix_len = 3 * _GATHER_GRANULE
seq_len = prefix_len + _GATHER_GRANULE
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
# Write the current chunk (stages it into the slot + persists the
# owned stripe), like the extend forward does before attention.
chunk_vals = _mla_value(
chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
pool.set_mla_kv_buffer(
layer,
chunk2_locs,
chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
)
# Chunked-prefix MHA style: fetch an arbitrary sub-range of the
# prefix through get_mla_kv_buffer.
sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3]
k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE)
expect = _mla_value(
sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
)
_check(
rank,
f"mla prefix read l{layer_id}",
k_nope[:, 0, :],
expect[:, :_GATHER_KV_LORA_RANK],
)
_check(
rank,
f"mla prefix rope l{layer_id}",
k_rope[:, 0, :],
expect[:, _GATHER_KV_LORA_RANK:],
)
# Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk]
# from get_key_buffer through the translated page table.
all_locs = req_to_token[0, :seq_len].long()
rows = pool.translate_loc_to_scratch(all_locs)
kv_scratch = pool.get_key_buffer(layer_id)
_check(
rank,
f"mla absorbed read l{layer_id}",
kv_scratch[rows, 0, :],
_mla_value(
all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
),
)
torch.distributed.barrier()
if rank == 0:
print("PASS: MLA page-interleave shard (attn-TP axis)")
def _run_mha(rank, world, port):
_dist_init(rank, world, port, attn_cp_size=world)
group = get_parallel().attn_cp_group
assert group.world_size == world
# Topology-first shard-group selection: with an active CP group, both
# GQA and MLA shard across CP (CP replicates KV for every attention
# type; the TP axis is only the no-CP MLA fallback).
assert get_kv_shard_group(use_mla_backend=False) is group
assert get_kv_shard_group(use_mla_backend=True) is group
spec = _gather_make_spec(shard_rank=group.rank_in_group)
pool = PageInterleaveMHATokenToKVPool(
_GATHER_SIZE,
page_size=_GATHER_PAGE_SIZE,
dtype=_GATHER_DTYPE,
head_num=_GATHER_HEAD_NUM,
head_dim=_GATHER_HEAD_DIM,
layer_num=_GATHER_LAYER_NUM,
device=f"cuda:{rank}",
enable_memory_saver=False,
start_layer=0,
end_layer=_GATHER_LAYER_NUM - 1,
enable_alt_stream=False,
shard_spec=spec,
shard_group=group,
)
device = pool.k_buffer[0].device
def kv_value(locs, layer_id, is_v):
base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0)
return (
base.view(-1, 1, 1)
+ torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01
+ torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001
).to(_GATHER_DTYPE)
# ---- chunk 1 (prefix-less batch): stage + owner-persist ----------------
chunk1_groups = [7, 3]
chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[
0
].long()
req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
pool.set_kv_buffer(
layer,
chunk1_locs,
kv_value(chunk1_locs, layer_id, False),
kv_value(chunk1_locs, layer_id, True),
)
# The current chunk must be readable through the scratch right away.
k_scratch = pool.get_key_buffer(layer_id)
rows = pool.translate_loc_to_scratch(chunk1_locs)
_check(
rank,
f"mha chunk stage l{layer_id}",
k_scratch[rows],
kv_value(chunk1_locs, layer_id, False),
)
torch.cuda.synchronize()
torch.distributed.barrier()
# ---- chunk 2: prefix gathered from peers via translated page table -----
seq_groups = chunk1_groups + [11]
prefix_len = 2 * _GATHER_GRANULE
seq_len = prefix_len + _GATHER_GRANULE
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
for layer_id in range(_GATHER_LAYER_NUM):
layer = SimpleNamespace(layer_id=layer_id)
pool.set_kv_buffer(
layer,
chunk2_locs,
kv_value(chunk2_locs, layer_id, False),
kv_value(chunk2_locs, layer_id, True),
)
all_locs = req_to_token[0, :seq_len].long()
rows = pool.translate_loc_to_scratch(all_locs)
k_scratch = pool.get_key_buffer(layer_id)
v_scratch = pool.get_value_buffer(layer_id)
_check(
rank,
f"mha seq read k l{layer_id}",
k_scratch[rows],
kv_value(all_locs, layer_id, False),
)
_check(
rank,
f"mha seq read v l{layer_id}",
v_scratch[rows],
kv_value(all_locs, layer_id, True),
)
torch.distributed.barrier()
if rank == 0:
print("PASS: MHA page-interleave shard (attn-CP axis)")
@unittest.skipIf(
torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices"
)
class TestPageInterleaveGatherMultiGpu(CustomTestCase):
"""Real pools, real NCCL, 2 ranks — one mp.spawn per phase.
Separate spawns (and separate ports) because each phase builds its own
process group with a different attention-CP width.
"""
def test_mla_shard_over_attention_tp(self):
mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True)
def test_mha_shard_over_attention_cp(self):
mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True)
if __name__ == "__main__":
unittest.main()