[Feat] KVShard: Support kv cache sharding for MTP (#40929)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -84,6 +84,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageShardSpec,
|
||||
get_kv_shard_group,
|
||||
get_shared_kv_shard_pool,
|
||||
make_page_shard_spec,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
@@ -1763,6 +1764,15 @@ class KVCacheConfigurator:
|
||||
is_dsv4_model: bool,
|
||||
req_to_token_pool: ReqToTokenPool,
|
||||
) -> KVCache:
|
||||
# MTP drafts sharing the target's sharded allocator need the same shard
|
||||
# layout with separate KV storage. Handle them before the normal pool
|
||||
# dispatch; targets and drafts without that allocator fall through.
|
||||
mtp_pool = self._build_mtp_kv_shard_pool(
|
||||
sizes=sizes, is_dsa_model=is_dsa_model, is_dsv4_model=is_dsv4_model
|
||||
)
|
||||
if mtp_pool is not None:
|
||||
return mtp_pool
|
||||
|
||||
# Page-granularity envelope layout for the MHA-shaped (full / SWA) pools,
|
||||
# selected by swapping in the PageMajorMHATokenToKVPool subclass. The
|
||||
# default keeps upstream's per-layer layout. The Mamba state pool is routed
|
||||
@@ -1889,6 +1899,115 @@ class KVCacheConfigurator:
|
||||
)
|
||||
return token_to_kv_pool
|
||||
|
||||
def _build_mtp_kv_shard_pool(
|
||||
self, *, sizes: _PoolSizes, is_dsa_model: bool, is_dsv4_model: bool
|
||||
) -> Optional[KVCache]:
|
||||
"""Build a draft pool in its target's logical space, with its own KV.
|
||||
|
||||
This is a pool-level opt-in through a shared sharded allocator. Target
|
||||
pool creation and speculative serving remain separate integration work.
|
||||
"""
|
||||
if not self.is_draft_worker:
|
||||
return None
|
||||
|
||||
target_pool = get_shared_kv_shard_pool(self)
|
||||
if target_pool is None:
|
||||
return None
|
||||
|
||||
if (
|
||||
self.spec_algorithm != SpeculativeAlgorithm.EAGLE
|
||||
or (self.model_config.num_nextn_predict_layers or 0) <= 0
|
||||
or get_spec().enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"Shared KV sharding supports only an EAGLE MTP draft runner."
|
||||
)
|
||||
if get_parallel().attn_dcp_size > 1:
|
||||
raise ValueError("MTP KV sharding is incompatible with DCP.")
|
||||
if (
|
||||
is_dsa_model
|
||||
or is_dsv4_model
|
||||
or self.is_hybrid_swa
|
||||
or self.mambaish_config is not None
|
||||
or self.sliding_window_size is not None
|
||||
or is_minimax_sparse(self.model_config.hf_config)
|
||||
or current_platform.is_out_of_tree()
|
||||
or _is_npu
|
||||
):
|
||||
raise ValueError("MTP KV sharding supports only dense MLA and MHA pools.")
|
||||
if (
|
||||
self.post_capture_kv_active
|
||||
or get_memory().enable_page_major_kv_layout
|
||||
or self.kv_cache_dtype_str == "mxfp8"
|
||||
or is_float4_e2m1fn_x2(self.kv_cache_dtype)
|
||||
):
|
||||
raise ValueError("MTP KV sharding requires the plain per-layer KV layout.")
|
||||
if (
|
||||
self.page_size != target_pool.page_size
|
||||
or self.pool_page_size != target_pool.page_size
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same physical page size."
|
||||
)
|
||||
if sizes.max_total_num_tokens != target_pool.size:
|
||||
raise ValueError(
|
||||
"MTP and target sharded KV pools must have the same per-rank capacity."
|
||||
)
|
||||
target_is_mla = isinstance(target_pool, PageInterleaveMLATokenToKVPool)
|
||||
if self.use_mla_backend != target_is_mla or not isinstance(
|
||||
target_pool,
|
||||
(PageInterleaveMLATokenToKVPool, PageInterleaveMHATokenToKVPool),
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same attention geometry."
|
||||
)
|
||||
fields = (
|
||||
("kv_lora_rank", "qk_rope_head_dim")
|
||||
if self.use_mla_backend
|
||||
else ("head_dim", "v_head_dim")
|
||||
)
|
||||
if (
|
||||
self.kv_cache_dtype != target_pool.dtype
|
||||
or any(
|
||||
getattr(self.model_config, name) != getattr(target_pool, name)
|
||||
for name in fields
|
||||
)
|
||||
or (
|
||||
not self.use_mla_backend
|
||||
and self.model_config.get_num_kv_heads(get_parallel().attn_tp_size)
|
||||
!= target_pool.head_num
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP and target KV pools must use the same KV geometry and dtype."
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
size=target_pool.size,
|
||||
page_size=target_pool.page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
layer_num=self.layer_info.num_effective_layers,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
start_layer=self.layer_info.start_layer,
|
||||
end_layer=self.layer_info.end_layer,
|
||||
shard_spec=target_pool.shard_spec,
|
||||
shard_group=target_pool.shard_group,
|
||||
)
|
||||
if self.use_mla_backend:
|
||||
return PageInterleaveMLATokenToKVPool(
|
||||
**kwargs,
|
||||
kv_lora_rank=self.model_config.kv_lora_rank,
|
||||
qk_rope_head_dim=self.model_config.qk_rope_head_dim,
|
||||
)
|
||||
return PageInterleaveMHATokenToKVPool(
|
||||
**kwargs,
|
||||
head_num=target_pool.head_num,
|
||||
head_dim=self.model_config.head_dim,
|
||||
v_head_dim=self.model_config.v_head_dim,
|
||||
enable_alt_stream=not get_disagg().enable_pdmux,
|
||||
)
|
||||
|
||||
def _build_dsv4_kv_pool(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -37,12 +37,18 @@ from typing import TYPE_CHECKING, Optional, Tuple
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.mem_cache import page_interleave_pool
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
PageInterleavePoolAllocator,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -127,11 +133,42 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
|
||||
return cp_group
|
||||
|
||||
|
||||
def get_shared_kv_shard_pool(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Optional[PageInterleaveKVPoolMixin]:
|
||||
"""Find a draft's sharded target through their shared logical allocator."""
|
||||
if not kvc.is_draft_worker:
|
||||
return None
|
||||
|
||||
allocator = kvc.token_to_kv_pool_allocator
|
||||
if not isinstance(allocator, PageInterleavePoolAllocator):
|
||||
return None
|
||||
|
||||
pool = allocator.get_kvcache()
|
||||
if not isinstance(pool, page_interleave_pool.PageInterleaveKVPoolMixin):
|
||||
raise ValueError("A sharded MTP draft requires the target's sharded KV pool.")
|
||||
if (
|
||||
allocator.shard_spec != pool.shard_spec
|
||||
or allocator.shard_size != pool.shard_spec.shard_size
|
||||
or allocator.page_size != pool.page_size
|
||||
or allocator.size != pool.size * allocator.shard_size
|
||||
):
|
||||
raise ValueError(
|
||||
"The shared allocator and target KV pool have different shard layouts."
|
||||
)
|
||||
return pool
|
||||
|
||||
|
||||
def get_kv_shard_group_info(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Tuple[Optional[int], int]:
|
||||
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
|
||||
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
|
||||
if kvc.is_draft_worker:
|
||||
pool = get_shared_kv_shard_pool(kvc)
|
||||
if pool is None:
|
||||
return None, 1
|
||||
return pool.shard_spec.shard_rank, pool.shard_spec.shard_size
|
||||
if not get_parallel().enable_kv_cache_sharding:
|
||||
return None, 1
|
||||
group = get_kv_shard_group(kvc.use_mla_backend)
|
||||
if group.world_size <= 1:
|
||||
@@ -163,7 +200,14 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
or chunk size. Both scratch slots are charged against HBM before sizing
|
||||
persistent KV; large model contexts therefore reduce persistent capacity.
|
||||
Admission still checks the actual batch's padded prefix and chunk spans.
|
||||
|
||||
MTP drafts inherit the target's exact bounds: both pools serve the same
|
||||
scheduled batch and logical locations, even if their model configs differ
|
||||
in context length.
|
||||
"""
|
||||
if kvc.is_draft_worker:
|
||||
pool = get_shared_kv_shard_pool(kvc)
|
||||
return pool.shard_spec if pool is not None else None
|
||||
shard_rank, shard_size = get_kv_shard_group_info(kvc)
|
||||
if shard_rank is None:
|
||||
return None
|
||||
@@ -181,11 +225,31 @@ def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
)
|
||||
|
||||
|
||||
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
def compute_page_shard_scratch_bytes(
|
||||
kvc: KVCacheConfigurator, *, include_mtp: bool = False
|
||||
) -> int:
|
||||
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
|
||||
the KV budget before pool sizing. Use the same spec as the pool builders:
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.
|
||||
|
||||
``include_mtp`` additionally budgets a target's single EAGLE MTP runner
|
||||
at the same KV geometry, as required by the draft pool builder. Its two
|
||||
scratch slots are reused across draft layers. Target sizing integration
|
||||
can opt into this estimate without changing the per-pool default.
|
||||
"""
|
||||
spec = make_page_shard_spec(kvc)
|
||||
if spec is None:
|
||||
return 0
|
||||
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
scratch_bytes = 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
if include_mtp:
|
||||
if (
|
||||
kvc.is_draft_worker
|
||||
or kvc.spec_algorithm != SpeculativeAlgorithm.EAGLE
|
||||
or (kvc.spec_aux_config.eagle_draft_num_layers or 0) <= 0
|
||||
or get_spec().enable_multi_layer_eagle
|
||||
):
|
||||
raise ValueError(
|
||||
"MTP scratch budgeting requires a target with one EAGLE MTP runner."
|
||||
)
|
||||
scratch_bytes *= 2
|
||||
return scratch_bytes
|
||||
|
||||
@@ -61,6 +61,7 @@ from sglang.kernels.ops.attention.dsa.dequant_k_cache import (
|
||||
)
|
||||
from sglang.kernels.ops.kvcache import dsa_index_k_cache
|
||||
from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from sglang.srt.mem_cache import page_interleave
|
||||
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
GPU_MEMORY_TYPE_KV_CACHE,
|
||||
@@ -70,15 +71,12 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
RadixAttention,
|
||||
unwrap_write_loc,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageInterleavePlacement,
|
||||
PageShardSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_mla_kv_buffer_triton
|
||||
from sglang.srt.utils import ceil_div, get_bool_env_var
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.page_interleave import PageShardSpec
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -123,7 +121,8 @@ class PageInterleaveKVPoolMixin:
|
||||
assert spec.chunk_tokens % spec.page_size == 0
|
||||
|
||||
self.shard_spec = spec
|
||||
self.placement = PageInterleavePlacement(spec)
|
||||
self.shard_group = shard_group
|
||||
self.placement = page_interleave.PageInterleavePlacement(spec)
|
||||
self.shard_rank = spec.shard_rank
|
||||
self.shard_size = spec.shard_size
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
|
||||
@@ -972,6 +972,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
extend_len = 1024
|
||||
prefix_len = context_len - extend_len
|
||||
kvc = SimpleNamespace(
|
||||
is_draft_worker=False,
|
||||
page_size=4,
|
||||
model_config=SimpleNamespace(context_len=context_len),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,687 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Multi-GPU tests for logical-page KV cache sharding: the real NCCL
|
||||
layer-ahead gather over a real 2-rank process group.
|
||||
|
||||
``test_page_interleave_shard.py`` pins the plan arithmetic on CPU stubs. These
|
||||
tests drive real pools, which is the only check that the plan the stubs
|
||||
validate actually addresses the bytes the collective delivers:
|
||||
|
||||
1. MLA pool sharded across the attention-TP group: replicated writes are
|
||||
owner-filtered into disjoint pool stripes; a later batch's chunked-prefix
|
||||
read (get_mla_kv_buffer) assembles the full prefix from all ranks via the
|
||||
layer-ahead NCCL allgather and must return the canonical bytes.
|
||||
2. MHA pool sharded across the attention-CP group: the post-allgather full
|
||||
chunk is staged into the scratch chunk region and owner-persisted; a later
|
||||
batch reads prefix+chunk through the translated page table (the scratch),
|
||||
and the assembled rows must match the canonical bytes.
|
||||
3. MTP draft pools sharing the target's sharded allocator, for MLA and MHA:
|
||||
exact KV bytes, separate target/draft storage and scratch plans, fragmented
|
||||
pages, partial tail pages, both MLA write APIs, and local/nonzero draft
|
||||
layer IDs across batches.
|
||||
|
||||
Registered as a base-b 2-gpu-large unit test; skips when fewer than 2 CUDA
|
||||
devices are visible. Run directly with:
|
||||
CUDA_VISIBLE_DEVICES=0,1 python3 \
|
||||
test/registered/unit/mem_cache/test_page_interleave_gather_multi_gpu.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.srt.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
from sglang.srt.mem_cache.page_interleave import PageShardSpec, get_kv_shard_group
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
PageInterleaveMHATokenToKVPool,
|
||||
PageInterleaveMLATokenToKVPool,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, get_parallel, publish
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase, publish_build_topology
|
||||
|
||||
register_cuda_ci(est_time=120, stage="base-b", runner_config="2-gpu-large")
|
||||
|
||||
_GATHER_WORLD = 2
|
||||
_GATHER_LAYER_NUM = 4
|
||||
_GATHER_PAGE_SIZE = 16
|
||||
_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE
|
||||
_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank
|
||||
_GATHER_KV_LORA_RANK = 128
|
||||
_GATHER_QK_ROPE = 32
|
||||
_GATHER_HEAD_NUM = 2
|
||||
_GATHER_HEAD_DIM = 32
|
||||
_GATHER_DTYPE = torch.bfloat16
|
||||
_MTP_SIZE = 32 * _GATHER_PAGE_SIZE # Physical token capacity per rank.
|
||||
_MTP_UNWRITTEN = -256
|
||||
|
||||
|
||||
def _mla_value(loc, dim):
|
||||
"""Deterministic canonical latent value for logical slot ``loc``."""
|
||||
loc = loc.to(torch.float32)
|
||||
return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to(
|
||||
_GATHER_DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _dist_init(rank, world, port, attn_cp_size):
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(port)
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world)
|
||||
os.environ.setdefault("no_proxy", "127.0.0.1,localhost")
|
||||
torch.cuda.set_device(rank)
|
||||
|
||||
init_distributed_environment(
|
||||
world_size=world,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{port}",
|
||||
backend="nccl",
|
||||
)
|
||||
# Publish the widths the groups below are about to be built at. The derived
|
||||
# quotients (attn_tp_size, attn_dcp_size, ...) are projected from these
|
||||
# leaves at publish; initialize_model_parallel no longer supplies them, and
|
||||
# MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path.
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size),
|
||||
role="scheduler",
|
||||
)
|
||||
publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank)
|
||||
initialize_model_parallel()
|
||||
|
||||
|
||||
def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4):
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
shard_size=_GATHER_WORLD,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE,
|
||||
chunk_tokens=chunk_groups * _GATHER_GRANULE,
|
||||
)
|
||||
|
||||
|
||||
def _fake_req_to_token(groups, seq_len, device):
|
||||
"""req_to_token row where sequence group j is allocator group groups[j]."""
|
||||
row = torch.zeros(
|
||||
(1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device
|
||||
)
|
||||
for j, q in enumerate(groups):
|
||||
row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange(
|
||||
q * _GATHER_GRANULE,
|
||||
(q + 1) * _GATHER_GRANULE,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
return row[:, :seq_len] if seq_len < row.shape[1] else row
|
||||
|
||||
|
||||
def _check(rank, name, got, expect, atol=0.0):
|
||||
ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0)
|
||||
max_err = (got.float() - expect.float()).abs().max().item()
|
||||
print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}")
|
||||
assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})"
|
||||
|
||||
|
||||
def _run_mla(rank, world, port):
|
||||
_dist_init(rank, world, port, attn_cp_size=1)
|
||||
|
||||
group = get_parallel().attn_tp_group
|
||||
assert group.world_size == world
|
||||
# Topology-first shard-group selection: no CP here, so MLA falls back to
|
||||
# the attn-TP axis, while GQA has no replicated axis (world_size 1).
|
||||
assert get_kv_shard_group(use_mla_backend=True) is group
|
||||
assert get_kv_shard_group(use_mla_backend=False).world_size == 1
|
||||
spec = _gather_make_spec(shard_rank=group.rank_in_group)
|
||||
|
||||
pool = PageInterleaveMLATokenToKVPool(
|
||||
_GATHER_SIZE,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
dtype=_GATHER_DTYPE,
|
||||
kv_lora_rank=_GATHER_KV_LORA_RANK,
|
||||
qk_rope_head_dim=_GATHER_QK_ROPE,
|
||||
layer_num=_GATHER_LAYER_NUM,
|
||||
device=f"cuda:{rank}",
|
||||
enable_memory_saver=False,
|
||||
start_layer=0,
|
||||
end_layer=_GATHER_LAYER_NUM - 1,
|
||||
shard_spec=spec,
|
||||
shard_group=group,
|
||||
)
|
||||
device = pool.kv_buffer[0].device
|
||||
|
||||
# ---- chunk 1: replicated write, owner-filtered persist -----------------
|
||||
# "Allocator" hands out fragmented groups (identical on every rank).
|
||||
chunk1_groups = [5, 2, 9]
|
||||
chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[
|
||||
0
|
||||
].long()
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
vals = _mla_value(
|
||||
chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
chunk1_locs,
|
||||
vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
|
||||
vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier()
|
||||
|
||||
# Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps)
|
||||
# on every rank, holding that rank's page of the group.
|
||||
for q in chunk1_groups:
|
||||
local_rows = torch.arange(
|
||||
q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device
|
||||
)
|
||||
owned_locs = (
|
||||
q * _GATHER_GRANULE
|
||||
+ group.rank_in_group * _GATHER_PAGE_SIZE
|
||||
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
|
||||
)
|
||||
got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE)
|
||||
expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE)
|
||||
_check(rank, f"mla owned stripe g{q}", got, expect)
|
||||
|
||||
# ---- chunk 2: prefix gather + staged chunk, both read styles -----------
|
||||
seq_groups = chunk1_groups + [12] # one new chunk group
|
||||
prefix_len = 3 * _GATHER_GRANULE
|
||||
seq_len = prefix_len + _GATHER_GRANULE
|
||||
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
|
||||
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
|
||||
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
# Write the current chunk (stages it into the slot + persists the
|
||||
# owned stripe), like the extend forward does before attention.
|
||||
chunk_vals = _mla_value(
|
||||
chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
chunk2_locs,
|
||||
chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
|
||||
chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
|
||||
)
|
||||
# Chunked-prefix MHA style: fetch an arbitrary sub-range of the
|
||||
# prefix through get_mla_kv_buffer.
|
||||
sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3]
|
||||
k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE)
|
||||
expect = _mla_value(
|
||||
sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mla prefix read l{layer_id}",
|
||||
k_nope[:, 0, :],
|
||||
expect[:, :_GATHER_KV_LORA_RANK],
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mla prefix rope l{layer_id}",
|
||||
k_rope[:, 0, :],
|
||||
expect[:, _GATHER_KV_LORA_RANK:],
|
||||
)
|
||||
# Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk]
|
||||
# from get_key_buffer through the translated page table.
|
||||
all_locs = req_to_token[0, :seq_len].long()
|
||||
rows = pool.translate_loc_to_scratch(all_locs)
|
||||
kv_scratch = pool.get_key_buffer(layer_id)
|
||||
_check(
|
||||
rank,
|
||||
f"mla absorbed read l{layer_id}",
|
||||
kv_scratch[rows, 0, :],
|
||||
_mla_value(
|
||||
all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
),
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
if rank == 0:
|
||||
print("PASS: MLA page-interleave shard (attn-TP axis)")
|
||||
|
||||
|
||||
def _run_mha(rank, world, port):
|
||||
_dist_init(rank, world, port, attn_cp_size=world)
|
||||
|
||||
group = get_parallel().attn_cp_group
|
||||
assert group.world_size == world
|
||||
# Topology-first shard-group selection: with an active CP group, both
|
||||
# GQA and MLA shard across CP (CP replicates KV for every attention
|
||||
# type; the TP axis is only the no-CP MLA fallback).
|
||||
assert get_kv_shard_group(use_mla_backend=False) is group
|
||||
assert get_kv_shard_group(use_mla_backend=True) is group
|
||||
spec = _gather_make_spec(shard_rank=group.rank_in_group)
|
||||
|
||||
pool = PageInterleaveMHATokenToKVPool(
|
||||
_GATHER_SIZE,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
dtype=_GATHER_DTYPE,
|
||||
head_num=_GATHER_HEAD_NUM,
|
||||
head_dim=_GATHER_HEAD_DIM,
|
||||
layer_num=_GATHER_LAYER_NUM,
|
||||
device=f"cuda:{rank}",
|
||||
enable_memory_saver=False,
|
||||
start_layer=0,
|
||||
end_layer=_GATHER_LAYER_NUM - 1,
|
||||
enable_alt_stream=False,
|
||||
shard_spec=spec,
|
||||
shard_group=group,
|
||||
)
|
||||
device = pool.k_buffer[0].device
|
||||
|
||||
def kv_value(locs, layer_id, is_v):
|
||||
base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0)
|
||||
return (
|
||||
base.view(-1, 1, 1)
|
||||
+ torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01
|
||||
+ torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001
|
||||
).to(_GATHER_DTYPE)
|
||||
|
||||
# ---- chunk 1 (prefix-less batch): stage + owner-persist ----------------
|
||||
chunk1_groups = [7, 3]
|
||||
chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[
|
||||
0
|
||||
].long()
|
||||
req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE])
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
pool.set_kv_buffer(
|
||||
layer,
|
||||
chunk1_locs,
|
||||
kv_value(chunk1_locs, layer_id, False),
|
||||
kv_value(chunk1_locs, layer_id, True),
|
||||
)
|
||||
# The current chunk must be readable through the scratch right away.
|
||||
k_scratch = pool.get_key_buffer(layer_id)
|
||||
rows = pool.translate_loc_to_scratch(chunk1_locs)
|
||||
_check(
|
||||
rank,
|
||||
f"mha chunk stage l{layer_id}",
|
||||
k_scratch[rows],
|
||||
kv_value(chunk1_locs, layer_id, False),
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier()
|
||||
|
||||
# ---- chunk 2: prefix gathered from peers via translated page table -----
|
||||
seq_groups = chunk1_groups + [11]
|
||||
prefix_len = 2 * _GATHER_GRANULE
|
||||
seq_len = prefix_len + _GATHER_GRANULE
|
||||
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
|
||||
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
|
||||
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
pool.set_kv_buffer(
|
||||
layer,
|
||||
chunk2_locs,
|
||||
kv_value(chunk2_locs, layer_id, False),
|
||||
kv_value(chunk2_locs, layer_id, True),
|
||||
)
|
||||
all_locs = req_to_token[0, :seq_len].long()
|
||||
rows = pool.translate_loc_to_scratch(all_locs)
|
||||
k_scratch = pool.get_key_buffer(layer_id)
|
||||
v_scratch = pool.get_value_buffer(layer_id)
|
||||
_check(
|
||||
rank,
|
||||
f"mha seq read k l{layer_id}",
|
||||
k_scratch[rows],
|
||||
kv_value(all_locs, layer_id, False),
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mha seq read v l{layer_id}",
|
||||
v_scratch[rows],
|
||||
kv_value(all_locs, layer_id, True),
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
if rank == 0:
|
||||
print("PASS: MHA page-interleave shard (attn-CP axis)")
|
||||
|
||||
|
||||
def _mtp_make_pool(rank, group, kind, start_layer, layer_num):
|
||||
kwargs = dict(
|
||||
size=_MTP_SIZE,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
dtype=_GATHER_DTYPE,
|
||||
layer_num=layer_num,
|
||||
device=f"cuda:{rank}",
|
||||
enable_memory_saver=False,
|
||||
start_layer=start_layer,
|
||||
end_layer=start_layer + layer_num - 1,
|
||||
shard_spec=PageShardSpec(
|
||||
shard_rank=group.rank_in_group,
|
||||
shard_size=_GATHER_WORLD,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
max_prefix_tokens=16 * _GATHER_WORLD * _GATHER_PAGE_SIZE,
|
||||
chunk_tokens=4 * _GATHER_PAGE_SIZE,
|
||||
),
|
||||
shard_group=group,
|
||||
)
|
||||
if kind == "mla":
|
||||
pool = PageInterleaveMLATokenToKVPool(
|
||||
**kwargs,
|
||||
kv_lora_rank=_GATHER_KV_LORA_RANK,
|
||||
qk_rope_head_dim=_GATHER_QK_ROPE,
|
||||
)
|
||||
else:
|
||||
pool = PageInterleaveMHATokenToKVPool(
|
||||
**kwargs,
|
||||
head_num=_GATHER_HEAD_NUM,
|
||||
head_dim=_GATHER_HEAD_DIM,
|
||||
enable_alt_stream=False,
|
||||
)
|
||||
for local_layer in range(layer_num):
|
||||
for buffer in _mtp_buffers(pool, kind, local_layer):
|
||||
buffer.fill_(_MTP_UNWRITTEN)
|
||||
return pool
|
||||
|
||||
|
||||
def _mtp_buffers(pool, kind, local_layer):
|
||||
if kind == "mla":
|
||||
return (pool.kv_buffer[local_layer],)
|
||||
return (pool.k_buffer[local_layer], pool.v_buffer[local_layer])
|
||||
|
||||
|
||||
def _mtp_make_draft_pool(target, allocator, kind, start_layer):
|
||||
# Skip model loading while using the production pool dispatch and sharing
|
||||
# the target allocator. A replicated draft pool must fail this test.
|
||||
kvc = KVCacheConfigurator.__new__(KVCacheConfigurator)
|
||||
kvc.device = target.device
|
||||
kvc.is_draft_worker = True
|
||||
kvc.page_size = _GATHER_PAGE_SIZE
|
||||
kvc.use_mla_backend = kind == "mla"
|
||||
kvc.kv_cache_dtype = _GATHER_DTYPE
|
||||
kvc.kv_cache_dtype_str = "bfloat16"
|
||||
kvc.post_capture_kv_active = False
|
||||
kvc.is_hybrid_swa = False
|
||||
kvc.sliding_window_size = None
|
||||
kvc.mambaish_config = None
|
||||
kvc.token_to_kv_pool_allocator = allocator
|
||||
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
|
||||
kvc.model_config = SimpleNamespace(
|
||||
num_nextn_predict_layers=1,
|
||||
hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]),
|
||||
kv_lora_rank=_GATHER_KV_LORA_RANK,
|
||||
qk_rope_head_dim=_GATHER_QK_ROPE,
|
||||
head_dim=_GATHER_HEAD_DIM,
|
||||
v_head_dim=_GATHER_HEAD_DIM,
|
||||
get_num_kv_heads=lambda *_args: _GATHER_HEAD_NUM,
|
||||
)
|
||||
kvc.layer_info = SimpleNamespace(
|
||||
start_layer=start_layer, end_layer=start_layer + 1, num_effective_layers=1
|
||||
)
|
||||
with get_context().override_server_args(
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
tp_size=_GATHER_WORLD,
|
||||
attn_cp_size=_GATHER_WORLD if kind == "mha" else 1,
|
||||
speculative_algorithm="EAGLE",
|
||||
enable_multi_layer_eagle=False,
|
||||
):
|
||||
pool = kvc._build_token_to_kv_pool(
|
||||
sizes=SimpleNamespace(max_total_num_tokens=target.size),
|
||||
is_dsa_model=False,
|
||||
is_dsv4_model=False,
|
||||
req_to_token_pool=None,
|
||||
)
|
||||
assert pool.shard_spec is target.shard_spec
|
||||
assert pool.shard_group is target.shard_group
|
||||
for buffer in _mtp_buffers(pool, kind, 0):
|
||||
buffer.fill_(_MTP_UNWRITTEN)
|
||||
return pool
|
||||
|
||||
|
||||
def _mtp_values(locs, width, tag, local_layer, is_v=False):
|
||||
"""Each location, feature, layer, and pool has distinguishable BF16 bytes.
|
||||
|
||||
Small integers survive BF16 exactly; high-valued logical slot IDs plus tiny
|
||||
feature offsets would round away the differences this test must detect.
|
||||
The first two columns encode the full logical slot, avoiding hash aliases.
|
||||
"""
|
||||
features = torch.arange(width, device=locs.device)
|
||||
values = (
|
||||
locs[:, None] * 37
|
||||
+ features[None, :] * 17
|
||||
+ tag * 73
|
||||
+ local_layer * 11
|
||||
+ int(is_v) * 97
|
||||
) % 251 - 125
|
||||
values[:, 0] = locs % 128
|
||||
values[:, 1] = locs // 128
|
||||
values[:, 2] = tag
|
||||
values[:, 3] = local_layer
|
||||
values[:, 4] = int(is_v)
|
||||
return values.to(_GATHER_DTYPE)
|
||||
|
||||
|
||||
def _mtp_canonical(kind, locs, tag, local_layer):
|
||||
if kind == "mla":
|
||||
return (
|
||||
_mtp_values(
|
||||
locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE, tag, local_layer
|
||||
).unsqueeze(1),
|
||||
)
|
||||
return tuple(
|
||||
_mtp_values(
|
||||
locs, _GATHER_HEAD_NUM * _GATHER_HEAD_DIM, tag, local_layer, is_v
|
||||
).reshape(-1, _GATHER_HEAD_NUM, _GATHER_HEAD_DIM)
|
||||
for is_v in (False, True)
|
||||
)
|
||||
|
||||
|
||||
def _mtp_assert_bytes(got, expected, label):
|
||||
assert got.shape == expected.shape, label
|
||||
assert got.dtype == expected.dtype == _GATHER_DTYPE, label
|
||||
assert torch.equal(
|
||||
got.contiguous().view(torch.uint8), expected.contiguous().view(torch.uint8)
|
||||
), f"{label}: KV bytes differ"
|
||||
|
||||
|
||||
def _mtp_logical_chain(device):
|
||||
# Owners alternate from rank 1, while local pages run out of order. Some
|
||||
# logical slots exceed the physical pool size, catching a replicated draft
|
||||
# indexing them raw.
|
||||
local_pages = torch.tensor([25, 3, 20, 6, 28, 4, 30, 8, 22], device=device)
|
||||
owners = (1 + torch.arange(local_pages.numel(), device=device)) % _GATHER_WORLD
|
||||
pages = local_pages * _GATHER_WORLD + owners
|
||||
return (
|
||||
pages[:, None] * _GATHER_PAGE_SIZE
|
||||
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
|
||||
).reshape(-1)
|
||||
|
||||
|
||||
def _mtp_write(pool, kind, locs, tag, local_layer, epoch):
|
||||
layer = SimpleNamespace(layer_id=pool.start_layer + local_layer)
|
||||
values = _mtp_canonical(kind, locs, tag, local_layer)
|
||||
if kind == "mha":
|
||||
pool.set_kv_buffer(layer, locs, *values)
|
||||
elif epoch % 2:
|
||||
# Both MLA write entry points must owner-filter and stage MTP rows.
|
||||
pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
locs,
|
||||
values[0][..., :_GATHER_KV_LORA_RANK],
|
||||
values[0][..., _GATHER_KV_LORA_RANK:],
|
||||
)
|
||||
else:
|
||||
pool.set_kv_buffer(
|
||||
layer, locs, values[0], values[0][..., :_GATHER_KV_LORA_RANK]
|
||||
)
|
||||
|
||||
|
||||
def _mtp_check_persisted(pool, kind, locs, tag, rank):
|
||||
owned_locs = locs[(locs // _GATHER_PAGE_SIZE) % _GATHER_WORLD == rank]
|
||||
local_rows = (
|
||||
owned_locs // (_GATHER_WORLD * _GATHER_PAGE_SIZE) * _GATHER_PAGE_SIZE
|
||||
+ owned_locs % _GATHER_PAGE_SIZE
|
||||
)
|
||||
for local_layer in range(pool.layer_num):
|
||||
canonical = _mtp_canonical(kind, owned_locs, tag, local_layer)
|
||||
for buffer, values in zip(_mtp_buffers(pool, kind, local_layer), canonical):
|
||||
expected = torch.full_like(buffer, _MTP_UNWRITTEN)
|
||||
expected[local_rows] = values
|
||||
_mtp_assert_bytes(
|
||||
buffer, expected, f"{kind} pool {tag} owned layer {local_layer}"
|
||||
)
|
||||
|
||||
|
||||
def _mtp_check_scratch(pool, kind, locs, prefix_len, tag, local_layer):
|
||||
layer_id = pool.start_layer + local_layer
|
||||
rows = pool.translate_loc_to_scratch(locs)
|
||||
expected = _mtp_canonical(kind, locs, tag, local_layer)
|
||||
_mtp_assert_bytes(
|
||||
pool.get_key_buffer(layer_id)[rows], expected[0], f"{kind} pool {tag} key"
|
||||
)
|
||||
_mtp_assert_bytes(
|
||||
pool.get_value_buffer(layer_id)[rows],
|
||||
expected[0][..., :_GATHER_KV_LORA_RANK] if kind == "mla" else expected[1],
|
||||
f"{kind} pool {tag} value",
|
||||
)
|
||||
if kind == "mla" and prefix_len:
|
||||
# Arbitrary prefix indices also exercise the chunked MLA consumer.
|
||||
subset = locs[3:prefix_len:7].clone()
|
||||
k_nope, k_rope = pool.get_mla_kv_buffer(
|
||||
SimpleNamespace(layer_id=layer_id), subset, _GATHER_DTYPE
|
||||
)
|
||||
reference = _mtp_canonical(kind, subset, tag, local_layer)[0]
|
||||
_mtp_assert_bytes(
|
||||
k_nope, reference[..., :_GATHER_KV_LORA_RANK], f"pool {tag} latent"
|
||||
)
|
||||
_mtp_assert_bytes(
|
||||
k_rope, reference[..., _GATHER_KV_LORA_RANK:], f"pool {tag} rope"
|
||||
)
|
||||
|
||||
|
||||
def _run_mtp(rank, port, kind):
|
||||
"""Check exact KV bytes and independent target/draft state across batches."""
|
||||
_dist_init(
|
||||
rank, _GATHER_WORLD, port, attn_cp_size=_GATHER_WORLD if kind == "mha" else 1
|
||||
)
|
||||
group = get_kv_shard_group(use_mla_backend=kind == "mla")
|
||||
assert group.world_size == _GATHER_WORLD
|
||||
target = _mtp_make_pool(
|
||||
rank, group, kind, start_layer=0, layer_num=_GATHER_LAYER_NUM
|
||||
)
|
||||
allocator = PageInterleavePoolAllocator(
|
||||
size=target.size,
|
||||
physical_page_size=_GATHER_PAGE_SIZE,
|
||||
shard_size=_GATHER_WORLD,
|
||||
dtype=_GATHER_DTYPE,
|
||||
device=target.device,
|
||||
kvcache=target,
|
||||
need_sort=False,
|
||||
shard_spec=target.shard_spec,
|
||||
)
|
||||
# DeepSeek's NextN uses local layer 0. Also cover an absolute/offset layer ID
|
||||
# so the one-layer gather terminates correctly on either scratch-slot parity.
|
||||
drafts = [
|
||||
_mtp_make_draft_pool(target, allocator, kind, start_layer=start)
|
||||
for start in (0, 61)
|
||||
]
|
||||
pools = [target, *drafts]
|
||||
for other in drafts:
|
||||
assert other._page_pos.data_ptr() != target._page_pos.data_ptr()
|
||||
assert other.kv_gather_comm is not target.kv_gather_comm
|
||||
for slot, target_slot in zip(other._slots, target._slots):
|
||||
for name in slot.tensors:
|
||||
assert (
|
||||
slot.tensors[name].data_ptr()
|
||||
!= target_slot.tensors[name].data_ptr()
|
||||
)
|
||||
|
||||
locs = _mtp_logical_chain(target.device)
|
||||
req_to_token = locs.to(torch.int32).unsqueeze(0)
|
||||
req_indices = torch.tensor([0], dtype=torch.int64, device=locs.device)
|
||||
# Each cached prefix is page aligned. The third batch writes only five
|
||||
# tokens of its tail page; the fourth recomputes and completes that page.
|
||||
batches = [(0, 48), (48, 96), (96, 133), (128, 144)]
|
||||
for epoch, (prefix_len, seq_len) in enumerate(batches, start=1):
|
||||
active_locs = locs[:seq_len]
|
||||
chunk_locs = locs[prefix_len:seq_len]
|
||||
target_epoch = target._epoch
|
||||
for tag, pool in enumerate(pools, start=1):
|
||||
pool.begin_shard_extend(req_to_token, req_indices, [prefix_len], [seq_len])
|
||||
assert pool._epoch == epoch
|
||||
for local_layer in range(pool.layer_num):
|
||||
_mtp_write(pool, kind, chunk_locs, tag, local_layer, epoch)
|
||||
_mtp_check_scratch(
|
||||
pool, kind, active_locs, prefix_len, tag, local_layer
|
||||
)
|
||||
_mtp_check_persisted(pool, kind, active_locs, tag, rank)
|
||||
# MTP metadata/writes/reads must leave the target's last resident
|
||||
# layer and gather plan usable, despite identical logical locations.
|
||||
assert target._epoch == target_epoch + 1
|
||||
_mtp_check_scratch(
|
||||
target, kind, active_locs, prefix_len, 1, _GATHER_LAYER_NUM - 1
|
||||
)
|
||||
if pool.layer_num == 1 and prefix_len:
|
||||
resident = pool._slots[pool.start_layer % 2]
|
||||
assert resident.resident_key == (pool.start_layer, epoch)
|
||||
assert pool._slots[(pool.start_layer + 1) % 2].resident_key is None
|
||||
if rank == 0:
|
||||
print(f"PASS: {kind.upper()} target/MTP prefix={prefix_len}, seq={seq_len}")
|
||||
|
||||
# Deactivating one draft's plan must not deactivate the other pools.
|
||||
drafts[0].end_shard_extend()
|
||||
assert not drafts[0]._shard_extend_active
|
||||
assert target._shard_extend_active and drafts[1]._shard_extend_active
|
||||
_mtp_check_scratch(drafts[1], kind, locs, 128, 3, 0)
|
||||
_mtp_check_scratch(target, kind, locs, 128, 1, _GATHER_LAYER_NUM - 1)
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier()
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices"
|
||||
)
|
||||
class TestPageInterleaveGatherMultiGpu(CustomTestCase):
|
||||
"""Real pools, real NCCL, 2 ranks — one mp.spawn per phase.
|
||||
|
||||
Separate spawns (and separate ports) because each phase builds its own
|
||||
process group with a different attention-CP width.
|
||||
"""
|
||||
|
||||
def test_mla_shard_over_attention_tp(self):
|
||||
mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
def test_mha_shard_over_attention_cp(self):
|
||||
mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
def test_mtp_mla_shard_over_attention_tp(self):
|
||||
mp.spawn(_run_mtp, args=(29813, "mla"), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
def test_mtp_mha_shard_over_attention_cp(self):
|
||||
mp.spawn(_run_mtp, args=(29814, "mha"), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -13,8 +13,7 @@
|
||||
# ==============================================================================
|
||||
"""Tests for logical-page KV cache sharding.
|
||||
|
||||
The CPU tests pin the pure arithmetic that rotated owner-classed
|
||||
allocation hangs on:
|
||||
Pins the pure arithmetic that rotated owner-classed allocation hangs on:
|
||||
|
||||
1. The placement bijection ``loc = Q*(N*ps) + r*ps + o`` — owner / local-row
|
||||
round-trip, disjoint equal partition across ranks.
|
||||
@@ -31,37 +30,31 @@ allocation hangs on:
|
||||
5. ``begin_shard_extend`` plan capture (page positions, padded send rows,
|
||||
owner-congruence guard) with the gather stubbed out, following the
|
||||
SimpleNamespace binding pattern of ``test_dsa_layer_shard_utils.py``.
|
||||
6. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's
|
||||
6. MTP draft pool construction, shared allocator/layout validation, and scratch
|
||||
sizing for separate target and draft pools.
|
||||
7. The P/D send filter ``filter_kv_indices_for_shard_rank`` and the sender's
|
||||
skip of a non-final chunk this rank owns no page of.
|
||||
7. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's
|
||||
8. Real sharded MLA pools: N ranks of stock-size pools hold N times one pool's
|
||||
tokens, each token stored on exactly one rank.
|
||||
|
||||
The GPU tests exercise real MLA and MHA pools and NCCL gathers on 2 ranks
|
||||
(``TestPageInterleaveGatherMultiGpu``). DSA indexer and attention coverage lives
|
||||
in ``test_page_sharded_dsa_attention.py``.
|
||||
GPU tests skip when too few CUDA devices are visible, so the same file runs
|
||||
in both CPU and CUDA CI suites.
|
||||
The real NCCL MLA/MHA/MTP gathers over 2 GPUs live in
|
||||
``test_page_interleave_gather_multi_gpu.py``. DSA indexer and attention
|
||||
coverage lives in ``test_page_sharded_dsa_attention.py``.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
import unittest.mock
|
||||
from array import array
|
||||
from contextlib import contextmanager
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.multiprocessing as mp
|
||||
from parameterized import parameterized_class
|
||||
|
||||
from sglang.srt.disaggregation.common.conn import CommonKVSender
|
||||
from sglang.srt.disaggregation.utils import filter_kv_indices_for_shard_rank
|
||||
from sglang.srt.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import ReqKvInfo
|
||||
from sglang.srt.mem_cache import page_interleave
|
||||
@@ -77,6 +70,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
)
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.common import _evict_until_allocatable
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
MHATokenToKVPool,
|
||||
MLATokenToKVPool,
|
||||
@@ -86,7 +80,8 @@ from sglang.srt.mem_cache.page_interleave import (
|
||||
PageInterleavePlacement,
|
||||
PageShardSpec,
|
||||
compute_page_shard_scratch_bytes,
|
||||
get_kv_shard_group,
|
||||
get_kv_shard_group_info,
|
||||
get_shared_kv_shard_pool,
|
||||
make_page_shard_spec,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
@@ -97,19 +92,14 @@ from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.runtime_context import get_parallel, publish
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.runtime_context import get_context, get_memory, get_parallel, get_spec
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils import ceil_div
|
||||
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.mem_cache_utils import finish_req
|
||||
from sglang.test.test_utils import (
|
||||
CustomTestCase,
|
||||
enter_scope,
|
||||
publish_build_topology,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase, enter_scope
|
||||
|
||||
register_cpu_ci(est_time=30, suite="base-a-test-cpu")
|
||||
register_cuda_ci(est_time=120, stage="base-c", runner_config="4-gpu-b200")
|
||||
|
||||
N = 4 # shard size
|
||||
PS = 16 # physical page size
|
||||
@@ -150,6 +140,7 @@ class TestPageShardScratchSizing(CustomTestCase):
|
||||
)
|
||||
kvc = SimpleNamespace(
|
||||
is_draft_worker=draft,
|
||||
token_to_kv_pool_allocator=None,
|
||||
use_mla_backend=use_mla,
|
||||
page_size=16,
|
||||
kv_cache_dtype=torch.bfloat16,
|
||||
@@ -182,6 +173,28 @@ class TestPageShardScratchSizing(CustomTestCase):
|
||||
self.assertEqual(spec.chunk_tokens, 48)
|
||||
self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0)
|
||||
|
||||
def test_target_with_mtp_budgets_one_extra_scratch_pair(self):
|
||||
with (
|
||||
self._fixture(context_len=100, chunk_tokens=64, shard_size=2) as kvc,
|
||||
get_context().override_server_args(enable_multi_layer_eagle=False),
|
||||
):
|
||||
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
|
||||
# Target configs can omit next-N layers; use the resolved draft count.
|
||||
kvc.model_config.num_nextn_predict_layers = None
|
||||
kvc.spec_aux_config = SimpleNamespace(eagle_draft_num_layers=1)
|
||||
expected = 2 * (8 * 128 + 64 + 16) * (16 + 8) * 2
|
||||
for layers in (1, 3):
|
||||
with self.subTest(layers=layers):
|
||||
kvc.spec_aux_config.eagle_draft_num_layers = layers
|
||||
self.assertEqual(compute_page_shard_scratch_bytes(kvc), expected)
|
||||
self.assertEqual(
|
||||
compute_page_shard_scratch_bytes(kvc, include_mtp=True),
|
||||
2 * expected,
|
||||
)
|
||||
kvc.spec_aux_config.eagle_draft_num_layers = None
|
||||
with self.assertRaisesRegex(ValueError, "EAGLE MTP"):
|
||||
compute_page_shard_scratch_bytes(kvc, include_mtp=True)
|
||||
|
||||
def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self):
|
||||
for use_mla in (True, False):
|
||||
with (
|
||||
@@ -211,6 +224,255 @@ class TestPageShardScratchSizing(CustomTestCase):
|
||||
self.assertEqual(estimated, actual)
|
||||
|
||||
|
||||
def _make_mtp_configurator(use_mla=True):
|
||||
# Keep real pool/allocator types for validation; allocate only CPU metadata.
|
||||
pool_class = (
|
||||
PageInterleaveMLATokenToKVPool if use_mla else PageInterleaveMHATokenToKVPool
|
||||
)
|
||||
target = pool_class.__new__(pool_class)
|
||||
target.shard_spec = PageShardSpec(1, 2, 16, 128, 64)
|
||||
target.shard_group = SimpleNamespace(rank_in_group=1, world_size=2)
|
||||
target.size, target.page_size, target.dtype = 128, 16, torch.bfloat16
|
||||
target.kv_lora_rank, target.qk_rope_head_dim = 8, 4
|
||||
target.head_num, target.head_dim, target.v_head_dim = 2, 4, 6
|
||||
kvc = KVCacheConfigurator.__new__(KVCacheConfigurator)
|
||||
kvc.token_to_kv_pool_allocator = PageInterleavePoolAllocator(
|
||||
size=target.size,
|
||||
physical_page_size=target.page_size,
|
||||
shard_size=target.shard_spec.shard_size,
|
||||
dtype=target.dtype,
|
||||
device="cpu",
|
||||
kvcache=target,
|
||||
need_sort=False,
|
||||
shard_spec=target.shard_spec,
|
||||
)
|
||||
kvc.device = "cpu"
|
||||
kvc.is_draft_worker = True
|
||||
kvc.page_size = target.page_size
|
||||
kvc.use_mla_backend = use_mla
|
||||
kvc.kv_cache_dtype = target.dtype
|
||||
kvc.kv_cache_dtype_str = "bfloat16"
|
||||
kvc.spec_algorithm = SpeculativeAlgorithm.EAGLE
|
||||
kvc.model_config = SimpleNamespace(
|
||||
num_nextn_predict_layers=1,
|
||||
context_len=9999,
|
||||
kv_lora_rank=8,
|
||||
qk_rope_head_dim=4,
|
||||
head_dim=4,
|
||||
v_head_dim=6,
|
||||
get_num_kv_heads=lambda tp: 4 // tp,
|
||||
hf_config=SimpleNamespace(architectures=["DeepseekV3ForCausalLM"]),
|
||||
)
|
||||
kvc.layer_info = SimpleNamespace(start_layer=5, end_layer=6, num_effective_layers=1)
|
||||
kvc.is_hybrid_swa = False
|
||||
kvc.sliding_window_size = None
|
||||
kvc.mambaish_config = None
|
||||
kvc.post_capture_kv_active = False
|
||||
return kvc, target
|
||||
|
||||
|
||||
def _build_mtp_pool(kvc, *, capacity=128, is_dsa_model=False, is_dsv4_model=False):
|
||||
return kvc._build_token_to_kv_pool(
|
||||
sizes=SimpleNamespace(max_total_num_tokens=capacity),
|
||||
is_dsa_model=is_dsa_model,
|
||||
is_dsv4_model=is_dsv4_model,
|
||||
req_to_token_pool=None,
|
||||
)
|
||||
|
||||
|
||||
class TestMTPKVShardConfig(CustomTestCase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
stack = ExitStack()
|
||||
self.addCleanup(stack.close)
|
||||
stack.enter_context(
|
||||
get_context().override_server_args(
|
||||
page_size=16, chunked_prefill_size=1024, enable_multi_layer_eagle=False
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
get_parallel().override(
|
||||
tp_size=2, attn_tp_size=2, moe_tp_size=2, attn_dcp_size=1
|
||||
)
|
||||
)
|
||||
|
||||
def test_draft_shares_target_layout_but_has_its_own_pool(self):
|
||||
for use_mla in (True, False):
|
||||
with self.subTest(use_mla=use_mla):
|
||||
kvc, target = _make_mtp_configurator(use_mla)
|
||||
pool_class = type(target)
|
||||
# Mock allocation only; preserve real types and distinct instances.
|
||||
with (
|
||||
unittest.mock.patch.object(
|
||||
pool_class, "__init__", return_value=None
|
||||
) as init,
|
||||
unittest.mock.patch.object(
|
||||
page_interleave,
|
||||
"get_kv_shard_group",
|
||||
side_effect=AssertionError,
|
||||
),
|
||||
):
|
||||
self.assertIs(get_shared_kv_shard_pool(kvc), target)
|
||||
self.assertEqual(get_kv_shard_group_info(kvc), (1, 2))
|
||||
self.assertIs(make_page_shard_spec(kvc), target.shard_spec)
|
||||
pool = _build_mtp_pool(kvc)
|
||||
self.assertIsInstance(pool, pool_class)
|
||||
self.assertIsNot(pool, target)
|
||||
kwargs = init.call_args.kwargs
|
||||
self.assertIs(kwargs["shard_spec"], target.shard_spec)
|
||||
self.assertIs(kwargs["shard_group"], target.shard_group)
|
||||
for name, expected in dict(
|
||||
size=128,
|
||||
page_size=16,
|
||||
dtype=torch.bfloat16,
|
||||
layer_num=1,
|
||||
start_layer=5,
|
||||
end_layer=6,
|
||||
).items():
|
||||
self.assertEqual(kwargs[name], expected, name)
|
||||
allocator = kvc.token_to_kv_pool_allocator
|
||||
self.assertIs(
|
||||
kvc._build_token_to_kv_pool_allocator(
|
||||
sizes=SimpleNamespace(max_total_num_tokens=128),
|
||||
token_to_kv_pool=pool,
|
||||
is_dsv4_model=False,
|
||||
req_to_token_pool=None,
|
||||
token_to_kv_pool_allocator=allocator,
|
||||
),
|
||||
allocator,
|
||||
)
|
||||
# Scratch uses inherited bounds despite different draft context/chunk sizes.
|
||||
row_bytes = (8 + 4) * 2 if use_mla else 2 * (4 + 6) * 2
|
||||
self.assertEqual(
|
||||
compute_page_shard_scratch_bytes(kvc),
|
||||
2 * (128 + 64 + 16) * row_bytes,
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "target"):
|
||||
compute_page_shard_scratch_bytes(kvc, include_mtp=True)
|
||||
|
||||
def test_without_shared_allocator_keeps_existing_pool(self):
|
||||
for draft, allocator in ((True, None), (True, object()), (False, None)):
|
||||
with self.subTest(draft=draft, allocator=allocator):
|
||||
kvc, _ = _make_mtp_configurator()
|
||||
kvc.is_draft_worker = draft
|
||||
kvc.token_to_kv_pool_allocator = allocator
|
||||
self.assertIsNone(get_shared_kv_shard_pool(kvc))
|
||||
if draft:
|
||||
self.assertEqual(get_kv_shard_group_info(kvc), (None, 1))
|
||||
self.assertIsNone(make_page_shard_spec(kvc))
|
||||
self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0)
|
||||
with unittest.mock.patch.object(
|
||||
KVCacheConfigurator, "_build_mla_kv_pool"
|
||||
) as build:
|
||||
self.assertIs(_build_mtp_pool(kvc), build.return_value)
|
||||
build.assert_called_once_with(
|
||||
max_total_num_tokens=128, mla_pool_class=MLATokenToKVPool
|
||||
)
|
||||
|
||||
def test_rejects_invalid_shared_allocator(self):
|
||||
kvc, _ = _make_mtp_configurator()
|
||||
for name, value, message in (
|
||||
("_kvcache", object(), "target's sharded KV pool"),
|
||||
("shard_spec", PageShardSpec(0, 2, 16, 128, 64), "different shard layouts"),
|
||||
("shard_size", 4, "different shard layouts"),
|
||||
("page_size", 32, "different shard layouts"),
|
||||
("size", 128, "different shard layouts"),
|
||||
):
|
||||
with (
|
||||
self.subTest(attribute=name),
|
||||
unittest.mock.patch.object(kvc.token_to_kv_pool_allocator, name, value),
|
||||
self.assertRaisesRegex(ValueError, message),
|
||||
):
|
||||
get_shared_kv_shard_pool(kvc)
|
||||
|
||||
def test_rejects_incompatible_pool_geometry(self):
|
||||
for use_mla in (True, False):
|
||||
kvc, _ = _make_mtp_configurator(use_mla)
|
||||
geometry = (
|
||||
("kv_lora_rank", "qk_rope_head_dim")
|
||||
if use_mla
|
||||
else ("head_dim", "v_head_dim")
|
||||
)
|
||||
cases = [
|
||||
(kvc, "page_size", 32, "physical page size"),
|
||||
(kvc, "kv_cache_dtype", torch.float16, "geometry and dtype"),
|
||||
(kvc, "use_mla_backend", not use_mla, "attention geometry"),
|
||||
*(
|
||||
(kvc.model_config, field, 16, "geometry and dtype")
|
||||
for field in geometry
|
||||
),
|
||||
]
|
||||
if not use_mla:
|
||||
cases.append(
|
||||
(
|
||||
kvc.model_config,
|
||||
"get_num_kv_heads",
|
||||
lambda tp: 4,
|
||||
"geometry and dtype",
|
||||
)
|
||||
)
|
||||
for obj, name, value, message in cases:
|
||||
with (
|
||||
self.subTest(use_mla=use_mla, attribute=name),
|
||||
unittest.mock.patch.object(obj, name, value),
|
||||
self.assertRaisesRegex(ValueError, message),
|
||||
):
|
||||
_build_mtp_pool(kvc)
|
||||
for capacity in (64, 256):
|
||||
with (
|
||||
self.subTest(use_mla=use_mla, capacity=capacity),
|
||||
self.assertRaisesRegex(ValueError, "same per-rank capacity"),
|
||||
):
|
||||
_build_mtp_pool(kvc, capacity=capacity)
|
||||
|
||||
def test_rejects_unsupported_draft_config(self):
|
||||
kvc, _ = _make_mtp_configurator()
|
||||
cases = [
|
||||
(kvc, "spec_algorithm", algorithm, "EAGLE MTP")
|
||||
for algorithm in (
|
||||
SpeculativeAlgorithm.EAGLE3,
|
||||
SpeculativeAlgorithm.DFLASH,
|
||||
SpeculativeAlgorithm.FROZEN_KV_MTP,
|
||||
)
|
||||
] + [
|
||||
(kvc.model_config, "num_nextn_predict_layers", None, "EAGLE MTP"),
|
||||
(kvc.model_config, "num_nextn_predict_layers", 0, "EAGLE MTP"),
|
||||
(kvc, "is_hybrid_swa", True, "dense MLA and MHA"),
|
||||
(kvc, "mambaish_config", object(), "dense MLA and MHA"),
|
||||
(kvc, "sliding_window_size", 128, "dense MLA and MHA"),
|
||||
(kvc, "post_capture_kv_active", True, "plain per-layer KV layout"),
|
||||
(kvc, "kv_cache_dtype_str", "mxfp8", "plain per-layer KV layout"),
|
||||
]
|
||||
for obj, name, value, message in cases:
|
||||
with (
|
||||
self.subTest(attribute=name, value=value),
|
||||
unittest.mock.patch.object(obj, name, value),
|
||||
self.assertRaisesRegex(ValueError, message),
|
||||
):
|
||||
_build_mtp_pool(kvc)
|
||||
for options in ({"is_dsa_model": True}, {"is_dsv4_model": True}):
|
||||
with (
|
||||
self.subTest(options=options),
|
||||
self.assertRaisesRegex(ValueError, "dense MLA and MHA"),
|
||||
):
|
||||
_build_mtp_pool(kvc, **options)
|
||||
for context, options, message in (
|
||||
(get_spec(), {"enable_multi_layer_eagle": True}, "EAGLE MTP"),
|
||||
(get_parallel(), {"attn_dcp_size": 2}, "incompatible with DCP"),
|
||||
(
|
||||
get_memory(),
|
||||
{"enable_page_major_kv_layout": True},
|
||||
"plain per-layer KV layout",
|
||||
),
|
||||
):
|
||||
with (
|
||||
self.subTest(options=options),
|
||||
context.override(**options),
|
||||
self.assertRaisesRegex(ValueError, message),
|
||||
):
|
||||
_build_mtp_pool(kvc)
|
||||
|
||||
|
||||
def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32):
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
@@ -1402,346 +1664,5 @@ class TestShardSendFilter(CustomTestCase):
|
||||
self.assertEqual(skips, expect_skip)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Multi-GPU: the real NCCL layer-ahead gather (2 GPUs).
|
||||
#
|
||||
# Everything above is pure arithmetic on a CPU stub. This section drives real
|
||||
# pools over a real process group, which is the only check that the plan the
|
||||
# stub validates actually addresses the bytes the collective delivers:
|
||||
#
|
||||
# 1. MLA pool sharded across the attention-TP group: replicated writes are
|
||||
# owner-filtered into disjoint pool stripes; a later batch's chunked-prefix
|
||||
# read (get_mla_kv_buffer) assembles the full prefix from all ranks via the
|
||||
# layer-ahead NCCL allgather and must return the canonical bytes.
|
||||
# 2. MHA pool sharded across the attention-CP group: the post-allgather full
|
||||
# chunk is staged into the scratch chunk region and owner-persisted; a later
|
||||
# batch reads prefix+chunk through the translated page table (the scratch),
|
||||
# and the assembled rows must match the canonical bytes.
|
||||
#
|
||||
# Skipped unless 2 CUDA devices are visible, so it is inert on the CPU runner
|
||||
# this file is registered to. Run it explicitly with:
|
||||
# CUDA_VISIBLE_DEVICES=0,1 python3 test/registered/unit/mem_cache/\
|
||||
# test_page_interleave_shard.py TestPageInterleaveGatherMultiGpu
|
||||
# =============================================================================
|
||||
|
||||
_GATHER_WORLD = 2
|
||||
_GATHER_LAYER_NUM = 4
|
||||
_GATHER_PAGE_SIZE = 16
|
||||
_GATHER_GRANULE = _GATHER_WORLD * _GATHER_PAGE_SIZE
|
||||
_GATHER_SIZE = _GATHER_PAGE_SIZE * 64 # physical token slots per rank
|
||||
_GATHER_KV_LORA_RANK = 128
|
||||
_GATHER_QK_ROPE = 32
|
||||
_GATHER_HEAD_NUM = 2
|
||||
_GATHER_HEAD_DIM = 32
|
||||
_GATHER_DTYPE = torch.bfloat16
|
||||
|
||||
|
||||
def _mla_value(loc, dim):
|
||||
"""Deterministic canonical latent value for logical slot ``loc``."""
|
||||
loc = loc.to(torch.float32)
|
||||
return (loc.unsqueeze(-1) + torch.arange(dim, device=loc.device) * 0.001).to(
|
||||
_GATHER_DTYPE
|
||||
)
|
||||
|
||||
|
||||
def _dist_init(rank, world, port, attn_cp_size):
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(port)
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world)
|
||||
os.environ.setdefault("no_proxy", "127.0.0.1,localhost")
|
||||
torch.cuda.set_device(rank)
|
||||
|
||||
init_distributed_environment(
|
||||
world_size=world,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=f"tcp://127.0.0.1:{port}",
|
||||
backend="nccl",
|
||||
)
|
||||
# Publish the widths the groups below are about to be built at. The derived
|
||||
# quotients (attn_tp_size, attn_dcp_size, ...) are projected from these
|
||||
# leaves at publish; initialize_model_parallel no longer supplies them, and
|
||||
# MLATokenToKVPool.set_mla_kv_buffer reads attn_dcp_size on the write path.
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", tp_size=world, attn_cp_size=attn_cp_size),
|
||||
role="scheduler",
|
||||
)
|
||||
publish_build_topology(tp_size=world, attn_cp_size=attn_cp_size, world_rank=rank)
|
||||
initialize_model_parallel()
|
||||
|
||||
|
||||
def _gather_make_spec(shard_rank, max_prefix_groups=16, chunk_groups=4):
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
shard_size=_GATHER_WORLD,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
max_prefix_tokens=max_prefix_groups * _GATHER_GRANULE,
|
||||
chunk_tokens=chunk_groups * _GATHER_GRANULE,
|
||||
)
|
||||
|
||||
|
||||
def _fake_req_to_token(groups, seq_len, device):
|
||||
"""req_to_token row where sequence group j is allocator group groups[j]."""
|
||||
row = torch.zeros(
|
||||
(1, len(groups) * _GATHER_GRANULE), dtype=torch.int32, device=device
|
||||
)
|
||||
for j, q in enumerate(groups):
|
||||
row[0, j * _GATHER_GRANULE : (j + 1) * _GATHER_GRANULE] = torch.arange(
|
||||
q * _GATHER_GRANULE,
|
||||
(q + 1) * _GATHER_GRANULE,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
return row[:, :seq_len] if seq_len < row.shape[1] else row
|
||||
|
||||
|
||||
def _check(rank, name, got, expect, atol=0.0):
|
||||
ok = torch.allclose(got.float(), expect.float(), atol=atol, rtol=0)
|
||||
max_err = (got.float() - expect.float()).abs().max().item()
|
||||
print(f"[rank {rank}] {name}: max_err={max_err:.6f} {'OK' if ok else 'FAIL'}")
|
||||
assert ok, f"[rank {rank}] {name} mismatch (max_err={max_err})"
|
||||
|
||||
|
||||
def _run_mla(rank, world, port):
|
||||
_dist_init(rank, world, port, attn_cp_size=1)
|
||||
|
||||
group = get_parallel().attn_tp_group
|
||||
assert group.world_size == world
|
||||
# Topology-first shard-group selection: no CP here, so MLA falls back to
|
||||
# the attn-TP axis, while GQA has no replicated axis (world_size 1).
|
||||
assert get_kv_shard_group(use_mla_backend=True) is group
|
||||
assert get_kv_shard_group(use_mla_backend=False).world_size == 1
|
||||
spec = _gather_make_spec(shard_rank=group.rank_in_group)
|
||||
|
||||
pool = PageInterleaveMLATokenToKVPool(
|
||||
_GATHER_SIZE,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
dtype=_GATHER_DTYPE,
|
||||
kv_lora_rank=_GATHER_KV_LORA_RANK,
|
||||
qk_rope_head_dim=_GATHER_QK_ROPE,
|
||||
layer_num=_GATHER_LAYER_NUM,
|
||||
device=f"cuda:{rank}",
|
||||
enable_memory_saver=False,
|
||||
start_layer=0,
|
||||
end_layer=_GATHER_LAYER_NUM - 1,
|
||||
shard_spec=spec,
|
||||
shard_group=group,
|
||||
)
|
||||
device = pool.kv_buffer[0].device
|
||||
|
||||
# ---- chunk 1: replicated write, owner-filtered persist -----------------
|
||||
# "Allocator" hands out fragmented groups (identical on every rank).
|
||||
chunk1_groups = [5, 2, 9]
|
||||
chunk1_locs = _fake_req_to_token(chunk1_groups, 3 * _GATHER_GRANULE, device)[
|
||||
0
|
||||
].long()
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
vals = _mla_value(
|
||||
chunk1_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
chunk1_locs,
|
||||
vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
|
||||
vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier()
|
||||
|
||||
# Pool holds only the owned stripe: group Q sits at local rows [Q*ps,(Q+1)*ps)
|
||||
# on every rank, holding that rank's page of the group.
|
||||
for q in chunk1_groups:
|
||||
local_rows = torch.arange(
|
||||
q * _GATHER_PAGE_SIZE, (q + 1) * _GATHER_PAGE_SIZE, device=device
|
||||
)
|
||||
owned_locs = (
|
||||
q * _GATHER_GRANULE
|
||||
+ group.rank_in_group * _GATHER_PAGE_SIZE
|
||||
+ torch.arange(_GATHER_PAGE_SIZE, device=device)
|
||||
)
|
||||
got = pool.kv_buffer[0][local_rows, 0, :].view(_GATHER_DTYPE)
|
||||
expect = _mla_value(owned_locs, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE)
|
||||
_check(rank, f"mla owned stripe g{q}", got, expect)
|
||||
|
||||
# ---- chunk 2: prefix gather + staged chunk, both read styles -----------
|
||||
seq_groups = chunk1_groups + [12] # one new chunk group
|
||||
prefix_len = 3 * _GATHER_GRANULE
|
||||
seq_len = prefix_len + _GATHER_GRANULE
|
||||
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
|
||||
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
|
||||
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
# Write the current chunk (stages it into the slot + persists the
|
||||
# owned stripe), like the extend forward does before attention.
|
||||
chunk_vals = _mla_value(
|
||||
chunk2_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
pool.set_mla_kv_buffer(
|
||||
layer,
|
||||
chunk2_locs,
|
||||
chunk_vals[:, :_GATHER_KV_LORA_RANK].unsqueeze(1),
|
||||
chunk_vals[:, _GATHER_KV_LORA_RANK:].unsqueeze(1),
|
||||
)
|
||||
# Chunked-prefix MHA style: fetch an arbitrary sub-range of the
|
||||
# prefix through get_mla_kv_buffer.
|
||||
sub = chunk1_locs[_GATHER_PAGE_SIZE // 2 : prefix_len - 3]
|
||||
k_nope, k_rope = pool.get_mla_kv_buffer(layer, sub, _GATHER_DTYPE)
|
||||
expect = _mla_value(
|
||||
sub + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mla prefix read l{layer_id}",
|
||||
k_nope[:, 0, :],
|
||||
expect[:, :_GATHER_KV_LORA_RANK],
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mla prefix rope l{layer_id}",
|
||||
k_rope[:, 0, :],
|
||||
expect[:, _GATHER_KV_LORA_RANK:],
|
||||
)
|
||||
# Absorbed-MLA style (what MLA-under-CP uses): read [prefix | chunk]
|
||||
# from get_key_buffer through the translated page table.
|
||||
all_locs = req_to_token[0, :seq_len].long()
|
||||
rows = pool.translate_loc_to_scratch(all_locs)
|
||||
kv_scratch = pool.get_key_buffer(layer_id)
|
||||
_check(
|
||||
rank,
|
||||
f"mla absorbed read l{layer_id}",
|
||||
kv_scratch[rows, 0, :],
|
||||
_mla_value(
|
||||
all_locs + layer_id * 1000, _GATHER_KV_LORA_RANK + _GATHER_QK_ROPE
|
||||
),
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
if rank == 0:
|
||||
print("PASS: MLA page-interleave shard (attn-TP axis)")
|
||||
|
||||
|
||||
def _run_mha(rank, world, port):
|
||||
_dist_init(rank, world, port, attn_cp_size=world)
|
||||
|
||||
group = get_parallel().attn_cp_group
|
||||
assert group.world_size == world
|
||||
# Topology-first shard-group selection: with an active CP group, both
|
||||
# GQA and MLA shard across CP (CP replicates KV for every attention
|
||||
# type; the TP axis is only the no-CP MLA fallback).
|
||||
assert get_kv_shard_group(use_mla_backend=False) is group
|
||||
assert get_kv_shard_group(use_mla_backend=True) is group
|
||||
spec = _gather_make_spec(shard_rank=group.rank_in_group)
|
||||
|
||||
pool = PageInterleaveMHATokenToKVPool(
|
||||
_GATHER_SIZE,
|
||||
page_size=_GATHER_PAGE_SIZE,
|
||||
dtype=_GATHER_DTYPE,
|
||||
head_num=_GATHER_HEAD_NUM,
|
||||
head_dim=_GATHER_HEAD_DIM,
|
||||
layer_num=_GATHER_LAYER_NUM,
|
||||
device=f"cuda:{rank}",
|
||||
enable_memory_saver=False,
|
||||
start_layer=0,
|
||||
end_layer=_GATHER_LAYER_NUM - 1,
|
||||
enable_alt_stream=False,
|
||||
shard_spec=spec,
|
||||
shard_group=group,
|
||||
)
|
||||
device = pool.k_buffer[0].device
|
||||
|
||||
def kv_value(locs, layer_id, is_v):
|
||||
base = locs.to(torch.float32) + layer_id * 1000 + (500000 if is_v else 0)
|
||||
return (
|
||||
base.view(-1, 1, 1)
|
||||
+ torch.arange(_GATHER_HEAD_NUM, device=device).view(1, -1, 1) * 0.01
|
||||
+ torch.arange(_GATHER_HEAD_DIM, device=device).view(1, 1, -1) * 0.0001
|
||||
).to(_GATHER_DTYPE)
|
||||
|
||||
# ---- chunk 1 (prefix-less batch): stage + owner-persist ----------------
|
||||
chunk1_groups = [7, 3]
|
||||
chunk1_locs = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)[
|
||||
0
|
||||
].long()
|
||||
req_to_token = _fake_req_to_token(chunk1_groups, 2 * _GATHER_GRANULE, device)
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [0], [2 * _GATHER_GRANULE])
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
pool.set_kv_buffer(
|
||||
layer,
|
||||
chunk1_locs,
|
||||
kv_value(chunk1_locs, layer_id, False),
|
||||
kv_value(chunk1_locs, layer_id, True),
|
||||
)
|
||||
# The current chunk must be readable through the scratch right away.
|
||||
k_scratch = pool.get_key_buffer(layer_id)
|
||||
rows = pool.translate_loc_to_scratch(chunk1_locs)
|
||||
_check(
|
||||
rank,
|
||||
f"mha chunk stage l{layer_id}",
|
||||
k_scratch[rows],
|
||||
kv_value(chunk1_locs, layer_id, False),
|
||||
)
|
||||
torch.cuda.synchronize()
|
||||
torch.distributed.barrier()
|
||||
|
||||
# ---- chunk 2: prefix gathered from peers via translated page table -----
|
||||
seq_groups = chunk1_groups + [11]
|
||||
prefix_len = 2 * _GATHER_GRANULE
|
||||
seq_len = prefix_len + _GATHER_GRANULE
|
||||
req_to_token = _fake_req_to_token(seq_groups, seq_len, device)
|
||||
chunk2_locs = req_to_token[0, prefix_len:seq_len].long()
|
||||
pool.begin_shard_extend(req_to_token, torch.tensor([0]), [prefix_len], [seq_len])
|
||||
|
||||
for layer_id in range(_GATHER_LAYER_NUM):
|
||||
layer = SimpleNamespace(layer_id=layer_id)
|
||||
pool.set_kv_buffer(
|
||||
layer,
|
||||
chunk2_locs,
|
||||
kv_value(chunk2_locs, layer_id, False),
|
||||
kv_value(chunk2_locs, layer_id, True),
|
||||
)
|
||||
all_locs = req_to_token[0, :seq_len].long()
|
||||
rows = pool.translate_loc_to_scratch(all_locs)
|
||||
k_scratch = pool.get_key_buffer(layer_id)
|
||||
v_scratch = pool.get_value_buffer(layer_id)
|
||||
_check(
|
||||
rank,
|
||||
f"mha seq read k l{layer_id}",
|
||||
k_scratch[rows],
|
||||
kv_value(all_locs, layer_id, False),
|
||||
)
|
||||
_check(
|
||||
rank,
|
||||
f"mha seq read v l{layer_id}",
|
||||
v_scratch[rows],
|
||||
kv_value(all_locs, layer_id, True),
|
||||
)
|
||||
|
||||
torch.distributed.barrier()
|
||||
if rank == 0:
|
||||
print("PASS: MHA page-interleave shard (attn-CP axis)")
|
||||
|
||||
|
||||
@unittest.skipIf(
|
||||
torch.cuda.device_count() < 2, "page-interleave gather needs 2 CUDA devices"
|
||||
)
|
||||
class TestPageInterleaveGatherMultiGpu(CustomTestCase):
|
||||
"""Real pools, real NCCL, 2 ranks — one mp.spawn per phase.
|
||||
|
||||
Separate spawns (and separate ports) because each phase builds its own
|
||||
process group with a different attention-CP width.
|
||||
"""
|
||||
|
||||
def test_mla_shard_over_attention_tp(self):
|
||||
mp.spawn(_run_mla, args=(_GATHER_WORLD, 29811), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
def test_mha_shard_over_attention_cp(self):
|
||||
mp.spawn(_run_mha, args=(_GATHER_WORLD, 29812), nprocs=_GATHER_WORLD, join=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user