[unified-memory] PD disaggregation for every unified pool shape (#37506)
This commit is contained in:
@@ -82,8 +82,10 @@ class Memory(msgspec.Struct):
|
||||
"Replace the statically-partitioned hybrid-model pools (full-attn KV + "
|
||||
"SWA/Mamba state) with one byte buffer split dynamically between "
|
||||
"sub-pools. Requires the Triton attention / linear-attn / Mamba "
|
||||
"backends; not yet compatible with PD disaggregation or speculative "
|
||||
"decoding.",
|
||||
"backends. PD disaggregation is supported over mooncake at equal "
|
||||
"attention TP with pp=1; not yet compatible with hierarchical / "
|
||||
"host-tiered KV cache, prefill cuda-graph capture, or speculative "
|
||||
"decoding other than DSPARK.",
|
||||
] = False
|
||||
enable_session_radix_cache: A[
|
||||
bool,
|
||||
|
||||
@@ -1416,7 +1416,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
decode_req.req.kv.req_pool_idx, window_start:seq_len
|
||||
]
|
||||
window_kv_indices_swa = (
|
||||
self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
|
||||
self.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
|
||||
window_kv_indices_full
|
||||
)
|
||||
)
|
||||
|
||||
@@ -957,6 +957,13 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
prefill_data_indices=prefill_kv_indices,
|
||||
dst_data_indices=dst_kv_indices,
|
||||
executor=executor,
|
||||
# The unified pool registers ONE region holding every layer's K and
|
||||
# V inside each page envelope. The MHA branch would half-split that
|
||||
# single region into K and V halves and compute num_kv_layers = 0,
|
||||
# transferring nothing at all; the flat branch addresses the region
|
||||
# as-is. MLA-unified already reaches the flat branch via
|
||||
# is_mla_backend, so this only adds the MHA-unified peer.
|
||||
force_flat=get_memory().enable_unified_memory,
|
||||
src_layer_ids=self.kv_args.kv_layer_ids,
|
||||
dst_layer_ids=dst_layer_ids,
|
||||
dst_device_data_indices=dst_device_kv_indices,
|
||||
@@ -1598,8 +1605,16 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
dst_data_indices=np.array(dst_indices_local, dtype=np.int32),
|
||||
executor=executor,
|
||||
state_type=st,
|
||||
force_flat=st
|
||||
in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED),
|
||||
# Two independent reasons to keep the flat layout.
|
||||
# QSA's per-layer list must not be half-split into K/V;
|
||||
# neither must a unified sub-pool's single region, which
|
||||
# holds every layer's K and V per slot envelope -- the
|
||||
# MHA branch would compute zero layers and ship nothing
|
||||
# (same reason as in `send_kvcache`).
|
||||
force_flat=(
|
||||
st in (StateType.QSA_PENDING, StateType.QSA_COMPRESSED)
|
||||
or get_memory().enable_unified_memory
|
||||
),
|
||||
src_layer_ids=src_state_layer_ids,
|
||||
dst_layer_ids=dst_state_layer_ids,
|
||||
)
|
||||
|
||||
@@ -1328,7 +1328,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
req.kv.req_pool_idx, window_start:seq_len
|
||||
]
|
||||
window_kv_indices_swa = (
|
||||
self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
|
||||
self.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
|
||||
window_kv_indices_full
|
||||
)
|
||||
)
|
||||
|
||||
@@ -346,6 +346,17 @@ class DeepSeekV4HiSparseTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
|
||||
return self.logical_attn_allocator.translate_loc_from_full_to_swa(kv_indices)
|
||||
|
||||
def translate_swa_indices_for_transfer(
|
||||
self, kv_indices: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Delegated like the read-path translate above: this composite is not a
|
||||
# SWA allocator itself, so it inherits neither the default nor an
|
||||
# override, and the PD payload path calls this on whatever allocator
|
||||
# the scheduler holds.
|
||||
return self.logical_attn_allocator.translate_swa_indices_for_transfer(
|
||||
kv_indices
|
||||
)
|
||||
|
||||
def full_available_size(self):
|
||||
return min(
|
||||
self.logical_attn_allocator.full_available_size(),
|
||||
|
||||
@@ -203,6 +203,19 @@ class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
assert self._kvcache.full_to_swa_index_mapping is not None
|
||||
return self._kvcache.translate_loc_from_full_to_swa(kv_indices)
|
||||
|
||||
def translate_swa_indices_for_transfer(
|
||||
self, kv_indices: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Sliding-window token ids as the PD transfer engine addresses them.
|
||||
|
||||
The sibling of `translate_kv_indices_for_transfer` for the SWA state
|
||||
component. On a static pool the sliding-window buffers are indexed by
|
||||
the same ids the kernels use, so the read-path translate IS the answer.
|
||||
A virtual-id pool must override: the transfer addresses raw bytes and
|
||||
needs PHYSICAL ids, not kernel-facing ones.
|
||||
"""
|
||||
return self.translate_loc_from_full_to_swa(kv_indices)
|
||||
|
||||
def alloc(self, need_size: int):
|
||||
assert self.page_size == 1
|
||||
if need_size > self.full_attn_allocator.available_size():
|
||||
|
||||
@@ -17,7 +17,7 @@ sub-pools of one `UnifiedKVPool`, and the tri-pool variant that adds mamba state
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import List, Optional, Sequence
|
||||
from typing import Callable, List, Optional, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
_flush_deferred_free_group,
|
||||
_full_tokens_before_mamba_recheck,
|
||||
_relieve_for_alloc,
|
||||
install_move_gate,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
from sglang.srt.utils.common import get_num_new_pages
|
||||
@@ -320,6 +321,46 @@ class UnifiedSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
"""Page-level physical->virtual table of the full sub-pool."""
|
||||
return self.full_attn_allocator.physical_to_virtual
|
||||
|
||||
def translate_kv_indices_for_transfer(
|
||||
self, kv_indices: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Virtual TOKEN ids -> full-sub-pool PHYSICAL token ids for the PD
|
||||
transfer engine.
|
||||
|
||||
PHYSICAL, not kernel-facing: the transfer registers page ENVELOPES (see
|
||||
`UnifiedMHATokenToKVPool.get_contiguous_buf_infos`). Without this
|
||||
override the base identity would put VIRTUAL ids on the wire, which
|
||||
address real bytes and so corrupt silently rather than fail.
|
||||
"""
|
||||
return self.full_attn_allocator.translate_kv_loc(kv_indices.to(torch.int64))
|
||||
|
||||
def translate_swa_indices_for_transfer(
|
||||
self, kv_indices: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Virtual TOKEN ids -> swa-sub-pool PHYSICAL token ids.
|
||||
|
||||
The SWA counterpart of the above. `translate_loc_from_full_to_swa`
|
||||
cannot serve here: it returns KERNEL-FACING ids (the physical page
|
||||
scaled by the sub-pool's per-page block count), which index the
|
||||
per-layer views, whereas the SWA state component is registered as whole
|
||||
page envelopes and addressed by physical page.
|
||||
"""
|
||||
return self.swa_attn_allocator.translate_kv_loc(kv_indices.to(torch.int64))
|
||||
|
||||
def _move_gate_targets(self):
|
||||
"""Every member a compaction gate must cover. A subclass that adds an
|
||||
end overrides THIS, and every gate widens with it."""
|
||||
return (self.full_attn_allocator, self.swa_attn_allocator)
|
||||
|
||||
def set_disagg_move_gate(self, gate: Callable[[], bool]) -> None:
|
||||
install_move_gate(
|
||||
self._move_gate_targets(),
|
||||
slot="disagg_move_gate",
|
||||
gate=gate,
|
||||
feature="PD disaggregation",
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def translate_kv_loc_for_kernel(
|
||||
self,
|
||||
loc: torch.Tensor,
|
||||
@@ -374,6 +415,52 @@ class UnifiedSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
self.swa_attn_allocator.alloc_with_virtual(new_virtual_pages)
|
||||
return v_tokens
|
||||
|
||||
def _extend_in_virtual_space(
|
||||
self,
|
||||
prefix_lens: torch.Tensor,
|
||||
prefix_lens_cpu: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
last_loc: torch.Tensor,
|
||||
extend_num_tokens: int,
|
||||
) -> Optional[Tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Run the full side's paged extend and report which virtual PAGES it
|
||||
newly took. Returns (virtual TOKEN ids, new virtual PAGE ids), or None
|
||||
when the joint capacity check cannot fund the allocation.
|
||||
|
||||
Both extend entries share this; they differ only in which of those pages
|
||||
the sliding-window side then binds.
|
||||
"""
|
||||
num_new_pages = get_num_new_pages(
|
||||
seq_lens=seq_lens_cpu,
|
||||
page_size=self.page_size,
|
||||
prefix_lens=prefix_lens_cpu,
|
||||
)
|
||||
need_tokens = num_new_pages * self.page_size
|
||||
if need_tokens > self.available_size():
|
||||
if not _relieve_for_alloc(self, need_tokens):
|
||||
return None
|
||||
|
||||
# Snapshot the virtual PAGES the kernel will consume; clone so swa keeps
|
||||
# its view after the slice is consumed.
|
||||
fa = self.full_attn_allocator
|
||||
new_virtual_pages = fa.free_virtual_ids[:num_new_pages].clone()
|
||||
|
||||
out_indices = fa.alloc_extend(
|
||||
prefix_lens,
|
||||
prefix_lens_cpu,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
last_loc,
|
||||
extend_num_tokens,
|
||||
num_new_pages=num_new_pages,
|
||||
)
|
||||
assert out_indices is not None, (
|
||||
"UnifiedSWA: full.alloc_extend returned None after joint pre-check "
|
||||
"passed — internal-state inconsistency"
|
||||
)
|
||||
return out_indices, new_virtual_pages
|
||||
|
||||
def alloc_extend(
|
||||
self,
|
||||
prefix_lens: torch.Tensor,
|
||||
@@ -386,37 +473,75 @@ class UnifiedSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
|
||||
"""Paged extend; returns virtual TOKEN ids. The same virtual page maps to
|
||||
full- and swa-physical, so swa binds exactly what the full kernel consumed."""
|
||||
with record_function("UnifiedSWAAlloc.alloc_extend"):
|
||||
num_new_pages = get_num_new_pages(
|
||||
seq_lens=seq_lens_cpu,
|
||||
page_size=self.page_size,
|
||||
prefix_lens=prefix_lens_cpu,
|
||||
)
|
||||
need_tokens = num_new_pages * self.page_size
|
||||
if need_tokens > self.available_size():
|
||||
if not _relieve_for_alloc(self, need_tokens):
|
||||
return None
|
||||
|
||||
# Snapshot the virtual PAGES the kernel will consume; clone so swa keeps
|
||||
# its view after the slice is consumed.
|
||||
fa = self.full_attn_allocator
|
||||
new_virtual_pages = fa.free_virtual_ids[:num_new_pages].clone()
|
||||
|
||||
out_indices = fa.alloc_extend(
|
||||
extended = self._extend_in_virtual_space(
|
||||
prefix_lens,
|
||||
prefix_lens_cpu,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
last_loc,
|
||||
extend_num_tokens,
|
||||
num_new_pages=num_new_pages,
|
||||
)
|
||||
assert out_indices is not None, (
|
||||
"UnifiedSWA.alloc_extend: full.alloc_extend returned None "
|
||||
"after joint pre-check passed — internal-state inconsistency"
|
||||
)
|
||||
if extended is None:
|
||||
return None
|
||||
out_indices, new_virtual_pages = extended
|
||||
self.swa_attn_allocator.alloc_with_virtual(new_virtual_pages)
|
||||
return out_indices # virtual TOKEN ids
|
||||
|
||||
def alloc_extend_swa_tail(
|
||||
self,
|
||||
prefix_lens: torch.Tensor,
|
||||
prefix_lens_cpu: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: torch.Tensor,
|
||||
last_loc: torch.Tensor,
|
||||
extend_num_tokens: int,
|
||||
swa_tail_len: int,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Decode-node prealloc: full KV for the whole sequence, sliding-window
|
||||
KV for the live window tail only.
|
||||
|
||||
The static composite allocates the two sides independently and records
|
||||
a full->swa index mapping. That is not representable here: the two
|
||||
sides SHARE one virtual id space (a virtual page names a full-physical
|
||||
page and, if bound, a swa-physical one), which is why
|
||||
`set_full_to_swa_mapping` is a no-op on this allocator and
|
||||
`translate_loc_from_full_to_swa` derives the swa id from the virtual id
|
||||
instead of a table. Running the static body would call `alloc_extend`
|
||||
on the swa sub-allocator, which asserts it is not the id owner.
|
||||
|
||||
The tail is expressed by binding swa for the TAIL's virtual pages only.
|
||||
A new page left unbound has no swa-physical page, which reads as the
|
||||
sink and is skipped by `free`'s `swa_v2p_page > 0` mask -- exactly the
|
||||
out-of-window state the ratchet produces via `free_swa`.
|
||||
|
||||
Admission is priced at the FULL side's page count, as plain
|
||||
`alloc_extend` is: pessimistic when the tail is short, but it reuses
|
||||
the composite's audited joint capacity path, and the bytes actually
|
||||
held still follow the tail.
|
||||
"""
|
||||
assert len(prefix_lens_cpu) == 1
|
||||
assert 0 <= swa_tail_len <= extend_num_tokens
|
||||
with record_function("UnifiedSWAAlloc.alloc_extend_swa_tail"):
|
||||
extended = self._extend_in_virtual_space(
|
||||
prefix_lens,
|
||||
prefix_lens_cpu,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
last_loc,
|
||||
extend_num_tokens,
|
||||
)
|
||||
if extended is None:
|
||||
return None
|
||||
out_indices, new_virtual_pages = extended
|
||||
if swa_tail_len > 0 and new_virtual_pages.numel() > 0:
|
||||
tail_pages = torch.unique(out_indices[-swa_tail_len:] // self.page_size)
|
||||
# Only NEW pages need binding; a tail page carried in from the
|
||||
# prefix is already bound on the swa side.
|
||||
to_bind = new_virtual_pages[torch.isin(new_virtual_pages, tail_pages)]
|
||||
if to_bind.numel() > 0:
|
||||
self.swa_attn_allocator.alloc_with_virtual(to_bind)
|
||||
return out_indices # virtual TOKEN ids
|
||||
|
||||
def alloc_decode(
|
||||
self,
|
||||
seq_lens: torch.Tensor,
|
||||
@@ -769,16 +894,9 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWATokenToKVPoolAllocator):
|
||||
binary search; the order matches the alloc path (full takes the high band).
|
||||
"""
|
||||
fa, sa = self.full_attn_allocator, self.swa_attn_allocator
|
||||
e_f, e_s = fa.entry_bytes_per_page, sa.entry_bytes_per_page
|
||||
e_f = fa.entry_bytes_per_page
|
||||
# full is grow-down: its chain gap IS the high band.
|
||||
b_high = fa._current_gap_bytes()
|
||||
if sa._is_frontier_transparent():
|
||||
b_low = 0
|
||||
else:
|
||||
b_low = max(
|
||||
0,
|
||||
sa._byte_low_frontier() - sa._chain_high_frontier_below_bytes(),
|
||||
)
|
||||
h_f = len(fa._free_phys_pages) if fa.lazy_compaction else 0
|
||||
h_s = sa._hole_pages()
|
||||
r_f = fa.num_pages - fa.min_page_index - fa._allocated_pages()
|
||||
@@ -819,6 +937,16 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWATokenToKVPoolAllocator):
|
||||
hi_n = mid - 1
|
||||
return lo_n * self.page_size
|
||||
|
||||
def _move_gate_targets(self):
|
||||
"""All three members. The mamba end compacts independently and its slot
|
||||
envelopes move as `StateType.MAMBA`, so leaving it out of a gate would
|
||||
let a conv/SSM slot relocate under an in-flight transfer."""
|
||||
return (
|
||||
self.full_attn_allocator,
|
||||
self.swa_attn_allocator,
|
||||
self.mamba_allocator,
|
||||
)
|
||||
|
||||
def _flush_targets(self):
|
||||
"""All three members, float FIRST: its zero-copy boundary absorption must
|
||||
land before the deficit math prices a relocation it already covered."""
|
||||
|
||||
@@ -31,6 +31,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
|
||||
_flush_deferred_free_group,
|
||||
_full_tokens_before_mamba_recheck,
|
||||
_relieve_for_alloc,
|
||||
install_move_gate,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
@@ -306,15 +307,20 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
)
|
||||
return self.full_attn_allocator.translate_kv_loc(kv_indices.to(torch.int64))
|
||||
|
||||
def _move_gate_targets(self):
|
||||
"""Every member a compaction gate must cover. The mamba end is gated
|
||||
even where its state is not itself transferred: the gate is about the
|
||||
MOVER, and the two ends compact as peers."""
|
||||
return (self.full_attn_allocator, self.mamba_allocator)
|
||||
|
||||
def set_disagg_move_gate(self, gate: Callable[[], bool]) -> None:
|
||||
"""Install the PD-disaggregation move gate on both sub-allocators."""
|
||||
assert self.lazy_compaction, (
|
||||
"PD disaggregation with the unified memory pool requires lazy "
|
||||
"compaction (eager free-path compaction moves pages under "
|
||||
"in-flight transfers)."
|
||||
install_move_gate(
|
||||
self._move_gate_targets(),
|
||||
slot="disagg_move_gate",
|
||||
gate=gate,
|
||||
feature="PD disaggregation",
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
self.full_attn_allocator.disagg_move_gate = gate
|
||||
self.mamba_allocator.disagg_move_gate = gate
|
||||
|
||||
def is_slot_allocated(self, slot: int) -> bool:
|
||||
return self.full_attn_allocator.is_slot_allocated(slot)
|
||||
|
||||
@@ -239,6 +239,31 @@ def _full_tokens_before_mamba_recheck(
|
||||
return -(-minimum_missing_bytes * dcp_size // full_allocator.entry_bytes)
|
||||
|
||||
|
||||
def install_move_gate(
|
||||
targets,
|
||||
*,
|
||||
slot: str,
|
||||
gate: Callable[[], bool],
|
||||
feature: str,
|
||||
lazy_compaction: bool,
|
||||
) -> None:
|
||||
"""Point every member of a composite at one compaction gate.
|
||||
|
||||
A gate that reaches only some members is not a weaker gate, it is no gate:
|
||||
the ungated end relocates its own pages under the same in-flight transfer.
|
||||
So the member list is stated once per composite (`_move_gate_targets`) and
|
||||
every gate installs over it, rather than each setter naming the members it
|
||||
happens to remember.
|
||||
"""
|
||||
assert lazy_compaction, (
|
||||
f"{feature} with the unified memory pool requires lazy compaction "
|
||||
"(eager free-path compaction moves pages under in-flight transfers)."
|
||||
)
|
||||
assert slot in ("disagg_move_gate", "host_transfer_move_gate"), slot
|
||||
for target in targets:
|
||||
setattr(target, slot, gate)
|
||||
|
||||
|
||||
class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
"""Allocator for one sub-pool over a `UnifiedKVPool`."""
|
||||
|
||||
|
||||
@@ -155,11 +155,21 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
)
|
||||
# Host-pool retraction transfers full and sliding-window components
|
||||
# only, so a model with recurrent state stays on cpu_tensor.
|
||||
supports_host_pool = not uses_ssm_state(
|
||||
tp_worker.model_runner.model_config
|
||||
) and (
|
||||
isinstance(kv_cache, MHATokenToKVPool)
|
||||
or (isinstance(kv_cache, SWAKVPool) and full_tokens_per_layer > 0)
|
||||
#
|
||||
# The unified pool is excluded for the same reason hierarchical cache is
|
||||
# (see `handle_unified_memory_pool`): the host-transfer path indexes the
|
||||
# device buffers with the ids it is handed, and under the unified pool
|
||||
# those are VIRTUAL. It also cannot be sized from `kv_cache.size`, which
|
||||
# is a KERNEL-FACING row count (`num_pages * 2 * layer_num * page_size`)
|
||||
# rather than a token capacity -- gpt-oss-20b reports 85M "tokens" and
|
||||
# asks for 418 GB of host memory per component.
|
||||
supports_host_pool = (
|
||||
not uses_ssm_state(tp_worker.model_runner.model_config)
|
||||
and not memory.enable_unified_memory
|
||||
and (
|
||||
isinstance(kv_cache, MHATokenToKVPool)
|
||||
or (isinstance(kv_cache, SWAKVPool) and full_tokens_per_layer > 0)
|
||||
)
|
||||
)
|
||||
schedule = get_schedule()
|
||||
priority_preemption = (
|
||||
|
||||
@@ -446,24 +446,12 @@ class KVCacheConfigurator:
|
||||
# from one byte buffer, then return. Gated to the target worker
|
||||
# (req_to_token_pool is None); supports hybrid Mamba and hybrid SWA (not DSV4).
|
||||
if get_memory().enable_unified_memory and req_to_token_pool is None:
|
||||
pd_enabled = get_disagg().disaggregation_mode != "null"
|
||||
is_dsv4 = is_deepseek_v4(self.model_config.hf_config)
|
||||
# Order matters: an Inkling-class model is BOTH mambaish and
|
||||
# hybrid-SWA, and the mamba pair would store every SWA layer's KV at
|
||||
# FULL lifetime -- its branch reads the HF config's
|
||||
# full_attention_layer_ids, which for Inkling is ALL layers.
|
||||
if self.mambaish_config is not None and self.is_hybrid_swa and not is_dsv4:
|
||||
if pd_enabled:
|
||||
# Same limitation as the 2-pool SWA branch below: the
|
||||
# tri-pool carries an SWA sub-pool, and there is no
|
||||
# whole-envelope transfer scheme for it.
|
||||
raise ValueError(
|
||||
"--enable-unified-memory with PD disaggregation does "
|
||||
"not support hybrid-SWA models yet (no whole-envelope "
|
||||
"transfer scheme for the SWA sub-pool); this model "
|
||||
"routes to the mamba+SWA tri-pool, which has one. Drop "
|
||||
"--enable-unified-memory or run without PD."
|
||||
)
|
||||
bundle = self._init_unified_mamba_swa_pools(
|
||||
max_num_reqs=sizes.max_running_requests,
|
||||
full_max_total_num_tokens=sizes.full_max_total_num_tokens,
|
||||
@@ -471,27 +459,12 @@ class KVCacheConfigurator:
|
||||
unified_total_bytes=sizes.unified_total_bytes,
|
||||
)
|
||||
elif self.mambaish_config is not None:
|
||||
if pd_enabled and not self.use_mla_backend:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory with PD disaggregation "
|
||||
"currently supports only MLA hybrid-Mamba models "
|
||||
"(e.g. kimi-linear); this model uses the MHA full-"
|
||||
"attention pool. Drop --enable-unified-memory or run "
|
||||
"without PD disaggregation."
|
||||
)
|
||||
bundle = self._init_unified_mamba_pools(
|
||||
max_num_reqs=sizes.max_running_requests,
|
||||
max_total_num_tokens=sizes.max_total_num_tokens,
|
||||
unified_total_bytes=sizes.unified_total_bytes,
|
||||
)
|
||||
elif self.is_hybrid_swa and not is_dsv4:
|
||||
if pd_enabled:
|
||||
raise ValueError(
|
||||
"--enable-unified-memory with PD disaggregation does "
|
||||
"not support hybrid-SWA models yet (no whole-envelope "
|
||||
"transfer scheme for the SWA sub-pool). Drop "
|
||||
"--enable-unified-memory or run without PD."
|
||||
)
|
||||
bundle = self._init_unified_swa_pools(
|
||||
max_num_reqs=sizes.max_running_requests,
|
||||
full_max_total_num_tokens=sizes.full_max_total_num_tokens,
|
||||
@@ -835,6 +808,13 @@ class KVCacheConfigurator:
|
||||
unified_total_bytes=(None if self.is_draft_worker else unified_total_bytes),
|
||||
# bs=1 feasibility floor input (context len is already passed).
|
||||
sliding_window_size=self.model_config.sliding_window_size,
|
||||
# Decode nodes hand out request rows to PREALLOCATED transfers on
|
||||
# top of the running set; the 2-pool mamba factory takes the same.
|
||||
decode_pre_alloc_size=(
|
||||
get_disagg().disaggregation_decode_extra_slots
|
||||
if get_disagg().disaggregation_mode == "decode"
|
||||
else 0
|
||||
),
|
||||
)
|
||||
|
||||
def _init_unified_swa_pools(
|
||||
@@ -863,12 +843,28 @@ class KVCacheConfigurator:
|
||||
extra_max_context_len = 4
|
||||
if get_spec().speculative_num_draft_tokens is not None:
|
||||
extra_max_context_len += get_spec().speculative_num_draft_tokens
|
||||
req_to_token_pool = ReqToTokenPool(
|
||||
size=max_num_reqs,
|
||||
max_context_len=self.model_config.context_len + extra_max_context_len,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
)
|
||||
if get_disagg().disaggregation_mode == "decode":
|
||||
# A decode node hands out request rows to PREALLOCATED transfers on
|
||||
# top of its running set, so it needs the extra-slot pool (and the
|
||||
# `pre_alloc_size` the scheduler's invariant checker reads). Mirrors
|
||||
# `_build_req_to_token_pool`'s decode branch; the mamba composite
|
||||
# already takes `decode_pre_alloc_size` the same way.
|
||||
from sglang.srt.disaggregation.decode import DecodeReqToTokenPool
|
||||
|
||||
req_to_token_pool = DecodeReqToTokenPool(
|
||||
size=max_num_reqs,
|
||||
max_context_len=self.model_config.context_len + extra_max_context_len,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
pre_alloc_size=get_disagg().disaggregation_decode_extra_slots,
|
||||
)
|
||||
else:
|
||||
req_to_token_pool = ReqToTokenPool(
|
||||
size=max_num_reqs,
|
||||
max_context_len=self.model_config.context_len + extra_max_context_len,
|
||||
device=self.device,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
)
|
||||
|
||||
head_num = self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
|
||||
@@ -624,10 +624,19 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
env[tgt_pages] = env[src_pages]
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"unified layout has no per-layer contiguous regions; "
|
||||
"KV transfer / disaggregation is unsupported."
|
||||
)
|
||||
"""PD-transfer registration: ONE entry, the raw buffer, addressed as
|
||||
``raw_ptr + physical_page_id * page_envelope_bytes``.
|
||||
|
||||
Same whole-envelope contract as `UnifiedMLATokenToKVPool`: the transfer
|
||||
item is one page across ALL layers and both K and V, because the
|
||||
per-layer views overlap inside the envelope and index in kernel-facing
|
||||
ids. A peer must therefore build an identical spec -- enforced on the
|
||||
wire by `_validate_envelope_kv_layout`.
|
||||
"""
|
||||
# The address formula omits the anchor; a nonzero one would mis-address.
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
raw = self._unified_buffer._raw
|
||||
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None):
|
||||
raise NotImplementedError(
|
||||
@@ -1882,6 +1891,7 @@ def init_unified_mamba_swa_pools(
|
||||
lazy_compaction: bool = False,
|
||||
unified_total_bytes: Optional[int] = None,
|
||||
sliding_window_size: Optional[int] = None,
|
||||
decode_pre_alloc_size: int = 0,
|
||||
) -> UnifiedPoolBundle:
|
||||
"""Build the TRI-pool unified-memory-pool stack for models with full KV +
|
||||
SWA KV + mamba/conv state (Inkling-class: `mambaish_config` AND
|
||||
@@ -2007,6 +2017,7 @@ def init_unified_mamba_swa_pools(
|
||||
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
||||
enable_overlap_schedule=not disable_overlap_schedule,
|
||||
start_layer=start_layer,
|
||||
pre_alloc_size=decode_pre_alloc_size,
|
||||
)
|
||||
allocator = UnifiedMambaSWATokenToKVPoolAllocator(
|
||||
unified_buffer=shared_pool,
|
||||
|
||||
@@ -9,6 +9,9 @@ from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
register_cuda_ci(est_time=236, stage="base-b", runner_config="2-gpu-large")
|
||||
|
||||
KIMI_LINEAR_MODEL = "yujiepan/kimi-linear-tiny-random"
|
||||
# Smallest in-tree GDN hybrid: MHA full attention + gated-delta-net linear
|
||||
# layers, i.e. the unified pool's MHA sub-pool rather than the MLA one.
|
||||
QWEN_GDN_MODEL = "Qwen/Qwen3.5-0.8B"
|
||||
SERVER_ENV = {"SGLANG_BATCH_INVARIANT_OPS_ENABLE_MM_DEEPGEMM": "0"}
|
||||
|
||||
# --attention-backend and --enable-deterministic-inference are deliberately
|
||||
@@ -63,5 +66,21 @@ class TestUnifiedMemoryDisaggregationChunkedPrefill(TestUnifiedMemoryDisaggregat
|
||||
extra_decode_args = _chunked_args
|
||||
|
||||
|
||||
class TestUnifiedMemoryDisaggregationMHA(TestUnifiedMemoryDisaggregation):
|
||||
"""The MHA full-attention sub-pool over the wire.
|
||||
|
||||
Kimi-Linear above exercises the MLA sub-pool, whose whole-envelope
|
||||
registration has always been the one PD supports. An MHA envelope is a
|
||||
different shape -- `2 * layer_num` row-blocks per page instead of
|
||||
`layer_num` -- and it reaches a different branch of
|
||||
`_send_kvcache_generic`: without `force_flat` the MHA branch halves the
|
||||
single registered region into K and V, computes `num_kv_layers = 0` and
|
||||
transfers NOTHING, which shows up as garbage decode rather than an error.
|
||||
Logprob parity against a non-PD unified reference is what catches that.
|
||||
"""
|
||||
|
||||
model = QWEN_GDN_MODEL
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""PD disaggregation for a hybrid-SWA model on the unified memory pool.
|
||||
|
||||
A hybrid-SWA model ships TWO attention components: the full-attention KV on the
|
||||
ordinary `kv_data_ptrs` channel and the sliding-window KV as `StateType.SWA`.
|
||||
Under `--enable-unified-memory` both are whole page envelopes into the SAME raw
|
||||
buffer, distinguished only by their per-page stride, and each is addressed by
|
||||
its OWN sub-pool's physical page id -- the full and SWA sides run independent
|
||||
compactions, so one virtual token names two unrelated physical pages.
|
||||
|
||||
That makes three ways to be silently wrong rather than loud:
|
||||
* shipping virtual ids (the base `translate_kv_indices_for_transfer` is the
|
||||
identity, and virtual ids address real bytes);
|
||||
* shipping the SWA side's KERNEL-FACING ids, which the read path uses, in
|
||||
place of its physical ones;
|
||||
* letting compaction relocate a page mid-transfer, which the SWA allocator
|
||||
had no `set_disagg_move_gate` to prevent.
|
||||
|
||||
Logprob parity against a non-PD unified reference catches all three; GSM8K on
|
||||
gpt-oss is too noisy to (single-server unified and static both score 0.570 at
|
||||
200 questions, and PD runs of each span 0.540-0.610).
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.pd_parity_kit import PDLogprobParityMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
from sglang.test.test_utils import DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE
|
||||
|
||||
register_cuda_ci(est_time=1200, stage="extra-a", runner_config="2-gpu-large")
|
||||
|
||||
UNIFIED_SWA_ARGS = [
|
||||
"--skip-tokenizer-init",
|
||||
"--random-seed",
|
||||
"1",
|
||||
"--enable-unified-memory",
|
||||
# gpt-oss uses attention sinks, which flashinfer does not support; triton
|
||||
# reads both sub-pools' per-layer views.
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--mem-fraction-static",
|
||||
"0.7",
|
||||
"--cuda-graph-backend-decode",
|
||||
"disabled",
|
||||
"--cuda-graph-backend-prefill",
|
||||
"disabled",
|
||||
]
|
||||
|
||||
|
||||
class TestUnifiedMemoryDisaggregationSWA(
|
||||
PDLogprobParityMixin, PDDisaggregationServerBase
|
||||
):
|
||||
"""1 prefill + 1 decode, both unified, vs a non-PD unified reference."""
|
||||
|
||||
model = DEFAULT_MODEL_NAME_FOR_TEST_MXFP4_WITH_MOE
|
||||
prefill_tp_size = 1
|
||||
decode_tp_size = 1
|
||||
decode_base_gpu_id = 1
|
||||
baseline_args = UNIFIED_SWA_ARGS
|
||||
extra_prefill_args = UNIFIED_SWA_ARGS
|
||||
extra_decode_args = UNIFIED_SWA_ARGS
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
"""PD disaggregation for a TRI-pool model on the unified memory pool.
|
||||
|
||||
Inkling is the only in-tree architecture that is both mambaish and hybrid-SWA,
|
||||
so one unified buffer carries three components with three independent
|
||||
compactions -- ``[conv state (up END) | swa (FLOAT) | full (down END)]`` -- and
|
||||
PD must ship all three per request: full KV on the ``kv_data_ptrs`` channel,
|
||||
sliding-window KV as ``StateType.SWA``, ShortConv state as ``StateType.MAMBA``
|
||||
(via the ``req_to_token_pool`` fallback, since the KV pool here is a
|
||||
``UnifiedSWAKVPool`` rather than a ``HybridLinearKVPool``).
|
||||
|
||||
Two failures this pins that the 2-pool cases cannot:
|
||||
|
||||
* the FLOAT sub-pool moves for reasons neither END does, so a move gate that
|
||||
reaches only full and swa still lets a conv slot relocate under an
|
||||
in-flight state transfer;
|
||||
* ``page_size > 1`` turns on the decode node's SWA-tail prealloc, whose
|
||||
static body allocates the swa side independently -- an assertion failure
|
||||
against this composite's single virtual id space, and, once that is
|
||||
handled, the first path that can bind the WRONG swa pages.
|
||||
|
||||
Logprob parity against a non-PD unified reference is the check: the tiny
|
||||
``test`` revision is undertrained, so answer quality carries no signal, but a
|
||||
dropped or misaddressed component moves logprobs immediately.
|
||||
"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.pd_parity_kit import PDLogprobParityMixin
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
|
||||
register_cuda_ci(est_time=900, stage="extra-a", runner_config="2-gpu-large")
|
||||
|
||||
_MODEL_PATH = os.environ.get("INKLING_TEST_MODEL_PATH", "thinkingmachines/Inkling")
|
||||
_MODEL_REVISION = os.environ.get("INKLING_TEST_MODEL_REVISION", "test")
|
||||
|
||||
# The unified radix tree is what merges the three components into one tree.
|
||||
SERVER_ENV = {"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}
|
||||
|
||||
UNIFIED_TRI_ARGS = [
|
||||
"--skip-tokenizer-init",
|
||||
"--random-seed",
|
||||
"1",
|
||||
"--enable-unified-memory",
|
||||
# Unified requires the Triton strided page-major read/write paths.
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--page-size",
|
||||
"128",
|
||||
"--mamba-radix-cache-strategy",
|
||||
"extra_buffer",
|
||||
"--swa-full-tokens-ratio",
|
||||
"0.1",
|
||||
"--mamba-full-memory-ratio",
|
||||
"0.1",
|
||||
"--mem-fraction-static",
|
||||
"0.5",
|
||||
# Inkling defaults to a FULL prefill graph, which unified rejects at boot.
|
||||
"--cuda-graph-backend-prefill",
|
||||
"disabled",
|
||||
"--revision",
|
||||
_MODEL_REVISION,
|
||||
]
|
||||
|
||||
|
||||
class TestUnifiedMemoryDisaggregationTriPool(
|
||||
PDLogprobParityMixin, PDDisaggregationServerBase
|
||||
):
|
||||
"""1 prefill + 1 decode, both unified, vs a non-PD unified reference."""
|
||||
|
||||
model = _MODEL_PATH
|
||||
extra_prefill_env = SERVER_ENV
|
||||
extra_decode_env = SERVER_ENV
|
||||
prefill_tp_size = 1
|
||||
decode_tp_size = 1
|
||||
decode_base_gpu_id = 1
|
||||
baseline_args = UNIFIED_TRI_ARGS
|
||||
extra_prefill_args = UNIFIED_TRI_ARGS
|
||||
extra_decode_args = UNIFIED_TRI_ARGS
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -186,5 +186,210 @@ class TestMoveGateRejectsNonPdNode(CustomTestCase):
|
||||
unified_memory_disagg_move_gate(scheduler)
|
||||
|
||||
|
||||
class TestUnifiedAllocatorsPublishTheTransferContract(CustomTestCase):
|
||||
"""Every unified composite allocator must OVERRIDE the two PD hooks.
|
||||
|
||||
`BaseTokenToKVPoolAllocator.translate_kv_indices_for_transfer` is the
|
||||
IDENTITY, and `set_disagg_move_gate` exists only where a composite defines
|
||||
it. Inheriting either is silent, not loud: identity puts VIRTUAL ids on the
|
||||
wire (they address real bytes, so the peer gets plausible garbage), and a
|
||||
missing gate lets lazy compaction relocate pages under in-flight RDMA.
|
||||
An AST-level check because instantiating these composites needs a GPU.
|
||||
"""
|
||||
|
||||
# Composites that own the full-side virtual ids and so must define the
|
||||
# transfer translate themselves.
|
||||
_COMPOSITES = (
|
||||
"UnifiedMambaTokenToKVPoolAllocator",
|
||||
"UnifiedSWATokenToKVPoolAllocator",
|
||||
)
|
||||
# Every composite must define the gate setter, including the tri-pool,
|
||||
# which inherits the SWA translates (same full side) but has a THIRD
|
||||
# member the 2-pool setter does not reach.
|
||||
_GATE_COMPOSITES = _COMPOSITES + ("UnifiedMambaSWATokenToKVPoolAllocator",)
|
||||
|
||||
@staticmethod
|
||||
def _own_methods(cls_name: str) -> Set[str]:
|
||||
"""Names this class defines ITSELF, inheritance excluded.
|
||||
|
||||
Resolved off the class object rather than by parsing a named module:
|
||||
these composites have already been moved once (out of
|
||||
`multi_ended_allocator` into `allocator/unified_*`), and a hardcoded
|
||||
module path turns that kind of move into a test failure that says
|
||||
nothing about the contract. `__dict__` needs no GPU -- it is the class
|
||||
body, not an instance.
|
||||
"""
|
||||
from sglang.srt.mem_cache.allocator import (
|
||||
unified_hybrid_swa,
|
||||
unified_mamba,
|
||||
)
|
||||
|
||||
for mod in (unified_mamba, unified_hybrid_swa):
|
||||
cls = getattr(mod, cls_name, None)
|
||||
if cls is not None:
|
||||
return set(vars(cls))
|
||||
raise AssertionError(f"class {cls_name} not found in the unified allocators")
|
||||
|
||||
def test_transfer_translate_is_not_inherited_identity(self):
|
||||
for name in self._COMPOSITES:
|
||||
with self.subTest(composite=name):
|
||||
self.assertIn(
|
||||
"translate_kv_indices_for_transfer",
|
||||
self._own_methods(name),
|
||||
f"{name} inherits the identity transfer translate; PD would "
|
||||
"ship VIRTUAL ids and corrupt KV without any error",
|
||||
)
|
||||
|
||||
# Every sub-allocator attribute a composite can hold. The stub carries all
|
||||
# of them regardless of composite, so the assertion is on what installation
|
||||
# REACHES rather than on what the stub was given.
|
||||
_MEMBER_ATTRS = ("full_attn_allocator", "swa_attn_allocator", "mamba_allocator")
|
||||
|
||||
# The members each composite's gate must reach. The tri-pool row is the one
|
||||
# that matters: it inherits the setter, so an enumeration written inside
|
||||
# that setter would silently leave the third member ungated.
|
||||
_EXPECTED_COVERAGE = {
|
||||
"UnifiedMambaTokenToKVPoolAllocator": {
|
||||
"full_attn_allocator",
|
||||
"mamba_allocator",
|
||||
},
|
||||
"UnifiedSWATokenToKVPoolAllocator": {
|
||||
"full_attn_allocator",
|
||||
"swa_attn_allocator",
|
||||
},
|
||||
"UnifiedMambaSWATokenToKVPoolAllocator": {
|
||||
"full_attn_allocator",
|
||||
"swa_attn_allocator",
|
||||
"mamba_allocator",
|
||||
},
|
||||
}
|
||||
|
||||
def _members_reached(self, cls_name: str, slot: str) -> Set[str]:
|
||||
"""Install one gate on a stub composite and report which members got it.
|
||||
|
||||
`object.__new__` skips `__init__` (which needs a GPU); the setter reads
|
||||
only `lazy_compaction` and the member attributes.
|
||||
"""
|
||||
from sglang.srt.mem_cache.allocator import unified_hybrid_swa, unified_mamba
|
||||
|
||||
cls = getattr(unified_mamba, cls_name, None) or getattr(
|
||||
unified_hybrid_swa, cls_name
|
||||
)
|
||||
alloc = object.__new__(cls)
|
||||
alloc.lazy_compaction = True
|
||||
for attr in self._MEMBER_ATTRS:
|
||||
member = type("_Member", (), {})()
|
||||
member.disagg_move_gate = None
|
||||
member.host_transfer_move_gate = None
|
||||
setattr(alloc, attr, member)
|
||||
|
||||
def gate() -> bool:
|
||||
return True
|
||||
|
||||
alloc.set_disagg_move_gate(gate)
|
||||
return {
|
||||
attr
|
||||
for attr in self._MEMBER_ATTRS
|
||||
if getattr(getattr(alloc, attr), slot) is gate
|
||||
}
|
||||
|
||||
def test_the_gate_reaches_every_member(self):
|
||||
"""A gate that reaches only some members is not a weaker gate, it is no
|
||||
gate: the ungated end relocates its own pages under the very transfer
|
||||
the gate was installed for.
|
||||
"""
|
||||
for name, expected in self._EXPECTED_COVERAGE.items():
|
||||
with self.subTest(composite=name):
|
||||
self.assertEqual(
|
||||
self._members_reached(name, "disagg_move_gate"),
|
||||
expected,
|
||||
f"{name}.disagg_move_gate does not cover every member",
|
||||
)
|
||||
|
||||
def test_gate_setters_do_not_enumerate_members_themselves(self):
|
||||
"""The structural half of the rule above: a setter that names its
|
||||
members is one a new member silently escapes. Installation must go
|
||||
through the shared helper, which drives off `_move_gate_targets`.
|
||||
"""
|
||||
import inspect
|
||||
|
||||
from sglang.srt.mem_cache.allocator import unified_hybrid_swa, unified_mamba
|
||||
|
||||
for name in self._EXPECTED_COVERAGE:
|
||||
cls = getattr(unified_mamba, name, None) or getattr(
|
||||
unified_hybrid_swa, name
|
||||
)
|
||||
if "set_disagg_move_gate" not in vars(cls):
|
||||
continue # inherited, and the inherited one is checked above
|
||||
with self.subTest(composite=name):
|
||||
body = inspect.getsource(cls.set_disagg_move_gate)
|
||||
self.assertIn("install_move_gate", body)
|
||||
self.assertNotIn("_move_gate = ", body)
|
||||
|
||||
def test_swa_composite_translates_the_swa_side_separately(self):
|
||||
"""The SWA sub-pool runs its OWN compaction, so a full-side physical id
|
||||
does not name the SWA page holding the same virtual token. The read-path
|
||||
`translate_loc_from_full_to_swa` cannot stand in either: it returns
|
||||
kernel-facing ids, and the transfer addresses raw page envelopes."""
|
||||
self.assertIn(
|
||||
"translate_swa_indices_for_transfer",
|
||||
self._own_methods("UnifiedSWATokenToKVPoolAllocator"),
|
||||
)
|
||||
|
||||
|
||||
class TestEverySwaAllocatorAnswersTheTransferTranslate(CustomTestCase):
|
||||
"""Any allocator with a full->SWA read translate needs the transfer sibling.
|
||||
|
||||
`_swa_payload` on both PD sides calls
|
||||
`translate_swa_indices_for_transfer` on whatever allocator the scheduler
|
||||
holds. Most get it by inheriting `SWATokenToKVPoolAllocator`, but a
|
||||
composite that merely DELEGATES the read translate (the DSV4 HiSparse
|
||||
allocator derives from `BaseTokenToKVPoolAllocator`) inherits neither the
|
||||
default nor an override, and PD aborts with an AttributeError the moment a
|
||||
sliding-window payload is built.
|
||||
|
||||
Derived from the live class tree rather than a hand-kept list: a list would
|
||||
pass forever the day someone adds the next delegating composite.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _allocator_classes():
|
||||
import importlib
|
||||
import inspect
|
||||
import pkgutil
|
||||
|
||||
import sglang.srt.mem_cache.allocator as pkg
|
||||
from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator
|
||||
|
||||
found = {}
|
||||
for mod_info in pkgutil.iter_modules(pkg.__path__):
|
||||
try:
|
||||
mod = importlib.import_module(
|
||||
f"sglang.srt.mem_cache.allocator.{mod_info.name}"
|
||||
)
|
||||
except Exception:
|
||||
continue # optional backends need hardware this runner may lack
|
||||
for _, cls in inspect.getmembers(mod, inspect.isclass):
|
||||
if issubclass(cls, BaseTokenToKVPoolAllocator):
|
||||
found[cls.__name__] = cls
|
||||
return found
|
||||
|
||||
def test_read_translate_implies_transfer_translate(self):
|
||||
classes = self._allocator_classes()
|
||||
# Guard the guard: an import failure that empties this set would make
|
||||
# the assertion below vacuous.
|
||||
self.assertIn("SWATokenToKVPoolAllocator", classes)
|
||||
for name, cls in sorted(classes.items()):
|
||||
if not hasattr(cls, "translate_loc_from_full_to_swa"):
|
||||
continue
|
||||
with self.subTest(allocator=name):
|
||||
self.assertTrue(
|
||||
hasattr(cls, "translate_swa_indices_for_transfer"),
|
||||
f"{name} translates full->SWA for reads but cannot answer "
|
||||
"translate_swa_indices_for_transfer; PD's _swa_payload "
|
||||
"calls it on whatever allocator the scheduler holds",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -18,9 +18,11 @@ import unittest
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.page_major import (
|
||||
build_mha_views,
|
||||
build_mla_views,
|
||||
build_page_major_mamba_views,
|
||||
mamba_entry_bytes,
|
||||
mha_entry_bytes,
|
||||
mla_entry_bytes,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -74,6 +76,118 @@ class TestMLAEnvelopeTransferAddressing(CustomTestCase):
|
||||
self.assertTrue(torch.equal(got, val), (page, layer, off))
|
||||
|
||||
|
||||
class TestMHAEnvelopeTransferAddressing(CustomTestCase):
|
||||
"""The MHA counterpart of the MLA case above.
|
||||
|
||||
An MHA page envelope holds ``2 * layer_num`` row-blocks (layer l's K at
|
||||
block 2l, its V at 2l+1). PD ships that whole envelope as one item, so a
|
||||
row written through ANY per-layer view must land inside its own page's
|
||||
``page_envelope_bytes`` block -- otherwise the transfer would carry a
|
||||
page's K but another page's V and every kernel would still read fine
|
||||
locally.
|
||||
"""
|
||||
|
||||
def test_page_envelope_matches_per_layer_views(self):
|
||||
layer_num, page_size, head_num, head_dim, num_pages = 3, 4, 2, 8, 6
|
||||
store_dtype = torch.bfloat16
|
||||
entry_bytes = mha_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=head_dim,
|
||||
itemsize=store_dtype.itemsize,
|
||||
)
|
||||
page_bytes = page_size * entry_bytes
|
||||
row_bytes = head_num * head_dim * store_dtype.itemsize
|
||||
self.assertEqual(page_bytes, page_size * 2 * layer_num * row_bytes)
|
||||
|
||||
# One page envelope of tail pad, as UnifiedKVPool allocates for MHA.
|
||||
raw = torch.zeros((num_pages + 1) * page_bytes, dtype=torch.uint8)
|
||||
k_views, v_views = build_mha_views(
|
||||
raw,
|
||||
layer_num=layer_num,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=head_dim,
|
||||
store_dtype=store_dtype,
|
||||
page_size=page_size,
|
||||
num_pages=num_pages,
|
||||
anchor_bytes=0,
|
||||
)
|
||||
|
||||
blocks = 2 * layer_num
|
||||
for page in range(num_pages):
|
||||
for layer in range(layer_num):
|
||||
for is_v, views in ((0, k_views), (1, v_views)):
|
||||
for pos in range(page_size):
|
||||
row = page * blocks * page_size + pos
|
||||
views[layer][row].fill_(1)
|
||||
(nz,) = torch.nonzero(raw, as_tuple=True)
|
||||
lo, hi = int(nz.min()), int(nz.max())
|
||||
self.assertGreaterEqual(
|
||||
lo,
|
||||
page * page_bytes,
|
||||
f"page={page} layer={layer} v={is_v} pos={pos} "
|
||||
"wrote below its page envelope",
|
||||
)
|
||||
self.assertLess(
|
||||
hi,
|
||||
(page + 1) * page_bytes,
|
||||
f"page={page} layer={layer} v={is_v} pos={pos} "
|
||||
"wrote past its page envelope",
|
||||
)
|
||||
views[layer][row].zero_()
|
||||
|
||||
def test_envelope_move_is_a_whole_page_copy(self):
|
||||
"""Relocating a page envelope must move every layer's K and V with it;
|
||||
this is what `UnifiedMHATokenToKVPool.move_kv_cache` relies on and what
|
||||
makes a physical page id a valid PD transfer index after compaction."""
|
||||
layer_num, page_size, head_num, head_dim, num_pages = 2, 2, 1, 4, 4
|
||||
store_dtype = torch.bfloat16
|
||||
entry_bytes = mha_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=head_dim,
|
||||
itemsize=store_dtype.itemsize,
|
||||
)
|
||||
page_bytes = page_size * entry_bytes
|
||||
raw = torch.zeros((num_pages + 1) * page_bytes, dtype=torch.uint8)
|
||||
k_views, v_views = build_mha_views(
|
||||
raw,
|
||||
layer_num=layer_num,
|
||||
head_num=head_num,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=head_dim,
|
||||
store_dtype=store_dtype,
|
||||
page_size=page_size,
|
||||
num_pages=num_pages,
|
||||
anchor_bytes=0,
|
||||
)
|
||||
blocks = 2 * layer_num
|
||||
# Distinct content in source page 1, every layer, K and V.
|
||||
for layer in range(layer_num):
|
||||
for pos in range(page_size):
|
||||
row = 1 * blocks * page_size + pos
|
||||
k_views[layer][row].fill_(layer + 1)
|
||||
v_views[layer][row].fill_(-(layer + 1))
|
||||
|
||||
env = raw[: num_pages * page_bytes].view(num_pages, page_bytes)
|
||||
env[3] = env[1]
|
||||
|
||||
for layer in range(layer_num):
|
||||
for pos in range(page_size):
|
||||
row = 3 * blocks * page_size + pos
|
||||
self.assertTrue(
|
||||
torch.all(k_views[layer][row] == layer + 1),
|
||||
f"K layer {layer} did not ride the envelope move",
|
||||
)
|
||||
self.assertTrue(
|
||||
torch.all(v_views[layer][row] == -(layer + 1)),
|
||||
f"V layer {layer} did not ride the envelope move",
|
||||
)
|
||||
|
||||
|
||||
class TestMambaEnvelopeTransferAddressing(CustomTestCase):
|
||||
def test_slot_envelope_is_self_contained(self):
|
||||
"""A slot's conv+temporal state for all layers must live exactly in
|
||||
|
||||
@@ -345,11 +345,12 @@ class TestUnifiedMHATokenToKVPool(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_transfer_entry_points_fail_loud(self):
|
||||
"""PD / CPU-copy entry points assume per-layer buffers indexed by TOKEN
|
||||
id and would silently mis-index the row space, so each must raise."""
|
||||
"""The entry points that assume per-layer buffers indexed by TOKEN id
|
||||
would silently mis-index against the row space (or hit a missing-attr
|
||||
AttributeError), so each must raise. `get_contiguous_buf_infos` is NOT
|
||||
among them: PD addresses this pool as whole page envelopes, pinned by
|
||||
`test_pd_registration_is_one_whole_envelope` below."""
|
||||
_, pool = _make_pool_and_kv(1)
|
||||
with self.assertRaises(NotImplementedError):
|
||||
pool.get_contiguous_buf_infos()
|
||||
with self.assertRaises(NotImplementedError):
|
||||
pool.get_cpu_copy(torch.tensor([1]))
|
||||
with self.assertRaises(NotImplementedError):
|
||||
@@ -357,6 +358,24 @@ class TestUnifiedMHATokenToKVPool(unittest.TestCase):
|
||||
with self.assertRaises(NotImplementedError):
|
||||
pool.set_kv_buffer_prefix_valid()
|
||||
|
||||
def test_pd_registration_is_one_whole_envelope(self):
|
||||
"""PD registers ONE region -- the whole raw buffer -- with the page
|
||||
envelope as the item, so the transfer engine addresses it as
|
||||
`raw_ptr + physical_page * page_envelope_bytes`. Per-layer regions
|
||||
would be wrong here: the per-layer views overlap inside the envelope
|
||||
and index in kernel-facing ids, not token ids."""
|
||||
kv, pool = _make_pool_and_kv(1)
|
||||
ptrs, lens, item_lens = pool.get_contiguous_buf_infos()
|
||||
self.assertEqual(len(ptrs), 1)
|
||||
self.assertEqual(len(lens), 1)
|
||||
self.assertEqual(len(item_lens), 1)
|
||||
self.assertEqual(ptrs[0], kv._raw.data_ptr())
|
||||
self.assertEqual(lens[0], kv._raw.numel())
|
||||
self.assertEqual(item_lens[0], pool._page_bytes)
|
||||
# The whole addressable page range must fit the registered region, or
|
||||
# the last page's write would run off the end of the RDMA mapping.
|
||||
self.assertLessEqual(pool._num_pages * item_lens[0], lens[0])
|
||||
|
||||
def test_hnd_env_cannot_hijack_layout(self):
|
||||
"""SGLANG_USE_HND_KVCACHE must not flip this pool's layout: HND indexes
|
||||
4-D while the per-layer views are 3-D, so the pinned label has to win."""
|
||||
|
||||
Reference in New Issue
Block a user