[unified-memory] Fuse the draft model's KV into the target's page envelope (mechanism) (#37628)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-08 00:54:19 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent 08d30f2300
commit 8b80808366
28 changed files with 2506 additions and 57 deletions
@@ -78,6 +78,17 @@ def assign_draft_cache_locs_contiguous(
)
@triton.jit
def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
"""Virtual token ids -> physical ones through the page-level v2p table,
as `translate_kv_loc` does. A negative id maps to the page-0 sink instead
of loading before `v2p`."""
i64 = ids.to(tl.int64)
vpage = tl.where(i64 < 0, 0, i64 // page_size)
phys = tl.load(v2p + vpage, mask=mask, other=0)
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
@triton.jit
def generate_draft_decode_kv_indices(
req_pool_indices,
@@ -86,6 +97,7 @@ def generate_draft_decode_kv_indices(
kv_indices,
kv_indptr,
positions,
v2p,
pool_len: tl.constexpr,
kv_indices_stride: tl.constexpr,
kv_indptr_stride: tl.constexpr,
@@ -96,6 +108,7 @@ def generate_draft_decode_kv_indices(
window_size: tl.constexpr = 0,
sink_size: tl.constexpr = 0,
NUM_STEPS: tl.constexpr = 0,
TRANSLATE: tl.constexpr = False,
):
# window_size > 0 restricts the draft (not the target) to sink_size prefix
# tokens + the most-recent window_size; window_size == 0 is the identity.
@@ -153,6 +166,8 @@ def generate_draft_decode_kv_indices(
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
tl.store(kv_ptr + copy_offset, data, mask=mask)
copy_offset += BLOCK_SIZE
else:
@@ -165,6 +180,8 @@ def generate_draft_decode_kv_indices(
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
tl.store(kv_ptr + copy_offset, data, mask=mask)
# Extension entries and kv_indptr belong to token block 0 alone; other
@@ -196,6 +213,11 @@ def generate_draft_decode_kv_indices(
mask=extend_offset < iters,
)
if TRANSLATE:
extend_data = _translate_token_ids(
extend_data, v2p, extend_offset < iters, page_size
)
tl.store(
kv_ptr + seq_len_w + extend_offset,
extend_data,
@@ -4647,6 +4647,7 @@ class AiterMultiStepDraftBackend:
self.req_to_token_pool = model_runner.req_to_token_pool
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
self.page_size = get_schedule().page_size
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int
@@ -4662,6 +4663,7 @@ class AiterMultiStepDraftBackend:
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
else 1
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
](
@@ -4671,6 +4673,7 @@ class AiterMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -4681,6 +4684,7 @@ class AiterMultiStepDraftBackend:
# A single token block is the historical launch; NUM_STEPS=0 keeps
# its 128-wide program instead of the token-block specialization.
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
TRANSLATE=v2p is not None,
)
for i in range(self.speculative_num_steps - 1):
@@ -3396,6 +3396,7 @@ class FlashAttentionMultiStepBackend:
self.model_runner = model_runner
self.topk = topk
self.speculative_num_steps = speculative_num_steps
self.kv_index_translator = model_runner.kv_index_translator
self.attn_backends = []
for i in range(self.speculative_num_steps - 1):
self.attn_backends.append(
@@ -2369,6 +2369,7 @@ class FlashInferMultiStepDraftBackend:
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
model_runner
)
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -2391,6 +2392,7 @@ class FlashInferMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -2400,6 +2402,7 @@ class FlashInferMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -2409,6 +2412,7 @@ class FlashInferMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -1218,6 +1218,7 @@ class FlashInferMLAMultiStepDraftBackend:
self.req_to_token_pool = model_runner.req_to_token_pool
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
self.page_size = get_schedule().page_size
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -1240,6 +1241,7 @@ class FlashInferMLAMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -1249,6 +1251,7 @@ class FlashInferMLAMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -1256,6 +1259,7 @@ class FlashInferMLAMultiStepDraftBackend:
next_power_of_2(self.speculative_num_steps),
next_power_of_2(bs),
self.page_size,
TRANSLATE=v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -612,6 +612,7 @@ class FlashMLAMultiStepDraftBackend:
)
self.topk = topk
self.speculative_num_steps = speculative_num_steps
self.kv_index_translator = model_runner.kv_index_translator
max_bs = model_runner.req_to_token_pool.size * self.topk
self.kv_indptr = torch.zeros(
(
@@ -2449,6 +2449,7 @@ class TritonMultiStepDraftBackend:
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
model_runner
)
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -2467,6 +2468,7 @@ class TritonMultiStepDraftBackend:
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
seq_lens_sum = num_seqs * self.max_context_len
v2p = self.kv_index_translator.full_flat_v2p()
generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -2476,6 +2478,7 @@ class TritonMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -2485,6 +2488,7 @@ class TritonMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
)
if call_fn is None:
@@ -570,16 +570,45 @@ class KVCacheConfigurator:
UnifiedSWAAllocatorBase,
),
):
draft_virtual_id_space = (
token_to_kv_pool_allocator.draft_virtual_id_space
)
alloc = token_to_kv_pool_allocator
placement = self._fused_draft_from_target_buffer(alloc)
if placement is not None:
# FUSED arm
from sglang.srt.mem_cache.layout.fused_draft import (
draft_swa_layer_ids,
)
from sglang.srt.mem_cache.unified_draft_pool import (
bind_fused_draft,
draft_kv_layer_ids,
)
assert req_to_token_pool is not None, (
"a draft worker shares the target's req_to_token_pool"
)
draft_pool = bind_fused_draft(
unified_buffer=alloc.unified_buffer,
host_allocator=alloc,
placement=placement,
runner=self.draft_model_idx or 0,
kv_layer_ids=draft_kv_layer_ids(self.model),
swa_layer_ids=draft_swa_layer_ids(self.model_config),
page_size=self.page_size,
)
return _InitializedPools(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool=draft_pool,
token_to_kv_pool_allocator=alloc,
unified_memory_pool=None,
)
# PRIVATE arm
draft_virtual_id_space = alloc.draft_virtual_id_space
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
"unified allocator virtual space smaller than the token "
f"budget: virtual_id_space={draft_virtual_id_space} < "
f"max_total_num_tokens={sizes.max_total_num_tokens}"
)
# Round UP to page alignment (paged draft backends view the
# pool as (-1, page_size, H, D); the virtual space is not aligned).
# pool as (-1, page_size, H, D); the space is not aligned).
page = max(int(self.pool_page_size or 1), 1)
draft_virtual_id_space = (
(draft_virtual_id_space + page - 1) // page * page
@@ -684,6 +713,33 @@ class KVCacheConfigurator:
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
)
def _fused_draft_from_target_buffer(self, alloc):
"""The placement this draft binds to, or None for the private-pool
fallback (the draft then owns a raw-virtual-indexed pool of its own).
Target boot declines a placement for legitimate geometry, so the
private arm is the rollback lever, not a boot-order bug."""
if not self.spec_algorithm.is_eagle():
return None
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
placement = alloc.unified_buffer.fused_draft
if placement is None:
logger.info(
"[unified-memory-pool] no fused draft placement on the target's "
"buffer; the draft binds a private pool over the virtual id space."
)
return None
# The region stores rows in the target's KV dtype; a draft that resolved
# its own would read and write them as something else.
if _store_dtype_for(self.kv_cache_dtype) != placement.region.store_dtype:
raise ValueError(
f"Fused draft KV: the draft resolved its KV cache dtype to "
f"{self.kv_cache_dtype}, but its region inside the target's pages "
f"stores {placement.region.store_dtype}. Set "
"--speculative-draft-kv-cache-dtype to the target's KV cache dtype."
)
return placement
def _init_unified_mamba_pools(
self,
*,
@@ -879,6 +935,127 @@ class KVCacheConfigurator:
),
)
def _fused_draft_decision(self):
"""Whether, and where, the EAGLE draft's layers fuse into the target's
sub-pools. Fusion applies only for: unified memory ON, a two-pool
hybrid-SWA target, an EAGLE-family algorithm whose draft config was
loaded at target boot and has no recurrent state of its own;
`place_fused_draft` then admits or declines the draft's layer kinds."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
draft_kv_profile,
place_fused_draft,
)
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
aux = self.spec_aux_config
if not (
get_memory().enable_unified_memory
and self.is_hybrid_swa
and self.mambaish_config is None
and not self.is_draft_worker
and self.spec_algorithm.is_eagle()
and aux.draft_kv_num_layers
and aux.draft_model_config is not None
and mambaish_config(aux.draft_model_config) is None
):
return FusedDraftDecision()
profile = draft_kv_profile(
aux.draft_model_config,
num_layers=int(aux.draft_kv_num_layers),
attn_tp_size=get_parallel().attn_tp_size,
)
num_runners = (
int(get_spec().speculative_num_steps)
if self.model_config.is_multi_layer_eagle
else 1
)
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves without the draft model, so a mismatch is
# declined here; `auto` follows the draft's quant config, which only
# the draft runner knows, and is checked when it binds.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
_, draft_kv_dtype = configure_kv_cache_dtype(
server_args_kv_cache_dtype=draft_kv_cache_dtype,
model=None,
model_dtype=aux.draft_model_config.dtype,
is_draft_worker=True,
is_dflash=False,
speculative_draft_attention_backend=None,
speculative_draft_kv_cache_dtype=draft_kv_cache_dtype,
)
if draft_kv_dtype != self.kv_cache_dtype:
return FusedDraftDecision(
declined=(
f"the draft's KV cache dtype ({draft_kv_dtype}) differs "
f"from the host's ({self.kv_cache_dtype})"
)
)
return place_fused_draft(
profile=profile,
num_runners=num_runners,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
)
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
pad) for the boot solve's cell model. Single source of truth: assembled
through the same spec the pool factory builds, so the priced entry and
the allocated entry cannot drift. None when the draft does not fuse
into that sub-pool."""
placement = self._fused_draft_decision().placement
if placement is None or sub_pool_name != "full":
return None
return self._full_host_spec(placement.region).entry_bytes()
def _full_host_spec(self, region):
from sglang.srt.mem_cache.unified_memory_pool import (
MHASubPoolSpec,
_store_dtype_for,
)
return MHASubPoolSpec(
name="full",
layer_num=len(self.layer_info.full_attention_layer_ids),
head_num=self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
grow_direction="down",
draft_region=region,
)
def _fused_draft_for_pool_factory(self):
"""Resolve ONCE per factory call (not inline) so the boot log reports
exactly the placement the factory is handed: a declined fusion and an
engaged one otherwise look identical from outside."""
decision = self._fused_draft_decision()
if decision.placement is None:
if decision.declined is not None:
logger.warning("fused draft KV disabled: %s", decision.declined)
return None
placement = decision.placement
region = placement.region
logger.info(
"[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d "
"kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s",
region.lane_num,
region.head_num,
region.head_dim,
region.resolved_v_head_dim(),
region.store_dtype,
region.entry_bytes(),
[
tuple(placement.lanes_for(r))
for r in range(len(placement.runner_lane_counts))
],
)
return placement
def _init_unified_swa_pools(
self,
*,
@@ -964,6 +1141,7 @@ class KVCacheConfigurator:
# charged, see `_check_bs1_feasibility_floor`.
model_context_len=self.model_config.context_len,
sliding_window_size=self.model_config.sliding_window_size,
fused_draft=self._fused_draft_for_pool_factory(),
)
return UnifiedPoolBundle(
unified_memory_pool=bundle.unified_memory_pool,
@@ -71,6 +71,7 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
)
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
from sglang.srt.runtime_context import get_parallel
@@ -112,13 +113,18 @@ class KVIndexTranslator:
self.page_size = page_size
self.device = device
self.is_translating = (
is_unified_target = (
isinstance(
token_to_kv_pool_allocator,
(UnifiedMambaTokenToKVPoolAllocator, UnifiedSWAAllocatorBase),
)
and token_to_kv_pool_allocator.get_kvcache() is token_to_kv_pool
)
host_allocator = fused_draft_host_allocator(token_to_kv_pool)
is_fused_draft = (
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
)
self.is_translating = is_unified_target or is_fused_draft
if self.is_translating:
alloc = token_to_kv_pool_allocator
self._capture_page_size = alloc.page_size
@@ -133,13 +139,17 @@ class KVIndexTranslator:
# DCP read ids stay WIDENED to the consumer: selecting this rank's
# share changes the length, so only the production site can do it.
self.defer_read_translate = get_parallel().attn_dcp_size > 1
if isinstance(alloc, UnifiedSWAAllocatorBase):
routes_window_layers = is_unified_target or isinstance(
token_to_kv_pool, BaseSWAKVPool
)
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
self._swa_v2p_table = alloc.swa_v2p_page_table
self._swa_write_loc_from_full = self._swa_write_loc_unified
else:
self._swa_v2p_table = None
self._swa_write_loc_from_full = None
else:
self._capture_page_size = page_size
self._full_v2p_table = None
self._full_p2v_table = None
self._translate_full = None
@@ -175,6 +185,14 @@ class KVIndexTranslator:
return self._full_v2p_table.numel() * self._capture_page_size
return max_token_pool_size + self.page_size
def full_flat_v2p(self) -> Optional[torch.Tensor]:
"""The full-side v2p page table for a kernel that translates flat ids
itself, or ``None`` when this runner does not translate (pass-through
and static pools)."""
if not self.is_translating:
return None
return self._full_v2p_table
# -- per-batch view --------------------------------------------------------
@property
@@ -0,0 +1,262 @@
"""Draft-model regions nested inside unified sub-pools' slot entries.
The nesting, outermost first:
sub-pool a named pool inside the unified pool, with its own grow direction
and frontier. HOST is the role it plays when its entries also
carry draft bytes; only "full" hosts a draft here.
slot one token entry of that sub-pool. The draft's KV becomes extra
parts of every slot, after the host's own parts at
`draft_offset_in_entry()`, so ONE slot id per token covers target
and draft: one allocation, one free, one whole-page relocation
carry both.
region the draft geometry inside the host's entries. A region is not a
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
lane one position in the region's layer dimension -- one draft layer of
one runner, present in every slot.
Runners are the draft's execution copies, and the two head shapes place them
oppositely. A replicated head gives EVERY runner all the draft's layers, so
each needs its own lanes or the runners clobber each other's KV (3 runners x
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
either direction. Each runner owns a contiguous run of lanes, and the runs
tile the region in runner order.
Three phases, three moments in boot, and what carries each:
PLAN on the TARGET, before any memory exists
KVCacheConfigurator._fused_draft_decision()
draft_kv_profile(draft_model_config) what the checkpoint asks for
place_fused_draft(profile, runners) admit, or decline with a reason
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
CARVE as the byte buffer is cut
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
init_unified_*_pools(fused_draft=...)
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
UnifiedKVPool.build_dense_draft_views()
BIND on each DRAFT runner, built after the target
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
bind_fused_draft(...) -> UnifiedDraftKVPool
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
A draft whose layers need another sub-pool kind declines to a private pool.
"""
from typing import List, Optional, Tuple
import msgspec
import torch
from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
sub-pool. The draft is a separate checkpoint, so its head geometry and
layer count differ from the host's; its K and V rows are two more parts of
the host entry, indexed by the host's physical token id."""
lane_num: int
head_num: int
head_dim: int
store_dtype: torch.dtype
v_head_dim: Optional[int] = None
def validate(self) -> None:
assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}"
assert self.head_num > 0, f"head_num must be positive; got {self.head_num}"
assert self.head_dim > 0, f"head_dim must be positive; got {self.head_dim}"
v = self.resolved_v_head_dim()
assert v > 0, f"v_head_dim must be positive; got {v}"
def resolved_v_head_dim(self) -> int:
return self.head_dim if self.v_head_dim is None else self.v_head_dim
def k_row_bytes(self) -> int:
return self.head_num * self.head_dim * self.store_dtype.itemsize
def v_row_bytes(self) -> int:
return self.head_num * self.resolved_v_head_dim() * self.store_dtype.itemsize
def entry_bytes(self) -> int:
"""Draft bytes per slot, before the host entry's alignment."""
return self.lane_num * (self.k_row_bytes() + self.v_row_bytes())
def parts(self, offset_bytes: int) -> Tuple[DensePart, DensePart]:
"""The draft's K and V parts, laid out from ``offset_bytes`` inside
the host entry."""
layer_stride = self.k_row_bytes() + self.v_row_bytes()
return (
DensePart(
name="draft_k",
offset_bytes=offset_bytes,
layer_stride_bytes=layer_stride,
layer_num=self.lane_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="draft_v",
offset_bytes=offset_bytes + self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.lane_num,
row_shape=(self.head_num, self.resolved_v_head_dim()),
dtype=self.store_dtype,
),
)
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
"""Where every draft runner's layers live inside the "full" sub-pool: the
region, and each runner's lane count in runner order. Built once on the
target, stored on the `UnifiedKVPool`, and read back by each draft
runner, so the two sides cannot disagree on a lane.
"""
region: DenseDraftRegion
runner_lane_counts: Tuple[int, ...]
def __post_init__(self):
assert self.runner_lane_counts, "a placement needs at least one draft runner"
assert sum(self.runner_lane_counts) == self.region.lane_num, (
f"runner lane counts {self.runner_lane_counts} must tile "
f"range({self.region.lane_num})"
)
def lanes_for(self, runner: int) -> range:
start = sum(self.runner_lane_counts[:runner])
return range(start, start + self.runner_lane_counts[runner])
class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
"""Per-GPU K/V row geometry of one kind of draft attention layer."""
head_num: int
head_dim: int
v_head_dim: int
class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True):
"""What the draft checkpoint asks of the host sub-pools.
``num_depths`` > 1 marks a per-depth head (one transformer block per MTP
depth, served by one runner each under multi-layer EAGLE); otherwise
every runner serves all ``num_layers`` layers.
"""
num_layers: int
full: DraftKVGeometry
swa_layer_ids: Tuple[int, ...] = ()
num_depths: int = 1
def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
"""The draft layer ids a hybrid-SWA pool routes to its swa side. This is
the pool-routing convention (`SWAKVPool.layers_mapping`), not a per-layer
kernel window: an attention-only draft window stays full-kind."""
mc = draft_model_config
if mc.is_hybrid_swa and not mc.is_deepseek_v4_arch:
return tuple(int(i) for i in mc.swa_attention_layer_ids)
return ()
def draft_kv_profile(
draft_model_config, *, num_layers: int, attn_tp_size: int
) -> DraftKVProfile:
"""The profile of a draft `ModelConfig`, heads divided by attn_tp the way
the target divides its own (drafts never join the DCP group)."""
mc = draft_model_config
num_depths = mc.num_nextn_predict_layers
return DraftKVProfile(
num_layers=int(num_layers),
full=DraftKVGeometry(
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
head_dim=int(mc.head_dim),
v_head_dim=int(mc.v_head_dim),
),
swa_layer_ids=draft_swa_layer_ids(mc),
num_depths=1 if num_depths is None else int(num_depths),
)
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
private pool. Neither means fusion simply does not apply."""
placement: Optional[FusedDraftPlacement] = None
declined: Optional[str] = None
def _runner_layer_counts(
profile: DraftKVProfile, num_runners: int
) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]:
"""Per runner, its (full, swa) layer counts; or why no runner layout exists."""
if profile.num_depths <= 1:
num_swa = len(profile.swa_layer_ids)
return [(profile.num_layers - num_swa, num_swa)] * num_runners, None
if num_runners == 1:
return None, (
f"a per-depth draft head ({profile.num_depths} depths) needs one "
"runner per depth (multi-layer EAGLE)"
)
if num_runners > profile.num_depths:
return None, (
f"{num_runners} draft runners exceed the head's {profile.num_depths} depths"
)
swa = set(profile.swa_layer_ids)
return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None
def place_fused_draft(
*,
profile: DraftKVProfile,
num_runners: int,
store_dtype: torch.dtype,
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host arm serves declines the whole draft to
its private pool."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
num_swa = sum(swa for _, swa in counts)
if num_swa:
return FusedDraftDecision(
declined=(
f"the draft has {num_swa} SWA layer(s) of its own, which the "
"fused dense pool cannot serve"
)
)
geometry = profile.full
if geometry.head_dim != geometry.v_head_dim:
return FusedDraftDecision(
declined=(
"the draft's K/V rows are asymmetric "
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), "
"which is not admitted yet"
)
)
row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return FusedDraftDecision(
declined=(
f"the draft's K/V rows are {row_bytes} B, not a multiple of the "
f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
)
)
full_counts = tuple(full for full, _ in counts)
region = DenseDraftRegion(
lane_num=sum(full_counts),
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
store_dtype=store_dtype,
)
return FusedDraftDecision(
placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts)
)
@@ -43,6 +43,11 @@ def align_entry_bytes(num_bytes: int) -> int:
return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES
def align_part_offset(num_bytes: int) -> int:
"""Round a part's byte offset inside the entry up to the row alignment."""
return -(-num_bytes // ROW_ALIGN_BYTES) * ROW_ALIGN_BYTES
class DensePart(msgspec.Struct, frozen=True, kw_only=True):
"""One row family inside the entry: ``layer_num`` rows of ``row_shape``,
layer ``l`` at ``offset_bytes + l * layer_stride_bytes``."""
@@ -0,0 +1,159 @@
"""KV pools a DRAFT runner binds over the draft lanes fused into the target's
unified pool: same pages, same slot ids, same v2p table as the target -- one
allocation, one free, one relocation."""
from typing import Any, Dict, List, Mapping, Optional, Sequence
import torch
from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
class UnifiedDraftKVPool(MHATokenToKVPool):
"""Dense draft KV over the draft parts of one host sub-pool's entries.
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
``layer_lanes`` maps each of this runner's layer ids to its region lane.
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
produced by the allocator's translate (the id-space choke point binds it,
see KVIndexTranslator); the pool exposes `host_allocator` for that
binding. Relocation needs no method here: compaction moves whole page
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
raises so a stray per-slot move fails loudly instead of corrupting the
fused layout.
"""
requires_physical_write_loc = True
def __init__(
self,
*,
unified_buffer: UnifiedKVPool,
host_sub_pool_name: str,
host_allocator,
layer_lanes: Mapping[int, int],
page_size: int = 1,
):
region = unified_buffer.require_draft_host_spec(host_sub_pool_name).draft_region
layer_ids = sorted(layer_lanes)
assert layer_ids, "UnifiedDraftKVPool binds at least one layer"
start_layer = layer_ids[0]
assert layer_ids == list(range(start_layer, start_layer + len(layer_ids))), (
f"fused draft layer ids must be contiguous; got {layer_ids}"
)
lanes = [layer_lanes[layer_id] for layer_id in layer_ids]
assert len(set(lanes)) == len(lanes) and all(
0 <= s < region.lane_num for s in lanes
), f"draft lanes {lanes} must be distinct and within range({region.lane_num})"
k_views, v_views = unified_buffer.build_dense_draft_views(host_sub_pool_name)
max_slots = unified_buffer.max_slots(host_sub_pool_name)
self._unified_buffer = unified_buffer
self._host_sub_pool_name = host_sub_pool_name
self.host_allocator = host_allocator
self.layer_lanes: Dict[int, int] = dict(layer_lanes)
self._k_views: List[torch.Tensor] = [k_views[s] for s in lanes]
self._v_views: List[torch.Tensor] = [v_views[s] for s in lanes]
num_pages = max_slots // page_size
super().__init__(
size=num_pages * page_size - page_size,
page_size=page_size,
dtype=region.store_dtype,
head_num=region.head_num,
head_dim=region.head_dim,
layer_num=len(layer_ids),
device=unified_buffer.device,
enable_memory_saver=False, # buffer owned by UnifiedKVPool
v_head_dim=region.resolved_v_head_dim(),
start_layer=start_layer,
end_layer=start_layer + len(layer_ids) - 1,
enable_kv_cache_copy=False,
kv_cache_layout="page_major",
)
def _create_buffers(self):
self.k_buffer = self._k_views
self.v_buffer = self._v_views
def _clear_buffers(self):
pass # lifetime owned by UnifiedKVPool
def get_kv_size_bytes(self):
return 0, 0 # fused into the host entries; UnifiedKVPool logs the total
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
raise NotImplementedError(
"fused draft KV relocates with the HOST pool's whole-page move; a "
"draft-side per-slot move would corrupt the fused layout"
)
def get_contiguous_buf_infos(self):
raise NotImplementedError(
"fused draft KV has no per-layer contiguous regions; KV transfer / "
"disaggregation is unsupported."
)
def get_cpu_copy(self, indices, mamba_indices=None):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]:
"""The host allocator a fused draft pool translates through, or None for
any other pool."""
if isinstance(token_to_kv_pool, UnifiedDraftKVPool):
return token_to_kv_pool.host_allocator
return None
def draft_kv_layer_ids(model) -> List[int]:
"""Layer ids owning attention KV in the BUILT draft model, in layer order.
Window layers count -- a window is a parameter of the attention layer;
linear and recurrent layers have no attention module and are absent."""
from sglang.srt.layers.radix_attention import RadixAttention
return sorted(
{m.layer_id for m in model.modules() if isinstance(m, RadixAttention)}
)
def bind_fused_draft(
*,
unified_buffer: UnifiedKVPool,
host_allocator,
placement: FusedDraftPlacement,
runner: int,
kv_layer_ids: Sequence[int],
swa_layer_ids: Sequence[int],
page_size: int,
) -> UnifiedDraftKVPool:
"""The KV pool draft runner ``runner`` binds over its fused slots.
The placement sized the lanes from the draft config; the model's real
layer ids fill them in layer order, so a count mismatch is a loud boot
failure, never a silent alias.
"""
swa = set(swa_layer_ids)
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa]
assert len(full_ids) == len(kv_layer_ids), (
f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused "
"SWA KV is not supported yet"
)
full_lanes = placement.lanes_for(runner)
assert len(full_ids) == len(full_lanes), (
f"draft runner {runner}: {len(full_ids)} full-attention layer(s) "
f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}"
)
return UnifiedDraftKVPool(
unified_buffer=unified_buffer,
host_sub_pool_name="full",
host_allocator=host_allocator,
layer_lanes=dict(zip(full_ids, full_lanes)),
page_size=page_size,
)
@@ -38,11 +38,16 @@ from sglang.kernels.ops.kvcache.zero_pages import zero_pages
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.layout.token_major import (
ROW_ALIGN_BYTES,
DenseEntryLayout,
DensePart,
align_entry_bytes,
align_part_offset,
build_dense_views,
build_mamba_entry_views,
)
@@ -113,6 +118,8 @@ class SubPoolSpec(ABC):
name: str
layer_num: int
grow_direction: str # "up" | "down" | "float"
# Fused draft region riding in this sub-pool's entries; None = unfused.
draft_region: Optional[DenseDraftRegion] = None
def __post_init__(self):
assert self.grow_direction in self._allowed_grow_directions, (
@@ -134,7 +141,17 @@ class SubPoolSpec(ABC):
@dataclass(frozen=True, kw_only=True)
class MHASubPoolSpec(SubPoolSpec):
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`."""
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`.
With `draft_region` set, the draft model's K and V rows are two more
parts of every slot's entry, after the host parts:
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
The slot stride stays the one entry, so host and draft views are indexed
by the same physical token id; only their offsets differ. `draft_region
is None` keeps the layout byte-identical to the unfused one.
"""
head_num: int
head_dim: int
@@ -159,6 +176,8 @@ class MHASubPoolSpec(SubPoolSpec):
assert self.v_head_dim > 0, (
f"v_head_dim must be positive; got {self.v_head_dim}"
)
if self.draft_region is not None:
self.draft_region.validate()
def k_row_bytes(self) -> int:
return self.head_num * self.head_dim * self.store_dtype.itemsize
@@ -166,9 +185,20 @@ class MHASubPoolSpec(SubPoolSpec):
def v_row_bytes(self) -> int:
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
def host_entry_bytes(self) -> int:
"""Host (target-only) bytes for one slot, before any draft parts."""
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
def draft_offset_in_entry(self) -> int:
"""Byte offset of the fused draft parts inside one slot's entry."""
assert self.draft_region is not None
return align_part_offset(self.host_entry_bytes())
def entry_bytes(self) -> int:
if self.draft_region is None:
return align_entry_bytes(self.host_entry_bytes())
return align_entry_bytes(
self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
)
def page_bytes(self, page_size: int) -> int:
@@ -179,27 +209,27 @@ class MHASubPoolSpec(SubPoolSpec):
is ``page_size`` such entries back to back."""
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
layer_stride = self.k_row_bytes() + self.v_row_bytes()
return DenseEntryLayout(
entry_bytes=self.entry_bytes(),
parts=(
DensePart(
name="k",
offset_bytes=0,
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="v",
offset_bytes=self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.v_head_dim),
dtype=self.store_dtype,
),
parts = (
DensePart(
name="k",
offset_bytes=0,
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="v",
offset_bytes=self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.v_head_dim),
dtype=self.store_dtype,
),
)
if self.draft_region is not None:
parts += self.draft_region.parts(self.draft_offset_in_entry())
return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts)
def get_dtype(self) -> torch.dtype:
return self.store_dtype
@@ -227,6 +257,9 @@ class MLASubPoolSpec(SubPoolSpec):
assert self.qk_rope_head_dim > 0, (
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
)
assert self.draft_region is None, (
"MLA sub-pools do not carry a fused draft region yet"
)
@property
def kv_cache_dim(self) -> int:
@@ -271,6 +304,9 @@ class MambaSubPoolSpec(SubPoolSpec):
def __post_init__(self):
super().__post_init__()
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
assert self.draft_region is None, (
"mamba state pages carry no fused draft region yet"
)
def conv_row_bytes(self, idx: int) -> int:
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
@@ -350,6 +386,7 @@ class UnifiedKVPool:
page_size: int = 1,
post_capture_active: bool = False,
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
fused_draft: Optional[FusedDraftPlacement] = None,
):
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
assert len(sub_pool_specs) >= 2, (
@@ -386,6 +423,17 @@ class UnifiedKVPool:
self.post_capture_active = post_capture_active
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
self._bs1_floor_terms = bs1_floor_terms or []
self.fused_draft = fused_draft
for spec in sub_pool_specs:
expected = (
fused_draft.region
if fused_draft is not None and spec.name == "full"
else None
)
assert spec.draft_region is expected, (
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
f"not match the fused draft placement's {expected}"
)
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
@@ -588,6 +636,17 @@ class UnifiedKVPool:
)
return s
def require_draft_host_spec(self, name: str) -> SubPoolSpec:
"""The sub-pool spec whose entries carry a fused draft region.
Kind-agnostic: any spec that resolves a `draft_region` also lays the
draft parts out in its `layout()`."""
s = self._specs_by_name[name]
assert s.draft_region is not None, (
f"sub-pool {name!r} carries no fused draft region"
)
return s
def max_slots(self, name: str) -> int:
return self._max_slots[name]
@@ -630,6 +689,27 @@ class UnifiedKVPool:
)
return k_views, v_views
def build_dense_draft_views(
self, sub_pool_name: str
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""Per-layer K/V views of the DRAFT parts fused into ``sub_pool_name``'s
entries: same pages, same slot ids, same v2p table as the host."""
spec = self.require_draft_host_spec(sub_pool_name)
layout = spec.layout()
page_size = self._page_size
num_slots = self.max_slots(sub_pool_name) // page_size * page_size
k_views, v_views = (
build_dense_views(
self._raw,
layout=layout,
part_name=name,
num_slots=num_slots,
anchor_bytes=self._anchor_bytes[sub_pool_name],
)
for name in ("draft_k", "draft_v")
)
return k_views, v_views
def _build_mla_views(
self,
spec: MLASubPoolSpec,
@@ -1899,8 +1979,15 @@ def init_unified_swa_pools(
lazy_compaction: bool = False,
model_context_len: Optional[int] = None,
sliding_window_size: Optional[int] = None,
fused_draft: Optional[FusedDraftPlacement] = None,
) -> UnifiedSWAPoolBundle:
"""Build the SWA-hybrid unified-memory-pool stack."""
"""Build the SWA-hybrid unified-memory-pool stack.
With ``fused_draft``, every entry of the "full" sub-pool carries the
draft model's K/V parts after the host parts, and each draft runner
binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a
pool of its own.
"""
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
@@ -1926,6 +2013,7 @@ def init_unified_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
)
swa_spec = MHASubPoolSpec(
name="swa",
@@ -1981,6 +2069,7 @@ def init_unified_swa_pools(
page_size=page_size,
post_capture_active=post_capture_active,
bs1_floor_terms=bs1_floor_terms,
fused_draft=fused_draft,
)
token_to_kv_pool = UnifiedSWAKVPool(
unified_buffer=shared_pool,
@@ -44,6 +44,11 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True):
dflash_target_layer_ids: Any = None
# DFLASH draft KV bytes/token; None when unresolved.
dflash_draft_cell_size_per_token: int | None = None
# The draft checkpoint read in draft mode, and its KV layer count: the
# fused-draft placement's input. `eagle_draft_num_layers` above keeps
# sizing a private draft pool exactly as before.
draft_model_config: Optional[ModelConfig] = None
draft_kv_num_layers: Optional[int] = None
def resolve_spec_aux_hidden_state_config(
@@ -85,24 +90,38 @@ def _resolve_eagle_aux_hidden_state(
):
return
draft_model_config = model_config
if get_spec().speculative_draft_model_path:
draft_model_config = ModelConfig.from_server_args(
server_args,
model_path=get_spec().speculative_draft_model_path,
model_revision=get_spec().speculative_draft_model_revision,
is_draft_model=True,
)
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
if num_nextn_predict_layers is not None:
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
elif get_spec().speculative_draft_model_path:
config.eagle_draft_num_layers = int(
draft_path = get_spec().speculative_draft_model_path
# The draft read in draft mode. A path-less NEXTN/MTP head is the TARGET
# checkpoint rewritten to its MTP form, which is where its
# `num_nextn_predict_layers` gets filled in.
draft_model_config = ModelConfig.from_server_args(
server_args,
model_path=draft_path,
model_revision=get_spec().speculative_draft_model_revision,
is_draft_model=True,
)
if draft_model_config.num_nextn_predict_layers is not None:
config.draft_kv_num_layers = int(draft_model_config.num_nextn_predict_layers)
elif draft_path is not None:
config.draft_kv_num_layers = int(
max(
draft_model_config.num_hidden_layers,
draft_model_config.num_attention_layers,
)
)
if config.draft_kv_num_layers is not None:
config.draft_model_config = draft_model_config
# The private draft pool's budget keeps its reading: a path-less run sizes
# the draft off the TARGET config. Reading the draft-mode config here would
# change the token budget of every path-less MTP deployment.
if not draft_path:
draft_model_config = model_config
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
if num_nextn_predict_layers is not None:
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
elif draft_path:
config.eagle_draft_num_layers = config.draft_kv_num_layers
else:
return
@@ -714,9 +714,27 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
self._draft_cell_size = _dflash_draft_cell_size(kvc)
self._fused_full_entry = kvc.fused_entry_bytes("full")
if self._fused_full_entry is not None:
self._draft_full_layers_num = 0
self._draft_swa_layers_num = 0
self._draft_swa_full_layers_num = 0
self._draft_cell_size = 0
self._recompute_cell_size()
def _full_cell_bytes(self) -> int:
"""Bytes per full-side token: the fused entry when the draft is placed
there, else the target's rows plus the private draft's full rows."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * (
self._full_layers_num + self._draft_full_layers_num
)
def _swa_cell_bytes(self) -> int:
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
def _recompute_cell_size(self) -> None:
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
@@ -730,12 +748,9 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
else:
self._cell_size = (
self._full_per_token
* (self._full_layers_num + self._draft_full_layers_num)
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
+ self._swa_full_tokens_ratio
* self._swa_per_token
* (self._swa_layers_num + self._draft_swa_layers_num)
+ self._swa_full_tokens_ratio * self._swa_cell_bytes()
+ self._draft_cell_size
)
@@ -747,9 +762,17 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
+ self._draft_cell_size
)
def _unified_full_bytes_per_token(self) -> int:
"""Bytes one full-side token takes in the unified pool: the fused
entry (host + draft + pad) when the draft lives in it, else the
target's own rows. A private draft pool is priced separately."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * self._full_layers_num
def _unified_pool_bytes(self, full_tokens: int, swa_tokens: int) -> int:
return (
full_tokens * self._full_per_token * self._full_layers_num
full_tokens * self._unified_full_bytes_per_token()
+ swa_tokens * self._swa_per_token * self._swa_layers_num
)
@@ -761,7 +784,7 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
) -> int:
"""Find the largest page-aligned full capacity whose allocations fit."""
draft_bytes_per_token = self._draft_pool_bytes_per_token()
target_full_bytes_per_token = self._full_per_token * self._full_layers_num
target_full_bytes_per_token = self._unified_full_bytes_per_token()
assert target_full_bytes_per_token > 0
def allocation_bytes(full_pages: int) -> int:
@@ -930,19 +953,14 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
) -> MemoryPoolConfig:
# SWA pool sized tightly from the cap; the rest of the budget goes to full.
swa_tokens = ceil_align(self._swa_cap, page_size)
fixed_swa_bytes = (
swa_tokens
* self._swa_per_token
* (self._swa_layers_num + self._draft_swa_layers_num)
)
if self._enable_unified_memory:
fixed_swa_bytes = swa_tokens * self._swa_cell_bytes()
if self._enable_unified_memory and self._draft_pool_bytes_per_token() > 0:
full_tokens = self._max_unified_full_tokens(
available_bytes, page_size, fixed_swa_tokens=swa_tokens
)
else:
full_cell_size = (
self._full_per_token
* (self._full_layers_num + self._draft_full_layers_num)
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
)
full_tokens = (
@@ -180,8 +180,15 @@ class BaseSpecWorker(ABC):
@property
def primary_draft_kv_pool(self) -> Optional[object]:
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
draft_runners = self._draft_model_runners()
return draft_runners[0].token_to_kv_pool if draft_runners else None
if not draft_runners:
return None
pool = draft_runners[0].token_to_kv_pool
if fused_draft_host_allocator(pool) is not None:
return None
return pool
@property
def target_worker(self) -> TpModelWorker:
@@ -389,6 +389,15 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self._configure_qsa_mtp_index_share()
self.tree_mask_mode = default_tree_mask_mode()
# Boot guard: every backend a draft forward reaches must carry its
# runner's translator or the index builders emit virtual ids.
translator = self.draft_runner.kv_index_translator
if translator.is_translating:
backends = [self.draft_attn_backend, self.draft_extend_attn_backend]
if self.draft_attn_backend is not None:
backends += self.draft_attn_backend.attn_backends
translator.bind_and_verify_backends(backends)
def _configure_qsa_mtp_index_share(self) -> None:
"""Reuse the draft-extend QSA selection across the MTP decode steps;
chain speculation only: with topk > 1 decode rows are not request-major."""
@@ -398,6 +398,13 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[
step
].attn_backend = self.draft_extend_attn_backend_list[-1]
# Boot guard: every backend a draft forward reaches must carry
# its runner's translator or the index builders emit virtual ids.
translator = self.draft_runner_list[step].kv_index_translator
if translator.is_translating:
translator.bind_and_verify_backends(
[self.draft_extend_attn_backend_list[-1]]
)
def _capture_cuda_graphs(self):
self.cuda_graph_runner = None
@@ -117,6 +117,7 @@ def _run(io, window_size=0, sink_size=0):
kv_indices,
kv_indptr,
io["positions"],
None, # v2p: no unified-pool translation
io["pool_len"],
kv_indices.shape[1],
kv_indptr.shape[1],
@@ -0,0 +1,172 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`generate_draft_decode_kv_indices` under the unified pool's translation.
BUG REGRESSION. The multi-step draft-decode kernel copied req_to_token values
verbatim into the per-step kv_indices. Under `--enable-unified-memory` those
values are VIRTUAL ids while the fused draft pool is indexed by physical
ids, so with `speculative_num_steps > 1` every draft decode step read the
wrong KV rows -- silently: bad drafts only depress acceptance, verify stays
correct. Pinned here: TRANSLATE=True must emit exactly `translate_kv_loc`'s
physical ids, and TRANSLATE=False must stay byte-identical to the historical
raw copy.
python -m pytest test/registered/kernels/test_generate_draft_decode_kv_indices_translate.py -v
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
_SENTINEL = -7
def _run_kernel(
*,
req_to_token,
seq_lens,
positions,
num_steps,
topk,
page_size,
v2p,
translate,
):
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_kv_indices,
)
from sglang.srt.utils import next_power_of_2
num_seqs = seq_lens.numel()
bs = num_seqs * topk
width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64)
kv_indices = torch.full(
(num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda"
)
kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda")
generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)](
torch.arange(num_seqs, dtype=torch.int64, device="cuda"),
req_to_token,
seq_lens,
kv_indices,
kv_indptr,
positions,
v2p if translate else None,
req_to_token.shape[1],
width,
kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(num_steps),
next_power_of_2(bs),
page_size,
TRANSLATE=translate,
)
return kv_indices, kv_indptr
class TestGenerateDraftDecodeKVIndicesTranslate(CustomTestCase):
def _check(self, *, page_size, num_steps=3, topk=1, unused_slots=False):
torch.manual_seed(7)
num_seqs, max_context = 3, 64
num_pages = max_context // page_size + 1
# Virtual ids: a shuffled page layout so translation is not identity.
perm = torch.randperm(num_pages, device="cuda")
v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda")
v2p[perm] = torch.arange(num_pages, device="cuda")
req_to_token = (
torch.arange(max_context, dtype=torch.int64, device="cuda")
.flip(0)
.repeat(num_seqs, 1)
)
if unused_slots:
# Negative entries inside the read window must land on the sink.
req_to_token[:, ::3] = -1
seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")[:num_seqs]
positions = seq_lens.repeat_interleave(topk)
common = dict(
req_to_token=req_to_token,
seq_lens=seq_lens,
positions=positions,
num_steps=num_steps,
topk=topk,
page_size=page_size,
v2p=v2p,
)
raw, raw_indptr = _run_kernel(translate=False, **common)
out, out_indptr = _run_kernel(translate=True, **common)
torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0)
written = raw != _SENTINEL
# Unwritten lanes stay untouched in both compilations.
self.assertTrue(bool((out[~written] == _SENTINEL).all()))
# Written lanes: translate_kv_loc's formula over the raw ids.
virt = raw[written]
page = torch.where(virt < 0, 0, virt // page_size)
expected = torch.where(
virt < 0,
0,
torch.clamp_min(v2p[page] * page_size + virt % page_size, 0),
)
torch.testing.assert_close(out[written], expected, rtol=0, atol=0)
def test_page_size_one(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=1)
def test_page_size_two(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=2)
def test_negative_ids_map_to_the_sink(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
for page_size in (1, 2):
self._check(page_size=page_size, unused_slots=True)
def test_translate_false_is_a_raw_copy(self):
"""The static-pool compilation must stay byte-identical to the
historical kernel: the emitted indices ARE the req_to_token values."""
if not torch.cuda.is_available():
self.skipTest("CUDA required")
torch.manual_seed(11)
num_seqs, max_context, page_size = 2, 32, 1
req_to_token = torch.randint(
0, 1 << 20, (num_seqs, max_context), dtype=torch.int64, device="cuda"
)
seq_lens = torch.tensor([4, 7], dtype=torch.int64, device="cuda")
raw, _ = _run_kernel(
req_to_token=req_to_token,
seq_lens=seq_lens,
positions=seq_lens,
num_steps=2,
topk=1,
page_size=page_size,
v2p=None,
translate=False,
)
written = raw != _SENTINEL
emitted = raw[written]
self.assertTrue(bool(torch.isin(emitted, req_to_token.flatten()).all()))
if __name__ == "__main__":
unittest.main()
@@ -60,6 +60,7 @@ def _run_draft(kern_inputs, topk, steps, page_size, nb, kw):
kv_i,
kv_p,
positions,
None, # v2p: no unified-pool translation
POOL_LEN,
kv_i.shape[1],
kv_p.shape[1],
@@ -0,0 +1,308 @@
"""Dense fused-draft geometry: the draft model's KV as two more parts of every
host slot's entry.
Derived-property pins for the fused entry layout
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
the draft parts start at the 16-B-aligned end of the host parts and the entry
rounds up to the 32-B entry alignment (never to an lcm of row widths), so the
draft's row width is free to differ from the host's; host and draft views
share the slot stride and are indexed by the same physical token id; and
writes through either family's views land inside their own part of their own
slot (compaction moves whole page envelopes, so confinement IS the correctness
of the fused move).
python -m pytest test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py -v
"""
import unittest
import torch
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.layout.token_major import (
ENTRY_ALIGN_BYTES,
ROW_ALIGN_BYTES,
align_entry_bytes,
align_part_offset,
)
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool
from sglang.srt.mem_cache.unified_memory_pool import (
MambaSubPoolSpec,
MHASubPoolSpec,
MLASubPoolSpec,
UnifiedKVPool,
UnifiedMHATokenToKVPool,
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
_DEV = "cpu"
_DTYPE = torch.bfloat16
_ITEM = _DTYPE.itemsize
def _host_spec(draft_region=None, layer_num=2, head_num=2, head_dim=4):
return MHASubPoolSpec(
name="full",
layer_num=layer_num,
head_num=head_num,
head_dim=head_dim,
store_dtype=_DTYPE,
grow_direction="down",
draft_region=draft_region,
)
def _draft_region():
# Deliberately a different row width from the host's (16 B): K rows of
# 48 B, V rows of 16 B -- asymmetric draft rows are ordinary parts.
return DenseDraftRegion(
lane_num=1, head_num=1, head_dim=24, v_head_dim=8, store_dtype=_DTYPE
)
def _placement(region):
return FusedDraftPlacement(region=region, runner_lane_counts=(region.lane_num,))
class TestFusedSpecMath(unittest.TestCase):
def test_unfused_spec_is_byte_identical_to_before(self):
s = _host_spec()
self.assertEqual(s.entry_bytes(), 2 * (16 + 16))
self.assertEqual(s.host_entry_bytes(), s.entry_bytes())
self.assertEqual([p.name for p in s.layout().parts], ["k", "v"])
def test_fused_entry_appends_the_draft_parts(self):
f = _host_spec(_draft_region())
r = f.draft_region
self.assertEqual(r.k_row_bytes(), 48)
self.assertEqual(r.v_row_bytes(), 16)
self.assertEqual(r.entry_bytes(), 64)
self.assertEqual(
f.draft_offset_in_entry(), align_part_offset(f.host_entry_bytes())
)
self.assertEqual(f.draft_offset_in_entry(), 64)
self.assertEqual(f.entry_bytes(), align_entry_bytes(64 + 64))
self.assertEqual(f.entry_bytes() % ENTRY_ALIGN_BYTES, 0)
layout = f.layout()
self.assertEqual(
[p.name for p in layout.parts], ["k", "v", "draft_k", "draft_v"]
)
self.assertEqual(layout.part("draft_k").offset_bytes, 64)
self.assertEqual(layout.part("draft_v").offset_bytes, 64 + 48)
self.assertEqual(layout.part("draft_k").layer_stride_bytes, 64)
def test_entry_alignment_pads_only_to_the_entry_quantum(self):
# host 32 B + draft 48 B = 80 B of rows -> one 96 B entry (16 B pad),
# not an lcm of the two row widths.
f = _host_spec(
DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, v_head_dim=16, store_dtype=_DTYPE
),
layer_num=1,
head_num=1,
head_dim=8,
)
self.assertEqual(f.host_entry_bytes(), 32)
self.assertEqual(f.draft_region.entry_bytes(), 48)
self.assertEqual(f.entry_bytes(), 96)
self.assertEqual(f.draft_offset_in_entry() % ROW_ALIGN_BYTES, 0)
f.layout() # parts fit and do not overlap
def test_draft_rows_must_be_row_aligned(self):
with self.assertRaises(AssertionError):
_host_spec(
DenseDraftRegion(lane_num=1, head_num=1, head_dim=3, store_dtype=_DTYPE)
).layout()
def test_only_page_envelope_kinds_accept_a_draft_region(self):
"""`draft_region` is a universal spec field (None = unfused), but a
kind without the fused entry layout must refuse one at construction;
a silently-carried region would never reach the layout."""
with self.assertRaises(AssertionError):
MLASubPoolSpec(
name="full",
layer_num=2,
kv_lora_rank=8,
qk_rope_head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="down",
draft_region=_draft_region(),
)
with self.assertRaises(AssertionError):
MambaSubPoolSpec(
name="mamba",
layer_num=1,
conv_state_shapes=((2, 2),),
conv_dtype=torch.float32,
temporal_state_shape=(2,),
temporal_dtype=torch.float32,
grow_direction="up",
draft_region=_draft_region(),
)
class TestFusedRegionConfinement(unittest.TestCase):
"""Writes through host/draft views must land in their own part of their
own slot."""
PS = 2
PAGES = 3
def _build(self):
spec = _host_spec(_draft_region())
swa = MHASubPoolSpec(
name="swa",
layer_num=1,
head_num=2,
head_dim=4,
store_dtype=_DTYPE,
grow_direction="up",
)
total = 8 * self.PS * spec.entry_bytes() + 8 * self.PS * swa.entry_bytes()
pool = UnifiedKVPool(
total_bytes=total,
sub_pool_specs=[spec, swa],
device=_DEV,
enable_memory_saver=False,
page_size=self.PS,
fused_draft=_placement(spec.draft_region),
)
hk, hv = pool.mha_views_for("full")
dk, dv = pool.build_dense_draft_views("full")
return spec, pool, (hk, hv), (dk, dv)
def test_host_and_draft_writes_confine_to_their_parts(self):
spec, pool, (hk, hv), (dk, dv) = self._build()
layout = spec.layout()
raw = pool._raw
entry = spec.entry_bytes()
families = (
("k", hk),
("v", hv),
("draft_k", dk),
("draft_v", dv),
)
for t in range(self.PAGES * self.PS):
for name, views in families:
part = layout.part(name)
for layer, view in enumerate(views):
raw.zero_()
view[t] = 1.0
nz = raw.nonzero().flatten()
lo = t * entry + part.layer_offset_bytes(layer)
hi = lo + part.row_bytes()
self.assertEqual(int(nz.min()), lo, f"{name}[{layer}] t={t}")
self.assertEqual(int(nz.max()), hi - 1, f"{name}[{layer}] t={t}")
# ... and inside the slot's page envelope.
page = t // self.PS
self.assertGreaterEqual(lo, page * self.PS * entry)
self.assertLessEqual(hi, (page + 1) * self.PS * entry)
def test_host_and_draft_views_share_the_slot_stride(self):
spec, _, (hk, hv), (dk, dv) = self._build()
stride = spec.entry_bytes()
for view in (*hk, *hv, *dk, *dv):
self.assertEqual(view.stride(0) * view.element_size(), stride)
self.assertEqual(view.shape[0], hk[0].shape[0])
self.assertEqual(tuple(dk[0].shape[1:]), (1, 24))
self.assertEqual(tuple(dv[0].shape[1:]), (1, 8))
def test_pool_helper_refuses_an_unfused_sub_pool(self):
_, pool, _, _ = self._build()
with self.assertRaises(AssertionError):
pool.build_dense_draft_views("swa") # no fused region there
class TestUnifiedDraftKVPool(unittest.TestCase):
PS = 2
PAGES = 8
def _pool(self):
spec = _host_spec(_draft_region())
swa = MHASubPoolSpec(
name="swa",
layer_num=1,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="up",
)
total = self.PAGES * self.PS * (spec.entry_bytes() + swa.entry_bytes())
return UnifiedKVPool(
total_bytes=total,
sub_pool_specs=[spec, swa],
device=_DEV,
enable_memory_saver=False,
page_size=self.PS,
fused_draft=_placement(spec.draft_region),
)
def _draft_pool(self, pool):
sentinel_allocator = object()
dp = UnifiedDraftKVPool(
unified_buffer=pool,
host_sub_pool_name="full",
host_allocator=sentinel_allocator,
layer_lanes={0: 0},
page_size=self.PS,
)
return dp, sentinel_allocator
def test_probe_surface_and_view_binding(self):
pool = self._pool()
dp, alloc = self._draft_pool(pool)
spec = pool.mha_spec("full")
self.assertIs(dp.host_allocator, alloc)
self.assertTrue(dp.requires_physical_write_loc)
self.assertEqual(len(dp.k_buffer), 1)
self.assertEqual(dp.k_buffer[0].shape[1:], (1, 24))
self.assertEqual(dp.v_buffer[0].shape[1:], (1, 8))
# Same slot space as the host: one row per physical token, the slot
# stride being the whole fused entry.
n_rows = pool.max_slots("full") // self.PS * self.PS
self.assertEqual(dp.k_buffer[0].shape[0], n_rows)
self.assertEqual(dp.size, n_rows - self.PS)
self.assertEqual(
dp.k_buffer[0].stride(0) * dp.k_buffer[0].element_size(), spec.entry_bytes()
)
def test_host_page_move_carries_the_draft_bytes(self):
# THE fused-layout property: compaction relocates whole page envelopes
# on the HOST pool; a draft marker written in page A must arrive at
# page B after host.move_kv_cache(B, A), with zero draft-side moves.
pool = self._pool()
dp, _ = self._draft_pool(pool)
host = UnifiedMHATokenToKVPool(
unified_buffer=pool, sub_pool_name="full", page_size=self.PS
)
src_page, dst_page = 3, 5
src_t, dst_t = src_page * self.PS, dst_page * self.PS
dp.k_buffer[0][src_t] = 7.0
self.assertEqual(float(dp.k_buffer[0][dst_t].sum()), 0.0)
ps = self.PS
to_tokens = lambda p: torch.arange(p * ps, (p + 1) * ps, dtype=torch.int64)
host.move_kv_cache(to_tokens(dst_page), to_tokens(src_page))
self.assertEqual(float(dp.k_buffer[0][dst_t].sum()), 7.0 * 24)
def test_draft_side_moves_and_transfers_fail_loudly(self):
pool = self._pool()
dp, _ = self._draft_pool(pool)
one = torch.zeros(self.PS, dtype=torch.int64)
with self.assertRaises(NotImplementedError):
dp.move_kv_cache(one, one)
with self.assertRaises(NotImplementedError):
dp.get_contiguous_buf_infos()
with self.assertRaises(NotImplementedError):
dp.get_cpu_copy(one)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,241 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`place_fused_draft`: which host/draft pairs fuse, and where each runner's
layers land.
Every decline is the private-pool fallback, so a wrong answer never fails
the boot -- it silently changes what the draft binds and what the boot solve
prices. Pinned:
- a replicated head under multi-layer EAGLE gets one lane RANGE per runner
(one shared region would let the runners clobber each other's KV);
- a per-depth head serves one depth per runner and needs one runner per
depth;
- a draft with SWA layers of its own, or asymmetric K/V rows, declines:
the fused arm binds one dense pool over the host's full slots;
- the profile divides the draft's heads by attn_tp, as the target does;
- a placement whose runner lane counts do not fill its region is refused;
- the priced entry counts the layers THIS runner owns, not the whole model's;
- only a two-pool hybrid-SWA target with a stateless draft is placed.
python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
DraftKVGeometry,
DraftKVProfile,
FusedDraftPlacement,
draft_kv_profile,
place_fused_draft,
)
from sglang.srt.runtime_context import get_parallel
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
_DTYPE = torch.bfloat16
def _profile(
*,
num_layers=1,
swa_layer_ids=(),
num_depths=1,
head_dim=64,
v_head_dim=64,
):
return DraftKVProfile(
num_layers=num_layers,
full=DraftKVGeometry(head_num=4, head_dim=head_dim, v_head_dim=v_head_dim),
swa_layer_ids=swa_layer_ids,
num_depths=num_depths,
)
def _place(profile, num_runners=1):
return place_fused_draft(
profile=profile,
num_runners=num_runners,
store_dtype=_DTYPE,
)
class TestPlaceFusedDraft(CustomTestCase):
def test_replicated_head_gets_one_slot_range_per_runner(self):
decision = _place(_profile(num_layers=2), num_runners=3)
placement = decision.placement
self.assertIsNotNone(placement, decision.declined)
self.assertEqual(placement.region.lane_num, 6)
for r in range(3):
self.assertEqual(placement.lanes_for(r), range(2 * r, 2 * r + 2))
def test_per_depth_head_serves_one_depth_per_runner(self):
decision = _place(_profile(num_layers=8, num_depths=8), num_runners=2)
placement = decision.placement
self.assertIsNotNone(placement, decision.declined)
self.assertEqual(placement.region.lane_num, 2)
self.assertEqual(placement.lanes_for(1), range(1, 2))
def test_per_depth_head_needs_one_runner_per_depth(self):
self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 1).placement)
self.assertIsNone(_place(_profile(num_layers=8, num_depths=8), 9).placement)
def test_swa_asymmetric_and_misaligned_drafts_decline(self):
for profile in (
_profile(swa_layer_ids=(0,)),
_profile(head_dim=64, v_head_dim=32),
# 4 heads x 5 dims x 2 B = 40 B: not a 16-B-aligned entry part.
_profile(head_dim=5, v_head_dim=5),
):
decision = _place(profile)
self.assertIsNone(decision.placement)
self.assertIsNotNone(decision.declined)
def test_region_carries_the_profile_geometry(self):
placement = _place(_profile()).placement
self.assertEqual(
placement.region,
DenseDraftRegion(
lane_num=1, head_num=4, head_dim=64, v_head_dim=64, store_dtype=_DTYPE
),
)
class TestDraftKVProfile(CustomTestCase):
def test_heads_are_divided_by_attn_tp(self):
mc = SimpleNamespace(
is_hybrid_swa=True,
is_deepseek_v4_arch=False,
swa_attention_layer_ids=[0],
num_nextn_predict_layers=None,
get_num_kv_heads=lambda tp: max(1, 8 // tp),
head_dim=64,
v_head_dim=32,
)
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=2)
self.assertEqual(
profile.full, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32)
)
self.assertEqual(profile.swa_layer_ids, (0,))
self.assertEqual(profile.num_depths, 1)
class TestFusedDraftPlacement(CustomTestCase):
def test_runner_lane_counts_must_fill_the_region(self):
region = DenseDraftRegion(
lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE
)
with self.assertRaises(AssertionError):
FusedDraftPlacement(region=region, runner_lane_counts=(1,))
with self.assertRaises(AssertionError):
FusedDraftPlacement(region=region, runner_lane_counts=())
class TestFusedEntryPricing(CustomTestCase):
"""The boot solve prices a fused entry through the same spec the pool
factory builds, so the two cannot drift. The factory builds the full
sub-pool from this runner's OWN layer slice; pricing from the whole-model
split over-counts every layer another pipeline rank holds, and the solve
then hands out fewer tokens than fit."""
def _configurator(self, *, owned, whole):
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
cfg = KVCacheConfigurator.__new__(KVCacheConfigurator)
cfg.layer_info = SimpleNamespace(full_attention_layer_ids=owned)
cfg.model_config = SimpleNamespace(
full_attention_layer_ids=whole,
head_dim=8,
get_num_kv_heads=lambda *_: 1,
)
cfg.kv_cache_dtype = _DTYPE
return cfg
def test_the_priced_entry_counts_only_this_runners_layers(self):
region = DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE
)
cfg = self._configurator(owned=[0, 1], whole=[0, 1, 2, 3])
with get_parallel().override(attn_tp_size=1, attn_dcp_size=1):
spec = cfg._full_host_spec(region)
self.assertEqual(spec.layer_num, 2)
class TestFusedDraftGate(CustomTestCase):
"""Only the two-pool hybrid-SWA factory takes a placement here. A target
that also carries recurrent state builds its pools unfused, so answering
a placement for it would price a fused entry the factory never allocates;
a draft with recurrent state needs a state pool the fused arm lacks."""
def _decide(self, *, target_state=None, draft_state=None, draft_kv_dtype=None):
from sglang.srt.mem_cache import kv_cache_configurator as kvc
cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator)
cfg.is_hybrid_swa = True
cfg.mambaish_config = target_state
cfg.is_draft_worker = False
cfg.spec_algorithm = SimpleNamespace(is_eagle=lambda: True)
cfg.model_config = SimpleNamespace(is_multi_layer_eagle=False)
cfg.kv_cache_dtype = _DTYPE
cfg.spec_aux_config = SimpleNamespace(
eagle_draft_num_layers=1,
draft_kv_num_layers=1,
draft_model_config=SimpleNamespace(
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
num_nextn_predict_layers=None,
get_num_kv_heads=lambda tp: 4,
head_dim=64,
v_head_dim=64,
dtype=torch.bfloat16,
),
)
memory = SimpleNamespace(enable_unified_memory=True)
spec = SimpleNamespace(
speculative_num_steps=1, speculative_draft_kv_cache_dtype=draft_kv_dtype
)
with (
patch.object(kvc, "get_memory", return_value=memory),
patch.object(kvc, "get_spec", return_value=spec),
patch.object(kvc, "mambaish_config", return_value=draft_state),
get_parallel().override(attn_tp_size=1),
):
return cfg._fused_draft_decision()
def test_a_two_pool_target_places_a_stateless_draft(self):
self.assertIsNotNone(self._decide().placement)
def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self):
"""A fused draft stores its rows in the host's KV dtype; an explicit
draft dtype that differs declines instead of mis-typing those rows."""
self.assertIsNotNone(self._decide(draft_kv_dtype="auto").placement)
declined = self._decide(draft_kv_dtype="fp8_e4m3")
self.assertIsNone(declined.placement)
self.assertIn("KV cache dtype", declined.declined)
def test_recurrent_state_on_either_side_keeps_the_private_pool(self):
state = SimpleNamespace()
for kw in ({"target_state": state}, {"draft_state": state}):
self.assertIsNone(self._decide(**kw).placement, kw)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,249 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""How a draft-shaped KVCacheConfigurator binds its KV under the unified pool.
DERIVED PROPERTY (binding dispatch). The fused-vs-private draft binding
dispatches on the spec algorithm, not the allocator kind: a non-EAGLE draft on
a unified SWA target takes the private arm, sized by the full sub-allocator's
VIRTUAL id space (`max_slots - 1`). The SWA allocator's `size_full` reports
the static token budget — smaller than the id space — so sizing by it would
put verify-window writes at high virtual ids out of bounds.
python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.mem_cache import kv_cache_configurator as kcc
from sglang.srt.mem_cache import unified_draft_pool
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class _FakeKVCache:
def __init__(self, max_slots):
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
self.allocator = None
def attach_allocator(self, allocator):
self.allocator = allocator
class _FakeUnifiedSWAKVPool:
def __init__(self, shared_pool):
self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full"))
self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa"))
self.full_to_swa_index_mapping = None
def attach_allocators(self, *, full_allocator, swa_allocator):
self._full_allocator = full_allocator
self._swa_allocator = swa_allocator
class _CapturedSizes(Exception):
"""Sentinel carrying the sizes handed to the token-pool build."""
def __init__(self, sizes):
self.sizes = sizes
_PS = 2
class TestDraftBindingDispatch(CustomTestCase):
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _swa_allocator(self, *, with_draft_region: bool, n_full=32, n_swa=16):
region = (
DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16
)
if with_draft_region
else None
)
full_spec = MHASubPoolSpec(
name="full",
layer_num=2,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="down",
draft_region=region,
)
swa_spec = MHASubPoolSpec(
name="swa",
layer_num=1,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="up",
)
total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes()
pool = UnifiedKVPool(
total_bytes=total,
sub_pool_specs=[full_spec, swa_spec],
device="cpu",
enable_memory_saver=False,
page_size=_PS,
fused_draft=(
FusedDraftPlacement(region=region, runner_lane_counts=(1,))
if region is not None
else None
),
)
return UnifiedSWATokenToKVPoolAllocator(
unified_buffer=pool,
kvcache=_FakeUnifiedSWAKVPool(pool),
device="cpu",
full_max_total_num_tokens=n_full,
swa_max_total_num_tokens=n_swa,
page_size=_PS,
need_sort=False,
forward_stream=None,
)
def _run(self, *, algorithm, alloc, max_total_num_tokens, kv_dtype=torch.bfloat16):
cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator)
cfg.kv_cache_dtype = kv_dtype
cfg.is_draft_worker = True
cfg.draft_model_idx = None
cfg.spec_algorithm = algorithm
cfg.page_size = _PS
cfg.is_hybrid_swa = False
cfg.is_hybrid_swa_mtp_draft = False
cfg.model = object()
cfg.model_config = SimpleNamespace(hf_config=None, is_hybrid_swa=False)
sizes = kcc._PoolSizes(
max_total_num_tokens=max_total_num_tokens,
max_running_requests=8,
full_max_total_num_tokens=max_total_num_tokens,
swa_max_total_num_tokens=16,
c4_max_total_num_tokens=0,
c128_max_total_num_tokens=0,
c4_state_pool_size=0,
c128_state_pool_size=0,
c4_state_dtype=None,
c128_state_dtype=None,
)
def _capture(self, *, sizes, **kw):
raise _CapturedSizes(sizes)
with (
patch.object(
kcc,
"get_memory",
return_value=SimpleNamespace(enable_unified_memory=True),
),
patch.object(
kcc,
"get_schedule",
return_value=SimpleNamespace(page_size=_PS),
),
patch.object(kcc, "is_deepseek_dsa", return_value=False),
patch.object(kcc, "is_deepseek_v4", return_value=False),
patch.object(
kcc.KVCacheConfigurator,
"_validate_prefill_only_disable_kv_cache_pool_family",
lambda self, *a, **kw: None,
),
patch.object(kcc.KVCacheConfigurator, "_build_token_to_kv_pool", _capture),
# The real scan walks the draft nn.Module for RadixAttention layers.
patch.object(unified_draft_pool, "draft_kv_layer_ids", return_value=[0]),
):
return cfg._init_pools(
sizes=sizes,
req_to_token_pool=object(),
token_to_kv_pool_allocator=alloc,
)
def test_non_eagle_draft_takes_the_private_arm_sized_by_the_id_space(self):
alloc = self._swa_allocator(with_draft_region=False)
id_space = alloc.full_attn_allocator.max_slots - 1
# The distinction under test only exists while budget < id space.
self.assertLess(alloc.size_full, id_space)
with self.assertRaises(_CapturedSizes) as caught:
self._run(
algorithm=SpeculativeAlgorithm.DSPARK,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
sized = caught.exception.sizes.max_total_num_tokens
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
def test_eagle_draft_still_binds_the_fused_pool(self):
alloc = self._swa_allocator(with_draft_region=True)
pools = self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
self.assertIs(pools.token_to_kv_pool_allocator, alloc)
def test_a_draft_kv_dtype_unlike_the_region_refuses_to_bind(self):
"""The region stores the target's KV dtype. A draft that resolved its
own -- `auto` picking up an fp8 quant config the target lacks, say --
must fail loudly rather than read those rows as another dtype."""
alloc = self._swa_allocator(with_draft_region=True)
with self.assertRaisesRegex(ValueError, "speculative-draft-kv-cache-dtype"):
self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
kv_dtype=torch.float8_e4m3fn,
)
def test_eagle_draft_without_a_placement_falls_back_to_the_private_arm(self):
"""Target boot declines a placement for legitimate configurations (a
draft with SWA layers of its own, asymmetric rows), so a
placement-less EAGLE draft binds the private pool -- sized by the id
space like any other private draft -- instead of failing the boot."""
alloc = self._swa_allocator(with_draft_region=False)
id_space = alloc.full_attn_allocator.max_slots - 1
with self.assertRaises(_CapturedSizes) as caught:
self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
sized = caught.exception.sizes.max_total_num_tokens
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,369 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""KVIndexTranslator's three id-space dispositions with a fused draft region.
1. TARGET runner (allocator's own kvcache): translates to physical ids.
2. FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to this allocator): the
SAME translate -- same pages, same v2p table, same physical ids (the draft
parts sit inside the host entry). A dense draft pool routes no window
layers, so it has no swa id space: the window index table falls back to
the one dense table and `sliding_window_write_loc_for()` answers None (a
separate-swa assumption here crashed the read side and left the write side
with nothing to derive).
3. PRIVATE-POOL draft (DSPARK/DFLASH shape: target's allocator, own
virtual-indexed buffer): strict passthrough — translating such a runner
would address a slot-count buffer with dense ids (OOB both directions),
so the no-op is load-bearing, not a default.
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
"""
import ast
import pathlib
import unittest
from types import SimpleNamespace
import torch
from sglang.srt.mem_cache import kv_index_translator
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
_DEV = "cpu"
_PS = 2
class _FakeKVCache:
def __init__(self, max_slots):
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
self.allocator = None
def attach_allocator(self, allocator):
self.allocator = allocator
def move_kv_cache(self, dst_loc, src_loc):
self.buf[dst_loc] = self.buf[src_loc].clone()
class _FakeUnifiedSWAKVPool:
def __init__(self, shared_pool):
self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full"))
self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa"))
self.full_to_swa_index_mapping = None
def attach_allocators(self, *, full_allocator, swa_allocator):
self._full_allocator = full_allocator
self._swa_allocator = swa_allocator
def _build(n_full=32, n_swa=16):
full_spec = MHASubPoolSpec(
name="full",
layer_num=2,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="down",
draft_region=DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16
),
)
swa_spec = MHASubPoolSpec(
name="swa",
layer_num=1,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="up",
)
total = n_full * full_spec.entry_bytes() + n_swa * swa_spec.entry_bytes()
pool = UnifiedKVPool(
total_bytes=total,
sub_pool_specs=[full_spec, swa_spec],
device=_DEV,
enable_memory_saver=False,
page_size=_PS,
fused_draft=FusedDraftPlacement(
region=full_spec.draft_region, runner_lane_counts=(1,)
),
)
kvcache = _FakeUnifiedSWAKVPool(pool)
allocator = UnifiedSWATokenToKVPoolAllocator(
unified_buffer=pool,
kvcache=kvcache,
device=_DEV,
full_max_total_num_tokens=n_full,
swa_max_total_num_tokens=n_swa,
page_size=_PS,
need_sort=False,
forward_stream=None,
)
draft_pool = UnifiedDraftKVPool(
unified_buffer=pool,
host_sub_pool_name="full",
host_allocator=allocator,
layer_lanes={0: 0},
page_size=_PS,
)
return pool, allocator, kvcache, draft_pool
def _source(allocator, pool_obj):
return KVIndexTranslator(
req_to_token=torch.zeros((4, 16), dtype=torch.int32),
token_to_kv_pool_allocator=allocator,
token_to_kv_pool=pool_obj,
page_size=_PS,
device=_DEV,
)
class TestKVIndexSourceDraftDisposition(unittest.TestCase):
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_target_runner_translates_to_physical_ids(self):
_, allocator, kvcache, _ = _build()
src = _source(allocator, kvcache)
self.assertTrue(src.is_translating)
v = allocator.alloc(2 * _PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
def test_fused_draft_runner_translates_to_the_same_physical_ids(self):
_, allocator, _, draft_pool = _build()
src = _source(allocator, draft_pool)
self.assertTrue(src.is_translating)
self.assertIsNone(src._swa_v2p_table) # single space: no swa id space
# rebind_write_loc translates to the HOST's physical ids: the draft
# parts live inside the same slots.
v = allocator.alloc(2 * _PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertIsNot(fb.out_cache_loc, v)
fa = allocator.full_attn_allocator
expected = torch.clamp_min(fa.virtual_to_physical[v // _PS] * _PS + v % _PS, 0)
torch.testing.assert_close(fb.out_cache_loc, expected, rtol=0, atol=0)
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
# A dense draft pool routes no window layers: no swa write loc.
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
def test_dense_fused_draft_has_no_window_write_loc(self):
"""`sliding_window_write_loc_for()` on a dense fused-draft batch answers
None: the pool routes no window layers, so there is nothing to
derive. The hybrid-SWA TARGET on the same allocator keeps a genuinely
derived swa loc, so the None can never paper over a real swa id
space."""
_, allocator, kvcache, draft_pool = _build()
src = _source(allocator, draft_pool)
v = allocator.alloc(_PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
# Target contrast: a real swa side derives a DIFFERENT loc.
tgt = _source(allocator, kvcache)
tv = allocator.alloc(_PS)
self.assertIsNotNone(tv)
tfb = SimpleNamespace(out_cache_loc=tv)
tgt.rebind_write_loc(tfb)
tswa = tgt.sliding_window_write_loc_for(tfb.out_cache_loc)
self.assertIsNotNone(tswa)
self.assertIsNot(tswa, tfb.out_cache_loc)
self.assertTrue(torch.equal(tswa, allocator.translate_loc_from_full_to_swa(tv)))
def test_private_pool_draft_stays_a_strict_passthrough(self):
_, allocator, _, _ = _build()
private_draft_pool = _FakeKVCache(64) # own buffer, not the allocator's
src = _source(allocator, private_draft_pool)
self.assertFalse(src.is_translating)
v = torch.arange(2 * _PS, dtype=torch.int64)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertIs(fb.out_cache_loc, v) # untouched, not even a copy
def test_foreign_allocator_draft_pool_is_not_enabled(self):
# A UnifiedDraftKVPool bound to a DIFFERENT allocator must not enable
# against this one (identity, not type, decides).
pool, allocator, kvcache, _ = _build()
_, other_alloc, _, other_draft = _build()
src = _source(allocator, other_draft)
self.assertFalse(src.is_translating)
def test_full_flat_v2p_per_disposition(self):
"""The flat-translate accessor must hand a kernel exactly what
`translate_kv_loc` would use: the shared v2p table on the target and
on the fused draft, and None on a pass-through runner (a translated
pass-through would address a private buffer with the target's ids)."""
_, allocator, kvcache, draft_pool = _build()
target = _source(allocator, kvcache)
self.assertIs(target.full_flat_v2p(), allocator.full_v2p_page_table)
draft = _source(allocator, draft_pool)
self.assertIs(draft.full_flat_v2p(), allocator.full_v2p_page_table)
passthrough = _source(allocator, _FakeKVCache(64))
self.assertIsNone(passthrough.full_flat_v2p())
def test_multi_step_containers_pass_the_v2p_table_to_the_kernel(self):
"""Every `generate_draft_decode_kv_indices` launch must thread the
runner's v2p table and a matching TRANSLATE flag. A launch
without them emits raw req_to_token values, which under the unified
pool are VIRTUAL ids the fused draft pool cannot address — the exact
silent-garbage bug this series fixed."""
import pathlib
import re
import sglang.srt.mem_cache.kv_index_translator as _kit
root = pathlib.Path(_kit.__file__).parent.parent / "layers" / "attention"
launching = {}
for path in sorted(root.glob("*.py")):
text = path.read_text()
launches = len(re.findall(r"generate_draft_decode_kv_indices\[", text))
if launches:
launching[path.name] = (
launches,
len(re.findall(r"TRANSLATE=v2p is not None", text)),
"full_flat_v2p" in text,
)
self.assertGreaterEqual(
len(launching), 4, f"launch sites disappeared: {sorted(launching)}"
)
for name, (launches, translated, has_accessor) in launching.items():
self.assertEqual(
launches,
translated,
f"{name}: {launches} kernel launch(es) but only {translated} "
"carry TRANSLATE=v2p is not None",
)
self.assertTrue(
has_accessor,
f"{name} launches the kernel without full_flat_v2p",
)
class TestDispositionBranchesAgree(unittest.TestCase):
"""Every `__init__` disposition must assign the SAME attribute set.
BUG REGRESSION. `KVIndexTranslator.__init__` picks a disposition --
translating (unified target or fused draft) or passthrough -- and later
methods read attributes off `self` unconditionally. When upstream adds an
attribute it naturally adds it to the branches IT knows about; a branch
added here is silently left short, and nothing fails until the missing
attribute is read at RUNTIME. That is exactly how the fused-draft branch
lost `_translate_write_full` and `defer_read_translate` across a rebase:
py_compile passes (the attribute is only ever read, never declared), the
undefined-NAME check passes (it is an attribute, not a bare name), and the
target and passthrough paths both work -- only a fused-draft forward raises
`AttributeError: 'KVIndexTranslator' object has no attribute
'_translate_write_full'`.
Comparing the branches against EACH OTHER needs no list to maintain: a new
attribute is covered the moment any one branch sets it.
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
"""
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _init_branch_assignments(self):
"""{branch index: {attr names it assigns}} for __init__'s if/elif/else."""
src = pathlib.Path(kv_index_translator.__file__).read_text()
tree = ast.parse(src)
cls = next(
n
for n in ast.walk(tree)
if isinstance(n, ast.ClassDef) and n.name == "KVIndexTranslator"
)
init = next(
n
for n in cls.body
if isinstance(n, ast.FunctionDef) and n.name == "__init__"
)
# The disposition chain is the `if` whose body assigns the id-space
# fields; find it by the attribute every disposition must set.
def assigns(stmts):
out = set()
for st in stmts:
for node in ast.walk(st):
if isinstance(node, ast.Assign):
for t in node.targets:
if (
isinstance(t, ast.Attribute)
and isinstance(t.value, ast.Name)
and t.value.id == "self"
):
out.add(t.attr)
return out
for node in init.body:
if not isinstance(node, ast.If):
continue
branches, cur = [], node
while True:
branches.append(assigns(cur.body))
if len(cur.orelse) == 1 and isinstance(cur.orelse[0], ast.If):
cur = cur.orelse[0]
continue
if cur.orelse:
branches.append(assigns(cur.orelse))
break
if len(branches) >= 2 and all("_full_v2p_table" in b for b in branches):
return branches
self.fail("could not locate the disposition if/elif/else in __init__")
def test_every_disposition_assigns_the_same_attributes(self):
branches = self._init_branch_assignments()
union = set().union(*branches)
missing = {i: sorted(union - b) for i, b in enumerate(branches) if union - b}
self.assertEqual(
missing,
{},
"a KVIndexTranslator.__init__ disposition does not assign every "
"attribute its siblings do; the branch raises AttributeError only "
"when that path is taken at runtime: " + repr(missing),
)
if __name__ == "__main__":
unittest.main()
@@ -179,6 +179,9 @@ def _make_model_runner(
eagle_draft_swa_num_layers=None,
dflash_draft_num_layers=None,
)
# Fused draft KV is off unless a test opts in (a bare MagicMock return
# would read as a truthy entry size and take the fused pricing branch).
mr.fused_entry_bytes.return_value = None
return mr
@@ -1430,5 +1433,106 @@ class TestSWAPoolFloor(CustomTestCase):
)
class TestFusedDraftPricing(unittest.TestCase):
"""Fused draft KV: the boot solve must price the EXACT fused entry.
The fused full-side entry (host + draft + alignment pad) replaces BOTH the
per-token full term and the per-draft-layer approximation; charging both
would over-reserve, charging only the approximation would under-reserve
and OOM at pool construction (the factory allocates tokens x fused entry).
"""
def _make(self, fused_entry, draft_layers, **runner_kwargs):
mr = _make_model_runner(
self,
is_hybrid_swa=True,
full_attention_layer_ids=list(range(4)),
swa_attention_layer_ids=list(range(4, 12)),
swa_num_kv_heads=4,
page_size=1,
swa_full_tokens_ratio=0.5,
**runner_kwargs,
)
mr.spec_algorithm.is_eagle.return_value = True
mr.spec_algorithm.is_standalone.return_value = False
mr.spec_algorithm.is_none.return_value = False
mr.spec_aux_config.eagle_draft_num_layers = draft_layers
mr.fused_entry_bytes.side_effect = lambda name: (
fused_entry if name == "full" else None
)
with mock_cpu_env():
from sglang.srt.model_executor.pool_configurator import (
create_memory_pool_configurator,
)
return mr, create_memory_pool_configurator(mr)
def _expected_cell(self, cfg, full_term):
return (
full_term
+ cfg._swa_per_token * cfg._draft_swa_full_layers_num
+ cfg._swa_full_tokens_ratio * cfg._swa_per_token * cfg._swa_layers_num
+ cfg._draft_cell_size
)
def test_unfused_keeps_the_draft_layer_approximation(self):
_, cfg = self._make(fused_entry=None, draft_layers=2)
self.assertEqual(cfg._draft_full_layers_num, 2)
self.assertEqual(
cfg._cell_size,
self._expected_cell(cfg, cfg._full_per_token * (4 + 2)),
)
def test_fused_entry_replaces_full_and_draft_terms(self):
fused_entry = 54_321
_, cfg = self._make(fused_entry=fused_entry, draft_layers=2)
self.assertEqual(cfg._draft_full_layers_num, 0)
self.assertEqual(cfg._cell_size, self._expected_cell(cfg, fused_entry))
def _assert_unified_bytes_price_the_fused_entry(self, cfg, config, available):
"""The bytes handed to the unified pool must buy every full token at
the fused entry -- the factory divides them by it -- and must not leave
a full token's worth of the budget unspent."""
fused_entry = cfg._fused_full_entry
swa_bytes = (
config.swa_max_total_num_tokens * cfg._swa_per_token * cfg._swa_layers_num
)
self.assertEqual(
config.unified_memory_pool_bytes,
config.full_max_total_num_tokens * fused_entry + swa_bytes,
)
self.assertLessEqual(config.unified_memory_pool_bytes, available)
self.assertGreater(
config.unified_memory_pool_bytes + 2 * fused_entry, available
)
def test_unified_solve_prices_the_fused_entry(self):
available = 50_000_000
_, cfg = self._make(
fused_entry=54_321, draft_layers=2, enable_unified_memory=True
)
config = cfg.calculate_pool_sizes(available, 1)
self._assert_unified_bytes_price_the_fused_entry(cfg, config, available)
def test_chunk_cap_solve_prices_the_fused_entry(self):
from sglang.srt.model_executor.pool_configurator import (
SWAChunkCapPoolConfigurator,
)
available = 50_000_000
_, cfg = self._make(
fused_entry=54_321,
draft_layers=2,
enable_unified_memory=True,
disable_radix_cache=True,
chunked_prefill_size=4,
sliding_window_size=8,
max_running_requests=2,
)
self.assertIsInstance(cfg, SWAChunkCapPoolConfigurator)
config = cfg.calculate_pool_sizes(available, 1)
self._assert_unified_bytes_price_the_fused_entry(cfg, config, available)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,191 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`resolve_spec_aux_hidden_state_config` runs before the TP group exists.
BUG REGRESSION. `ModelRunner.init_spec_aux_hidden_state()` runs BEFORE
`init_torch_distributed()`, so the attention-TP group does not exist while
this resolver runs. A `get_parallel().attn_tp_size` read inside it therefore
raises `AssertionError: attention tensor model parallel group is not
initialized` and kills the server at startup -- for EVERY EAGLE-family run
with a draft path, unified pool or not. It is invisible to CPU unit gates
(they never boot a runner) and was caught only by a GPU boot guard.
The resolver therefore records the draft's `ModelConfig` whole, and the
per-GPU geometry is derived at pool-build time (`draft_kv_profile`), where
the group exists -- the same point the target divides its own heads.
python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py -v
"""
import ast
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
_SRC = Path(_mod.__file__)
def _function_named(tree, name):
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == name:
return node
raise AssertionError(f"{name} not found in {_SRC}")
def _config(*, num_nextn_predict_layers, num_hidden_layers=1):
return SimpleNamespace(
num_nextn_predict_layers=num_nextn_predict_layers,
num_hidden_layers=num_hidden_layers,
num_attention_layers=num_hidden_layers,
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
)
def _spec_algorithm(*, eagle: bool, eagle3: bool = False):
return SimpleNamespace(
is_eagle=lambda: eagle,
is_standalone=lambda: False,
is_eagle3=lambda: eagle3,
)
class TestResolverTouchesNoParallelState(unittest.TestCase):
def test_no_parallel_state_in_the_eagle_resolver(self):
"""AST pin: the EAGLE resolver runs pre-distributed-init, so any
parallel-state read in it -- `get_parallel()` directly, or a per-GPU
head count that divides by it -- is a boot crash, not a wrong number."""
tree = ast.parse(_SRC.read_text())
fn = _function_named(tree, "_resolve_eagle_aux_hidden_state")
called = {
n.func.id
for n in ast.walk(fn)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
self.assertNotIn(
"get_parallel",
called,
"_resolve_eagle_aux_hidden_state reads parallel state, but it runs "
"before init_torch_distributed() -- the attention-TP group does not "
"exist yet and the server dies at startup.",
)
methods = {
n.func.attr
for n in ast.walk(fn)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
}
self.assertNotIn("get_num_kv_heads", methods)
imported = {
alias.name
for n in ast.walk(fn)
if isinstance(n, ast.ImportFrom)
for alias in n.names
}
self.assertNotIn("get_parallel", imported)
def _resolve_with(self, *, draft_path, target, draft):
"""Run the EAGLE resolver with the config factory stubbed: `draft`
answers the draft-mode read, `target` is the runner's own config."""
config = _mod.SpecAuxHiddenStateConfig()
spec = SimpleNamespace(
speculative_draft_model_path=draft_path,
speculative_draft_model_revision=None,
)
with (
patch.object(_mod, "get_spec", return_value=spec),
patch.object(
_mod.ModelConfig, "from_server_args", return_value=draft
) as factory,
):
_mod._resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=target,
spec_algorithm=_spec_algorithm(eagle=True),
is_draft_worker=False,
)
self.assertTrue(factory.call_args.kwargs["is_draft_model"])
return config
def test_path_less_nextn_fuses_without_resizing_the_private_pool(self):
"""Every `num_nextn_predict_layers = 1` assignment in model_config.py
is guarded by `if is_draft_model`, so a path-less MTP target answers
None for it. The fused placement must read the draft-mode config, or a
path-less NEXTN head never fuses; the private-pool budget must keep
reading the target, or every path-less MTP deployment's token budget
changes."""
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=1)
config = self._resolve_with(draft_path=None, target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 1)
self.assertIsNone(config.eagle_draft_num_layers)
def test_draft_path_sizes_both_from_the_draft_checkpoint(self):
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2)
config = self._resolve_with(draft_path="/draft", target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 2)
self.assertEqual(config.eagle_draft_num_layers, 2)
def test_the_draft_config_is_recorded_whole(self):
"""The aux config carries the draft `ModelConfig`, never a pre-divided
geometry: a per-GPU count recorded here would be computed without the
TP group (every unit fixture runs at tp=1, where that is invisible)."""
fields = _mod.SpecAuxHiddenStateConfig.__struct_fields__
self.assertIn("draft_model_config", fields)
self.assertNotIn("eagle_draft_total_kv_heads", fields)
self.assertNotIn("eagle_draft_kv_head_num", fields)
class TestResolverIsInertOffTheTargetEagleArm(unittest.TestCase):
"""BUG REGRESSION. The draft-config block is guarded by "EAGLE-family
target"; a rebase left the hybrid-SWA read outside that guard, so every
draft worker and every spec-off runner raised `UnboundLocalError` at
boot, and the EAGLE3 arm's try/except masked the same read into a silent
`None`. Off the target EAGLE arm the resolver must return untouched."""
def _resolve(self, *, spec_algorithm, is_draft_worker):
config = _mod.SpecAuxHiddenStateConfig()
_mod._resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=None,
spec_algorithm=spec_algorithm,
is_draft_worker=is_draft_worker,
)
return config
def test_draft_workers_and_spec_off_runners_resolve_nothing(self):
untouched = _mod.SpecAuxHiddenStateConfig()
for spec_algorithm, is_draft_worker in (
(_spec_algorithm(eagle=True), True),
(_spec_algorithm(eagle=True, eagle3=True), True),
(_spec_algorithm(eagle=False), False),
):
config = self._resolve(
spec_algorithm=spec_algorithm, is_draft_worker=is_draft_worker
)
self.assertEqual(config, untouched)
if __name__ == "__main__":
unittest.main()
@@ -269,6 +269,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
worker.draft_runner = SimpleNamespace(
attn_backend=existing_backend,
model_config=SimpleNamespace(hf_config=SimpleNamespace()),
kv_index_translator=SimpleNamespace(is_translating=False),
)
worker.topk = 1
worker.speculative_num_steps = 2
@@ -294,6 +295,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
worker.draft_runner = SimpleNamespace(
attn_backend=existing_backend,
model_config=SimpleNamespace(hf_config=SimpleNamespace()),
kv_index_translator=SimpleNamespace(is_translating=False),
)
worker.topk = 1
worker.speculative_num_steps = 2