[unified-memory] Fuse the draft model's KV into the target's page envelope (mechanism) (#37628)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-08 00:54:19 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent 08d30f2300
commit 8b80808366
28 changed files with 2506 additions and 57 deletions
@@ -78,6 +78,17 @@ def assign_draft_cache_locs_contiguous(
)
@triton.jit
def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
"""Virtual token ids -> physical ones through the page-level v2p table,
as `translate_kv_loc` does. A negative id maps to the page-0 sink instead
of loading before `v2p`."""
i64 = ids.to(tl.int64)
vpage = tl.where(i64 < 0, 0, i64 // page_size)
phys = tl.load(v2p + vpage, mask=mask, other=0)
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
@triton.jit
def generate_draft_decode_kv_indices(
req_pool_indices,
@@ -86,6 +97,7 @@ def generate_draft_decode_kv_indices(
kv_indices,
kv_indptr,
positions,
v2p,
pool_len: tl.constexpr,
kv_indices_stride: tl.constexpr,
kv_indptr_stride: tl.constexpr,
@@ -96,6 +108,7 @@ def generate_draft_decode_kv_indices(
window_size: tl.constexpr = 0,
sink_size: tl.constexpr = 0,
NUM_STEPS: tl.constexpr = 0,
TRANSLATE: tl.constexpr = False,
):
# window_size > 0 restricts the draft (not the target) to sink_size prefix
# tokens + the most-recent window_size; window_size == 0 is the identity.
@@ -153,6 +166,8 @@ def generate_draft_decode_kv_indices(
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
tl.store(kv_ptr + copy_offset, data, mask=mask)
copy_offset += BLOCK_SIZE
else:
@@ -165,6 +180,8 @@ def generate_draft_decode_kv_indices(
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
tl.store(kv_ptr + copy_offset, data, mask=mask)
# Extension entries and kv_indptr belong to token block 0 alone; other
@@ -196,6 +213,11 @@ def generate_draft_decode_kv_indices(
mask=extend_offset < iters,
)
if TRANSLATE:
extend_data = _translate_token_ids(
extend_data, v2p, extend_offset < iters, page_size
)
tl.store(
kv_ptr + seq_len_w + extend_offset,
extend_data,
@@ -4647,6 +4647,7 @@ class AiterMultiStepDraftBackend:
self.req_to_token_pool = model_runner.req_to_token_pool
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
self.page_size = get_schedule().page_size
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int
@@ -4662,6 +4663,7 @@ class AiterMultiStepDraftBackend:
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
else 1
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
](
@@ -4671,6 +4673,7 @@ class AiterMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -4681,6 +4684,7 @@ class AiterMultiStepDraftBackend:
# A single token block is the historical launch; NUM_STEPS=0 keeps
# its 128-wide program instead of the token-block specialization.
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
TRANSLATE=v2p is not None,
)
for i in range(self.speculative_num_steps - 1):
@@ -3396,6 +3396,7 @@ class FlashAttentionMultiStepBackend:
self.model_runner = model_runner
self.topk = topk
self.speculative_num_steps = speculative_num_steps
self.kv_index_translator = model_runner.kv_index_translator
self.attn_backends = []
for i in range(self.speculative_num_steps - 1):
self.attn_backends.append(
@@ -2369,6 +2369,7 @@ class FlashInferMultiStepDraftBackend:
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
model_runner
)
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -2391,6 +2392,7 @@ class FlashInferMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -2400,6 +2402,7 @@ class FlashInferMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -2409,6 +2412,7 @@ class FlashInferMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -1218,6 +1218,7 @@ class FlashInferMLAMultiStepDraftBackend:
self.req_to_token_pool = model_runner.req_to_token_pool
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
self.page_size = get_schedule().page_size
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -1240,6 +1241,7 @@ class FlashInferMLAMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -1249,6 +1251,7 @@ class FlashInferMLAMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -1256,6 +1259,7 @@ class FlashInferMLAMultiStepDraftBackend:
next_power_of_2(self.speculative_num_steps),
next_power_of_2(bs),
self.page_size,
TRANSLATE=v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -612,6 +612,7 @@ class FlashMLAMultiStepDraftBackend:
)
self.topk = topk
self.speculative_num_steps = speculative_num_steps
self.kv_index_translator = model_runner.kv_index_translator
max_bs = model_runner.req_to_token_pool.size * self.topk
self.kv_indptr = torch.zeros(
(
@@ -2449,6 +2449,7 @@ class TritonMultiStepDraftBackend:
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
model_runner
)
self.kv_index_translator = model_runner.kv_index_translator
def common_template(
self,
@@ -2467,6 +2468,7 @@ class TritonMultiStepDraftBackend:
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
seq_lens_sum = num_seqs * self.max_context_len
v2p = self.kv_index_translator.full_flat_v2p()
generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
@@ -2476,6 +2478,7 @@ class TritonMultiStepDraftBackend:
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
@@ -2485,6 +2488,7 @@ class TritonMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
)
if call_fn is None:
@@ -570,16 +570,45 @@ class KVCacheConfigurator:
UnifiedSWAAllocatorBase,
),
):
draft_virtual_id_space = (
token_to_kv_pool_allocator.draft_virtual_id_space
)
alloc = token_to_kv_pool_allocator
placement = self._fused_draft_from_target_buffer(alloc)
if placement is not None:
# FUSED arm
from sglang.srt.mem_cache.layout.fused_draft import (
draft_swa_layer_ids,
)
from sglang.srt.mem_cache.unified_draft_pool import (
bind_fused_draft,
draft_kv_layer_ids,
)
assert req_to_token_pool is not None, (
"a draft worker shares the target's req_to_token_pool"
)
draft_pool = bind_fused_draft(
unified_buffer=alloc.unified_buffer,
host_allocator=alloc,
placement=placement,
runner=self.draft_model_idx or 0,
kv_layer_ids=draft_kv_layer_ids(self.model),
swa_layer_ids=draft_swa_layer_ids(self.model_config),
page_size=self.page_size,
)
return _InitializedPools(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool=draft_pool,
token_to_kv_pool_allocator=alloc,
unified_memory_pool=None,
)
# PRIVATE arm
draft_virtual_id_space = alloc.draft_virtual_id_space
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
"unified allocator virtual space smaller than the token "
f"budget: virtual_id_space={draft_virtual_id_space} < "
f"max_total_num_tokens={sizes.max_total_num_tokens}"
)
# Round UP to page alignment (paged draft backends view the
# pool as (-1, page_size, H, D); the virtual space is not aligned).
# pool as (-1, page_size, H, D); the space is not aligned).
page = max(int(self.pool_page_size or 1), 1)
draft_virtual_id_space = (
(draft_virtual_id_space + page - 1) // page * page
@@ -684,6 +713,33 @@ class KVCacheConfigurator:
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
)
def _fused_draft_from_target_buffer(self, alloc):
"""The placement this draft binds to, or None for the private-pool
fallback (the draft then owns a raw-virtual-indexed pool of its own).
Target boot declines a placement for legitimate geometry, so the
private arm is the rollback lever, not a boot-order bug."""
if not self.spec_algorithm.is_eagle():
return None
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
placement = alloc.unified_buffer.fused_draft
if placement is None:
logger.info(
"[unified-memory-pool] no fused draft placement on the target's "
"buffer; the draft binds a private pool over the virtual id space."
)
return None
# The region stores rows in the target's KV dtype; a draft that resolved
# its own would read and write them as something else.
if _store_dtype_for(self.kv_cache_dtype) != placement.region.store_dtype:
raise ValueError(
f"Fused draft KV: the draft resolved its KV cache dtype to "
f"{self.kv_cache_dtype}, but its region inside the target's pages "
f"stores {placement.region.store_dtype}. Set "
"--speculative-draft-kv-cache-dtype to the target's KV cache dtype."
)
return placement
def _init_unified_mamba_pools(
self,
*,
@@ -879,6 +935,127 @@ class KVCacheConfigurator:
),
)
def _fused_draft_decision(self):
"""Whether, and where, the EAGLE draft's layers fuse into the target's
sub-pools. Fusion applies only for: unified memory ON, a two-pool
hybrid-SWA target, an EAGLE-family algorithm whose draft config was
loaded at target boot and has no recurrent state of its own;
`place_fused_draft` then admits or declines the draft's layer kinds."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
draft_kv_profile,
place_fused_draft,
)
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
aux = self.spec_aux_config
if not (
get_memory().enable_unified_memory
and self.is_hybrid_swa
and self.mambaish_config is None
and not self.is_draft_worker
and self.spec_algorithm.is_eagle()
and aux.draft_kv_num_layers
and aux.draft_model_config is not None
and mambaish_config(aux.draft_model_config) is None
):
return FusedDraftDecision()
profile = draft_kv_profile(
aux.draft_model_config,
num_layers=int(aux.draft_kv_num_layers),
attn_tp_size=get_parallel().attn_tp_size,
)
num_runners = (
int(get_spec().speculative_num_steps)
if self.model_config.is_multi_layer_eagle
else 1
)
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves without the draft model, so a mismatch is
# declined here; `auto` follows the draft's quant config, which only
# the draft runner knows, and is checked when it binds.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
_, draft_kv_dtype = configure_kv_cache_dtype(
server_args_kv_cache_dtype=draft_kv_cache_dtype,
model=None,
model_dtype=aux.draft_model_config.dtype,
is_draft_worker=True,
is_dflash=False,
speculative_draft_attention_backend=None,
speculative_draft_kv_cache_dtype=draft_kv_cache_dtype,
)
if draft_kv_dtype != self.kv_cache_dtype:
return FusedDraftDecision(
declined=(
f"the draft's KV cache dtype ({draft_kv_dtype}) differs "
f"from the host's ({self.kv_cache_dtype})"
)
)
return place_fused_draft(
profile=profile,
num_runners=num_runners,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
)
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
pad) for the boot solve's cell model. Single source of truth: assembled
through the same spec the pool factory builds, so the priced entry and
the allocated entry cannot drift. None when the draft does not fuse
into that sub-pool."""
placement = self._fused_draft_decision().placement
if placement is None or sub_pool_name != "full":
return None
return self._full_host_spec(placement.region).entry_bytes()
def _full_host_spec(self, region):
from sglang.srt.mem_cache.unified_memory_pool import (
MHASubPoolSpec,
_store_dtype_for,
)
return MHASubPoolSpec(
name="full",
layer_num=len(self.layer_info.full_attention_layer_ids),
head_num=self.model_config.get_num_kv_heads(
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
),
head_dim=self.model_config.head_dim,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
grow_direction="down",
draft_region=region,
)
def _fused_draft_for_pool_factory(self):
"""Resolve ONCE per factory call (not inline) so the boot log reports
exactly the placement the factory is handed: a declined fusion and an
engaged one otherwise look identical from outside."""
decision = self._fused_draft_decision()
if decision.placement is None:
if decision.declined is not None:
logger.warning("fused draft KV disabled: %s", decision.declined)
return None
placement = decision.placement
region = placement.region
logger.info(
"[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d "
"kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s",
region.lane_num,
region.head_num,
region.head_dim,
region.resolved_v_head_dim(),
region.store_dtype,
region.entry_bytes(),
[
tuple(placement.lanes_for(r))
for r in range(len(placement.runner_lane_counts))
],
)
return placement
def _init_unified_swa_pools(
self,
*,
@@ -964,6 +1141,7 @@ class KVCacheConfigurator:
# charged, see `_check_bs1_feasibility_floor`.
model_context_len=self.model_config.context_len,
sliding_window_size=self.model_config.sliding_window_size,
fused_draft=self._fused_draft_for_pool_factory(),
)
return UnifiedPoolBundle(
unified_memory_pool=bundle.unified_memory_pool,
@@ -71,6 +71,7 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
)
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
from sglang.srt.runtime_context import get_parallel
@@ -112,13 +113,18 @@ class KVIndexTranslator:
self.page_size = page_size
self.device = device
self.is_translating = (
is_unified_target = (
isinstance(
token_to_kv_pool_allocator,
(UnifiedMambaTokenToKVPoolAllocator, UnifiedSWAAllocatorBase),
)
and token_to_kv_pool_allocator.get_kvcache() is token_to_kv_pool
)
host_allocator = fused_draft_host_allocator(token_to_kv_pool)
is_fused_draft = (
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
)
self.is_translating = is_unified_target or is_fused_draft
if self.is_translating:
alloc = token_to_kv_pool_allocator
self._capture_page_size = alloc.page_size
@@ -133,13 +139,17 @@ class KVIndexTranslator:
# DCP read ids stay WIDENED to the consumer: selecting this rank's
# share changes the length, so only the production site can do it.
self.defer_read_translate = get_parallel().attn_dcp_size > 1
if isinstance(alloc, UnifiedSWAAllocatorBase):
routes_window_layers = is_unified_target or isinstance(
token_to_kv_pool, BaseSWAKVPool
)
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
self._swa_v2p_table = alloc.swa_v2p_page_table
self._swa_write_loc_from_full = self._swa_write_loc_unified
else:
self._swa_v2p_table = None
self._swa_write_loc_from_full = None
else:
self._capture_page_size = page_size
self._full_v2p_table = None
self._full_p2v_table = None
self._translate_full = None
@@ -175,6 +185,14 @@ class KVIndexTranslator:
return self._full_v2p_table.numel() * self._capture_page_size
return max_token_pool_size + self.page_size
def full_flat_v2p(self) -> Optional[torch.Tensor]:
"""The full-side v2p page table for a kernel that translates flat ids
itself, or ``None`` when this runner does not translate (pass-through
and static pools)."""
if not self.is_translating:
return None
return self._full_v2p_table
# -- per-batch view --------------------------------------------------------
@property
@@ -0,0 +1,262 @@
"""Draft-model regions nested inside unified sub-pools' slot entries.
The nesting, outermost first:
sub-pool a named pool inside the unified pool, with its own grow direction
and frontier. HOST is the role it plays when its entries also
carry draft bytes; only "full" hosts a draft here.
slot one token entry of that sub-pool. The draft's KV becomes extra
parts of every slot, after the host's own parts at
`draft_offset_in_entry()`, so ONE slot id per token covers target
and draft: one allocation, one free, one whole-page relocation
carry both.
region the draft geometry inside the host's entries. A region is not a
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
lane one position in the region's layer dimension -- one draft layer of
one runner, present in every slot.
Runners are the draft's execution copies, and the two head shapes place them
oppositely. A replicated head gives EVERY runner all the draft's layers, so
each needs its own lanes or the runners clobber each other's KV (3 runners x
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
either direction. Each runner owns a contiguous run of lanes, and the runs
tile the region in runner order.
Three phases, three moments in boot, and what carries each:
PLAN on the TARGET, before any memory exists
KVCacheConfigurator._fused_draft_decision()
draft_kv_profile(draft_model_config) what the checkpoint asks for
place_fused_draft(profile, runners) admit, or decline with a reason
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
CARVE as the byte buffer is cut
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
init_unified_*_pools(fused_draft=...)
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
UnifiedKVPool.build_dense_draft_views()
BIND on each DRAFT runner, built after the target
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
bind_fused_draft(...) -> UnifiedDraftKVPool
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
A draft whose layers need another sub-pool kind declines to a private pool.
"""
from typing import List, Optional, Tuple
import msgspec
import torch
from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
sub-pool. The draft is a separate checkpoint, so its head geometry and
layer count differ from the host's; its K and V rows are two more parts of
the host entry, indexed by the host's physical token id."""
lane_num: int
head_num: int
head_dim: int
store_dtype: torch.dtype
v_head_dim: Optional[int] = None
def validate(self) -> None:
assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}"
assert self.head_num > 0, f"head_num must be positive; got {self.head_num}"
assert self.head_dim > 0, f"head_dim must be positive; got {self.head_dim}"
v = self.resolved_v_head_dim()
assert v > 0, f"v_head_dim must be positive; got {v}"
def resolved_v_head_dim(self) -> int:
return self.head_dim if self.v_head_dim is None else self.v_head_dim
def k_row_bytes(self) -> int:
return self.head_num * self.head_dim * self.store_dtype.itemsize
def v_row_bytes(self) -> int:
return self.head_num * self.resolved_v_head_dim() * self.store_dtype.itemsize
def entry_bytes(self) -> int:
"""Draft bytes per slot, before the host entry's alignment."""
return self.lane_num * (self.k_row_bytes() + self.v_row_bytes())
def parts(self, offset_bytes: int) -> Tuple[DensePart, DensePart]:
"""The draft's K and V parts, laid out from ``offset_bytes`` inside
the host entry."""
layer_stride = self.k_row_bytes() + self.v_row_bytes()
return (
DensePart(
name="draft_k",
offset_bytes=offset_bytes,
layer_stride_bytes=layer_stride,
layer_num=self.lane_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="draft_v",
offset_bytes=offset_bytes + self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.lane_num,
row_shape=(self.head_num, self.resolved_v_head_dim()),
dtype=self.store_dtype,
),
)
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
"""Where every draft runner's layers live inside the "full" sub-pool: the
region, and each runner's lane count in runner order. Built once on the
target, stored on the `UnifiedKVPool`, and read back by each draft
runner, so the two sides cannot disagree on a lane.
"""
region: DenseDraftRegion
runner_lane_counts: Tuple[int, ...]
def __post_init__(self):
assert self.runner_lane_counts, "a placement needs at least one draft runner"
assert sum(self.runner_lane_counts) == self.region.lane_num, (
f"runner lane counts {self.runner_lane_counts} must tile "
f"range({self.region.lane_num})"
)
def lanes_for(self, runner: int) -> range:
start = sum(self.runner_lane_counts[:runner])
return range(start, start + self.runner_lane_counts[runner])
class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
"""Per-GPU K/V row geometry of one kind of draft attention layer."""
head_num: int
head_dim: int
v_head_dim: int
class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True):
"""What the draft checkpoint asks of the host sub-pools.
``num_depths`` > 1 marks a per-depth head (one transformer block per MTP
depth, served by one runner each under multi-layer EAGLE); otherwise
every runner serves all ``num_layers`` layers.
"""
num_layers: int
full: DraftKVGeometry
swa_layer_ids: Tuple[int, ...] = ()
num_depths: int = 1
def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
"""The draft layer ids a hybrid-SWA pool routes to its swa side. This is
the pool-routing convention (`SWAKVPool.layers_mapping`), not a per-layer
kernel window: an attention-only draft window stays full-kind."""
mc = draft_model_config
if mc.is_hybrid_swa and not mc.is_deepseek_v4_arch:
return tuple(int(i) for i in mc.swa_attention_layer_ids)
return ()
def draft_kv_profile(
draft_model_config, *, num_layers: int, attn_tp_size: int
) -> DraftKVProfile:
"""The profile of a draft `ModelConfig`, heads divided by attn_tp the way
the target divides its own (drafts never join the DCP group)."""
mc = draft_model_config
num_depths = mc.num_nextn_predict_layers
return DraftKVProfile(
num_layers=int(num_layers),
full=DraftKVGeometry(
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
head_dim=int(mc.head_dim),
v_head_dim=int(mc.v_head_dim),
),
swa_layer_ids=draft_swa_layer_ids(mc),
num_depths=1 if num_depths is None else int(num_depths),
)
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
private pool. Neither means fusion simply does not apply."""
placement: Optional[FusedDraftPlacement] = None
declined: Optional[str] = None
def _runner_layer_counts(
profile: DraftKVProfile, num_runners: int
) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]:
"""Per runner, its (full, swa) layer counts; or why no runner layout exists."""
if profile.num_depths <= 1:
num_swa = len(profile.swa_layer_ids)
return [(profile.num_layers - num_swa, num_swa)] * num_runners, None
if num_runners == 1:
return None, (
f"a per-depth draft head ({profile.num_depths} depths) needs one "
"runner per depth (multi-layer EAGLE)"
)
if num_runners > profile.num_depths:
return None, (
f"{num_runners} draft runners exceed the head's {profile.num_depths} depths"
)
swa = set(profile.swa_layer_ids)
return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None
def place_fused_draft(
*,
profile: DraftKVProfile,
num_runners: int,
store_dtype: torch.dtype,
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host arm serves declines the whole draft to
its private pool."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
num_swa = sum(swa for _, swa in counts)
if num_swa:
return FusedDraftDecision(
declined=(
f"the draft has {num_swa} SWA layer(s) of its own, which the "
"fused dense pool cannot serve"
)
)
geometry = profile.full
if geometry.head_dim != geometry.v_head_dim:
return FusedDraftDecision(
declined=(
"the draft's K/V rows are asymmetric "
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), "
"which is not admitted yet"
)
)
row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return FusedDraftDecision(
declined=(
f"the draft's K/V rows are {row_bytes} B, not a multiple of the "
f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
)
)
full_counts = tuple(full for full, _ in counts)
region = DenseDraftRegion(
lane_num=sum(full_counts),
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
store_dtype=store_dtype,
)
return FusedDraftDecision(
placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts)
)
@@ -43,6 +43,11 @@ def align_entry_bytes(num_bytes: int) -> int:
return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES
def align_part_offset(num_bytes: int) -> int:
"""Round a part's byte offset inside the entry up to the row alignment."""
return -(-num_bytes // ROW_ALIGN_BYTES) * ROW_ALIGN_BYTES
class DensePart(msgspec.Struct, frozen=True, kw_only=True):
"""One row family inside the entry: ``layer_num`` rows of ``row_shape``,
layer ``l`` at ``offset_bytes + l * layer_stride_bytes``."""
@@ -0,0 +1,159 @@
"""KV pools a DRAFT runner binds over the draft lanes fused into the target's
unified pool: same pages, same slot ids, same v2p table as the target -- one
allocation, one free, one relocation."""
from typing import Any, Dict, List, Mapping, Optional, Sequence
import torch
from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
class UnifiedDraftKVPool(MHATokenToKVPool):
"""Dense draft KV over the draft parts of one host sub-pool's entries.
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
``layer_lanes`` maps each of this runner's layer ids to its region lane.
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
produced by the allocator's translate (the id-space choke point binds it,
see KVIndexTranslator); the pool exposes `host_allocator` for that
binding. Relocation needs no method here: compaction moves whole page
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
raises so a stray per-slot move fails loudly instead of corrupting the
fused layout.
"""
requires_physical_write_loc = True
def __init__(
self,
*,
unified_buffer: UnifiedKVPool,
host_sub_pool_name: str,
host_allocator,
layer_lanes: Mapping[int, int],
page_size: int = 1,
):
region = unified_buffer.require_draft_host_spec(host_sub_pool_name).draft_region
layer_ids = sorted(layer_lanes)
assert layer_ids, "UnifiedDraftKVPool binds at least one layer"
start_layer = layer_ids[0]
assert layer_ids == list(range(start_layer, start_layer + len(layer_ids))), (
f"fused draft layer ids must be contiguous; got {layer_ids}"
)
lanes = [layer_lanes[layer_id] for layer_id in layer_ids]
assert len(set(lanes)) == len(lanes) and all(
0 <= s < region.lane_num for s in lanes
), f"draft lanes {lanes} must be distinct and within range({region.lane_num})"
k_views, v_views = unified_buffer.build_dense_draft_views(host_sub_pool_name)
max_slots = unified_buffer.max_slots(host_sub_pool_name)
self._unified_buffer = unified_buffer
self._host_sub_pool_name = host_sub_pool_name
self.host_allocator = host_allocator
self.layer_lanes: Dict[int, int] = dict(layer_lanes)
self._k_views: List[torch.Tensor] = [k_views[s] for s in lanes]
self._v_views: List[torch.Tensor] = [v_views[s] for s in lanes]
num_pages = max_slots // page_size
super().__init__(
size=num_pages * page_size - page_size,
page_size=page_size,
dtype=region.store_dtype,
head_num=region.head_num,
head_dim=region.head_dim,
layer_num=len(layer_ids),
device=unified_buffer.device,
enable_memory_saver=False, # buffer owned by UnifiedKVPool
v_head_dim=region.resolved_v_head_dim(),
start_layer=start_layer,
end_layer=start_layer + len(layer_ids) - 1,
enable_kv_cache_copy=False,
kv_cache_layout="page_major",
)
def _create_buffers(self):
self.k_buffer = self._k_views
self.v_buffer = self._v_views
def _clear_buffers(self):
pass # lifetime owned by UnifiedKVPool
def get_kv_size_bytes(self):
return 0, 0 # fused into the host entries; UnifiedKVPool logs the total
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
raise NotImplementedError(
"fused draft KV relocates with the HOST pool's whole-page move; a "
"draft-side per-slot move would corrupt the fused layout"
)
def get_contiguous_buf_infos(self):
raise NotImplementedError(
"fused draft KV has no per-layer contiguous regions; KV transfer / "
"disaggregation is unsupported."
)
def get_cpu_copy(self, indices, mamba_indices=None):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]:
"""The host allocator a fused draft pool translates through, or None for
any other pool."""
if isinstance(token_to_kv_pool, UnifiedDraftKVPool):
return token_to_kv_pool.host_allocator
return None
def draft_kv_layer_ids(model) -> List[int]:
"""Layer ids owning attention KV in the BUILT draft model, in layer order.
Window layers count -- a window is a parameter of the attention layer;
linear and recurrent layers have no attention module and are absent."""
from sglang.srt.layers.radix_attention import RadixAttention
return sorted(
{m.layer_id for m in model.modules() if isinstance(m, RadixAttention)}
)
def bind_fused_draft(
*,
unified_buffer: UnifiedKVPool,
host_allocator,
placement: FusedDraftPlacement,
runner: int,
kv_layer_ids: Sequence[int],
swa_layer_ids: Sequence[int],
page_size: int,
) -> UnifiedDraftKVPool:
"""The KV pool draft runner ``runner`` binds over its fused slots.
The placement sized the lanes from the draft config; the model's real
layer ids fill them in layer order, so a count mismatch is a loud boot
failure, never a silent alias.
"""
swa = set(swa_layer_ids)
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa]
assert len(full_ids) == len(kv_layer_ids), (
f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused "
"SWA KV is not supported yet"
)
full_lanes = placement.lanes_for(runner)
assert len(full_ids) == len(full_lanes), (
f"draft runner {runner}: {len(full_ids)} full-attention layer(s) "
f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}"
)
return UnifiedDraftKVPool(
unified_buffer=unified_buffer,
host_sub_pool_name="full",
host_allocator=host_allocator,
layer_lanes=dict(zip(full_ids, full_lanes)),
page_size=page_size,
)
@@ -38,11 +38,16 @@ from sglang.kernels.ops.kvcache.zero_pages import zero_pages
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.layout.token_major import (
ROW_ALIGN_BYTES,
DenseEntryLayout,
DensePart,
align_entry_bytes,
align_part_offset,
build_dense_views,
build_mamba_entry_views,
)
@@ -113,6 +118,8 @@ class SubPoolSpec(ABC):
name: str
layer_num: int
grow_direction: str # "up" | "down" | "float"
# Fused draft region riding in this sub-pool's entries; None = unfused.
draft_region: Optional[DenseDraftRegion] = None
def __post_init__(self):
assert self.grow_direction in self._allowed_grow_directions, (
@@ -134,7 +141,17 @@ class SubPoolSpec(ABC):
@dataclass(frozen=True, kw_only=True)
class MHASubPoolSpec(SubPoolSpec):
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`."""
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`.
With `draft_region` set, the draft model's K and V rows are two more
parts of every slot's entry, after the host parts:
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
The slot stride stays the one entry, so host and draft views are indexed
by the same physical token id; only their offsets differ. `draft_region
is None` keeps the layout byte-identical to the unfused one.
"""
head_num: int
head_dim: int
@@ -159,6 +176,8 @@ class MHASubPoolSpec(SubPoolSpec):
assert self.v_head_dim > 0, (
f"v_head_dim must be positive; got {self.v_head_dim}"
)
if self.draft_region is not None:
self.draft_region.validate()
def k_row_bytes(self) -> int:
return self.head_num * self.head_dim * self.store_dtype.itemsize
@@ -166,9 +185,20 @@ class MHASubPoolSpec(SubPoolSpec):
def v_row_bytes(self) -> int:
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
def host_entry_bytes(self) -> int:
"""Host (target-only) bytes for one slot, before any draft parts."""
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
def draft_offset_in_entry(self) -> int:
"""Byte offset of the fused draft parts inside one slot's entry."""
assert self.draft_region is not None
return align_part_offset(self.host_entry_bytes())
def entry_bytes(self) -> int:
if self.draft_region is None:
return align_entry_bytes(self.host_entry_bytes())
return align_entry_bytes(
self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
)
def page_bytes(self, page_size: int) -> int:
@@ -179,27 +209,27 @@ class MHASubPoolSpec(SubPoolSpec):
is ``page_size`` such entries back to back."""
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
layer_stride = self.k_row_bytes() + self.v_row_bytes()
return DenseEntryLayout(
entry_bytes=self.entry_bytes(),
parts=(
DensePart(
name="k",
offset_bytes=0,
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="v",
offset_bytes=self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.v_head_dim),
dtype=self.store_dtype,
),
parts = (
DensePart(
name="k",
offset_bytes=0,
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="v",
offset_bytes=self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.v_head_dim),
dtype=self.store_dtype,
),
)
if self.draft_region is not None:
parts += self.draft_region.parts(self.draft_offset_in_entry())
return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts)
def get_dtype(self) -> torch.dtype:
return self.store_dtype
@@ -227,6 +257,9 @@ class MLASubPoolSpec(SubPoolSpec):
assert self.qk_rope_head_dim > 0, (
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
)
assert self.draft_region is None, (
"MLA sub-pools do not carry a fused draft region yet"
)
@property
def kv_cache_dim(self) -> int:
@@ -271,6 +304,9 @@ class MambaSubPoolSpec(SubPoolSpec):
def __post_init__(self):
super().__post_init__()
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
assert self.draft_region is None, (
"mamba state pages carry no fused draft region yet"
)
def conv_row_bytes(self, idx: int) -> int:
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
@@ -350,6 +386,7 @@ class UnifiedKVPool:
page_size: int = 1,
post_capture_active: bool = False,
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
fused_draft: Optional[FusedDraftPlacement] = None,
):
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
assert len(sub_pool_specs) >= 2, (
@@ -386,6 +423,17 @@ class UnifiedKVPool:
self.post_capture_active = post_capture_active
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
self._bs1_floor_terms = bs1_floor_terms or []
self.fused_draft = fused_draft
for spec in sub_pool_specs:
expected = (
fused_draft.region
if fused_draft is not None and spec.name == "full"
else None
)
assert spec.draft_region is expected, (
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
f"not match the fused draft placement's {expected}"
)
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
@@ -588,6 +636,17 @@ class UnifiedKVPool:
)
return s
def require_draft_host_spec(self, name: str) -> SubPoolSpec:
"""The sub-pool spec whose entries carry a fused draft region.
Kind-agnostic: any spec that resolves a `draft_region` also lays the
draft parts out in its `layout()`."""
s = self._specs_by_name[name]
assert s.draft_region is not None, (
f"sub-pool {name!r} carries no fused draft region"
)
return s
def max_slots(self, name: str) -> int:
return self._max_slots[name]
@@ -630,6 +689,27 @@ class UnifiedKVPool:
)
return k_views, v_views
def build_dense_draft_views(
self, sub_pool_name: str
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""Per-layer K/V views of the DRAFT parts fused into ``sub_pool_name``'s
entries: same pages, same slot ids, same v2p table as the host."""
spec = self.require_draft_host_spec(sub_pool_name)
layout = spec.layout()
page_size = self._page_size
num_slots = self.max_slots(sub_pool_name) // page_size * page_size
k_views, v_views = (
build_dense_views(
self._raw,
layout=layout,
part_name=name,
num_slots=num_slots,
anchor_bytes=self._anchor_bytes[sub_pool_name],
)
for name in ("draft_k", "draft_v")
)
return k_views, v_views
def _build_mla_views(
self,
spec: MLASubPoolSpec,
@@ -1899,8 +1979,15 @@ def init_unified_swa_pools(
lazy_compaction: bool = False,
model_context_len: Optional[int] = None,
sliding_window_size: Optional[int] = None,
fused_draft: Optional[FusedDraftPlacement] = None,
) -> UnifiedSWAPoolBundle:
"""Build the SWA-hybrid unified-memory-pool stack."""
"""Build the SWA-hybrid unified-memory-pool stack.
With ``fused_draft``, every entry of the "full" sub-pool carries the
draft model's K/V parts after the host parts, and each draft runner
binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a
pool of its own.
"""
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
@@ -1926,6 +2013,7 @@ def init_unified_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
)
swa_spec = MHASubPoolSpec(
name="swa",
@@ -1981,6 +2069,7 @@ def init_unified_swa_pools(
page_size=page_size,
post_capture_active=post_capture_active,
bs1_floor_terms=bs1_floor_terms,
fused_draft=fused_draft,
)
token_to_kv_pool = UnifiedSWAKVPool(
unified_buffer=shared_pool,
@@ -44,6 +44,11 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True):
dflash_target_layer_ids: Any = None
# DFLASH draft KV bytes/token; None when unresolved.
dflash_draft_cell_size_per_token: int | None = None
# The draft checkpoint read in draft mode, and its KV layer count: the
# fused-draft placement's input. `eagle_draft_num_layers` above keeps
# sizing a private draft pool exactly as before.
draft_model_config: Optional[ModelConfig] = None
draft_kv_num_layers: Optional[int] = None
def resolve_spec_aux_hidden_state_config(
@@ -85,24 +90,38 @@ def _resolve_eagle_aux_hidden_state(
):
return
draft_model_config = model_config
if get_spec().speculative_draft_model_path:
draft_model_config = ModelConfig.from_server_args(
server_args,
model_path=get_spec().speculative_draft_model_path,
model_revision=get_spec().speculative_draft_model_revision,
is_draft_model=True,
)
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
if num_nextn_predict_layers is not None:
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
elif get_spec().speculative_draft_model_path:
config.eagle_draft_num_layers = int(
draft_path = get_spec().speculative_draft_model_path
# The draft read in draft mode. A path-less NEXTN/MTP head is the TARGET
# checkpoint rewritten to its MTP form, which is where its
# `num_nextn_predict_layers` gets filled in.
draft_model_config = ModelConfig.from_server_args(
server_args,
model_path=draft_path,
model_revision=get_spec().speculative_draft_model_revision,
is_draft_model=True,
)
if draft_model_config.num_nextn_predict_layers is not None:
config.draft_kv_num_layers = int(draft_model_config.num_nextn_predict_layers)
elif draft_path is not None:
config.draft_kv_num_layers = int(
max(
draft_model_config.num_hidden_layers,
draft_model_config.num_attention_layers,
)
)
if config.draft_kv_num_layers is not None:
config.draft_model_config = draft_model_config
# The private draft pool's budget keeps its reading: a path-less run sizes
# the draft off the TARGET config. Reading the draft-mode config here would
# change the token budget of every path-less MTP deployment.
if not draft_path:
draft_model_config = model_config
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
if num_nextn_predict_layers is not None:
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
elif draft_path:
config.eagle_draft_num_layers = config.draft_kv_num_layers
else:
return
@@ -714,9 +714,27 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
self._draft_cell_size = _dflash_draft_cell_size(kvc)
self._fused_full_entry = kvc.fused_entry_bytes("full")
if self._fused_full_entry is not None:
self._draft_full_layers_num = 0
self._draft_swa_layers_num = 0
self._draft_swa_full_layers_num = 0
self._draft_cell_size = 0
self._recompute_cell_size()
def _full_cell_bytes(self) -> int:
"""Bytes per full-side token: the fused entry when the draft is placed
there, else the target's rows plus the private draft's full rows."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * (
self._full_layers_num + self._draft_full_layers_num
)
def _swa_cell_bytes(self) -> int:
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
def _recompute_cell_size(self) -> None:
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
@@ -730,12 +748,9 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
else:
self._cell_size = (
self._full_per_token
* (self._full_layers_num + self._draft_full_layers_num)
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
+ self._swa_full_tokens_ratio
* self._swa_per_token
* (self._swa_layers_num + self._draft_swa_layers_num)
+ self._swa_full_tokens_ratio * self._swa_cell_bytes()
+ self._draft_cell_size
)
@@ -747,9 +762,17 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
+ self._draft_cell_size
)
def _unified_full_bytes_per_token(self) -> int:
"""Bytes one full-side token takes in the unified pool: the fused
entry (host + draft + pad) when the draft lives in it, else the
target's own rows. A private draft pool is priced separately."""
if self._fused_full_entry is not None:
return self._fused_full_entry
return self._full_per_token * self._full_layers_num
def _unified_pool_bytes(self, full_tokens: int, swa_tokens: int) -> int:
return (
full_tokens * self._full_per_token * self._full_layers_num
full_tokens * self._unified_full_bytes_per_token()
+ swa_tokens * self._swa_per_token * self._swa_layers_num
)
@@ -761,7 +784,7 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
) -> int:
"""Find the largest page-aligned full capacity whose allocations fit."""
draft_bytes_per_token = self._draft_pool_bytes_per_token()
target_full_bytes_per_token = self._full_per_token * self._full_layers_num
target_full_bytes_per_token = self._unified_full_bytes_per_token()
assert target_full_bytes_per_token > 0
def allocation_bytes(full_pages: int) -> int:
@@ -930,19 +953,14 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
) -> MemoryPoolConfig:
# SWA pool sized tightly from the cap; the rest of the budget goes to full.
swa_tokens = ceil_align(self._swa_cap, page_size)
fixed_swa_bytes = (
swa_tokens
* self._swa_per_token
* (self._swa_layers_num + self._draft_swa_layers_num)
)
if self._enable_unified_memory:
fixed_swa_bytes = swa_tokens * self._swa_cell_bytes()
if self._enable_unified_memory and self._draft_pool_bytes_per_token() > 0:
full_tokens = self._max_unified_full_tokens(
available_bytes, page_size, fixed_swa_tokens=swa_tokens
)
else:
full_cell_size = (
self._full_per_token
* (self._full_layers_num + self._draft_full_layers_num)
self._full_cell_bytes()
+ self._swa_per_token * self._draft_swa_full_layers_num
)
full_tokens = (
@@ -180,8 +180,15 @@ class BaseSpecWorker(ABC):
@property
def primary_draft_kv_pool(self) -> Optional[object]:
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
draft_runners = self._draft_model_runners()
return draft_runners[0].token_to_kv_pool if draft_runners else None
if not draft_runners:
return None
pool = draft_runners[0].token_to_kv_pool
if fused_draft_host_allocator(pool) is not None:
return None
return pool
@property
def target_worker(self) -> TpModelWorker:
@@ -389,6 +389,15 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self._configure_qsa_mtp_index_share()
self.tree_mask_mode = default_tree_mask_mode()
# Boot guard: every backend a draft forward reaches must carry its
# runner's translator or the index builders emit virtual ids.
translator = self.draft_runner.kv_index_translator
if translator.is_translating:
backends = [self.draft_attn_backend, self.draft_extend_attn_backend]
if self.draft_attn_backend is not None:
backends += self.draft_attn_backend.attn_backends
translator.bind_and_verify_backends(backends)
def _configure_qsa_mtp_index_share(self) -> None:
"""Reuse the draft-extend QSA selection across the MTP decode steps;
chain speculation only: with topk > 1 decode rows are not request-major."""
@@ -398,6 +398,13 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[
step
].attn_backend = self.draft_extend_attn_backend_list[-1]
# Boot guard: every backend a draft forward reaches must carry
# its runner's translator or the index builders emit virtual ids.
translator = self.draft_runner_list[step].kv_index_translator
if translator.is_translating:
translator.bind_and_verify_backends(
[self.draft_extend_attn_backend_list[-1]]
)
def _capture_cuda_graphs(self):
self.cuda_graph_runner = None