[unified-memory] Fuse the draft model's KV into the target's page envelope (mechanism) (#37628)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
08d30f2300
commit
8b80808366
@@ -78,6 +78,17 @@ def assign_draft_cache_locs_contiguous(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
|
||||
"""Virtual token ids -> physical ones through the page-level v2p table,
|
||||
as `translate_kv_loc` does. A negative id maps to the page-0 sink instead
|
||||
of loading before `v2p`."""
|
||||
i64 = ids.to(tl.int64)
|
||||
vpage = tl.where(i64 < 0, 0, i64 // page_size)
|
||||
phys = tl.load(v2p + vpage, mask=mask, other=0)
|
||||
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
|
||||
|
||||
|
||||
@triton.jit
|
||||
def generate_draft_decode_kv_indices(
|
||||
req_pool_indices,
|
||||
@@ -86,6 +97,7 @@ def generate_draft_decode_kv_indices(
|
||||
kv_indices,
|
||||
kv_indptr,
|
||||
positions,
|
||||
v2p,
|
||||
pool_len: tl.constexpr,
|
||||
kv_indices_stride: tl.constexpr,
|
||||
kv_indptr_stride: tl.constexpr,
|
||||
@@ -96,6 +108,7 @@ def generate_draft_decode_kv_indices(
|
||||
window_size: tl.constexpr = 0,
|
||||
sink_size: tl.constexpr = 0,
|
||||
NUM_STEPS: tl.constexpr = 0,
|
||||
TRANSLATE: tl.constexpr = False,
|
||||
):
|
||||
# window_size > 0 restricts the draft (not the target) to sink_size prefix
|
||||
# tokens + the most-recent window_size; window_size == 0 is the identity.
|
||||
@@ -153,6 +166,8 @@ def generate_draft_decode_kv_indices(
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
copy_offset += BLOCK_SIZE
|
||||
else:
|
||||
@@ -165,6 +180,8 @@ def generate_draft_decode_kv_indices(
|
||||
recent_start + copy_offset - s_eff,
|
||||
)
|
||||
data = tl.load(token_pool_ptr + src, mask=mask)
|
||||
if TRANSLATE:
|
||||
data = _translate_token_ids(data, v2p, mask, page_size)
|
||||
tl.store(kv_ptr + copy_offset, data, mask=mask)
|
||||
|
||||
# Extension entries and kv_indptr belong to token block 0 alone; other
|
||||
@@ -196,6 +213,11 @@ def generate_draft_decode_kv_indices(
|
||||
mask=extend_offset < iters,
|
||||
)
|
||||
|
||||
if TRANSLATE:
|
||||
extend_data = _translate_token_ids(
|
||||
extend_data, v2p, extend_offset < iters, page_size
|
||||
)
|
||||
|
||||
tl.store(
|
||||
kv_ptr + seq_len_w + extend_offset,
|
||||
extend_data,
|
||||
|
||||
@@ -4647,6 +4647,7 @@ class AiterMultiStepDraftBackend:
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
|
||||
self.page_size = get_schedule().page_size
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
|
||||
def common_template(
|
||||
self, forward_batch: ForwardBatch, kv_indices_buffer: torch.Tensor, call_fn: int
|
||||
@@ -4662,6 +4663,7 @@ class AiterMultiStepDraftBackend:
|
||||
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
|
||||
else 1
|
||||
)
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
|
||||
](
|
||||
@@ -4671,6 +4673,7 @@ class AiterMultiStepDraftBackend:
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
@@ -4681,6 +4684,7 @@ class AiterMultiStepDraftBackend:
|
||||
# A single token block is the historical launch; NUM_STEPS=0 keeps
|
||||
# its 128-wide program instead of the token-block specialization.
|
||||
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
|
||||
TRANSLATE=v2p is not None,
|
||||
)
|
||||
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
|
||||
@@ -3396,6 +3396,7 @@ class FlashAttentionMultiStepBackend:
|
||||
self.model_runner = model_runner
|
||||
self.topk = topk
|
||||
self.speculative_num_steps = speculative_num_steps
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
self.attn_backends = []
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
self.attn_backends.append(
|
||||
|
||||
@@ -2369,6 +2369,7 @@ class FlashInferMultiStepDraftBackend:
|
||||
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
|
||||
model_runner
|
||||
)
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
|
||||
def common_template(
|
||||
self,
|
||||
@@ -2391,6 +2392,7 @@ class FlashInferMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
@@ -2400,6 +2402,7 @@ class FlashInferMultiStepDraftBackend:
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
@@ -2409,6 +2412,7 @@ class FlashInferMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -1218,6 +1218,7 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
self.req_to_token_pool = model_runner.req_to_token_pool
|
||||
self.pool_len = model_runner.req_to_token_pool.req_to_token.shape[1]
|
||||
self.page_size = get_schedule().page_size
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
|
||||
def common_template(
|
||||
self,
|
||||
@@ -1240,6 +1241,7 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
seq_lens_sum=seq_lens_sum,
|
||||
)
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
self.generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
@@ -1249,6 +1251,7 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
@@ -1256,6 +1259,7 @@ class FlashInferMLAMultiStepDraftBackend:
|
||||
next_power_of_2(self.speculative_num_steps),
|
||||
next_power_of_2(bs),
|
||||
self.page_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
)
|
||||
|
||||
assert forward_batch.spec_info is not None
|
||||
|
||||
@@ -612,6 +612,7 @@ class FlashMLAMultiStepDraftBackend:
|
||||
)
|
||||
self.topk = topk
|
||||
self.speculative_num_steps = speculative_num_steps
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
max_bs = model_runner.req_to_token_pool.size * self.topk
|
||||
self.kv_indptr = torch.zeros(
|
||||
(
|
||||
|
||||
@@ -2449,6 +2449,7 @@ class TritonMultiStepDraftBackend:
|
||||
self.draft_window_size, self.draft_sink_size = resolve_draft_decode_window(
|
||||
model_runner
|
||||
)
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
|
||||
def common_template(
|
||||
self,
|
||||
@@ -2467,6 +2468,7 @@ class TritonMultiStepDraftBackend:
|
||||
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
|
||||
seq_lens_sum = num_seqs * self.max_context_len
|
||||
|
||||
v2p = self.kv_index_translator.full_flat_v2p()
|
||||
generate_draft_decode_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
@@ -2476,6 +2478,7 @@ class TritonMultiStepDraftBackend:
|
||||
kv_indices_buffer,
|
||||
self.kv_indptr,
|
||||
forward_batch.positions,
|
||||
v2p,
|
||||
self.pool_len,
|
||||
kv_indices_buffer.shape[1],
|
||||
self.kv_indptr.shape[1],
|
||||
@@ -2485,6 +2488,7 @@ class TritonMultiStepDraftBackend:
|
||||
self.page_size,
|
||||
self.draft_window_size,
|
||||
self.draft_sink_size,
|
||||
TRANSLATE=v2p is not None,
|
||||
)
|
||||
|
||||
if call_fn is None:
|
||||
|
||||
@@ -570,16 +570,45 @@ class KVCacheConfigurator:
|
||||
UnifiedSWAAllocatorBase,
|
||||
),
|
||||
):
|
||||
draft_virtual_id_space = (
|
||||
token_to_kv_pool_allocator.draft_virtual_id_space
|
||||
)
|
||||
alloc = token_to_kv_pool_allocator
|
||||
placement = self._fused_draft_from_target_buffer(alloc)
|
||||
if placement is not None:
|
||||
# FUSED arm
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
draft_swa_layer_ids,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_draft_pool import (
|
||||
bind_fused_draft,
|
||||
draft_kv_layer_ids,
|
||||
)
|
||||
|
||||
assert req_to_token_pool is not None, (
|
||||
"a draft worker shares the target's req_to_token_pool"
|
||||
)
|
||||
draft_pool = bind_fused_draft(
|
||||
unified_buffer=alloc.unified_buffer,
|
||||
host_allocator=alloc,
|
||||
placement=placement,
|
||||
runner=self.draft_model_idx or 0,
|
||||
kv_layer_ids=draft_kv_layer_ids(self.model),
|
||||
swa_layer_ids=draft_swa_layer_ids(self.model_config),
|
||||
page_size=self.page_size,
|
||||
)
|
||||
return _InitializedPools(
|
||||
req_to_token_pool=req_to_token_pool,
|
||||
token_to_kv_pool=draft_pool,
|
||||
token_to_kv_pool_allocator=alloc,
|
||||
unified_memory_pool=None,
|
||||
)
|
||||
# PRIVATE arm
|
||||
draft_virtual_id_space = alloc.draft_virtual_id_space
|
||||
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
|
||||
"unified allocator virtual space smaller than the token "
|
||||
f"budget: virtual_id_space={draft_virtual_id_space} < "
|
||||
f"max_total_num_tokens={sizes.max_total_num_tokens}"
|
||||
)
|
||||
# Round UP to page alignment (paged draft backends view the
|
||||
# pool as (-1, page_size, H, D); the virtual space is not aligned).
|
||||
# pool as (-1, page_size, H, D); the space is not aligned).
|
||||
page = max(int(self.pool_page_size or 1), 1)
|
||||
draft_virtual_id_space = (
|
||||
(draft_virtual_id_space + page - 1) // page * page
|
||||
@@ -684,6 +713,33 @@ class KVCacheConfigurator:
|
||||
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
|
||||
)
|
||||
|
||||
def _fused_draft_from_target_buffer(self, alloc):
|
||||
"""The placement this draft binds to, or None for the private-pool
|
||||
fallback (the draft then owns a raw-virtual-indexed pool of its own).
|
||||
Target boot declines a placement for legitimate geometry, so the
|
||||
private arm is the rollback lever, not a boot-order bug."""
|
||||
if not self.spec_algorithm.is_eagle():
|
||||
return None
|
||||
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
|
||||
|
||||
placement = alloc.unified_buffer.fused_draft
|
||||
if placement is None:
|
||||
logger.info(
|
||||
"[unified-memory-pool] no fused draft placement on the target's "
|
||||
"buffer; the draft binds a private pool over the virtual id space."
|
||||
)
|
||||
return None
|
||||
# The region stores rows in the target's KV dtype; a draft that resolved
|
||||
# its own would read and write them as something else.
|
||||
if _store_dtype_for(self.kv_cache_dtype) != placement.region.store_dtype:
|
||||
raise ValueError(
|
||||
f"Fused draft KV: the draft resolved its KV cache dtype to "
|
||||
f"{self.kv_cache_dtype}, but its region inside the target's pages "
|
||||
f"stores {placement.region.store_dtype}. Set "
|
||||
"--speculative-draft-kv-cache-dtype to the target's KV cache dtype."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _init_unified_mamba_pools(
|
||||
self,
|
||||
*,
|
||||
@@ -879,6 +935,127 @@ class KVCacheConfigurator:
|
||||
),
|
||||
)
|
||||
|
||||
def _fused_draft_decision(self):
|
||||
"""Whether, and where, the EAGLE draft's layers fuse into the target's
|
||||
sub-pools. Fusion applies only for: unified memory ON, a two-pool
|
||||
hybrid-SWA target, an EAGLE-family algorithm whose draft config was
|
||||
loaded at target boot and has no recurrent state of its own;
|
||||
`place_fused_draft` then admits or declines the draft's layer kinds."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
draft_kv_profile,
|
||||
place_fused_draft,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import _store_dtype_for
|
||||
|
||||
aux = self.spec_aux_config
|
||||
if not (
|
||||
get_memory().enable_unified_memory
|
||||
and self.is_hybrid_swa
|
||||
and self.mambaish_config is None
|
||||
and not self.is_draft_worker
|
||||
and self.spec_algorithm.is_eagle()
|
||||
and aux.draft_kv_num_layers
|
||||
and aux.draft_model_config is not None
|
||||
and mambaish_config(aux.draft_model_config) is None
|
||||
):
|
||||
return FusedDraftDecision()
|
||||
profile = draft_kv_profile(
|
||||
aux.draft_model_config,
|
||||
num_layers=int(aux.draft_kv_num_layers),
|
||||
attn_tp_size=get_parallel().attn_tp_size,
|
||||
)
|
||||
num_runners = (
|
||||
int(get_spec().speculative_num_steps)
|
||||
if self.model_config.is_multi_layer_eagle
|
||||
else 1
|
||||
)
|
||||
# A fused draft stores its rows in the host's KV dtype. An explicit
|
||||
# draft dtype resolves without the draft model, so a mismatch is
|
||||
# declined here; `auto` follows the draft's quant config, which only
|
||||
# the draft runner knows, and is checked when it binds.
|
||||
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
|
||||
if draft_kv_cache_dtype not in (None, "auto"):
|
||||
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
|
||||
|
||||
_, draft_kv_dtype = configure_kv_cache_dtype(
|
||||
server_args_kv_cache_dtype=draft_kv_cache_dtype,
|
||||
model=None,
|
||||
model_dtype=aux.draft_model_config.dtype,
|
||||
is_draft_worker=True,
|
||||
is_dflash=False,
|
||||
speculative_draft_attention_backend=None,
|
||||
speculative_draft_kv_cache_dtype=draft_kv_cache_dtype,
|
||||
)
|
||||
if draft_kv_dtype != self.kv_cache_dtype:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft's KV cache dtype ({draft_kv_dtype}) differs "
|
||||
f"from the host's ({self.kv_cache_dtype})"
|
||||
)
|
||||
)
|
||||
return place_fused_draft(
|
||||
profile=profile,
|
||||
num_runners=num_runners,
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
)
|
||||
|
||||
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
|
||||
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
|
||||
pad) for the boot solve's cell model. Single source of truth: assembled
|
||||
through the same spec the pool factory builds, so the priced entry and
|
||||
the allocated entry cannot drift. None when the draft does not fuse
|
||||
into that sub-pool."""
|
||||
placement = self._fused_draft_decision().placement
|
||||
if placement is None or sub_pool_name != "full":
|
||||
return None
|
||||
return self._full_host_spec(placement.region).entry_bytes()
|
||||
|
||||
def _full_host_spec(self, region):
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
MHASubPoolSpec,
|
||||
_store_dtype_for,
|
||||
)
|
||||
|
||||
return MHASubPoolSpec(
|
||||
name="full",
|
||||
layer_num=len(self.layer_info.full_attention_layer_ids),
|
||||
head_num=self.model_config.get_num_kv_heads(
|
||||
get_parallel().attn_tp_size, get_parallel().attn_dcp_size
|
||||
),
|
||||
head_dim=self.model_config.head_dim,
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
grow_direction="down",
|
||||
draft_region=region,
|
||||
)
|
||||
|
||||
def _fused_draft_for_pool_factory(self):
|
||||
"""Resolve ONCE per factory call (not inline) so the boot log reports
|
||||
exactly the placement the factory is handed: a declined fusion and an
|
||||
engaged one otherwise look identical from outside."""
|
||||
decision = self._fused_draft_decision()
|
||||
if decision.placement is None:
|
||||
if decision.declined is not None:
|
||||
logger.warning("fused draft KV disabled: %s", decision.declined)
|
||||
return None
|
||||
placement = decision.placement
|
||||
region = placement.region
|
||||
logger.info(
|
||||
"[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d "
|
||||
"kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s",
|
||||
region.lane_num,
|
||||
region.head_num,
|
||||
region.head_dim,
|
||||
region.resolved_v_head_dim(),
|
||||
region.store_dtype,
|
||||
region.entry_bytes(),
|
||||
[
|
||||
tuple(placement.lanes_for(r))
|
||||
for r in range(len(placement.runner_lane_counts))
|
||||
],
|
||||
)
|
||||
return placement
|
||||
|
||||
def _init_unified_swa_pools(
|
||||
self,
|
||||
*,
|
||||
@@ -964,6 +1141,7 @@ class KVCacheConfigurator:
|
||||
# charged, see `_check_bs1_feasibility_floor`.
|
||||
model_context_len=self.model_config.context_len,
|
||||
sliding_window_size=self.model_config.sliding_window_size,
|
||||
fused_draft=self._fused_draft_for_pool_factory(),
|
||||
)
|
||||
return UnifiedPoolBundle(
|
||||
unified_memory_pool=bundle.unified_memory_pool,
|
||||
|
||||
@@ -71,6 +71,7 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
|
||||
)
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@@ -112,13 +113,18 @@ class KVIndexTranslator:
|
||||
self.page_size = page_size
|
||||
self.device = device
|
||||
|
||||
self.is_translating = (
|
||||
is_unified_target = (
|
||||
isinstance(
|
||||
token_to_kv_pool_allocator,
|
||||
(UnifiedMambaTokenToKVPoolAllocator, UnifiedSWAAllocatorBase),
|
||||
)
|
||||
and token_to_kv_pool_allocator.get_kvcache() is token_to_kv_pool
|
||||
)
|
||||
host_allocator = fused_draft_host_allocator(token_to_kv_pool)
|
||||
is_fused_draft = (
|
||||
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
|
||||
)
|
||||
self.is_translating = is_unified_target or is_fused_draft
|
||||
if self.is_translating:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
self._capture_page_size = alloc.page_size
|
||||
@@ -133,13 +139,17 @@ class KVIndexTranslator:
|
||||
# DCP read ids stay WIDENED to the consumer: selecting this rank's
|
||||
# share changes the length, so only the production site can do it.
|
||||
self.defer_read_translate = get_parallel().attn_dcp_size > 1
|
||||
if isinstance(alloc, UnifiedSWAAllocatorBase):
|
||||
routes_window_layers = is_unified_target or isinstance(
|
||||
token_to_kv_pool, BaseSWAKVPool
|
||||
)
|
||||
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
|
||||
self._swa_v2p_table = alloc.swa_v2p_page_table
|
||||
self._swa_write_loc_from_full = self._swa_write_loc_unified
|
||||
else:
|
||||
self._swa_v2p_table = None
|
||||
self._swa_write_loc_from_full = None
|
||||
else:
|
||||
self._capture_page_size = page_size
|
||||
self._full_v2p_table = None
|
||||
self._full_p2v_table = None
|
||||
self._translate_full = None
|
||||
@@ -175,6 +185,14 @@ class KVIndexTranslator:
|
||||
return self._full_v2p_table.numel() * self._capture_page_size
|
||||
return max_token_pool_size + self.page_size
|
||||
|
||||
def full_flat_v2p(self) -> Optional[torch.Tensor]:
|
||||
"""The full-side v2p page table for a kernel that translates flat ids
|
||||
itself, or ``None`` when this runner does not translate (pass-through
|
||||
and static pools)."""
|
||||
if not self.is_translating:
|
||||
return None
|
||||
return self._full_v2p_table
|
||||
|
||||
# -- per-batch view --------------------------------------------------------
|
||||
|
||||
@property
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""Draft-model regions nested inside unified sub-pools' slot entries.
|
||||
|
||||
The nesting, outermost first:
|
||||
|
||||
sub-pool a named pool inside the unified pool, with its own grow direction
|
||||
and frontier. HOST is the role it plays when its entries also
|
||||
carry draft bytes; only "full" hosts a draft here.
|
||||
slot one token entry of that sub-pool. The draft's KV becomes extra
|
||||
parts of every slot, after the host's own parts at
|
||||
`draft_offset_in_entry()`, so ONE slot id per token covers target
|
||||
and draft: one allocation, one free, one whole-page relocation
|
||||
carry both.
|
||||
region the draft geometry inside the host's entries. A region is not a
|
||||
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
|
||||
lane one position in the region's layer dimension -- one draft layer of
|
||||
one runner, present in every slot.
|
||||
|
||||
Runners are the draft's execution copies, and the two head shapes place them
|
||||
oppositely. A replicated head gives EVERY runner all the draft's layers, so
|
||||
each needs its own lanes or the runners clobber each other's KV (3 runners x
|
||||
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
|
||||
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
|
||||
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
|
||||
either direction. Each runner owns a contiguous run of lanes, and the runs
|
||||
tile the region in runner order.
|
||||
|
||||
Three phases, three moments in boot, and what carries each:
|
||||
|
||||
PLAN on the TARGET, before any memory exists
|
||||
KVCacheConfigurator._fused_draft_decision()
|
||||
draft_kv_profile(draft_model_config) what the checkpoint asks for
|
||||
place_fused_draft(profile, runners) admit, or decline with a reason
|
||||
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
|
||||
|
||||
CARVE as the byte buffer is cut
|
||||
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
|
||||
init_unified_*_pools(fused_draft=...)
|
||||
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
|
||||
UnifiedKVPool.build_dense_draft_views()
|
||||
|
||||
BIND on each DRAFT runner, built after the target
|
||||
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
|
||||
bind_fused_draft(...) -> UnifiedDraftKVPool
|
||||
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
|
||||
|
||||
A draft whose layers need another sub-pool kind declines to a private pool.
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
|
||||
|
||||
|
||||
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
|
||||
sub-pool. The draft is a separate checkpoint, so its head geometry and
|
||||
layer count differ from the host's; its K and V rows are two more parts of
|
||||
the host entry, indexed by the host's physical token id."""
|
||||
|
||||
lane_num: int
|
||||
head_num: int
|
||||
head_dim: int
|
||||
store_dtype: torch.dtype
|
||||
v_head_dim: Optional[int] = None
|
||||
|
||||
def validate(self) -> None:
|
||||
assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}"
|
||||
assert self.head_num > 0, f"head_num must be positive; got {self.head_num}"
|
||||
assert self.head_dim > 0, f"head_dim must be positive; got {self.head_dim}"
|
||||
v = self.resolved_v_head_dim()
|
||||
assert v > 0, f"v_head_dim must be positive; got {v}"
|
||||
|
||||
def resolved_v_head_dim(self) -> int:
|
||||
return self.head_dim if self.v_head_dim is None else self.v_head_dim
|
||||
|
||||
def k_row_bytes(self) -> int:
|
||||
return self.head_num * self.head_dim * self.store_dtype.itemsize
|
||||
|
||||
def v_row_bytes(self) -> int:
|
||||
return self.head_num * self.resolved_v_head_dim() * self.store_dtype.itemsize
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
"""Draft bytes per slot, before the host entry's alignment."""
|
||||
return self.lane_num * (self.k_row_bytes() + self.v_row_bytes())
|
||||
|
||||
def parts(self, offset_bytes: int) -> Tuple[DensePart, DensePart]:
|
||||
"""The draft's K and V parts, laid out from ``offset_bytes`` inside
|
||||
the host entry."""
|
||||
layer_stride = self.k_row_bytes() + self.v_row_bytes()
|
||||
return (
|
||||
DensePart(
|
||||
name="draft_k",
|
||||
offset_bytes=offset_bytes,
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.lane_num,
|
||||
row_shape=(self.head_num, self.head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
DensePart(
|
||||
name="draft_v",
|
||||
offset_bytes=offset_bytes + self.k_row_bytes(),
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.lane_num,
|
||||
row_shape=(self.head_num, self.resolved_v_head_dim()),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Where every draft runner's layers live inside the "full" sub-pool: the
|
||||
region, and each runner's lane count in runner order. Built once on the
|
||||
target, stored on the `UnifiedKVPool`, and read back by each draft
|
||||
runner, so the two sides cannot disagree on a lane.
|
||||
"""
|
||||
|
||||
region: DenseDraftRegion
|
||||
runner_lane_counts: Tuple[int, ...]
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.runner_lane_counts, "a placement needs at least one draft runner"
|
||||
assert sum(self.runner_lane_counts) == self.region.lane_num, (
|
||||
f"runner lane counts {self.runner_lane_counts} must tile "
|
||||
f"range({self.region.lane_num})"
|
||||
)
|
||||
|
||||
def lanes_for(self, runner: int) -> range:
|
||||
start = sum(self.runner_lane_counts[:runner])
|
||||
return range(start, start + self.runner_lane_counts[runner])
|
||||
|
||||
|
||||
class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-GPU K/V row geometry of one kind of draft attention layer."""
|
||||
|
||||
head_num: int
|
||||
head_dim: int
|
||||
v_head_dim: int
|
||||
|
||||
|
||||
class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""What the draft checkpoint asks of the host sub-pools.
|
||||
|
||||
``num_depths`` > 1 marks a per-depth head (one transformer block per MTP
|
||||
depth, served by one runner each under multi-layer EAGLE); otherwise
|
||||
every runner serves all ``num_layers`` layers.
|
||||
"""
|
||||
|
||||
num_layers: int
|
||||
full: DraftKVGeometry
|
||||
swa_layer_ids: Tuple[int, ...] = ()
|
||||
num_depths: int = 1
|
||||
|
||||
|
||||
def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
|
||||
"""The draft layer ids a hybrid-SWA pool routes to its swa side. This is
|
||||
the pool-routing convention (`SWAKVPool.layers_mapping`), not a per-layer
|
||||
kernel window: an attention-only draft window stays full-kind."""
|
||||
mc = draft_model_config
|
||||
if mc.is_hybrid_swa and not mc.is_deepseek_v4_arch:
|
||||
return tuple(int(i) for i in mc.swa_attention_layer_ids)
|
||||
return ()
|
||||
|
||||
|
||||
def draft_kv_profile(
|
||||
draft_model_config, *, num_layers: int, attn_tp_size: int
|
||||
) -> DraftKVProfile:
|
||||
"""The profile of a draft `ModelConfig`, heads divided by attn_tp the way
|
||||
the target divides its own (drafts never join the DCP group)."""
|
||||
mc = draft_model_config
|
||||
num_depths = mc.num_nextn_predict_layers
|
||||
return DraftKVProfile(
|
||||
num_layers=int(num_layers),
|
||||
full=DraftKVGeometry(
|
||||
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
|
||||
head_dim=int(mc.head_dim),
|
||||
v_head_dim=int(mc.v_head_dim),
|
||||
),
|
||||
swa_layer_ids=draft_swa_layer_ids(mc),
|
||||
num_depths=1 if num_depths is None else int(num_depths),
|
||||
)
|
||||
|
||||
|
||||
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
|
||||
private pool. Neither means fusion simply does not apply."""
|
||||
|
||||
placement: Optional[FusedDraftPlacement] = None
|
||||
declined: Optional[str] = None
|
||||
|
||||
|
||||
def _runner_layer_counts(
|
||||
profile: DraftKVProfile, num_runners: int
|
||||
) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]:
|
||||
"""Per runner, its (full, swa) layer counts; or why no runner layout exists."""
|
||||
if profile.num_depths <= 1:
|
||||
num_swa = len(profile.swa_layer_ids)
|
||||
return [(profile.num_layers - num_swa, num_swa)] * num_runners, None
|
||||
if num_runners == 1:
|
||||
return None, (
|
||||
f"a per-depth draft head ({profile.num_depths} depths) needs one "
|
||||
"runner per depth (multi-layer EAGLE)"
|
||||
)
|
||||
if num_runners > profile.num_depths:
|
||||
return None, (
|
||||
f"{num_runners} draft runners exceed the head's {profile.num_depths} depths"
|
||||
)
|
||||
swa = set(profile.swa_layer_ids)
|
||||
return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None
|
||||
|
||||
|
||||
def place_fused_draft(
|
||||
*,
|
||||
profile: DraftKVProfile,
|
||||
num_runners: int,
|
||||
store_dtype: torch.dtype,
|
||||
) -> FusedDraftDecision:
|
||||
"""Assign every draft layer of every runner to the host sub-pool whose
|
||||
lifetime covers what the layer reads: a full-attention layer rides in
|
||||
``"full"``. A layer kind no host arm serves declines the whole draft to
|
||||
its private pool."""
|
||||
counts, reason = _runner_layer_counts(profile, num_runners)
|
||||
if counts is None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
num_swa = sum(swa for _, swa in counts)
|
||||
if num_swa:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft has {num_swa} SWA layer(s) of its own, which the "
|
||||
"fused dense pool cannot serve"
|
||||
)
|
||||
)
|
||||
geometry = profile.full
|
||||
if geometry.head_dim != geometry.v_head_dim:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
"the draft's K/V rows are asymmetric "
|
||||
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}), "
|
||||
"which is not admitted yet"
|
||||
)
|
||||
)
|
||||
row_bytes = geometry.head_num * geometry.head_dim * store_dtype.itemsize
|
||||
if row_bytes % ROW_ALIGN_BYTES:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft's K/V rows are {row_bytes} B, not a multiple of the "
|
||||
f"{ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
|
||||
)
|
||||
)
|
||||
full_counts = tuple(full for full, _ in counts)
|
||||
region = DenseDraftRegion(
|
||||
lane_num=sum(full_counts),
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
store_dtype=store_dtype,
|
||||
)
|
||||
return FusedDraftDecision(
|
||||
placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts)
|
||||
)
|
||||
@@ -43,6 +43,11 @@ def align_entry_bytes(num_bytes: int) -> int:
|
||||
return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES
|
||||
|
||||
|
||||
def align_part_offset(num_bytes: int) -> int:
|
||||
"""Round a part's byte offset inside the entry up to the row alignment."""
|
||||
return -(-num_bytes // ROW_ALIGN_BYTES) * ROW_ALIGN_BYTES
|
||||
|
||||
|
||||
class DensePart(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One row family inside the entry: ``layer_num`` rows of ``row_shape``,
|
||||
layer ``l`` at ``offset_bytes + l * layer_stride_bytes``."""
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""KV pools a DRAFT runner binds over the draft lanes fused into the target's
|
||||
unified pool: same pages, same slot ids, same v2p table as the target -- one
|
||||
allocation, one free, one relocation."""
|
||||
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
|
||||
|
||||
class UnifiedDraftKVPool(MHATokenToKVPool):
|
||||
"""Dense draft KV over the draft parts of one host sub-pool's entries.
|
||||
|
||||
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
|
||||
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
|
||||
``layer_lanes`` maps each of this runner's layer ids to its region lane.
|
||||
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
|
||||
produced by the allocator's translate (the id-space choke point binds it,
|
||||
see KVIndexTranslator); the pool exposes `host_allocator` for that
|
||||
binding. Relocation needs no method here: compaction moves whole page
|
||||
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
|
||||
raises so a stray per-slot move fails loudly instead of corrupting the
|
||||
fused layout.
|
||||
"""
|
||||
|
||||
requires_physical_write_loc = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
unified_buffer: UnifiedKVPool,
|
||||
host_sub_pool_name: str,
|
||||
host_allocator,
|
||||
layer_lanes: Mapping[int, int],
|
||||
page_size: int = 1,
|
||||
):
|
||||
region = unified_buffer.require_draft_host_spec(host_sub_pool_name).draft_region
|
||||
layer_ids = sorted(layer_lanes)
|
||||
assert layer_ids, "UnifiedDraftKVPool binds at least one layer"
|
||||
start_layer = layer_ids[0]
|
||||
assert layer_ids == list(range(start_layer, start_layer + len(layer_ids))), (
|
||||
f"fused draft layer ids must be contiguous; got {layer_ids}"
|
||||
)
|
||||
lanes = [layer_lanes[layer_id] for layer_id in layer_ids]
|
||||
assert len(set(lanes)) == len(lanes) and all(
|
||||
0 <= s < region.lane_num for s in lanes
|
||||
), f"draft lanes {lanes} must be distinct and within range({region.lane_num})"
|
||||
k_views, v_views = unified_buffer.build_dense_draft_views(host_sub_pool_name)
|
||||
max_slots = unified_buffer.max_slots(host_sub_pool_name)
|
||||
|
||||
self._unified_buffer = unified_buffer
|
||||
self._host_sub_pool_name = host_sub_pool_name
|
||||
self.host_allocator = host_allocator
|
||||
self.layer_lanes: Dict[int, int] = dict(layer_lanes)
|
||||
self._k_views: List[torch.Tensor] = [k_views[s] for s in lanes]
|
||||
self._v_views: List[torch.Tensor] = [v_views[s] for s in lanes]
|
||||
num_pages = max_slots // page_size
|
||||
|
||||
super().__init__(
|
||||
size=num_pages * page_size - page_size,
|
||||
page_size=page_size,
|
||||
dtype=region.store_dtype,
|
||||
head_num=region.head_num,
|
||||
head_dim=region.head_dim,
|
||||
layer_num=len(layer_ids),
|
||||
device=unified_buffer.device,
|
||||
enable_memory_saver=False, # buffer owned by UnifiedKVPool
|
||||
v_head_dim=region.resolved_v_head_dim(),
|
||||
start_layer=start_layer,
|
||||
end_layer=start_layer + len(layer_ids) - 1,
|
||||
enable_kv_cache_copy=False,
|
||||
kv_cache_layout="page_major",
|
||||
)
|
||||
|
||||
def _create_buffers(self):
|
||||
self.k_buffer = self._k_views
|
||||
self.v_buffer = self._v_views
|
||||
|
||||
def _clear_buffers(self):
|
||||
pass # lifetime owned by UnifiedKVPool
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
return 0, 0 # fused into the host entries; UnifiedKVPool logs the total
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
raise NotImplementedError(
|
||||
"fused draft KV relocates with the HOST pool's whole-page move; a "
|
||||
"draft-side per-slot move would corrupt the fused layout"
|
||||
)
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"fused draft KV has no per-layer contiguous regions; KV transfer / "
|
||||
"disaggregation is unsupported."
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None):
|
||||
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
|
||||
|
||||
def load_cpu_copy(self, kv_cache_cpu, indices, mamba_indices=None):
|
||||
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
|
||||
|
||||
|
||||
def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]:
|
||||
"""The host allocator a fused draft pool translates through, or None for
|
||||
any other pool."""
|
||||
if isinstance(token_to_kv_pool, UnifiedDraftKVPool):
|
||||
return token_to_kv_pool.host_allocator
|
||||
return None
|
||||
|
||||
|
||||
def draft_kv_layer_ids(model) -> List[int]:
|
||||
"""Layer ids owning attention KV in the BUILT draft model, in layer order.
|
||||
Window layers count -- a window is a parameter of the attention layer;
|
||||
linear and recurrent layers have no attention module and are absent."""
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
|
||||
return sorted(
|
||||
{m.layer_id for m in model.modules() if isinstance(m, RadixAttention)}
|
||||
)
|
||||
|
||||
|
||||
def bind_fused_draft(
|
||||
*,
|
||||
unified_buffer: UnifiedKVPool,
|
||||
host_allocator,
|
||||
placement: FusedDraftPlacement,
|
||||
runner: int,
|
||||
kv_layer_ids: Sequence[int],
|
||||
swa_layer_ids: Sequence[int],
|
||||
page_size: int,
|
||||
) -> UnifiedDraftKVPool:
|
||||
"""The KV pool draft runner ``runner`` binds over its fused slots.
|
||||
|
||||
The placement sized the lanes from the draft config; the model's real
|
||||
layer ids fill them in layer order, so a count mismatch is a loud boot
|
||||
failure, never a silent alias.
|
||||
"""
|
||||
swa = set(swa_layer_ids)
|
||||
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa]
|
||||
assert len(full_ids) == len(kv_layer_ids), (
|
||||
f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused "
|
||||
"SWA KV is not supported yet"
|
||||
)
|
||||
full_lanes = placement.lanes_for(runner)
|
||||
assert len(full_ids) == len(full_lanes), (
|
||||
f"draft runner {runner}: {len(full_ids)} full-attention layer(s) "
|
||||
f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}"
|
||||
)
|
||||
return UnifiedDraftKVPool(
|
||||
unified_buffer=unified_buffer,
|
||||
host_sub_pool_name="full",
|
||||
host_allocator=host_allocator,
|
||||
layer_lanes=dict(zip(full_ids, full_lanes)),
|
||||
page_size=page_size,
|
||||
)
|
||||
@@ -38,11 +38,16 @@ from sglang.kernels.ops.kvcache.zero_pages import zero_pages
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
DenseDraftRegion,
|
||||
FusedDraftPlacement,
|
||||
)
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
ROW_ALIGN_BYTES,
|
||||
DenseEntryLayout,
|
||||
DensePart,
|
||||
align_entry_bytes,
|
||||
align_part_offset,
|
||||
build_dense_views,
|
||||
build_mamba_entry_views,
|
||||
)
|
||||
@@ -113,6 +118,8 @@ class SubPoolSpec(ABC):
|
||||
name: str
|
||||
layer_num: int
|
||||
grow_direction: str # "up" | "down" | "float"
|
||||
# Fused draft region riding in this sub-pool's entries; None = unfused.
|
||||
draft_region: Optional[DenseDraftRegion] = None
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.grow_direction in self._allowed_grow_directions, (
|
||||
@@ -134,7 +141,17 @@ class SubPoolSpec(ABC):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MHASubPoolSpec(SubPoolSpec):
|
||||
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`."""
|
||||
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`.
|
||||
|
||||
With `draft_region` set, the draft model's K and V rows are two more
|
||||
parts of every slot's entry, after the host parts:
|
||||
|
||||
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
|
||||
|
||||
The slot stride stays the one entry, so host and draft views are indexed
|
||||
by the same physical token id; only their offsets differ. `draft_region
|
||||
is None` keeps the layout byte-identical to the unfused one.
|
||||
"""
|
||||
|
||||
head_num: int
|
||||
head_dim: int
|
||||
@@ -159,6 +176,8 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
assert self.v_head_dim > 0, (
|
||||
f"v_head_dim must be positive; got {self.v_head_dim}"
|
||||
)
|
||||
if self.draft_region is not None:
|
||||
self.draft_region.validate()
|
||||
|
||||
def k_row_bytes(self) -> int:
|
||||
return self.head_num * self.head_dim * self.store_dtype.itemsize
|
||||
@@ -166,9 +185,20 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
def v_row_bytes(self) -> int:
|
||||
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
|
||||
|
||||
def host_entry_bytes(self) -> int:
|
||||
"""Host (target-only) bytes for one slot, before any draft parts."""
|
||||
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
|
||||
|
||||
def draft_offset_in_entry(self) -> int:
|
||||
"""Byte offset of the fused draft parts inside one slot's entry."""
|
||||
assert self.draft_region is not None
|
||||
return align_part_offset(self.host_entry_bytes())
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
if self.draft_region is None:
|
||||
return align_entry_bytes(self.host_entry_bytes())
|
||||
return align_entry_bytes(
|
||||
self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
|
||||
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
|
||||
)
|
||||
|
||||
def page_bytes(self, page_size: int) -> int:
|
||||
@@ -179,27 +209,27 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
is ``page_size`` such entries back to back."""
|
||||
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
|
||||
layer_stride = self.k_row_bytes() + self.v_row_bytes()
|
||||
return DenseEntryLayout(
|
||||
entry_bytes=self.entry_bytes(),
|
||||
parts=(
|
||||
DensePart(
|
||||
name="k",
|
||||
offset_bytes=0,
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
DensePart(
|
||||
name="v",
|
||||
offset_bytes=self.k_row_bytes(),
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.v_head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
parts = (
|
||||
DensePart(
|
||||
name="k",
|
||||
offset_bytes=0,
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
DensePart(
|
||||
name="v",
|
||||
offset_bytes=self.k_row_bytes(),
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.v_head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
)
|
||||
if self.draft_region is not None:
|
||||
parts += self.draft_region.parts(self.draft_offset_in_entry())
|
||||
return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts)
|
||||
|
||||
def get_dtype(self) -> torch.dtype:
|
||||
return self.store_dtype
|
||||
@@ -227,6 +257,9 @@ class MLASubPoolSpec(SubPoolSpec):
|
||||
assert self.qk_rope_head_dim > 0, (
|
||||
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
|
||||
)
|
||||
assert self.draft_region is None, (
|
||||
"MLA sub-pools do not carry a fused draft region yet"
|
||||
)
|
||||
|
||||
@property
|
||||
def kv_cache_dim(self) -> int:
|
||||
@@ -271,6 +304,9 @@ class MambaSubPoolSpec(SubPoolSpec):
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
|
||||
assert self.draft_region is None, (
|
||||
"mamba state pages carry no fused draft region yet"
|
||||
)
|
||||
|
||||
def conv_row_bytes(self, idx: int) -> int:
|
||||
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
|
||||
@@ -350,6 +386,7 @@ class UnifiedKVPool:
|
||||
page_size: int = 1,
|
||||
post_capture_active: bool = False,
|
||||
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
|
||||
fused_draft: Optional[FusedDraftPlacement] = None,
|
||||
):
|
||||
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
|
||||
assert len(sub_pool_specs) >= 2, (
|
||||
@@ -386,6 +423,17 @@ class UnifiedKVPool:
|
||||
self.post_capture_active = post_capture_active
|
||||
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
|
||||
self._bs1_floor_terms = bs1_floor_terms or []
|
||||
self.fused_draft = fused_draft
|
||||
for spec in sub_pool_specs:
|
||||
expected = (
|
||||
fused_draft.region
|
||||
if fused_draft is not None and spec.name == "full"
|
||||
else None
|
||||
)
|
||||
assert spec.draft_region is expected, (
|
||||
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
|
||||
f"not match the fused draft placement's {expected}"
|
||||
)
|
||||
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
@@ -588,6 +636,17 @@ class UnifiedKVPool:
|
||||
)
|
||||
return s
|
||||
|
||||
def require_draft_host_spec(self, name: str) -> SubPoolSpec:
|
||||
"""The sub-pool spec whose entries carry a fused draft region.
|
||||
|
||||
Kind-agnostic: any spec that resolves a `draft_region` also lays the
|
||||
draft parts out in its `layout()`."""
|
||||
s = self._specs_by_name[name]
|
||||
assert s.draft_region is not None, (
|
||||
f"sub-pool {name!r} carries no fused draft region"
|
||||
)
|
||||
return s
|
||||
|
||||
def max_slots(self, name: str) -> int:
|
||||
return self._max_slots[name]
|
||||
|
||||
@@ -630,6 +689,27 @@ class UnifiedKVPool:
|
||||
)
|
||||
return k_views, v_views
|
||||
|
||||
def build_dense_draft_views(
|
||||
self, sub_pool_name: str
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
"""Per-layer K/V views of the DRAFT parts fused into ``sub_pool_name``'s
|
||||
entries: same pages, same slot ids, same v2p table as the host."""
|
||||
spec = self.require_draft_host_spec(sub_pool_name)
|
||||
layout = spec.layout()
|
||||
page_size = self._page_size
|
||||
num_slots = self.max_slots(sub_pool_name) // page_size * page_size
|
||||
k_views, v_views = (
|
||||
build_dense_views(
|
||||
self._raw,
|
||||
layout=layout,
|
||||
part_name=name,
|
||||
num_slots=num_slots,
|
||||
anchor_bytes=self._anchor_bytes[sub_pool_name],
|
||||
)
|
||||
for name in ("draft_k", "draft_v")
|
||||
)
|
||||
return k_views, v_views
|
||||
|
||||
def _build_mla_views(
|
||||
self,
|
||||
spec: MLASubPoolSpec,
|
||||
@@ -1899,8 +1979,15 @@ def init_unified_swa_pools(
|
||||
lazy_compaction: bool = False,
|
||||
model_context_len: Optional[int] = None,
|
||||
sliding_window_size: Optional[int] = None,
|
||||
fused_draft: Optional[FusedDraftPlacement] = None,
|
||||
) -> UnifiedSWAPoolBundle:
|
||||
"""Build the SWA-hybrid unified-memory-pool stack."""
|
||||
"""Build the SWA-hybrid unified-memory-pool stack.
|
||||
|
||||
With ``fused_draft``, every entry of the "full" sub-pool carries the
|
||||
draft model's K/V parts after the host parts, and each draft runner
|
||||
binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a
|
||||
pool of its own.
|
||||
"""
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
)
|
||||
@@ -1926,6 +2013,7 @@ def init_unified_swa_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="down",
|
||||
draft_region=None if fused_draft is None else fused_draft.region,
|
||||
)
|
||||
swa_spec = MHASubPoolSpec(
|
||||
name="swa",
|
||||
@@ -1981,6 +2069,7 @@ def init_unified_swa_pools(
|
||||
page_size=page_size,
|
||||
post_capture_active=post_capture_active,
|
||||
bs1_floor_terms=bs1_floor_terms,
|
||||
fused_draft=fused_draft,
|
||||
)
|
||||
token_to_kv_pool = UnifiedSWAKVPool(
|
||||
unified_buffer=shared_pool,
|
||||
|
||||
@@ -44,6 +44,11 @@ class SpecAuxHiddenStateConfig(msgspec.Struct, kw_only=True):
|
||||
dflash_target_layer_ids: Any = None
|
||||
# DFLASH draft KV bytes/token; None when unresolved.
|
||||
dflash_draft_cell_size_per_token: int | None = None
|
||||
# The draft checkpoint read in draft mode, and its KV layer count: the
|
||||
# fused-draft placement's input. `eagle_draft_num_layers` above keeps
|
||||
# sizing a private draft pool exactly as before.
|
||||
draft_model_config: Optional[ModelConfig] = None
|
||||
draft_kv_num_layers: Optional[int] = None
|
||||
|
||||
|
||||
def resolve_spec_aux_hidden_state_config(
|
||||
@@ -85,24 +90,38 @@ def _resolve_eagle_aux_hidden_state(
|
||||
):
|
||||
return
|
||||
|
||||
draft_model_config = model_config
|
||||
if get_spec().speculative_draft_model_path:
|
||||
draft_model_config = ModelConfig.from_server_args(
|
||||
server_args,
|
||||
model_path=get_spec().speculative_draft_model_path,
|
||||
model_revision=get_spec().speculative_draft_model_revision,
|
||||
is_draft_model=True,
|
||||
)
|
||||
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
|
||||
if num_nextn_predict_layers is not None:
|
||||
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
|
||||
elif get_spec().speculative_draft_model_path:
|
||||
config.eagle_draft_num_layers = int(
|
||||
draft_path = get_spec().speculative_draft_model_path
|
||||
# The draft read in draft mode. A path-less NEXTN/MTP head is the TARGET
|
||||
# checkpoint rewritten to its MTP form, which is where its
|
||||
# `num_nextn_predict_layers` gets filled in.
|
||||
draft_model_config = ModelConfig.from_server_args(
|
||||
server_args,
|
||||
model_path=draft_path,
|
||||
model_revision=get_spec().speculative_draft_model_revision,
|
||||
is_draft_model=True,
|
||||
)
|
||||
if draft_model_config.num_nextn_predict_layers is not None:
|
||||
config.draft_kv_num_layers = int(draft_model_config.num_nextn_predict_layers)
|
||||
elif draft_path is not None:
|
||||
config.draft_kv_num_layers = int(
|
||||
max(
|
||||
draft_model_config.num_hidden_layers,
|
||||
draft_model_config.num_attention_layers,
|
||||
)
|
||||
)
|
||||
if config.draft_kv_num_layers is not None:
|
||||
config.draft_model_config = draft_model_config
|
||||
|
||||
# The private draft pool's budget keeps its reading: a path-less run sizes
|
||||
# the draft off the TARGET config. Reading the draft-mode config here would
|
||||
# change the token budget of every path-less MTP deployment.
|
||||
if not draft_path:
|
||||
draft_model_config = model_config
|
||||
num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers
|
||||
if num_nextn_predict_layers is not None:
|
||||
config.eagle_draft_num_layers = int(num_nextn_predict_layers)
|
||||
elif draft_path:
|
||||
config.eagle_draft_num_layers = config.draft_kv_num_layers
|
||||
else:
|
||||
return
|
||||
|
||||
|
||||
@@ -714,9 +714,27 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
|
||||
self._draft_cell_size = _dflash_draft_cell_size(kvc)
|
||||
self._fused_full_entry = kvc.fused_entry_bytes("full")
|
||||
if self._fused_full_entry is not None:
|
||||
self._draft_full_layers_num = 0
|
||||
self._draft_swa_layers_num = 0
|
||||
self._draft_swa_full_layers_num = 0
|
||||
self._draft_cell_size = 0
|
||||
|
||||
self._recompute_cell_size()
|
||||
|
||||
def _full_cell_bytes(self) -> int:
|
||||
"""Bytes per full-side token: the fused entry when the draft is placed
|
||||
there, else the target's rows plus the private draft's full rows."""
|
||||
if self._fused_full_entry is not None:
|
||||
return self._fused_full_entry
|
||||
return self._full_per_token * (
|
||||
self._full_layers_num + self._draft_full_layers_num
|
||||
)
|
||||
|
||||
def _swa_cell_bytes(self) -> int:
|
||||
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
|
||||
|
||||
def _recompute_cell_size(self) -> None:
|
||||
# Bytes per token of max_total_num_tokens: full_tokens when hybrid, else
|
||||
# swa_tokens with no ratio, since all-SWA has no full pool to relate to.
|
||||
@@ -730,12 +748,9 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
else:
|
||||
self._cell_size = (
|
||||
self._full_per_token
|
||||
* (self._full_layers_num + self._draft_full_layers_num)
|
||||
self._full_cell_bytes()
|
||||
+ self._swa_per_token * self._draft_swa_full_layers_num
|
||||
+ self._swa_full_tokens_ratio
|
||||
* self._swa_per_token
|
||||
* (self._swa_layers_num + self._draft_swa_layers_num)
|
||||
+ self._swa_full_tokens_ratio * self._swa_cell_bytes()
|
||||
+ self._draft_cell_size
|
||||
)
|
||||
|
||||
@@ -747,9 +762,17 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
+ self._draft_cell_size
|
||||
)
|
||||
|
||||
def _unified_full_bytes_per_token(self) -> int:
|
||||
"""Bytes one full-side token takes in the unified pool: the fused
|
||||
entry (host + draft + pad) when the draft lives in it, else the
|
||||
target's own rows. A private draft pool is priced separately."""
|
||||
if self._fused_full_entry is not None:
|
||||
return self._fused_full_entry
|
||||
return self._full_per_token * self._full_layers_num
|
||||
|
||||
def _unified_pool_bytes(self, full_tokens: int, swa_tokens: int) -> int:
|
||||
return (
|
||||
full_tokens * self._full_per_token * self._full_layers_num
|
||||
full_tokens * self._unified_full_bytes_per_token()
|
||||
+ swa_tokens * self._swa_per_token * self._swa_layers_num
|
||||
)
|
||||
|
||||
@@ -761,7 +784,7 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
) -> int:
|
||||
"""Find the largest page-aligned full capacity whose allocations fit."""
|
||||
draft_bytes_per_token = self._draft_pool_bytes_per_token()
|
||||
target_full_bytes_per_token = self._full_per_token * self._full_layers_num
|
||||
target_full_bytes_per_token = self._unified_full_bytes_per_token()
|
||||
assert target_full_bytes_per_token > 0
|
||||
|
||||
def allocation_bytes(full_pages: int) -> int:
|
||||
@@ -930,19 +953,14 @@ class SWAChunkCapPoolConfigurator(HybridSWAPoolConfigurator):
|
||||
) -> MemoryPoolConfig:
|
||||
# SWA pool sized tightly from the cap; the rest of the budget goes to full.
|
||||
swa_tokens = ceil_align(self._swa_cap, page_size)
|
||||
fixed_swa_bytes = (
|
||||
swa_tokens
|
||||
* self._swa_per_token
|
||||
* (self._swa_layers_num + self._draft_swa_layers_num)
|
||||
)
|
||||
if self._enable_unified_memory:
|
||||
fixed_swa_bytes = swa_tokens * self._swa_cell_bytes()
|
||||
if self._enable_unified_memory and self._draft_pool_bytes_per_token() > 0:
|
||||
full_tokens = self._max_unified_full_tokens(
|
||||
available_bytes, page_size, fixed_swa_tokens=swa_tokens
|
||||
)
|
||||
else:
|
||||
full_cell_size = (
|
||||
self._full_per_token
|
||||
* (self._full_layers_num + self._draft_full_layers_num)
|
||||
self._full_cell_bytes()
|
||||
+ self._swa_per_token * self._draft_swa_full_layers_num
|
||||
)
|
||||
full_tokens = (
|
||||
|
||||
@@ -180,8 +180,15 @@ class BaseSpecWorker(ABC):
|
||||
|
||||
@property
|
||||
def primary_draft_kv_pool(self) -> Optional[object]:
|
||||
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
|
||||
|
||||
draft_runners = self._draft_model_runners()
|
||||
return draft_runners[0].token_to_kv_pool if draft_runners else None
|
||||
if not draft_runners:
|
||||
return None
|
||||
pool = draft_runners[0].token_to_kv_pool
|
||||
if fused_draft_host_allocator(pool) is not None:
|
||||
return None
|
||||
return pool
|
||||
|
||||
@property
|
||||
def target_worker(self) -> TpModelWorker:
|
||||
|
||||
@@ -389,6 +389,15 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
self._configure_qsa_mtp_index_share()
|
||||
self.tree_mask_mode = default_tree_mask_mode()
|
||||
|
||||
# Boot guard: every backend a draft forward reaches must carry its
|
||||
# runner's translator or the index builders emit virtual ids.
|
||||
translator = self.draft_runner.kv_index_translator
|
||||
if translator.is_translating:
|
||||
backends = [self.draft_attn_backend, self.draft_extend_attn_backend]
|
||||
if self.draft_attn_backend is not None:
|
||||
backends += self.draft_attn_backend.attn_backends
|
||||
translator.bind_and_verify_backends(backends)
|
||||
|
||||
def _configure_qsa_mtp_index_share(self) -> None:
|
||||
"""Reuse the draft-extend QSA selection across the MTP decode steps;
|
||||
chain speculation only: with topk > 1 decode rows are not request-major."""
|
||||
|
||||
@@ -398,6 +398,13 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
self.draft_runner_list[
|
||||
step
|
||||
].attn_backend = self.draft_extend_attn_backend_list[-1]
|
||||
# Boot guard: every backend a draft forward reaches must carry
|
||||
# its runner's translator or the index builders emit virtual ids.
|
||||
translator = self.draft_runner_list[step].kv_index_translator
|
||||
if translator.is_translating:
|
||||
translator.bind_and_verify_backends(
|
||||
[self.draft_extend_attn_backend_list[-1]]
|
||||
)
|
||||
|
||||
def _capture_cuda_graphs(self):
|
||||
self.cuda_graph_runner = None
|
||||
|
||||
Reference in New Issue
Block a user