[Feat] unified-memory: Fuse drafts with sliding-window and recurrent-state layers (#43326)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-18 05:39:09 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent 19cb24f1e1
commit 3edb156dec
50 changed files with 4117 additions and 354 deletions
@@ -559,7 +559,9 @@ def _fwd_kernel(
mask=(mask_m[:, None] & mask_n[None, :]),
other=0,
)
final_mask &= custom_mask
# The mask buffer is int8; narrow it to i1, since a score_mod hands
# final_mask to tl.load, which rejects an i8 mask.
final_mask &= custom_mask != 0
if SLIDING_WINDOW_SIZE > 0:
# Add mask where q_id <= kv_id + sliding_window_size
# q_id = prefix_len + cur_m, kv_id = cur_n
@@ -749,7 +751,8 @@ def _fwd_kernel(
mask=(mask_m[:, None] & mask_n[None, :]),
other=0,
)
custom_mask &= mask_m[:, None] & mask_n[None, :]
# int8 -> i1 before the &=; see the prefix loop above.
custom_mask = (custom_mask != 0) & mask_m[:, None] & mask_n[None, :]
final_mask &= custom_mask
elif IS_CAUSAL:
mask_causual = (cur_block_m * BLOCK_M + offs_m[:, None]) >= (
@@ -1349,7 +1352,8 @@ def _fwd_kernel_unified(
mask=(mask_m[:, None] & mask_n[None, :]),
other=0,
)
final_mask &= custom_mask
# int8 -> i1 before the &=; see `_fwd_kernel`'s prefix loop.
final_mask &= custom_mask != 0
# Apply causal mask for extend part
if IS_CAUSAL and not USE_CUSTOM_MASK:
@@ -17,6 +17,7 @@ from sglang.kernels.spec import (
_TRITON_KERNELS = [
("cache_locs", "assign_extend_cache_locs_func"),
("cache_locs", "generate_draft_decode_kv_indices"),
("cache_locs", "generate_draft_decode_window_kv_indices"),
("eagle", "fill_bonus_tokens"),
("eagle", "fill_accept_out_cache_loc"),
("gather_spec_extras", "gather_spec_extras"),
@@ -265,6 +265,124 @@ def generate_draft_decode_kv_indices(
tl.store(kv_indptr + zid, base + zid * iters)
@triton.jit
def generate_draft_decode_window_kv_indices(
req_pool_indices,
req_to_token,
paged_kernel_lens,
window_kv_indices,
window_kv_indptr,
positions,
# Runtime, as in generate_draft_decode_kv_indices.
row_stride,
kv_indices_stride: tl.constexpr,
kv_indptr_stride: tl.constexpr,
bs_upper: tl.constexpr,
iter_upper: tl.constexpr,
num_tokens_upper: tl.constexpr,
page_size: tl.constexpr,
window: tl.constexpr,
ENTRY_PAGE_SIZE: tl.constexpr = 1,
# The source's sliding-window page table (`KVIndexTable.v2p`).
v2p=None,
TRANSLATE: tl.constexpr = False,
):
"""Per-step sliding-window read rail of the multi-step draft.
Step ``iters`` of branch ``(bid, topk_id)`` attends ``seq_len + iters``
tokens: the committed prefix plus the draft's own chain in the slots
`generate_draft_decode_kv_indices` emits. A window layer reads the last
``min(seq_len + iters, window)`` of them, so this emits that tail only,
prefix ids first and chain ids after, from the sliding-window read source.
"""
BLOCK_SIZE: tl.constexpr = 128
iters = tl.program_id(axis=0)
bid = tl.program_id(axis=1)
topk_id = tl.program_id(axis=2)
num_steps = tl.num_programs(axis=0)
num_seqs = tl.num_programs(axis=1)
topk = tl.num_programs(axis=2)
window_kv_indices += kv_indices_stride * iters
window_kv_indptr += kv_indptr_stride * iters
iters += 1
seq_len = tl.load(paged_kernel_lens + bid)
total = seq_len + iters
window_len = tl.minimum(total, window)
# Chain position of the first token in the window; the prefix part covers
# [start, seq_len), the chain part [max(start - seq_len, 0), iters).
start = total - window_len
# This branch's output offset: the window lengths of the branches before
# it, each clipped like this one.
zid = bid * topk + topk_id
bs_offset = tl.arange(0, num_tokens_upper)
in_front = bs_offset < zid
prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0)
prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0)
base = tl.sum(prev_window)
out_ptr = window_kv_indices + base
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
prefix_start = tl.minimum(start, seq_len)
prefix_len = seq_len - prefix_start
kv_offset = tl.arange(0, BLOCK_SIZE)
num_loop = tl.cdiv(prefix_len, BLOCK_SIZE)
for _ in range(num_loop):
mask = kv_offset < prefix_len
data = _load_token_ids(
token_pool_ptr,
prefix_start + kv_offset,
mask,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
tl.store(out_ptr + kv_offset, data, mask=mask)
kv_offset += BLOCK_SIZE
chain_start = tl.maximum(start - seq_len, 0)
extend_offset = tl.arange(0, iter_upper)
chain_step = chain_start + extend_offset
chain_mask = chain_step < iters
# The same chain slots `generate_draft_decode_kv_indices` emits.
if page_size == 1 or topk == 1:
chain_pos = seq_len + topk_id * num_steps + chain_step
else:
last_page_len = seq_len % page_size
num_new_pages_per_topk = (
last_page_len + num_steps + page_size - 1
) // page_size
prefix_base = seq_len // page_size * page_size
chain_base = (
prefix_base + topk_id * num_new_pages_per_topk * page_size + last_page_len
)
chain_pos = chain_base + chain_step
extend_data = _load_token_ids(
token_pool_ptr,
chain_pos,
chain_mask,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
tl.store(out_ptr + prefix_len + extend_offset, extend_data, mask=chain_mask)
# Update window_kv_indptr: program 0 writes the total at [num_seqs * topk],
# every other program its own start; [0] stays the buffer's zero.
if zid == 0:
zid = num_seqs * topk
in_front = bs_offset < zid
prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0)
prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0)
base = tl.sum(prev_window)
tl.store(window_kv_indptr + zid, base)
@triton.jit
def align_evict_mask_to_page_size(
seq_lens,
@@ -607,7 +607,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
assert _mc.is_hybrid_swa or mambaish_config(_mc) is not None, (
"--enable-unified-memory + EAGLE/EAGLE3 requires a unified "
"target (hybrid-SWA or a mamba hybrid): the draft's KV lives "
"fused inside the full-attention page envelope."
"fused inside the target's sub-pool entries."
)
# The target verifies on its own pages: the MLA family on an MLA host.
eagle_allowed = (
@@ -998,6 +998,12 @@ class ModelConfig:
and self.hf_config.architectures[0] == "InklingForConditionalGeneration"
):
self.hf_config.architectures[0] = "InklingForConditionalGenerationMTP"
if self.hf_text_config.mtp_local_layer_ids:
# A local MTP depth keeps the trunk's window unless it declares
# its own: the draft backend builds its window rail only if set.
self.sliding_window_size = (
self.hf_text_config.mtp_local_extent or self.sliding_window_size
)
if (
is_draft_model
and self.hf_config.architectures[0] == "GigaChat35ForCausalLM"
@@ -1,5 +1,6 @@
from __future__ import annotations
import contextlib
from dataclasses import dataclass
from typing import TYPE_CHECKING, List, Optional
@@ -41,10 +42,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
)
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule, get_spec
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
from sglang.srt.speculative.spec_utils import (
draft_kv_indices_buffer_width,
draft_kv_indices_used_len,
generate_draft_decode_kv_indices,
generate_draft_decode_window_kv_indices,
resolve_draft_decode_window,
)
from sglang.srt.utils import (
@@ -719,6 +722,22 @@ class TritonAttnBackend(AttentionBackend):
kv_indptr = self._fill_kv_indptr_and_indices(
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
)
if self.sliding_window_size is not None and self.sliding_window_size > 0:
# A window layer reads the window-clipped prefix, as the eager
# draft-extend path builds it.
_, _, _, self.cuda_graph_window_kv_offsets[:bs] = (
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
kv_lens,
bs,
token_to_kv_pool=self.token_to_kv_pool,
window_kv_indices=self.cuda_graph_window_kv_indices,
)
)
return qo_indptr, kv_indptr, num_tokens_per_req
def init_forward_metadata_out_graph(
@@ -744,6 +763,19 @@ class TritonAttnBackend(AttentionBackend):
# Multi-step spec decode: kv buffers come from spec_info, not the
# cuda-graph pool, so replay is not involved.
if forward_mode.is_decode_or_idle() and spec_info is not None:
# This step's swa write ids, in the buffer the multi-step
# backend refills from the step's columns before replay.
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
forward_batch, in_capture=True
)
assert not (
self.use_sliding_window_kv_pool and swa_out_cache_loc is None
), (
"multi-step spec decode on a sliding-window KV pool could "
"not derive swa_out_cache_loc; the window layers' writes "
"have no target. The per-step out_cache_loc must fit "
"cuda_graph_swa_out_cache_loc."
)
self.forward_metadata = ForwardMetadata(
attn_logits=self.cuda_graph_attn_logits,
attn_lse=self.cuda_graph_attn_lse,
@@ -754,11 +786,18 @@ class TritonAttnBackend(AttentionBackend):
qo_indptr=None,
custom_mask=None,
mask_indptr=None,
window_kv_indptr=self.window_kv_indptr,
window_kv_indices=None,
# The per-step window rail lives in the multi-step
# backend's persistent buffers, refilled before replay.
window_kv_indptr=(
spec_info.window_kv_indptr
if spec_info.window_kv_indices is not None
else self.window_kv_indptr
),
window_kv_indices=spec_info.window_kv_indices,
window_num_kv_splits=None,
window_kv_offsets=None,
swa_attn_logits=self.cuda_graph_swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
lean_Mp=self.cuda_graph_lean_Mp,
lean_Lp=self.cuda_graph_lean_Lp,
lean_Op=self.cuda_graph_lean_Op,
@@ -800,8 +839,9 @@ class TritonAttnBackend(AttentionBackend):
self, forward_batch: ForwardBatch, in_capture: bool = False
) -> Optional[torch.Tensor]:
"""Refill the SWA write-target buffer with the batch's write ids in the
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
multi-step draft) so the captured store reads fresh slots on replay.
sliding-window sub-pool, returning the [:n] view (None for non-SWA) so
the captured store reads fresh slots on replay. A multi-step draft step
hands in a batch bound to its own columns of the window.
"""
if not self.use_sliding_window_kv_pool:
return None
@@ -892,6 +932,18 @@ class TritonAttnBackend(AttentionBackend):
else:
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
bs = kv_indptr.shape[0] - 1
if spec_info.window_kv_indices is not None:
# The multi-step draft backend built the per-step window
# rail (the draft's own chain, window-clipped).
window_kv_indptr = spec_info.window_kv_indptr
window_kv_indices = spec_info.window_kv_indices
window_num_kv_splits = torch.empty(
(bs,), dtype=torch.int32, device=self.device
)
self.get_num_kv_splits(
window_num_kv_splits,
window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs],
)
attn_logits = torch.empty(
(bs, self.num_head, self.max_kv_splits, self.v_head_dim),
@@ -1317,10 +1369,13 @@ class TritonAttnBackend(AttentionBackend):
qo_indptr=self.qo_indptr[: bs + 1],
custom_mask=None,
mask_indptr=None,
window_kv_indptr=self.window_kv_indptr,
window_kv_indices=None,
window_kv_indptr=(
self.window_kv_indptr[: bs + 1] if swa else self.window_kv_indptr
),
window_kv_indices=self.cuda_graph_window_kv_indices if swa else None,
# Extend reads through qo_indptr, never through kv splits.
window_num_kv_splits=None,
window_kv_offsets=None,
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
swa_out_cache_loc=swa_out_cache_loc,
)
else:
@@ -1750,6 +1805,13 @@ class TritonAttnBackend(AttentionBackend):
kv_indptr = self.forward_metadata.window_kv_indptr
kv_indices = self.forward_metadata.window_kv_indices
window_kv_offsets = self.forward_metadata.window_kv_offsets
# A null read rail reaches Triton as `None + offsets` and fails as
# an unreadable IR-build error; say what is actually missing.
assert kv_indices is not None, (
f"layer {layer.layer_id} is a sliding-window layer but this "
"extend forward built no window_kv_indices; every mode that "
"reaches forward_extend must fill the window rail."
)
else:
sliding_window_size = -1
kv_indptr = self.forward_metadata.kv_indptr
@@ -2234,6 +2296,18 @@ class TritonAttnBackend(AttentionBackend):
logits_soft_cap = logit_capping_mod(layer.logit_capping_method, layer.logit_cap)
if save_kv_cache:
swa_loc = self.forward_metadata.swa_out_cache_loc
if (
self.is_draft_runner
and forward_batch.spec_info is not None
and swa_loc is not None
):
# KVWriteLoc keeps a longer swa_loc's prefix, but a draft step's
# slice of the window is not a prefix of it.
assert swa_loc.shape[0] == forward_batch.out_cache_loc.shape[0], (
f"draft step swa_out_cache_loc has {swa_loc.shape[0]} "
f"entries for {forward_batch.out_cache_loc.shape[0]} tokens"
)
if self.use_mla:
if layer.k_scale is not None:
# MLATokenToKVPool doesn't accept scale parameters; k is unused
@@ -2265,6 +2339,14 @@ class TritonAttnBackend(AttentionBackend):
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
kv_indptr = self.forward_metadata.window_kv_indptr
kv_indices = self.forward_metadata.window_kv_indices
# A null read rail reaches Triton as `None + offsets` and fails as
# an unreadable IR-build error; say what is actually missing.
assert kv_indices is not None, (
f"layer {layer.layer_id} is a sliding-window layer but this "
"forward built no window_kv_indices; a multi-step draft gets the "
"per-step window rail whenever its model has a window "
"(TritonMultiStepDraftBackend.window_rail)."
)
else:
kv_indptr = self.forward_metadata.kv_indptr
kv_indices = self.forward_metadata.kv_indices
@@ -2451,15 +2533,35 @@ class TritonMultiStepDraftBackend:
model_runner
)
self.kv_index_translator = model_runner.kv_index_translator
self.token_to_kv_pool = model_runner.token_to_kv_pool
# Each draft step's window layers read a window-clipped rail of their
# own: the draft's chain is not part of the committed prefix yet.
self.sliding_window_size = model_runner.sliding_window_size
self.window_rail = (
self.sliding_window_size is not None and self.sliding_window_size > 0
)
self.window_kv_indptr = (
torch.zeros(
(self.speculative_num_steps, max_bs + 1),
dtype=torch.int32,
device=model_runner.device,
)
if self.window_rail
else None
)
self.cuda_graph_window_kv_indices = None
def common_template(
self,
forward_batch: ForwardBatch,
kv_indices_buffer: Optional[torch.Tensor],
call_fn: int,
window_kv_indices_buffer: Optional[torch.Tensor] = None,
):
if kv_indices_buffer is None:
kv_indices_buffer = self.cuda_graph_kv_indices
if window_kv_indices_buffer is None:
window_kv_indices_buffer = self.cuda_graph_window_kv_indices
num_seqs = forward_batch.batch_size
bs = self.topk * num_seqs
@@ -2496,6 +2598,8 @@ class TritonMultiStepDraftBackend:
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
if self.window_rail:
self._fill_window_rail(forward_batch, window_kv_indices_buffer, bs)
if call_fn is None:
return
@@ -2505,8 +2609,95 @@ class TritonMultiStepDraftBackend:
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
]
if self.window_rail:
forward_batch.spec_info.window_kv_indptr = self.window_kv_indptr[
i, : bs + 1
]
forward_batch.spec_info.window_kv_indices = window_kv_indices_buffer[i][
: bs * self.sliding_window_size
]
call_fn(i, forward_batch)
@contextlib.contextmanager
def _step_writes(self, forward_batch, step: int):
"""Bind draft step ``step``'s slice of the batch's write ids, as
`EagleWorkerV2.draft_forward` slices them, and its columns of the
iteration's write window, while a per-step backend derives its write
targets (the swa ids included) from the batch."""
loc, cols = forward_batch.out_cache_loc, forward_batch.kv_loc_cols
if loc is None or not self.attn_backends[step].use_sliding_window_kv_pool:
yield
return
num_steps = self.speculative_num_steps
if cols is None:
# The draft's own window, each row laid out (topk, num_steps).
step_cols = slice(step, None, num_steps)
else:
# A chain draft writes the leading columns of the verify window.
assert self.topk == 1 and isinstance(cols, slice) and cols.step is None
step_cols = slice(cols.start + step, cols.start + step + 1)
forward_batch.out_cache_loc = per_step_draft_out_cache_loc(
loc, forward_batch.batch_size, self.topk, num_steps
)[step]
forward_batch.kv_loc_cols = step_cols
try:
yield
finally:
forward_batch.out_cache_loc, forward_batch.kv_loc_cols = loc, cols
def _fill_window_rail(
self,
forward_batch: ForwardBatch,
window_kv_indices_buffer: torch.Tensor,
bs: int,
) -> None:
"""Emit every step's window-clipped read ids into the per-step rows
of ``window_kv_indices_buffer`` and their indptr into
``self.window_kv_indptr``. The reader selects FULL or SWA storage;
the layer's window determines which positions to gather."""
num_seqs = forward_batch.batch_size
width = bs * self.sliding_window_size
assert window_kv_indices_buffer.shape[1] >= width, (
f"window rail row {window_kv_indices_buffer.shape[1]} < "
f"{bs} branch(es) x window {self.sliding_window_size}"
)
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
kind=self.kv_index_translator.window_kv_kind,
)
generate_draft_decode_window_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
src.row_ids,
src.ids,
forward_batch.seq_lens,
window_kv_indices_buffer,
self.window_kv_indptr,
forward_batch.positions,
src.row_stride,
window_kv_indices_buffer.shape[1],
self.window_kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(self.speculative_num_steps),
next_power_of_2(bs),
self.page_size,
self.sliding_window_size,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
if (
not src.is_translated
and self.kv_index_translator.window_kv_kind == IdSpaceKind.SLIDING_WINDOW
):
# Static pool: the legacy full->swa translate, as
# update_sliding_window_buffer applies it to the target's window.
for i in range(self.speculative_num_steps):
row = window_kv_indices_buffer[i][:width]
row.copy_(self.token_to_kv_pool.translate_loc_from_full_to_swa(row))
def init_forward_metadata(self, forward_batch: ForwardBatch):
kv_indices_width = draft_kv_indices_buffer_width(
forward_batch.batch_size, self.topk, self.max_context_len
@@ -2516,6 +2707,17 @@ class TritonMultiStepDraftBackend:
dtype=torch.int64,
device=self.device,
)
window_kv_indices = None
if self.window_rail:
# zeros, not empty: a static pool translates the whole row.
window_kv_indices = torch.zeros(
(
self.speculative_num_steps,
forward_batch.batch_size * self.topk * self.sliding_window_size,
),
dtype=torch.int64,
device=self.device,
)
def call_fn(i, forward_batch):
forward_batch.spec_info.kv_indptr = (
@@ -2524,9 +2726,22 @@ class TritonMultiStepDraftBackend:
forward_batch.spec_info.kv_indices = (
forward_batch.spec_info.kv_indices.clone()
)
self.attn_backends[i].init_forward_metadata(forward_batch)
if self.window_rail:
forward_batch.spec_info.window_kv_indptr = (
forward_batch.spec_info.window_kv_indptr.clone()
)
forward_batch.spec_info.window_kv_indices = (
forward_batch.spec_info.window_kv_indices.clone()
)
with self._step_writes(forward_batch, i):
self.attn_backends[i].init_forward_metadata(forward_batch)
self.common_template(forward_batch, kv_indices, call_fn)
self.common_template(
forward_batch,
kv_indices,
call_fn,
window_kv_indices_buffer=window_kv_indices,
)
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
kv_indices_width = draft_kv_indices_buffer_width(
@@ -2543,6 +2758,15 @@ class TritonMultiStepDraftBackend:
dtype=torch.int32,
device=self.device,
)
if self.window_rail:
self.cuda_graph_window_kv_indices = torch.zeros(
(
self.speculative_num_steps,
max_bs * self.topk * self.sliding_window_size,
),
dtype=torch.int64,
device=self.device,
)
for i in range(self.speculative_num_steps - 1):
self.attn_backends[i].init_cuda_graph_state(
@@ -2567,9 +2791,10 @@ class TritonMultiStepDraftBackend:
)
def call_fn(i, _forward_batch):
self.attn_backends[i].init_forward_metadata_out_graph(
inner_fb, in_capture=True
)
with self._step_writes(inner_fb, i):
self.attn_backends[i].init_forward_metadata_out_graph(
inner_fb, in_capture=True
)
self.common_template(forward_batch, None, call_fn)
else:
@@ -2587,6 +2812,14 @@ class TritonMultiStepDraftBackend:
self.attn_backends[-1].cuda_graph_num_kv_splits[:num_token],
forward_batch.seq_lens[:bs],
)
if self.attn_backends[0].use_sliding_window_kv_pool:
# Each captured step writes its window layers through its own
# backend's swa buffer: refill it from that step's columns.
for i in range(self.speculative_num_steps - 1):
with self._step_writes(forward_batch, i):
self.attn_backends[i]._fill_cuda_graph_swa_out_cache_loc(
forward_batch
)
def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None:
for attn_backend in self.attn_backends:
@@ -2640,12 +2873,7 @@ def update_sliding_window_buffer(
indptr=window_kv_indptr,
out=window_kv_indices,
kv_start_idx=window_kv_start_idx,
kind=(
IdSpaceKind.SLIDING_WINDOW
if translator.reads_are_translated
and isinstance(token_to_kv_pool, BaseSWAKVPool)
else IdSpaceKind.FULL
),
kind=translator.window_kv_kind,
token_mapping=token_mapping,
)
if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool):
+5 -10
View File
@@ -3976,7 +3976,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
def maybe_evict_swa(self):
if self.tree_cache.supports_swa():
sliding_window_size = self.tree_cache.sliding_window_size
retain_window = self.tree_cache.swa_retain_window
# Auxiliary windows check their own cursors and prefix locks.
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
@@ -3985,9 +3985,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
)
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
swa_evict_due = self._swa_eviction_trigger(
sliding_window_size, eviction_interval
)
swa_evict_due = self._swa_eviction_trigger(retain_window, eviction_interval)
self.token_to_kv_pool_allocator.free_group_begin()
for idx, req in enumerate(self.reqs):
if self.forward_mode.is_decode():
@@ -4009,12 +4007,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
),
)
# Past the window the request no longer reads its prefill's SWA;
# release that part of the tree lock so SWA LRU can reclaim it.
if (
release_leaf_lock
and req.decode_batch_idx >= sliding_window_size
):
# Past the retain window the request no longer reads its prefill's
# SWA; release that part of the tree lock so SWA LRU can reclaim it.
if release_leaf_lock and req.decode_batch_idx >= retain_window:
self.tree_cache.release_swa_prefix_lock(req)
elif (
self.forward_mode.is_extend()
+5
View File
@@ -590,6 +590,11 @@ class Scheduler(
if self.draft_worker is not None
else None
),
draft_swa_window=(
self.draft_worker.draft_swa_window
if self.draft_worker is not None
else 0
),
)
self.is_hybrid_swa = result.is_hybrid_swa
self.is_hybrid_ssm = result.is_hybrid_ssm
@@ -391,6 +391,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
storage_prefetch_retries: Optional[StoragePrefetchRetries] = None
# Set by caches that publish KV placement events; None means they don't.
kv_events: Optional[KVCacheEventRecorder] = None
# Tokens back from the committed length a fused draft reads in the swa
# sub-pool (CacheInitParams.draft_swa_window); 0 when it reads none.
draft_swa_window: int = 0
def init_metrics_collector(self):
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
@@ -663,6 +666,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def supports_auxiliary_swa(self) -> bool:
return False
@property
def swa_retain_window(self) -> Optional[int]:
# What the frees and unlocks of a running request's SWA keep behind it;
# every other SWA rule uses sliding_window_size.
if self.sliding_window_size is None:
return None
return max(self.sliding_window_size, self.draft_swa_window)
def evict_sliding_windows(
self, req: Req, pre_len: int, *, eviction_interval: int = 1
) -> None:
@@ -672,7 +683,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
free_swa_out_of_window_slots(
req,
pre_len,
sliding_window_size=self.sliding_window_size,
sliding_window_size=self.swa_retain_window,
page_size=self.page_size,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
@@ -58,6 +58,9 @@ class CacheInitParams:
# paged SWA pool only a draft uses), so it must not gate prefix matching.
swa_gates_prefix_match: bool = True
decoder_swa_bounded_replay: bool = False
# Tokens back from the committed length that a fused draft reads in the
# swa sub-pool (BaseSpecWorker.draft_swa_window); 0 when it reads none.
draft_swa_window: int = 0
# Time-to-live for cache entries in seconds. If None, TTL is disabled.
cache_ttl_seconds: Optional[float] = None
@@ -70,3 +73,12 @@ class CacheInitParams:
mtp_draft_device_pools: tuple[object, ...] = ()
mtp_draft_mamba_pools: tuple[object, ...] = ()
@property
def swa_retain_window(self) -> Optional[int]:
# Cache history required by every reader of the SWA sub-pool. Retention,
# tree locks, prefix matching and host transfers must use the same span.
# The model's attention window remains sliding_window_size.
if self.sliding_window_size is None:
return None
return max(self.sliding_window_size, self.draft_swa_window)
@@ -300,6 +300,7 @@ def build_kv_cache(
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
dynamo_kv_event_format: bool = False,
kv_event_lora_names: Optional[KvEventLoraNames] = None,
draft_swa_window: int = 0,
) -> KVCacheBuildResult:
parallel = get_parallel()
sliding_window_size: Optional[int] = None
@@ -324,6 +325,9 @@ def build_kv_cache(
full_tokens_per_layer, swa_tokens_per_layer = (
tp_worker.get_tokens_per_layer_info()
)
assert draft_swa_window == 0 or sliding_window_size is not None, (
"a fused draft reads an swa sub-pool this target does not build"
)
req_to_token_pool, token_to_kv_pool_allocator = tp_worker.get_memory_pool()
mtp_draft_device_pools = tp_worker.model_runner.mtp_draft_device_pools
@@ -398,9 +402,17 @@ def build_kv_cache(
chunked_prefill_size=effective_chunked_prefill_size,
sliding_window_size=sliding_window_size,
swa_gates_prefix_match=swa_gates_prefix_match,
draft_swa_window=draft_swa_window,
mtp_draft_device_pools=mtp_draft_device_pools,
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
)
if params.swa_retain_window != sliding_window_size:
logger.info(
"SWA eviction keeps %d tokens back: the fused draft reads past the "
"%d-token window.",
params.swa_retain_window,
sliding_window_size,
)
tree_context = TreeCacheBuildContext(
server_args=server_args,
@@ -638,12 +638,8 @@ class KVCacheConfigurator:
alloc = token_to_kv_pool_allocator
placement = self._fused_draft_from_target_buffer(alloc)
if placement is not None:
from sglang.srt.mem_cache.layout.fused_draft import (
draft_swa_layer_ids,
)
from sglang.srt.mem_cache.unified_draft_pool import (
bind_fused_draft,
draft_kv_layer_ids,
draft_state_layer_classes,
)
@@ -651,27 +647,29 @@ class KVCacheConfigurator:
req_to_token_pool = self._build_req_to_token_pool(
max_num_reqs=sizes.max_running_requests
)
runner = self.draft_model_idx or 0
# The target placed this draft from its config; the built
# model is the ground truth for whether it carries state.
state_layers = draft_state_layer_classes(self.model)
if state_layers:
if state_layers and not placement.lanes_for(runner, "mamba"):
raise ValueError(
"Fused draft KV: the draft model has recurrent / "
f"linear-attention layers ({', '.join(state_layers)}) "
"that the fused region gives no state pool."
"that the fused placement gives no state lanes."
)
draft_pool = bind_fused_draft(
binding = bind_fused_draft(
placement=placement,
runner=runner,
unified_buffer=alloc.unified_buffer,
host_allocator=alloc,
placement=placement,
runner=self.draft_model_idx or 0,
kv_layer_ids=draft_kv_layer_ids(self.model),
swa_layer_ids=draft_swa_layer_ids(self.model_config),
model=self.model,
model_config=self.model_config,
req_to_token_pool=req_to_token_pool,
page_size=self.page_size,
)
return _InitializedPools(
req_to_token_pool=req_to_token_pool,
token_to_kv_pool=draft_pool,
req_to_token_pool=binding.req_to_token_pool,
token_to_kv_pool=binding.token_to_kv_pool,
token_to_kv_pool_allocator=alloc,
unified_memory_pool=None,
)
@@ -823,14 +821,20 @@ class KVCacheConfigurator:
# The region holds rows in the target's KV dtype; a draft that resolved
# its own would read and write them as something else. Compare the KV
# dtypes, not their storage: every fp8 flavor is stored as uint8.
region_kv_dtype = placement.region.resolved_kv_dtype()
if self.kv_cache_dtype != region_kv_dtype:
raise ValueError(
f"Fused draft KV: the draft resolved its KV cache dtype to "
f"{self.kv_cache_dtype}, but its region inside the target's pages "
f"holds {region_kv_dtype}. Set "
"--speculative-draft-kv-cache-dtype to the target's KV cache dtype."
)
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
for host in placement.hosts():
if HOST_KINDS[host].family != "dense":
continue
region_kv_dtype = placement.region(host).resolved_kv_dtype()
if self.kv_cache_dtype != region_kv_dtype:
raise ValueError(
f"Fused draft KV: the draft resolved its KV cache dtype to "
f"{self.kv_cache_dtype}, but its region inside the target's "
f"pages holds {region_kv_dtype}. Set "
"--speculative-draft-kv-cache-dtype to the target's KV cache "
"dtype."
)
return placement
def _init_unified_mamba_pools(
@@ -1038,6 +1042,15 @@ class KVCacheConfigurator:
swa_v_head_dim=mc.swa_v_head_dim,
)
def _unified_host_names(self) -> tuple:
"""The sub-pools the unified factory for this model builds, by name."""
names = ["full"]
if self.is_hybrid_swa:
names.append("swa")
if self.mambaish_config is not None:
names.append("mamba")
return tuple(names)
def _fused_draft_decision(self):
"""Whether, and where, the draft's layers fuse into the target's
sub-pools. An empty decision means fusion does not apply (unified
@@ -1048,6 +1061,7 @@ class KVCacheConfigurator:
to the kernel."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
PlacementContext,
draft_kv_profile,
place_fused_draft,
)
@@ -1166,11 +1180,24 @@ class KVCacheConfigurator:
return place_fused_draft(
profile=profile,
num_runners=num_runners,
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
ctx=PlacementContext(
host_names=self._unified_host_names(),
target_window=self.model_config.sliding_window_size,
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
draft_backends=self._draft_attention_backends(),
),
store_dtype=_store_dtype_for(self.kv_cache_dtype),
kv_dtype=self.kv_cache_dtype,
)
def _draft_attention_backends(self) -> tuple:
"""The backends the draft worker may run on: its explicit one, else
the target's, which it inherits."""
explicit = get_spec().speculative_draft_attention_backend
if explicit is not None:
return (explicit,)
return tuple(sorted(b for b in attention_backends() if b is not None))
def _draft_backends_carry_v_head_dim(self) -> bool:
"""Whether every resolved attention backend, the draft's included,
carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The
@@ -1187,10 +1214,69 @@ class KVCacheConfigurator:
pad), built from the same spec as the pool factory's so the priced and
allocated entries agree; None when the draft does not fuse into that
sub-pool."""
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
placement = self._fused_draft_decision().placement
if placement is None or sub_pool_name != "full":
if placement is None:
return None
return self._full_host_spec(placement.region).entry_bytes()
region = placement.region(sub_pool_name)
if region is None:
return None
builders = {
"full": self._full_host_spec,
"swa": self._swa_host_spec,
"mamba": self._mamba_host_spec,
}
assert sub_pool_name in HOST_KINDS, (
f"sub-pool {sub_pool_name!r} is not a registered host kind"
)
assert sub_pool_name in builders, (
f"no fused host spec builder for sub-pool {sub_pool_name!r}"
)
return builders[sub_pool_name](region).entry_bytes()
def _mamba_host_spec(self, region):
from sglang.srt.mem_cache.unified_memory_pool import MambaSubPoolSpec
assert self.mambaish_config is not None, (
"only a mamba-ish host builds a mamba sub-pool"
)
cp = self.mambaish_config.mamba2_cache_params
# This runner's OWN state layers, as the mamba factory slices them.
layer_ids = [
i
for i in cp.layers
if self.layer_info.start_layer <= i < self.layer_info.end_layer
]
return MambaSubPoolSpec(
name="mamba",
layer_num=len(layer_ids),
conv_state_shapes=tuple(tuple(int(x) for x in s) for s in cp.shape.conv),
conv_dtype=cp.dtype.conv,
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
temporal_dtype=cp.dtype.temporal,
grow_direction="up",
draft_region=region,
)
def _swa_host_spec(self, region):
from sglang.srt.mem_cache.unified_memory_pool import (
MHASubPoolSpec,
_store_dtype_for,
)
assert self.is_hybrid_swa, "only a hybrid-SWA host builds an swa sub-pool"
geometry = self._unified_swa_head_geometry()
return MHASubPoolSpec(
name="swa",
layer_num=len(self.layer_info.swa_attention_layer_ids),
head_num=geometry.swa_head_num,
head_dim=geometry.swa_head_dim,
v_head_dim=geometry.swa_v_head_dim,
store_dtype=_store_dtype_for(self.kv_cache_dtype),
grow_direction="up",
draft_region=region,
)
def _full_host_spec(self, region):
from sglang.srt.mem_cache.unified_memory_pool import (
@@ -1292,31 +1378,28 @@ class KVCacheConfigurator:
def _fused_draft_for_pool_factory(self, decision):
"""The placement a pool factory is handed. It logs the decision it
returns, so the boot log shows whether fusion engaged or why not."""
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
if decision.placement is None:
if decision.declined is not None:
logger.warning("fused draft KV disabled: %s", decision.declined)
return None
placement = decision.placement
region = placement.region
kv_dtype = region.resolved_kv_dtype()
logger.info(
"[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d "
"kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s",
region.lane_num,
region.head_num,
region.head_dim,
region.resolved_v_head_dim(),
(
kv_dtype
if kv_dtype == region.store_dtype
else f"{kv_dtype} (stored as {region.store_dtype})"
),
region.entry_bytes(),
[
tuple(placement.lanes_for(r))
for r in range(len(placement.runner_lane_counts))
],
)
if decision.note is not None:
logger.info(
"[unified-memory-pool] fused draft placement: %s", decision.note
)
for host in placement.hosts():
logger.info(
"[unified-memory-pool] %s",
HOST_KINDS[host].describe(
region=placement.region(host),
lanes=[
tuple(placement.lanes_for(r, host))
for r in range(len(placement.runners))
],
),
)
return placement
def _init_unified_swa_pools(
@@ -2956,9 +3039,15 @@ class KVCacheConfigurator:
else:
max_stage_mamba_layers = len(all_mamba_layers)
pp_layer_scale = max_stage_mamba_layers / max(len(all_mamba_layers), 1)
stage_per_req = int(
config.mamba2_cache_params.mamba_cache_per_req * pp_layer_scale
# A fused draft state block widens every state slot: price the entry
# the unified factory carves out (host + draft + pad), not the host's.
fused_state_entry = self.fused_entry_bytes("mamba")
per_req = (
config.mamba2_cache_params.mamba_cache_per_req
if fused_state_entry is None
else fused_state_entry
)
stage_per_req = int(per_req * pp_layer_scale)
has_spec_dec = not self.spec_algorithm.is_none()
# ReplaySSM drops the per-step intermediate_ssm scratch, so its mamba budget
@@ -226,6 +226,19 @@ class KVIndexTranslator:
None when its pool has no such sub-pool."""
return self._spaces.get(kind)
@property
def window_kv_kind(self) -> IdSpaceKind:
"""Storage used by window attention, independent of its attention mask.
A fused window draft may live in FULL when SWA cannot host its window.
Such a draft still clips its reads, but uses FULL physical addresses.
"""
return (
IdSpaceKind.SLIDING_WINDOW
if IdSpaceKind.SLIDING_WINDOW in self._spaces
else IdSpaceKind.FULL
)
def write_ids(
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
) -> Optional[torch.Tensor]:
@@ -406,7 +419,7 @@ class KVIndexTranslator:
reads stay virtual (a static pool, or DCP)."""
if not self._reads_translated(kind):
return self._passthrough_table(req_pool_indices)
if plan.has_read_table(kind):
if plan.has_read_table(kind, reader=self):
return self._plan_table(plan, kind=kind, rows=bs)
return KVIndexTable(
ids=self.req_to_token,
@@ -444,7 +457,7 @@ class KVIndexTranslator:
"a translating reader must read through the plan of its own "
"req_to_token rows"
)
return plan.read_table(kind=kind, rows=rows, into=into)
return plan.read_table(kind=kind, rows=rows, into=into, reader=self)
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
return KVIndexTable(
@@ -474,7 +487,7 @@ class KVIndexTranslator:
(a static pool, or DCP) for the caller to finish; a static SWA pool can
fuse its full->swa table into the gather as ``token_mapping``."""
bs = int(seq_lens.numel())
if self._reads_translated(kind) and not plan.has_read_table(kind):
if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
"req_to_token rows"
@@ -534,7 +547,7 @@ class KVIndexTranslator:
"DCP, where the caller selects this rank's share itself)"
)
rows = out.shape[0]
first = not plan.has_read_table(kind)
first = not plan.has_read_table(kind, reader=self)
table = self._plan_table(
plan, kind=kind, rows=rows, into=out if first else None
)
+19 -4
View File
@@ -293,9 +293,14 @@ class KVLocPlan:
# -- reads -----------------------------------------------------------------
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
def has_read_table(
self,
kind: IdSpaceKind = IdSpaceKind.FULL,
*,
reader: Optional[KVIndexTranslator] = None,
) -> bool:
"""Whether a reader has had the ``kind`` table built yet."""
return self._source.space(kind).key in self._read_tables
return self._read_space(kind, reader).key in self._read_tables
def read_table(
self,
@@ -303,14 +308,17 @@ class KVLocPlan:
kind: IdSpaceKind = IdSpaceKind.FULL,
rows: Optional[int] = None,
into: Optional[torch.Tensor] = None,
reader: Optional[KVIndexTranslator] = None,
) -> KVIndexTable:
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
when a table reader first asks (in place in ``into``, that reader's
capture-stable table, when given) and shared by every later reader.
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
sink, appended by copying, never by translating again. Where reads stay
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
space = self._source.space(kind)
virtual (a static pool, or DCP) it is the `req_to_token` passthrough.
``reader`` names the sub-pool, as in `write_ids`; the plan's own
runner by default."""
space = self._read_space(kind, reader)
table = self._read_tables.get(space.key)
if table is None or (
table.is_translated and rows is not None and table.ids.shape[0] < rows
@@ -329,6 +337,13 @@ class KVLocPlan:
)
return table
def _read_space(
self, kind: IdSpaceKind, reader: Optional[KVIndexTranslator]
) -> IdSpace:
# A later depth of a multi-layer draft can route window layers that
# the depth whose runner built the plan does not.
return (self._source if reader is None else reader).space(kind)
def is_read_by(self, reader: KVIndexTranslator) -> bool:
"""Whether `reader` reads this plan's tables: it indexes the plan's
physical ids and gathers the same `req_to_token` rows. A reader with a
+534 -113
View File
@@ -2,35 +2,53 @@
The nesting, outermost first:
sub-pool a named pool inside the unified pool, with its own grow direction
and frontier. HOST is the role it plays when its entries also
carry draft bytes; only "full" hosts a draft here.
sub-pool a named pool inside the unified pool ("full", "swa", "mamba"),
with its own grow direction and frontier. HOST is the role it
plays when its entries also carry draft bytes.
slot one token entry of that sub-pool. The draft's KV is extra parts of
every slot, after the host's own parts, so ONE slot id per token
covers target and draft: one allocation, one free, one whole-page
relocation carry both.
region the draft geometry inside the host's entries. A region is not a
region the draft geometry inside one host's entries. A region is not a
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
lane one position in the region's layer dimension -- one draft layer of
one runner, present in every slot.
range the contiguous lanes one runner owns in one host.
A lane is not a layer. Runners are the draft's execution copies: a
replicated head gives EVERY runner all the draft's layers, on lanes of its own
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
while a per-depth head (one block per predicted position) gives each runner
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
contiguous run of lanes, and the runs tile the region in runner order.
ONE depth (8 layers across 2 runners = 2 lanes). The runners' ranges tile
each region in runner order, with no gap and no overlap.
A draft whose layers need another sub-pool kind declines fusion as a whole.
A draft LAYER kind (`LAYER_*`) is what the draft checkpoint has; a HOST kind
(`HostKind`, keyed by sub-pool name) is what a sub-pool can carry in its
entries. `place_fused_draft` consults the `HOST_KINDS` registry, and the draft
worker binds each host through its family's binder
(`unified_draft_pool.DRAFT_BINDERS`). When a kind's host turns it away, its
layers FOLD into that host's `fallback_host` if the one-region-one-geometry
rule allows.
"""
from typing import List, Optional, Tuple
import math
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
import msgspec
import torch
from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
LAYER_FULL = "full"
LAYER_WINDOW = "window"
LAYER_STATE = "state"
_LAYER_KIND_NOUN = {
LAYER_FULL: "full-attention",
LAYER_WINDOW: "sliding-window",
LAYER_STATE: "recurrent-state",
}
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
@@ -94,28 +112,6 @@ class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
)
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
"""Where every draft runner's layers live inside the "full" sub-pool: the
region, and each runner's lane count in runner order. Built once on the
target, stored on the `UnifiedKVPool`, and read back by each draft
runner, so the two sides cannot disagree on a lane.
"""
region: DenseDraftRegion
runner_lane_counts: Tuple[int, ...]
def __post_init__(self):
assert self.runner_lane_counts, "a placement needs at least one draft runner"
assert sum(self.runner_lane_counts) == self.region.lane_num, (
f"runner lane counts {self.runner_lane_counts} must tile "
f"range({self.region.lane_num})"
)
def lanes_for(self, runner: int) -> range:
start = sum(self.runner_lane_counts[:runner])
return range(start, start + self.runner_lane_counts[runner])
class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
"""Per-GPU K/V row geometry of one kind of draft attention layer."""
@@ -124,37 +120,345 @@ class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
v_head_dim: int
class DraftStateGeometry(msgspec.Struct, frozen=True, kw_only=True):
"""Per-GPU shapes of one recurrent-state layer of the draft: one conv
tensor per stream plus the temporal state, as `MambaPool.State` lays them
out. Every stream is carried, even ones a depth never touches, because
the backend indexes ``conv[stream]`` by absolute stream number."""
conv_state_shapes: Tuple[Tuple[int, ...], ...]
conv_dtype: torch.dtype
temporal_state_shape: Tuple[int, ...]
temporal_dtype: torch.dtype
def conv_row_bytes(self, idx: int) -> int:
return math.prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
def temporal_row_bytes(self) -> int:
return math.prod(self.temporal_state_shape) * self.temporal_dtype.itemsize
def layer_bytes(self) -> int:
conv = sum(self.conv_row_bytes(i) for i in range(len(self.conv_state_shapes)))
return conv + self.temporal_row_bytes()
class DraftStateRegion(msgspec.Struct, frozen=True, kw_only=True):
"""The draft's recurrent state fused into every slot of a host state
sub-pool: one block per fused (runner, layer), laid out stream-major like
the host's own block, after it. The host's whole-entry clear and copy then
carry the draft's state with the target's."""
lane_num: int
state: DraftStateGeometry
def validate(self) -> None:
assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}"
assert len(self.state.conv_state_shapes) > 0, (
"conv_state_shapes must be non-empty"
)
def entry_bytes(self) -> int:
"""Draft state bytes per slot, before the host entry's alignment."""
return self.lane_num * self.state.layer_bytes()
DraftRegion = Union[DenseDraftRegion, DraftStateRegion]
class DraftLayerSet(msgspec.Struct, frozen=True, kw_only=True):
"""The draft's layers of one kind: layer ids for a replicated head, depth
ids for a per-depth head, with the per-GPU geometry they share."""
layer_ids: Tuple[int, ...]
geometry: Any
# LAYER_WINDOW only: how many tokens back the layers read.
window: Optional[int] = None
class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True):
"""What the draft checkpoint asks of the host sub-pools.
"""What the draft checkpoint asks of the host sub-pools, by layer kind.
``num_depths`` > 1 marks a per-depth head (one transformer block per MTP
depth, served by one runner each under multi-layer EAGLE); otherwise
every runner serves all ``num_layers`` layers.
depth, served by one runner each under multi-layer EAGLE), whose layer
ids are depth ids; otherwise every runner serves all ``num_layers``
layers. A kind is present only when the draft has layers of it.
"""
num_layers: int
full: DraftKVGeometry
swa_layer_ids: Tuple[int, ...] = ()
num_depths: int = 1
num_state_layers: int = 0
kinds: Dict[str, DraftLayerSet] = {}
def draft_state_layer_num(draft_model_config) -> int:
"""Recurrent-state layers the draft HEAD owns -- not the trunk's.
class PlacementContext(msgspec.Struct, frozen=True, kw_only=True):
"""Target-side facts a host kind consults when placing a layer set:
the sub-pools the host builds, the target's window, whether every
resolved backend carries v_head_dim, and the backends the draft worker
may run on (its explicit one, else the target's)."""
A plain NEXTN head of a linear-attention trunk inherits the trunk's
config class, so `mamba2_cache_params` lists the TRUNK's state layers
while the head itself is a full-attention block. Only a conv-chain MTP
head declares `mtp_local_layer_ids`, so that is what separates them.
host_names: Tuple[str, ...]
target_window: Optional[int] = None
asymmetric_rows_ok: bool = False
draft_backends: Tuple[str, ...] = ()
class HostKind(msgspec.Struct, frozen=True, kw_only=True):
"""A unified sub-pool that carries one kind of draft layer in its entries.
``serves`` is the layer kind this host is the first choice for and
``family`` picks the draft worker's binder. ``fallback_host`` carries
``serves`` when this host cannot, under the one-region-one-geometry fold
rule.
"""
from sglang.srt.configs.hybrid_arch import mambaish_config
mambaish = mambaish_config(draft_model_config)
if mambaish is None:
return 0
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
return 0
return len(mambaish.mamba2_cache_params.layers)
name: str
serves: str
family: str
fallback_host: Optional[str] = None
def prerequisite(
self, *, layers: DraftLayerSet, ctx: PlacementContext
) -> Optional[str]:
"""What the runtime must carry for ``layers`` to be served fused at
all, whichever host takes them; a reason here declines the draft."""
return None
def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]:
"""None when this host's entries can carry ``layers``, else why not."""
if self.name not in ctx.host_names:
return f"the host has no {self.name!r} sub-pool"
return None
def region_for(
self,
*,
layers: DraftLayerSet,
lane_num: int,
store_dtype: torch.dtype,
kv_dtype: Optional[torch.dtype],
) -> DraftRegion:
raise NotImplementedError
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
"""The boot-log line for ``region`` and each runner's lanes in it."""
raise NotImplementedError
class DenseHostKind(HostKind):
"""A host whose entries take the draft's K/V rows as two more dense parts."""
def region_for(
self,
*,
layers: DraftLayerSet,
lane_num: int,
store_dtype: torch.dtype,
kv_dtype: Optional[torch.dtype],
) -> DenseDraftRegion:
geometry = layers.geometry
assert isinstance(geometry, DraftKVGeometry), geometry
return DenseDraftRegion(
lane_num=lane_num,
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
store_dtype=store_dtype,
kv_dtype=kv_dtype,
)
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
kv_dtype = region.resolved_kv_dtype()
stored = (
kv_dtype
if kv_dtype == region.store_dtype
else f"{kv_dtype} (stored as {region.store_dtype})"
)
return (
f"fused draft region in {self.name!r}: {region.lane_num} lane(s) x "
f"{region.head_num} kv head(s) x {region.head_dim}/"
f"{region.resolved_v_head_dim()} k/v head_dim @ {stored} "
f"= {region.entry_bytes()} B/token; runner lanes {list(lanes)}"
)
# Registration order is the order placements report their hosts in.
HOST_KINDS: Dict[str, HostKind] = {}
def host_for(layer_kind: str) -> Optional[HostKind]:
"""The registered host that serves ``layer_kind`` first, if any."""
for kind in HOST_KINDS.values():
if kind.serves == layer_kind:
return kind
return None
def register_host_kind(kind: HostKind) -> HostKind:
assert kind.name not in HOST_KINDS, f"host kind {kind.name!r} is already registered"
primary = host_for(kind.serves)
assert primary is None, (
f"layer kind {kind.serves!r} is already served by host {primary.name!r}"
)
assert kind.fallback_host is None or kind.fallback_host in HOST_KINDS, (
f"host kind {kind.name!r} falls back to unregistered host "
f"{kind.fallback_host!r}"
)
HOST_KINDS[kind.name] = kind
return kind
FULL_HOST = register_host_kind(
DenseHostKind(name="full", serves=LAYER_FULL, family="dense")
)
# Multi-step draft backends that build the per-step sliding-window read and
# write rails a fused window layer needs (TritonMultiStepDraftBackend).
WINDOW_RAIL_BACKENDS = frozenset({"triton"})
class WindowHostKind(DenseHostKind):
"""The swa sub-pool: a draft window layer rides there when its window fits
the target's; eviction keeps any further reach the draft worker declares."""
def prerequisite(
self, *, layers: DraftLayerSet, ctx: PlacementContext
) -> Optional[str]:
unrailed = sorted(set(ctx.draft_backends) - WINDOW_RAIL_BACKENDS)
if unrailed:
return (
f"the draft's attention backend {unrailed} carries no per-step "
f"sliding-window rail (only {sorted(WINDOW_RAIL_BACKENDS)} does)"
)
return None
def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]:
reason = super().admits(layers=layers, ctx=ctx)
if reason is not None:
return reason
if layers.window is None:
return "the draft declares no sliding window size"
if ctx.target_window is None:
return "the target declares no sliding window size"
if layers.window > ctx.target_window:
return (
f"its window {layers.window} exceeds the target's window "
f"{ctx.target_window}"
)
return None
SWA_HOST = register_host_kind(
WindowHostKind(
name="swa", serves=LAYER_WINDOW, family="dense", fallback_host="full"
)
)
class StateHostKind(HostKind):
"""The mamba sub-pool: its entries take the draft's recurrent-state block
after the host's streams, so the slot that is the request's carries both."""
def region_for(
self,
*,
layers: DraftLayerSet,
lane_num: int,
store_dtype: torch.dtype,
kv_dtype: Optional[torch.dtype],
) -> DraftStateRegion:
geometry = layers.geometry
assert isinstance(geometry, DraftStateGeometry), geometry
return DraftStateRegion(lane_num=lane_num, state=geometry)
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
state = region.state
return (
f"fused draft state in {self.name!r}: {region.lane_num} lane(s) x "
f"{len(state.conv_state_shapes)} conv stream(s) @ {state.conv_dtype} + "
f"temporal {tuple(state.temporal_state_shape)} @ {state.temporal_dtype} "
f"= {region.entry_bytes()} B/slot; runner lanes {list(lanes)}"
)
MAMBA_HOST = register_host_kind(
StateHostKind(name="mamba", serves=LAYER_STATE, family="state")
)
class RunnerLanes(msgspec.Struct, frozen=True, kw_only=True):
"""One draft runner's lanes inside each host region: host -> (start, count)."""
ranges: Dict[str, Tuple[int, int]] = {}
def range_for(self, host: str) -> range:
start, count = self.ranges.get(host, (0, 0))
return range(start, start + count)
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
"""Where every draft runner's layers live inside the host sub-pools.
One region per host sub-pool that carries draft lanes plus each runner's
lane range into it. Built once on the target, stored on the
`UnifiedKVPool`, and read back by each draft runner, so the two sides
cannot disagree on a lane.
"""
runners: Tuple[RunnerLanes, ...]
regions: Dict[str, DraftRegion] = {}
def __post_init__(self):
assert len(self.runners) > 0, "a placement needs at least one draft runner"
hosts = set(self.regions).union(*(r.ranges for r in self.runners))
for host in sorted(hosts):
assert host in HOST_KINDS, (
f"no host kind is registered for sub-pool {host!r}"
)
region = self.regions.get(host)
lanes = [s for r in self.runners for s in r.range_for(host)]
if region is None:
assert not lanes, f"host {host!r} has draft lanes but no region"
continue
assert lanes == list(range(region.lane_num)), (
f"host {host!r}: runner lane ranges {lanes} must tile "
f"range({region.lane_num}) in runner order"
)
def region(self, host: str) -> Optional[DraftRegion]:
return self.regions.get(host)
def hosts(self) -> Tuple[str, ...]:
"""The hosts carrying draft lanes, in registry order."""
return tuple(h for h in HOST_KINDS if h in self.regions)
def lanes_for(self, runner: int, host: str) -> range:
return self.runners[runner].range_for(host)
@classmethod
def from_counts(
cls,
*,
counts: Mapping[str, Sequence[int]],
regions: Mapping[str, DraftRegion],
) -> "FusedDraftPlacement":
"""Tile each host's region with the runners' layer counts, in runner order."""
num_runners = {len(per_runner) for per_runner in counts.values()}
assert len(num_runners) == 1, (
f"every host needs one layer count per runner; got {dict(counts)}"
)
ranges: List[Dict[str, Tuple[int, int]]] = [
{} for _ in range(num_runners.pop())
]
for host, per_runner in counts.items():
assert host in regions, f"host {host!r} has layer counts but no region"
start = 0
for runner, count in enumerate(per_runner):
if count:
ranges[runner][host] = (start, count)
start += count
return cls(
runners=tuple(RunnerLanes(ranges=r) for r in ranges),
regions=dict(regions),
)
def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
@@ -167,6 +471,25 @@ def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
return ()
def draft_state_layer_ids(draft_model_config, *, num_depths: int) -> Tuple[int, ...]:
"""The draft depth ids that own a recurrent-state block of their own.
Only a per-depth conv-chain MTP head carries state: the baseline serves
it with a state cache cloned beside the target's. A plain NEXTN head of
a linear-attention trunk inherits the trunk's mamba-ish config class but
is a full-attention block; its config lists the TRUNK's state layers,
and the head carries none.
"""
from sglang.srt.configs.hybrid_arch import mambaish_config
mc = draft_model_config
if mambaish_config(mc) is None or mc.num_nextn_predict_layers is None:
return ()
if getattr(mc.hf_text_config, "mtp_local_layer_ids", None) is None:
return ()
return tuple(range(num_depths))
def draft_kv_profile(
draft_model_config,
*,
@@ -177,37 +500,76 @@ def draft_kv_profile(
"""The profile of a draft `ModelConfig`, heads divided by attn_tp the way
the target divides its own (drafts never join the DCP group).
``num_depths`` overrides the config's `num_nextn_predict_layers`."""
from sglang.srt.configs.hybrid_arch import mambaish_config
mc = draft_model_config
if num_depths is None:
num_depths = mc.num_nextn_predict_layers
return DraftKVProfile(
num_layers=int(num_layers),
full=DraftKVGeometry(
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
head_dim=int(mc.head_dim),
v_head_dim=int(mc.v_head_dim),
),
swa_layer_ids=draft_swa_layer_ids(mc),
num_depths=1 if num_depths is None else int(num_depths),
num_state_layers=draft_state_layer_num(mc),
num_depths = 1 if num_depths is None else int(num_depths)
num_layers = int(num_layers)
window_ids = draft_swa_layer_ids(mc)
assert set(window_ids) <= set(range(num_layers)), (
f"draft window layer ids {window_ids} outside range({num_layers})"
)
kinds: Dict[str, DraftLayerSet] = {}
full_ids = tuple(i for i in range(num_layers) if i not in window_ids)
if full_ids:
kinds[LAYER_FULL] = DraftLayerSet(
layer_ids=full_ids,
geometry=DraftKVGeometry(
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
head_dim=int(mc.head_dim),
v_head_dim=int(mc.v_head_dim),
),
)
if window_ids:
kinds[LAYER_WINDOW] = DraftLayerSet(
layer_ids=window_ids,
geometry=DraftKVGeometry(
head_num=int(mc.get_swa_num_kv_heads(attn_tp_size)),
head_dim=int(mc.swa_head_dim),
v_head_dim=int(mc.swa_v_head_dim),
),
window=(
None if mc.sliding_window_size is None else int(mc.sliding_window_size)
),
)
state_ids = draft_state_layer_ids(mc, num_depths=num_depths)
if state_ids:
cp = mambaish_config(mc).mamba2_cache_params
kinds[LAYER_STATE] = DraftLayerSet(
layer_ids=state_ids,
geometry=DraftStateGeometry(
conv_state_shapes=tuple(
tuple(int(x) for x in s) for s in cp.shape.conv
),
conv_dtype=cp.dtype.conv,
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
temporal_dtype=cp.dtype.temporal,
),
)
return DraftKVProfile(num_layers=num_layers, num_depths=num_depths, kinds=kinds)
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
fuse. Neither means fusion does not apply."""
fuse. Neither means fusion does not apply. ``note`` says why a placed
layer kind did not get its first-choice host."""
placement: Optional[FusedDraftPlacement] = None
declined: Optional[str] = None
note: Optional[str] = None
def _runner_layer_counts(
profile: DraftKVProfile, num_runners: int
) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]:
"""Per runner, its (full, swa) layer counts; or why no runner layout exists."""
) -> Tuple[Optional[Dict[str, List[int]]], Optional[str]]:
"""Per layer kind, each runner's layer count; or why no runner layout exists."""
if profile.num_depths <= 1:
num_swa = len(profile.swa_layer_ids)
return [(profile.num_layers - num_swa, num_swa)] * num_runners, None
return {
kind: [len(layers.layer_ids)] * num_runners
for kind, layers in profile.kinds.items()
}, None
if num_runners == 1:
return None, (
f"a per-depth draft head ({profile.num_depths} depths) needs one "
@@ -217,69 +579,128 @@ def _runner_layer_counts(
return None, (
f"{num_runners} draft runners exceed the head's {profile.num_depths} depths"
)
swa = set(profile.swa_layer_ids)
return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None
return {
kind: [1 if r in layers.layer_ids else 0 for r in range(num_runners)]
for kind, layers in profile.kinds.items()
}, None
def _kv_rows_declined(
geometry: Any, *, store_dtype: torch.dtype, asymmetric_rows_ok: bool
) -> Optional[str]:
"""Why one kind's K/V rows cannot be fused entry parts, or None."""
if not isinstance(geometry, DraftKVGeometry):
return None
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
return (
"the draft's K/V rows are asymmetric "
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
"and a resolved attention backend does not carry v_head_dim "
"through to the kernel"
)
for dim in (geometry.head_dim, geometry.v_head_dim):
row_bytes = geometry.head_num * dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return (
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
)
return None
def _fold_refused(
profile: DraftKVProfile, *, kind: str, into: HostKind
) -> Optional[str]:
"""One region holds one row geometry: ``kind`` folds into ``into`` only
when the draft has no layers of the kind ``into`` serves, or both kinds
share a geometry."""
own = profile.kinds.get(into.serves)
if own is None or own.geometry == profile.kinds[kind].geometry:
return None
return (
f"its {_LAYER_KIND_NOUN[kind]} layers' rows differ from its "
f"{_LAYER_KIND_NOUN[into.serves]} layers' rows, so they cannot share the "
f"{into.name!r} sub-pool"
)
def place_fused_draft(
*,
profile: DraftKVProfile,
num_runners: int,
ctx: PlacementContext,
store_dtype: torch.dtype,
asymmetric_rows_ok: bool,
kv_dtype: Optional[torch.dtype] = None,
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host sub-pool serves declines the whole draft,
as do asymmetric K/V rows unless the caller vouches that every attention
backend carries v_head_dim through to the kernel."""
"""Assign every draft layer of every runner to a host sub-pool through the
registry: a layer kind goes to the host that serves it, else to that
host's fallback under the fold rule. A kind no host takes declines the
whole draft, as does a kind whose runtime prerequisite is missing,
asymmetric K/V rows unless ``ctx`` vouches that every attention backend
carries v_head_dim through to the kernel, and rows off the entry-part
alignment."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
if profile.num_state_layers:
return FusedDraftDecision(
declined=(
f"the draft has {profile.num_state_layers} recurrent-state "
"layer(s) of its own, which no host state pool carries"
)
for kind, layers in profile.kinds.items():
what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)"
host = host_for(kind)
if host is None:
return FusedDraftDecision(declined=f"{what}: no host kind serves them")
reason = host.prerequisite(layers=layers, ctx=ctx)
if reason is not None:
return FusedDraftDecision(declined=f"{what}: {reason}")
reason = _kv_rows_declined(
layers.geometry,
store_dtype=store_dtype,
asymmetric_rows_ok=ctx.asymmetric_rows_ok,
)
num_swa = sum(swa for _, swa in counts)
if num_swa:
return FusedDraftDecision(
declined=(
f"the draft has {num_swa} SWA layer(s) of its own, which the "
"fused dense pool cannot serve"
if reason is not None:
return FusedDraftDecision(declined=reason)
host_counts: Dict[str, List[int]] = {}
host_layers: Dict[str, DraftLayerSet] = {}
notes: List[str] = []
for kind, layers in profile.kinds.items():
host = host_for(kind)
what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)"
reason = host.admits(layers=layers, ctx=ctx)
if reason is not None:
fallback = (
None if host.fallback_host is None else HOST_KINDS[host.fallback_host]
)
)
geometry = profile.full
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
return FusedDraftDecision(
declined=(
"the draft's K/V rows are asymmetric "
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
"and a resolved attention backend does not carry v_head_dim "
"through to the kernel"
)
)
for dim in (geometry.head_dim, geometry.v_head_dim):
row_bytes = geometry.head_num * dim * store_dtype.itemsize
if row_bytes % ROW_ALIGN_BYTES:
return FusedDraftDecision(
declined=(
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
fold = None
if fallback is not None:
fold = _fold_refused(profile, kind=kind, into=fallback)
if fold is None:
fold = fallback.admits(layers=layers, ctx=ctx)
if fallback is None or fold is not None:
if fold is not None:
reason = f"{reason}, and {fold}"
return FusedDraftDecision(
declined=(
f"{what} cannot ride in the {host.name!r} sub-pool: {reason}"
)
)
)
full_counts = tuple(full for full, _ in counts)
region = DenseDraftRegion(
lane_num=sum(full_counts),
head_num=geometry.head_num,
head_dim=geometry.head_dim,
v_head_dim=geometry.v_head_dim,
store_dtype=store_dtype,
kv_dtype=kv_dtype,
)
notes.append(f"{what} ride in the {fallback.name!r} sub-pool: {reason}")
host = fallback
per_runner = host_counts.setdefault(host.name, [0] * num_runners)
for runner, count in enumerate(counts[kind]):
per_runner[runner] += count
# The fold rule made every kind sharing a host share its geometry.
host_layers.setdefault(host.name, layers)
placed = {host: c for host, c in host_counts.items() if sum(c) > 0}
regions = {
host: HOST_KINDS[host].region_for(
layers=host_layers[host],
lane_num=sum(c),
store_dtype=store_dtype,
kv_dtype=kv_dtype,
)
for host, c in placed.items()
}
return FusedDraftDecision(
placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts)
placement=FusedDraftPlacement.from_counts(counts=placed, regions=regions),
note="; ".join(notes) or None,
)
@@ -13,7 +13,7 @@ so every per-layer view is ``(num_slots, *row_shape)`` with slot stride
is the pool region's offset inside the raw buffer.
"""
from typing import List, Sequence, Tuple
from typing import List, Optional, Sequence, Tuple
import msgspec
import torch
@@ -179,21 +179,26 @@ def build_mamba_entry_views(
temporal_dtype: torch.dtype,
max_slots: int,
anchor_bytes: int = 0,
entry_bytes: Optional[int] = None,
offset_bytes: int = 0,
) -> Tuple[List[torch.Tensor], torch.Tensor]:
"""Per-slot envelope views over ``raw`` for Mamba state.
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
[temporal rows × layers]``. Each returned view has shape
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
Layout of one block: ``[conv[0] rows × layers][conv[1] rows × layers]...
[temporal rows × layers]``, starting ``offset_bytes`` into a slot's entry
of ``entry_bytes`` (the block's own size when omitted, i.e. a standalone
envelope). Each returned view has shape ``(num_layers, max_slots,
*inner_shape)`` matching ``MambaPool.State.conv[i]`` / ``.temporal``.
Mamba state is always token-granular (page_size == 1).
"""
entry_bytes = mamba_entry_bytes(
layer_num=layer_num,
conv_state_shapes=conv_state_shapes,
conv_dtype=conv_dtype,
temporal_state_shape=temporal_state_shape,
temporal_dtype=temporal_dtype,
)
if entry_bytes is None:
entry_bytes = mamba_entry_bytes(
layer_num=layer_num,
conv_state_shapes=conv_state_shapes,
conv_dtype=conv_dtype,
temporal_state_shape=temporal_state_shape,
temporal_dtype=temporal_dtype,
)
conv_itemsize = conv_dtype.itemsize
assert entry_bytes % conv_itemsize == 0, (
@@ -207,7 +212,7 @@ def build_mamba_entry_views(
as_conv_dtype = raw.view(conv_dtype)
conv_slot_stride_elems = entry_bytes // conv_itemsize
offset_bytes_within_entry = 0
offset_bytes_within_entry = offset_bytes
conv_views: List[torch.Tensor] = []
for shape in conv_state_shapes:
inner_shape_bytes = _prod(shape) * conv_itemsize
@@ -1971,6 +1971,8 @@ class KVWriteLoc:
# swa_loc / full_loc are resolved once at metadata-init from the full
# (padded) out_cache_loc; piecewise/DP-padded paths later narrow loc per
# layer, so slice these pre-resolved locs to match (same per-token order).
# A multi-step draft step's slice is NOT a prefix of the all-steps loc;
# its backend derives per-step locs and never relies on this.
if self.swa_loc is not None and self.swa_loc.shape[0] != self.loc.shape[0]:
self.swa_loc = self.swa_loc[: self.loc.shape[0]]
if self.full_loc is not None and self.full_loc.shape[0] != self.loc.shape[0]:
@@ -23,6 +23,7 @@ class PureSWARadixCache(RadixCache):
def __init__(self, params: CacheInitParams):
super().__init__(params)
self.sliding_window_size = params.sliding_window_size
self.draft_swa_window = params.draft_swa_window
def supports_swa(self) -> bool:
assert self.sliding_window_size is not None, (
@@ -468,7 +468,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
enable_hicache=False,
write_through_threshold=256,
device=str(self.device),
swa_sliding_window_size=params.sliding_window_size,
# Rust's tree window has the same cache-history contract as
# SWAComponent, including the fused draft's front rows.
swa_sliding_window_size=params.swa_retain_window,
swa_req_ring=not params.swa_gates_prefix_match,
enable_kv_cache_events=params.enable_kv_cache_events,
dynamo_kv_event_format=params.dynamo_kv_event_format,
@@ -216,11 +216,12 @@ class SWAKVPool(BaseSWAKVPool):
def get_v_head_dim(self):
# The FULL side's dim, as HybridLinearKVPool.get_v_head_dim(): a caller
# asking a pool for "the" v_head_dim wants the full-attention geometry.
# A window-only pool (a sliding-window MTP depth) has no full side;
# its swa side is the only geometry there is.
# `start_layer`, not 0, so pipeline parallelism (start_layer > 0) works,
# and because layer 0 need not be a full-attention layer.
return self.full_kv_pool.get_value_buffer(self.full_kv_pool.start_layer).shape[
-1
]
lead = self.full_kv_pool if self.full_layer_nums > 0 else self.swa_kv_pool
return lead.get_value_buffer(lead.start_layer).shape[-1]
def get_kv_buffer(self, layer_id: int):
self._wait_for_layer(layer_id)
@@ -88,7 +88,9 @@ class SWAComponent(TreeComponent):
super().__init__(cache, params)
self._session_leaf_covered_len: dict[str, dict[UnifiedTreeNode, int]] = {}
self.sliding_window_size = params.sliding_window_size
# This is the tree's required history, including widened draft reads;
# it does not change the model's attention window.
self.sliding_window_size = params.swa_retain_window
self.gates_prefix_match = params.swa_gates_prefix_match
self.full_window_pages = (
self.sliding_window_size + params.page_size - 1
+493 -33
View File
@@ -1,14 +1,37 @@
"""KV pools a DRAFT runner binds over the draft lanes fused into the target's
unified pool: same pages, same slot ids, same v2p table as the target -- one
allocation, one free, one relocation."""
allocation, one free, one relocation.
from typing import Any, Dict, List, Mapping, Optional, Sequence
`DRAFT_BINDERS` maps a `HostKind.family` to the binder that builds the
runner's pool over every host of that family; registering a binder is how a
new host kind reaches the draft worker.
"""
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
import msgspec
import torch
from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.mem_cache.layout.fused_draft import (
HOST_KINDS,
DraftStateRegion,
FusedDraftPlacement,
draft_state_layer_ids,
draft_swa_layer_ids,
)
from sglang.srt.mem_cache.memory_pool import (
KVWriteLoc,
MambaPool,
MHATokenToKVPool,
unwrap_write_loc,
write_loc_is_physical,
)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.mem_cache.unified_memory_pool import (
UnifiedHybridReqToTokenPool,
UnifiedKVPool,
)
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
class UnifiedDraftKVPool(MHATokenToKVPool):
@@ -102,10 +125,314 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
class UnifiedDraftSWAKVPool(SWAKVPool):
"""A draft's KV pool when it has sliding-window layers: one dense draft
pool per host sub-pool it fuses into ("full" and/or "swa"), routed per
layer exactly like the target's `UnifiedSWAKVPool`, so the attention
backends run their swa rail for it. Inherits `SWAKVPool` for `isinstance`
only; never calls its `__init__` (it would allocate static pools)."""
requires_physical_write_loc = True
def __init__(
self,
*,
unified_buffer: UnifiedKVPool,
host_allocator,
page_size: int,
full_layer_lanes: Mapping[int, int],
swa_layer_lanes: Mapping[int, int],
):
assert swa_layer_lanes, (
"UnifiedDraftSWAKVPool binds at least one window layer; a full-only "
"draft binds UnifiedDraftKVPool"
)
self.unified_buffer = unified_buffer
self.host_allocator = host_allocator
self.page_size = page_size
self.device = unified_buffer.device
self.layer_transfer_counter = None
self.full_layer_nums = len(full_layer_lanes)
self.swa_layer_nums = len(swa_layer_lanes)
self.layer_num = self.full_layer_nums + self.swa_layer_nums
self.start_layer = min([*full_layer_lanes, *swa_layer_lanes])
self.size = unified_buffer.max_slots("full") - 1
self.size_swa = unified_buffer.max_slots("swa") - 1
self.full_kv_pool: Optional[UnifiedDraftKVPool] = None
if full_layer_lanes:
self.full_kv_pool = self._side_pool("full", full_layer_lanes)
self.swa_kv_pool = self._side_pool("swa", swa_layer_lanes)
lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool
self.dtype = lead.dtype
self.head_num = lead.head_num
self.head_dim = lead.head_dim
# {layer_id: (per-side index, is_swa_layer)}, sides indexed in layer order.
self.layers_mapping: Dict[int, Tuple[int, bool]] = {}
for idx, layer_id in enumerate(sorted(full_layer_lanes)):
self.layers_mapping[layer_id] = (idx, False)
for idx, layer_id in enumerate(sorted(swa_layer_lanes)):
self.layers_mapping[layer_id] = (idx, True)
# None so dispatch routes through the host's v2p tables, never a
# registered mapping.
self.full_to_swa_index_mapping: Optional[torch.Tensor] = None
self.enable_custom_mem_pool = False
self.custom_mem_pool = None
self.dsa_kv_cache_store_fp8 = False
self.kv_cache_dim = None
self.index_head_dim = None
self.mem_usage = 0.0 # fused into the host entries
def _side_pool(
self, host_sub_pool_name: str, layer_lanes: Mapping[int, int]
) -> UnifiedDraftKVPool:
# The side is indexed 0..n-1 in layer order (`layer_id_override`).
return UnifiedDraftKVPool(
unified_buffer=self.unified_buffer,
host_sub_pool_name=host_sub_pool_name,
host_allocator=self.host_allocator,
layer_lanes={
idx: layer_lanes[layer_id]
for idx, layer_id in enumerate(sorted(layer_lanes))
},
page_size=self.page_size,
)
def _side(self, layer_id: int) -> Tuple[UnifiedDraftKVPool, int]:
pool_layer_id, is_swa = self.layers_mapping[layer_id]
pool = self.swa_kv_pool if is_swa else self.full_kv_pool
assert pool is not None, layer_id
return pool, pool_layer_id
# -- KVCache surface a side of None cannot answer --
@property
def post_capture_active(self) -> bool:
return False # the host buffer is fully backed at boot
@property
def post_capture_backed_bytes(self) -> int:
return 0
def finalize_backing(self, config) -> None:
return
def register_layer_transfer_counter(self, layer_transfer_counter):
self.layer_transfer_counter = layer_transfer_counter
# -- BaseSWAKVPool ABC surface --
def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
return # the host's swa v2p table IS the mapping
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
"""Virtual token ids -> swa-physical token ids, through the host."""
return self.host_allocator.translate_loc_from_full_to_swa(kv_indices)
def get_state_buf_infos(self):
raise NotImplementedError(
"fused draft KV has no per-layer contiguous regions; KV transfer / "
"disaggregation is unsupported."
)
# -- size/info --
def get_kv_size_bytes(self):
return 0, 0 # fused into the host entries; UnifiedKVPool logs the total
def get_contiguous_buf_infos(self):
raise NotImplementedError(
"fused draft KV has no per-layer contiguous regions; KV transfer / "
"disaggregation is unsupported."
)
def get_v_head_dim(self):
lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool
return lead.get_value_buffer(lead.start_layer).shape[-1]
# -- buffer accessors --
def get_key_buffer(self, layer_id: int):
pool, pool_layer_id = self._side(layer_id)
return pool.get_key_buffer(pool_layer_id)
def get_value_buffer(self, layer_id: int):
pool, pool_layer_id = self._side(layer_id)
return pool.get_value_buffer(pool_layer_id)
def get_kv_buffer(self, layer_id: int):
pool, pool_layer_id = self._side(layer_id)
return pool.get_kv_buffer(pool_layer_id)
# -- kv writing --
def set_kv_buffer(
self,
layer,
loc_info,
cache_k: torch.Tensor,
cache_v: torch.Tensor,
k_scale: float = 1.0,
v_scale: float = 1.0,
):
"""Route to the right side. Both locs are physical already (the
backend derives `swa_loc` once per forward); never translates here.
The full side writes through `loc`: the capture-stable `full_loc`
alias is sized for the target's writes, not a per-step draft slice."""
loc, swa_loc, _ = unwrap_write_loc(loc_info)
physical = write_loc_is_physical(loc_info)
pool, pool_layer_id = self._side(layer.layer_id)
if pool is self.swa_kv_pool:
assert swa_loc is not None, (
"UnifiedDraftSWAKVPool.set_kv_buffer: window layer received no "
"swa_loc; the attention backend must bundle "
"forward_metadata.swa_out_cache_loc."
)
side_loc = swa_loc
else:
side_loc = loc
pool.set_kv_buffer(
None,
KVWriteLoc(side_loc, physical=physical),
cache_k,
cache_v,
k_scale,
v_scale,
layer_id_override=pool_layer_id,
)
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
raise NotImplementedError(
"fused draft KV relocates with the HOST pool's whole-page move; a "
"draft-side per-slot move would corrupt the fused layout"
)
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
def load_cpu_copy(
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
):
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
class UnifiedDraftMambaPool(MambaPool):
"""A draft runner's recurrent state as views of the draft block fused into
the host's state entries: `mamba_cache.conv[stream][layer]` is laid out
exactly as a private `MambaPool`'s. Slot ids are the target's, translated
by the shared allocator. Pure view: the host's whole-entry clear and copy
already cover the draft block, so the block ops here act on the draft block alone, and it
never calls `MambaPool.__init__` (no buffers of its own)."""
def __init__(
self,
*,
unified_buffer: UnifiedKVPool,
sub_pool_name: str,
layer_lanes: Mapping[int, int],
):
spec = unified_buffer.mamba_spec(sub_pool_name)
region = spec.draft_region
assert isinstance(region, DraftStateRegion), (
f"UnifiedDraftMambaPool: sub-pool {sub_pool_name!r} carries no fused "
"draft state region"
)
layer_ids = sorted(layer_lanes)
assert layer_ids, "UnifiedDraftMambaPool binds at least one layer"
lanes = [layer_lanes[layer_id] for layer_id in layer_ids]
# A runner's lanes are one contiguous range, so its layers are one
# strided slice of the region's views (a gather would copy).
start = lanes[0]
assert lanes == list(range(start, start + len(lanes))) and (
0 <= start and start + len(lanes) <= region.lane_num
), (
f"draft state lanes {lanes} must be a contiguous range within "
f"{region.lane_num}"
)
conv_views, temporal_view = unified_buffer.build_draft_state_views(
sub_pool_name
)
stop = start + len(lanes)
self._unified_buffer = unified_buffer
self._sub_pool_name = sub_pool_name
self.layer_lanes: Dict[int, int] = dict(layer_lanes)
self.mamba_layer_ids = layer_ids
self.num_mamba_layers = len(layer_ids)
self._max_size = unified_buffer.max_slots(sub_pool_name) - 1
self.size = self._max_size
self.device = unified_buffer.device
self.memory_saver_adapter = TorchMemorySaverAdapter.create(enable=False)
self.enable_custom_mem_pool = False
self.custom_mem_pool = None
# Same disabled-state attributes `UnifiedMambaPool` replicates for the
# base class's unconditional reads.
self.enable_linear_replayssm = False
self.linear_replayssm_cache_len = 16
self.replayssm_write_pos = None
self.replayssm_is_kda = False
self.enable_linear_replayssm_spec = False
self.replayssm_spec_fold = False
self.replayssm_cache_base = None
self.replayssm_is_flush = None
self.debug_memory_pool = False
self.conv_shard_groups = None
self.conv_slice_axis = spec.conv_slice_axis
self.mamba_cache = self.State(
conv=[view[start:stop] for view in conv_views],
temporal=temporal_view[start:stop],
)
self.mem_usage = 0.0 # fused into the host entries
def clear_slots(self, indices: torch.Tensor):
"""Zero THIS runner's state block in the given slots.
The block only, never the whole entry: the host's own streams share
the entry and must survive, since a new request takes over the
draft's block while the target's state for that slot is still live.
The host's whole-entry clear covers the block too, but only on a
plain EXTEND forward: ModelRunner skips it on decode, target-verify
and draft-extend, the modes the draft needs it in.
"""
for view in self.mamba_cache.conv:
view[:, indices] = 0
self.mamba_cache.temporal[:, indices] = 0
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
"""Copy-on-write THIS runner's state block, block only (see above)."""
for view in self.mamba_cache.conv:
view[:, dst_indices] = view[:, src_indices]
self.mamba_cache.temporal[:, dst_indices] = self.mamba_cache.temporal[
:, src_indices
]
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
raise NotImplementedError(
"fused draft state relocates with the HOST pool's whole-entry move"
)
def get_contiguous_buf_infos(self):
raise NotImplementedError(
"fused draft state has no contiguous region of its own; state "
"transfer / disaggregation is unsupported."
)
def get_cpu_copy(self, indices):
raise NotImplementedError(
"CPU offloading is unsupported for fused draft state."
)
def load_cpu_copy(self, mamba_cache_cpu, indices):
raise NotImplementedError(
"CPU offloading is unsupported for fused draft state."
)
def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]:
"""The host allocator a fused draft pool translates through, or None for
any other pool."""
if isinstance(token_to_kv_pool, UnifiedDraftKVPool):
if isinstance(token_to_kv_pool, (UnifiedDraftKVPool, UnifiedDraftSWAKVPool)):
return token_to_kv_pool.host_allocator
return None
@@ -123,8 +450,8 @@ def draft_kv_layer_ids(model) -> List[int]:
def draft_state_layer_classes(model) -> List[str]:
"""Class names of the BUILT draft model's recurrent / linear-attention
modules. A fused draft has KV lanes only, so any of these would run with
no state pool of its own."""
modules. A runner the placement gives no state lanes would run any of
these with no state pool of its own."""
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
@@ -137,37 +464,170 @@ def draft_state_layer_classes(model) -> List[str]:
)
class FusedDraftBinding(msgspec.Struct, frozen=True, kw_only=True):
"""What a draft runner binds: its KV pool over the fused lanes and the
request table it reads."""
token_to_kv_pool: Any = None
req_to_token_pool: Any = None
class BindContext(msgspec.Struct, frozen=True, kw_only=True):
"""One runner's inputs to every binder."""
placement: FusedDraftPlacement
runner: int
unified_buffer: Any
host_allocator: Any
model: Any
model_config: Any
page_size: int
def hosts_of(self, family: str) -> Tuple[str, ...]:
"""This runner's hosts of ``family`` that hold lanes for it."""
return tuple(
host
for host in self.placement.hosts()
if HOST_KINDS[host].family == family
and len(self.placement.lanes_for(self.runner, host)) > 0
)
Binder = Callable[[BindContext, FusedDraftBinding], FusedDraftBinding]
DRAFT_BINDERS: Dict[str, Binder] = {}
def register_draft_binder(family: str, binder: Binder) -> Binder:
assert family not in DRAFT_BINDERS, (
f"a draft binder is already registered for host family {family!r}"
)
DRAFT_BINDERS[family] = binder
return binder
def bind_fused_draft(
*,
unified_buffer: UnifiedKVPool,
host_allocator,
placement: FusedDraftPlacement,
runner: int,
kv_layer_ids: Sequence[int],
swa_layer_ids: Sequence[int],
unified_buffer: UnifiedKVPool,
host_allocator,
model,
model_config,
req_to_token_pool,
page_size: int,
) -> UnifiedDraftKVPool:
"""The KV pool draft runner ``runner`` binds over its fused slots.
The placement sized the lanes from the draft config; the model's real
layer ids fill them in layer order, so a count mismatch is a loud boot
failure, never a silent alias.
"""
swa = set(swa_layer_ids)
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa]
assert len(full_ids) == len(kv_layer_ids), (
f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused "
"SWA KV is not supported yet"
)
full_lanes = placement.lanes_for(runner)
assert len(full_ids) == len(full_lanes), (
f"draft runner {runner}: {len(full_ids)} full-attention layer(s) "
f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}"
)
return UnifiedDraftKVPool(
) -> FusedDraftBinding:
"""Bind draft runner ``runner`` to its fused lanes: one binder per host
family that holds lanes for it, in registry order, each refining the
binding the previous one returned."""
ctx = BindContext(
placement=placement,
runner=runner,
unified_buffer=unified_buffer,
host_sub_pool_name="full",
host_allocator=host_allocator,
layer_lanes=dict(zip(full_ids, full_lanes)),
model=model,
model_config=model_config,
page_size=page_size,
)
families: List[str] = []
for host in placement.hosts():
family = HOST_KINDS[host].family
if family not in families and len(placement.lanes_for(runner, host)) > 0:
families.append(family)
assert families, f"draft runner {runner} holds no lane in any host"
binding = FusedDraftBinding(req_to_token_pool=req_to_token_pool)
for family in families:
assert family in DRAFT_BINDERS, (
f"no draft binder is registered for host family {family!r}"
)
binding = DRAFT_BINDERS[family](ctx, binding)
assert binding.token_to_kv_pool is not None, (
f"draft runner {runner}: no binder produced a KV pool"
)
return binding
def _bind_dense(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding:
"""This runner's attention layers, in layer order, over its lanes in the
dense hosts. The placement sized the lanes from the draft config; the
model's real layer ids fill them, so a count mismatch is a loud boot
failure, never a silent alias. Window layers follow the placement: they
bind the swa sub-pool when it holds a region for them, else the full
sub-pool, where a full lifetime covers any window."""
hosts = ctx.hosts_of("dense")
assert set(hosts) <= {"full", "swa"}, f"unknown dense host(s) in {hosts}"
kv_layer_ids = draft_kv_layer_ids(ctx.model)
full_lanes = ctx.placement.lanes_for(ctx.runner, "full")
swa_lanes = ctx.placement.lanes_for(ctx.runner, "swa")
if not swa_lanes:
full_ids, swa_ids = list(kv_layer_ids), []
elif not full_lanes:
full_ids, swa_ids = [], list(kv_layer_ids)
else:
# Both kinds in one runner: a replicated head, whose config lists every
# window layer (a per-depth runner's config is clipped to its block).
window = set(draft_swa_layer_ids(ctx.model_config))
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in window]
swa_ids = [layer_id for layer_id in kv_layer_ids if layer_id in window]
assert len(full_ids) == len(full_lanes) and len(swa_ids) == len(swa_lanes), (
f"draft runner {ctx.runner}: layers full={full_ids} swa={swa_ids} vs placed "
f"lanes full={list(full_lanes)} swa={list(swa_lanes)}"
)
if not swa_ids:
pool = UnifiedDraftKVPool(
unified_buffer=ctx.unified_buffer,
host_sub_pool_name="full",
host_allocator=ctx.host_allocator,
layer_lanes=dict(zip(full_ids, full_lanes)),
page_size=ctx.page_size,
)
else:
pool = UnifiedDraftSWAKVPool(
unified_buffer=ctx.unified_buffer,
host_allocator=ctx.host_allocator,
page_size=ctx.page_size,
full_layer_lanes=dict(zip(full_ids, full_lanes)),
swa_layer_lanes=dict(zip(swa_ids, swa_lanes)),
)
return msgspec.structs.replace(binding, token_to_kv_pool=pool)
register_draft_binder("dense", _bind_dense)
def _runner_state_layer_ids(ctx: BindContext) -> Tuple[int, ...]:
"""The recurrent-state layer ids this runner serves: its own depth for a
per-depth head, every state layer otherwise."""
mc = ctx.model_config
num_depths = mc.num_nextn_predict_layers
num_depths = 1 if num_depths is None else int(num_depths)
layer_ids = draft_state_layer_ids(mc, num_depths=num_depths)
if num_depths > 1:
return (ctx.runner,) if ctx.runner in layer_ids else ()
return layer_ids
def _bind_state(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding:
"""This runner's recurrent-state block: its request table is the target's
(same request->slot mappings, same slot allocator) with a `mamba_pool`
view keyed by the runner's own state layer ids."""
hosts = ctx.hosts_of("state")
assert hosts == ("mamba",), (
f"the state binder serves the 'mamba' host alone; got {hosts}"
)
assert isinstance(binding.req_to_token_pool, UnifiedHybridReqToTokenPool), (
"a fused draft state block needs the target's unified state pool"
)
layer_ids = _runner_state_layer_ids(ctx)
lanes = ctx.placement.lanes_for(ctx.runner, "mamba")
assert len(layer_ids) == len(lanes), (
f"draft runner {ctx.runner}: state layers {list(layer_ids)} vs placed "
f"lanes {list(lanes)}"
)
req_to_token_pool = binding.req_to_token_pool.clone_for_fused_draft(
layer_lanes=dict(zip(layer_ids, lanes))
)
return msgspec.structs.replace(binding, req_to_token_pool=req_to_token_pool)
register_draft_binder("state", _bind_state)
@@ -25,6 +25,7 @@ compaction only mutates those (no reference rewriting).
from __future__ import annotations
import copy
import logging
from abc import ABC, abstractmethod
from dataclasses import dataclass
@@ -40,6 +41,8 @@ from sglang.srt.environ import envs
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
DraftRegion,
DraftStateRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.layout.token_major import (
@@ -119,7 +122,9 @@ class SubPoolSpec(ABC):
layer_num: int
grow_direction: str # "up" | "down" | "float"
# Fused draft region riding in this sub-pool's entries; None = unfused.
draft_region: Optional[DenseDraftRegion] = None
# Each kind accepts the region kind its entry can lay out, after the host
# block at `draft_offset_in_entry()`.
draft_region: Optional[DraftRegion] = None
def __post_init__(self):
assert self.grow_direction in self._allowed_grow_directions, (
@@ -176,6 +181,9 @@ class MHASubPoolSpec(SubPoolSpec):
f"v_head_dim must be positive; got {self.v_head_dim}"
)
if self.draft_region is not None:
assert isinstance(self.draft_region, DenseDraftRegion), (
"dense entries carry dense draft K/V rows, never a state region"
)
self.draft_region.validate()
def k_row_bytes(self) -> int:
@@ -258,6 +266,9 @@ class MLASubPoolSpec(SubPoolSpec):
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
)
if self.draft_region is not None:
assert isinstance(self.draft_region, DenseDraftRegion), (
"dense entries carry dense draft K/V rows, never a state region"
)
self.draft_region.validate()
@property
@@ -305,7 +316,15 @@ class MLASubPoolSpec(SubPoolSpec):
@dataclass(frozen=True, kw_only=True)
class MambaSubPoolSpec(SubPoolSpec):
"""Per-slot layout of one Mamba-shaped sub-pool."""
"""Per-slot layout of one Mamba-shaped sub-pool.
With `draft_region` (a `DraftStateRegion`) set, the draft's state block
follows the host's inside every slot's entry:
[ conv[0] x L | ... | temporal x L | draft conv[0] x Ld | ... | pad ]
Unfused, the entry keeps its raw (unaligned) size, byte-identical to before.
"""
conv_state_shapes: Tuple[Tuple[int, ...], ...] # one shape per conv tensor
conv_dtype: torch.dtype
@@ -316,9 +335,11 @@ class MambaSubPoolSpec(SubPoolSpec):
def __post_init__(self):
super().__post_init__()
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
assert self.draft_region is None, (
"mamba state pages carry no fused draft region yet"
)
if self.draft_region is not None:
assert isinstance(self.draft_region, DraftStateRegion), (
"mamba state entries carry a draft STATE region, never dense K/V rows"
)
self.draft_region.validate()
def conv_row_bytes(self, idx: int) -> int:
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
@@ -326,13 +347,26 @@ class MambaSubPoolSpec(SubPoolSpec):
def temporal_row_bytes(self) -> int:
return _prod(self.temporal_state_shape) * self.temporal_dtype.itemsize
def entry_bytes(self) -> int:
def host_entry_bytes(self) -> int:
"""Host (target-only) bytes for one slot, before any draft block."""
total = 0
for i in range(len(self.conv_state_shapes)):
total += self.layer_num * self.conv_row_bytes(i)
total += self.layer_num * self.temporal_row_bytes()
return total
def draft_offset_in_entry(self) -> int:
"""Byte offset of the fused draft state block inside one slot's entry."""
assert self.draft_region is not None
return align_part_offset(self.host_entry_bytes())
def entry_bytes(self) -> int:
if self.draft_region is None:
return self.host_entry_bytes()
return align_entry_bytes(
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
)
def get_dtype(self) -> torch.dtype:
return self.conv_dtype # representative state dtype; matches MambaPool.dtype
@@ -432,11 +466,7 @@ class UnifiedKVPool:
self._bs1_floor_terms = bs1_floor_terms or []
self.fused_draft = fused_draft
for spec in sub_pool_specs:
expected = (
fused_draft.region
if fused_draft is not None and spec.name == "full"
else None
)
expected = None if fused_draft is None else fused_draft.region(spec.name)
assert spec.draft_region is expected, (
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
f"not match the fused draft placement's {expected}"
@@ -749,6 +779,30 @@ class UnifiedKVPool:
temporal_dtype=spec.temporal_dtype,
max_slots=max_slots,
anchor_bytes=anchor_bytes,
entry_bytes=spec.entry_bytes(),
)
def build_draft_state_views(
self, sub_pool_name: str
) -> Tuple[List[torch.Tensor], torch.Tensor]:
"""Per-layer conv/temporal views of the DRAFT state block fused into
``sub_pool_name``'s entries: same slots, same v2p table as the host."""
spec = self.mamba_spec(sub_pool_name)
region = spec.draft_region
assert isinstance(region, DraftStateRegion), (
f"sub-pool {sub_pool_name!r} carries no fused draft state region"
)
return build_mamba_entry_views(
self._raw,
layer_num=region.lane_num,
conv_state_shapes=region.state.conv_state_shapes,
conv_dtype=region.state.conv_dtype,
temporal_state_shape=region.state.temporal_state_shape,
temporal_dtype=region.state.temporal_dtype,
max_slots=self.max_slots(sub_pool_name),
anchor_bytes=self._anchor_bytes[sub_pool_name],
entry_bytes=spec.entry_bytes(),
offset_bytes=spec.draft_offset_in_entry(),
)
@@ -1126,8 +1180,29 @@ class UnifiedMambaPool(MambaPool):
self.num_mamba_layers,
)
# Inherited MambaPool state ops (copy_from/clear_slots/get_cpu_copy/load_cpu_copy)
# take PHYSICAL slot ids; callers translate via the slot allocator first.
# State ops take PHYSICAL slot ids and act on whole entries, so a fused
# draft state block is cleared, copied and moved with the host streams.
def _entry_view(self) -> torch.Tensor:
spec = self._unified_buffer.mamba_spec(self._sub_pool_name)
entry_bytes = spec.entry_bytes()
anchor = self._unified_buffer.anchor_bytes(self._sub_pool_name)
num_slots = self._max_size + 1
raw = self._unified_buffer._raw
entries = raw[anchor : anchor + num_slots * entry_bytes]
return entries.view(num_slots, entry_bytes)
def clear_slots(self, indices: torch.Tensor):
self._entry_view()[indices] = 0
for sibling in self._slot_siblings:
sibling.reset_slots(indices)
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
assert self.replayssm_write_pos is None
entries = self._entry_view()
entries[dst_indices] = entries[src_indices]
for sibling in self._slot_siblings:
sibling.copy_slots(src_indices, dst_indices)
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
# Compaction moves disjoint physical slots. Copy their whole envelopes
@@ -1143,6 +1218,8 @@ class UnifiedMambaPool(MambaPool):
self._unified_buffer.max_slots(self._sub_pool_name),
self._unified_buffer.mamba_spec(self._sub_pool_name).entry_bytes(),
)
if self.replayssm_write_pos is not None:
self.replayssm_write_pos[tgt_loc] = 0
for sibling in self._slot_siblings:
sibling.copy_slots(src_loc, tgt_loc)
@@ -1407,6 +1484,27 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
)
)
def clone_for_fused_draft(
self, *, layer_lanes: Dict[int, int]
) -> UnifiedHybridReqToTokenPool:
"""Shallow copy for a draft runner whose recurrent state is fused into
this pool's state entries: it shares the request->slot mappings and
the slot allocator, and its `mamba_pool` is a view of the draft block
keyed by the runner's own layer ids."""
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftMambaPool
clone = copy.copy(self)
clone.mamba_pool = UnifiedDraftMambaPool(
unified_buffer=self._unified_buffer,
sub_pool_name=self._mamba_sub_pool_name,
layer_lanes=layer_lanes,
)
clone.mamba_map = {
layer_id: idx for idx, layer_id in enumerate(sorted(layer_lanes))
}
clone.mamba_ckpt_pool = None
return clone
@property
def mamba_v2p_table(self) -> Optional[torch.Tensor]:
"""This pool's ids ARE virtual; page_size is 1, so the translate is the
@@ -1552,7 +1650,7 @@ def init_unified_mamba_pools(
qk_rope_head_dim=qk_rope_head_dim,
store_dtype=store_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
draft_region=None if fused_draft is None else fused_draft.region("full"),
)
else:
full_spec = MHASubPoolSpec(
@@ -1564,7 +1662,7 @@ def init_unified_mamba_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
draft_region=None if fused_draft is None else fused_draft.region("full"),
)
cp = mamba2_cache_params
mamba_spec = MambaSubPoolSpec(
@@ -1576,6 +1674,7 @@ def init_unified_mamba_pools(
temporal_dtype=cp.dtype.temporal,
conv_slice_axis=getattr(cp.shape, "conv_slice_axis", 0),
grow_direction="up",
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
)
if unified_total_bytes is not None:
# PROFILED byte budget for the token side (captured pre-ratio-floor);
@@ -2051,10 +2150,10 @@ def init_unified_swa_pools(
) -> UnifiedSWAPoolBundle:
"""Build the SWA-hybrid unified-memory-pool stack.
With ``fused_draft``, every entry of the "full" sub-pool carries the
draft model's K/V parts after the host parts, and each draft runner
binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a
pool of its own.
With ``fused_draft``, every entry of a sub-pool the placement names
carries the draft model's K/V parts after the host parts, and each draft
runner binds a `UnifiedDraftKVPool` over its own lanes instead of
allocating a pool of its own.
"""
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
@@ -2081,7 +2180,7 @@ def init_unified_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
draft_region=None if fused_draft is None else fused_draft.region("full"),
)
swa_spec = MHASubPoolSpec(
name="swa",
@@ -2092,6 +2191,7 @@ def init_unified_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="up",
draft_region=None if fused_draft is None else fused_draft.region("swa"),
)
legacy_allocator_capacities = {}
if total_bytes is None:
@@ -2251,8 +2351,9 @@ def init_unified_mamba_swa_pools(
Sizing inputs are the same token counts the 2-pool factories take (ratio-
fed until the byte configurator lands); the buffer budget is their byte
sum and the runtime split floats. With ``fused_draft``, every entry of the
"full" sub-pool carries the draft's parts, as in the 2-pool factories.
sum and the runtime split floats. With ``fused_draft``, every entry of a
sub-pool the placement names carries the draft's parts, as in the 2-pool
factories.
"""
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedMambaSWATokenToKVPoolAllocator,
@@ -2279,7 +2380,7 @@ def init_unified_mamba_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="down",
draft_region=None if fused_draft is None else fused_draft.region,
draft_region=None if fused_draft is None else fused_draft.region("full"),
)
swa_spec = MHASubPoolSpec(
name="swa",
@@ -2290,6 +2391,7 @@ def init_unified_mamba_swa_pools(
store_dtype=store_dtype,
kv_cache_dtype=kv_cache_dtype,
grow_direction="float",
draft_region=None if fused_draft is None else fused_draft.region("swa"),
)
cp = mamba2_cache_params
mamba_spec = MambaSubPoolSpec(
@@ -2300,6 +2402,7 @@ def init_unified_mamba_swa_pools(
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
temporal_dtype=cp.dtype.temporal,
grow_direction="up",
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
)
if unified_total_bytes is not None:
# PROFILED byte budget for the token side (captured pre-ratio-floor);
@@ -227,11 +227,12 @@ class UnifiedRadixCache(BasePrefixCache):
self.enable_mamba_extra_buffer = (
params.enable_mamba_extra_buffer if self.is_mamba_enabled else False
)
# SWA window size (None when SWA is not enabled).
# Required cache history, shared by retention, budgeting and transfers.
self._sliding_window_size = (
params.sliding_window_size if self.is_swa_enabled else None
params.swa_retain_window if self.is_swa_enabled else None
)
self.decoder_swa_bounded_replay = params.decoder_swa_bounded_replay
self.draft_swa_window = params.draft_swa_window
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
# evictable leaves) and drives the components' tree-level hooks.
self._tree_core_backend = select_tree_core_backend(params)
@@ -739,7 +739,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
self._draft_cell_size = _dflash_draft_cell_size(kvc)
self._fused_full_entry = kvc.fused_entry_bytes("full")
if self._fused_full_entry is not None:
self._fused_swa_entry = kvc.fused_entry_bytes("swa")
if self._fused_full_entry is not None or self._fused_swa_entry is not None:
self._draft_full_layers_num = 0
self._draft_swa_layers_num = 0
self._draft_swa_full_layers_num = 0
@@ -757,6 +758,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
)
def _swa_cell_bytes(self) -> int:
if self._fused_swa_entry is not None:
return self._fused_swa_entry
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
def _recompute_cell_size(self) -> None:
+10
View File
@@ -1841,6 +1841,16 @@ class InklingForConditionalGenerationMTP(nn.Module):
hidden_states_before_norm=hidden_states_before_norm,
)
def get_attention_sliding_window_size(self) -> Optional[int]:
"""The trunk's convention (`InklingForConditionalGeneration`): the
backend window is one less than the extent, so the head reads back
the same span the trunk's window layers do. A banded depth runs at
its own extent, every other depth at the trunk's window."""
text_config = self.text_config
if self.draft_model_idx in text_config.mtp_local_layer_ids:
return text_config.mtp_local_extent - 1
return text_config.sliding_window_size - 1
def get_embed_and_head(self):
return self.model.embed_tokens.weight, self.lm_head.weight
@@ -158,6 +158,12 @@ class BaseSpecWorker(ABC):
def hicache_draft_plan(self) -> HiCacheDraftPlan:
return self._hicache_draft_plan
@property
def draft_swa_window(self) -> int:
"""Tokens back from the committed length that the draft reads in the
target's swa sub-pool; 0 when the target's own window covers them."""
return 0
def weight_update_runners(self) -> list[tuple[str, ModelRunner]]:
"""(role, runner) pairs weight ops apply to; [] when the draft owns no weights."""
# NGRAM and UNO have no draft model
@@ -177,6 +177,10 @@ class EagleDraftInput(SpecInput):
# shape: (b + 1,)
kv_indptr: torch.Tensor = None
kv_indices: torch.Tensor = None
# The window-clipped counterpart of the pair above, for the draft's
# sliding-window layers; None on a pool with no window side.
window_kv_indptr: torch.Tensor = None
window_kv_indices: torch.Tensor = None
num_tokens_per_req: int = -1
num_tokens_for_logprob_per_req: int = -1
@@ -34,6 +34,7 @@ from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftSWAKVPool
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
@@ -587,14 +588,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
return
seen = set()
for runner in self.draft_runner_list:
pool = runner.req_to_token_pool.mamba_pool
req_pool = runner.req_to_token_pool
pool = req_pool.mamba_pool
if id(pool) in seen:
continue
seen.add(id(pool))
# The pool is a PHYSICAL store; a unified req pool hands out
# virtual slot ids (identity on a static pool).
if clear is not None and len(clear) > 0:
pool.clear_slots(clear)
pool.clear_slots(req_pool.translate_mamba_indices(clear))
if cow_src is not None and len(cow_src) > 0:
pool.copy_from(cow_src, cow_dst)
pool.copy_from(
req_pool.translate_mamba_indices(cow_src),
req_pool.translate_mamba_indices(cow_dst),
)
forward_batch.mamba_clear_indices = None
forward_batch.mamba_cow_src_indices = None
forward_batch.mamba_cow_dst_indices = None
@@ -1092,6 +1099,20 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
),
)
@property
def draft_swa_window(self) -> int:
# The widened extend starts front rows below the committed length, and
# each depth reads its own window back from there.
draft = self._draft_worker
windows = [
runner.sliding_window_size
for runner in draft.draft_runner_list
if isinstance(runner.token_to_kv_pool, UnifiedDraftSWAKVPool)
]
if not windows:
return 0
return max(windows) + draft.draft_extend_num_front_tokens
def forward_batch_generation(
self,
batch: ScheduleBatch,
@@ -411,6 +411,11 @@ class SpecInput(ABC):
future_dsa_topk_indices_available: bool = False
dsa_seed_topk_capture: Optional[torch.Tensor] = None
# A multi-step draft's per-step sliding-window read rail; only the
# multi-step draft backend of a sliding-window KV pool builds it.
window_kv_indptr: Optional[torch.Tensor] = None
window_kv_indices: Optional[torch.Tensor] = None
def __init__(self, spec_input_type: SpecInputType):
self.spec_input_type = spec_input_type
@@ -23,6 +23,9 @@ from sglang.kernels.ops.speculative.cache_locs import (
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_kv_indices as generate_draft_decode_kv_indices,
)
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_window_kv_indices as generate_draft_decode_window_kv_indices,
)
from sglang.kernels.ops.speculative.cache_locs import (
get_src_tgt_cache_loc as get_src_tgt_cache_loc,
)