[Feat] unified-memory: Fuse drafts with sliding-window and recurrent-state layers (#43326)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
19cb24f1e1
commit
3edb156dec
@@ -559,7 +559,9 @@ def _fwd_kernel(
|
||||
mask=(mask_m[:, None] & mask_n[None, :]),
|
||||
other=0,
|
||||
)
|
||||
final_mask &= custom_mask
|
||||
# The mask buffer is int8; narrow it to i1, since a score_mod hands
|
||||
# final_mask to tl.load, which rejects an i8 mask.
|
||||
final_mask &= custom_mask != 0
|
||||
if SLIDING_WINDOW_SIZE > 0:
|
||||
# Add mask where q_id <= kv_id + sliding_window_size
|
||||
# q_id = prefix_len + cur_m, kv_id = cur_n
|
||||
@@ -749,7 +751,8 @@ def _fwd_kernel(
|
||||
mask=(mask_m[:, None] & mask_n[None, :]),
|
||||
other=0,
|
||||
)
|
||||
custom_mask &= mask_m[:, None] & mask_n[None, :]
|
||||
# int8 -> i1 before the &=; see the prefix loop above.
|
||||
custom_mask = (custom_mask != 0) & mask_m[:, None] & mask_n[None, :]
|
||||
final_mask &= custom_mask
|
||||
elif IS_CAUSAL:
|
||||
mask_causual = (cur_block_m * BLOCK_M + offs_m[:, None]) >= (
|
||||
@@ -1349,7 +1352,8 @@ def _fwd_kernel_unified(
|
||||
mask=(mask_m[:, None] & mask_n[None, :]),
|
||||
other=0,
|
||||
)
|
||||
final_mask &= custom_mask
|
||||
# int8 -> i1 before the &=; see `_fwd_kernel`'s prefix loop.
|
||||
final_mask &= custom_mask != 0
|
||||
|
||||
# Apply causal mask for extend part
|
||||
if IS_CAUSAL and not USE_CUSTOM_MASK:
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.kernels.spec import (
|
||||
_TRITON_KERNELS = [
|
||||
("cache_locs", "assign_extend_cache_locs_func"),
|
||||
("cache_locs", "generate_draft_decode_kv_indices"),
|
||||
("cache_locs", "generate_draft_decode_window_kv_indices"),
|
||||
("eagle", "fill_bonus_tokens"),
|
||||
("eagle", "fill_accept_out_cache_loc"),
|
||||
("gather_spec_extras", "gather_spec_extras"),
|
||||
|
||||
@@ -265,6 +265,124 @@ def generate_draft_decode_kv_indices(
|
||||
tl.store(kv_indptr + zid, base + zid * iters)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def generate_draft_decode_window_kv_indices(
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
paged_kernel_lens,
|
||||
window_kv_indices,
|
||||
window_kv_indptr,
|
||||
positions,
|
||||
# Runtime, as in generate_draft_decode_kv_indices.
|
||||
row_stride,
|
||||
kv_indices_stride: tl.constexpr,
|
||||
kv_indptr_stride: tl.constexpr,
|
||||
bs_upper: tl.constexpr,
|
||||
iter_upper: tl.constexpr,
|
||||
num_tokens_upper: tl.constexpr,
|
||||
page_size: tl.constexpr,
|
||||
window: tl.constexpr,
|
||||
ENTRY_PAGE_SIZE: tl.constexpr = 1,
|
||||
# The source's sliding-window page table (`KVIndexTable.v2p`).
|
||||
v2p=None,
|
||||
TRANSLATE: tl.constexpr = False,
|
||||
):
|
||||
"""Per-step sliding-window read rail of the multi-step draft.
|
||||
|
||||
Step ``iters`` of branch ``(bid, topk_id)`` attends ``seq_len + iters``
|
||||
tokens: the committed prefix plus the draft's own chain in the slots
|
||||
`generate_draft_decode_kv_indices` emits. A window layer reads the last
|
||||
``min(seq_len + iters, window)`` of them, so this emits that tail only,
|
||||
prefix ids first and chain ids after, from the sliding-window read source.
|
||||
"""
|
||||
BLOCK_SIZE: tl.constexpr = 128
|
||||
iters = tl.program_id(axis=0)
|
||||
bid = tl.program_id(axis=1)
|
||||
topk_id = tl.program_id(axis=2)
|
||||
|
||||
num_steps = tl.num_programs(axis=0)
|
||||
num_seqs = tl.num_programs(axis=1)
|
||||
topk = tl.num_programs(axis=2)
|
||||
|
||||
window_kv_indices += kv_indices_stride * iters
|
||||
window_kv_indptr += kv_indptr_stride * iters
|
||||
iters += 1
|
||||
|
||||
seq_len = tl.load(paged_kernel_lens + bid)
|
||||
total = seq_len + iters
|
||||
window_len = tl.minimum(total, window)
|
||||
# Chain position of the first token in the window; the prefix part covers
|
||||
# [start, seq_len), the chain part [max(start - seq_len, 0), iters).
|
||||
start = total - window_len
|
||||
|
||||
# This branch's output offset: the window lengths of the branches before
|
||||
# it, each clipped like this one.
|
||||
zid = bid * topk + topk_id
|
||||
bs_offset = tl.arange(0, num_tokens_upper)
|
||||
in_front = bs_offset < zid
|
||||
prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0)
|
||||
prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0)
|
||||
base = tl.sum(prev_window)
|
||||
out_ptr = window_kv_indices + base
|
||||
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
|
||||
|
||||
prefix_start = tl.minimum(start, seq_len)
|
||||
prefix_len = seq_len - prefix_start
|
||||
kv_offset = tl.arange(0, BLOCK_SIZE)
|
||||
num_loop = tl.cdiv(prefix_len, BLOCK_SIZE)
|
||||
for _ in range(num_loop):
|
||||
mask = kv_offset < prefix_len
|
||||
data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
prefix_start + kv_offset,
|
||||
mask,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
tl.store(out_ptr + kv_offset, data, mask=mask)
|
||||
kv_offset += BLOCK_SIZE
|
||||
|
||||
chain_start = tl.maximum(start - seq_len, 0)
|
||||
extend_offset = tl.arange(0, iter_upper)
|
||||
chain_step = chain_start + extend_offset
|
||||
chain_mask = chain_step < iters
|
||||
# The same chain slots `generate_draft_decode_kv_indices` emits.
|
||||
if page_size == 1 or topk == 1:
|
||||
chain_pos = seq_len + topk_id * num_steps + chain_step
|
||||
else:
|
||||
last_page_len = seq_len % page_size
|
||||
num_new_pages_per_topk = (
|
||||
last_page_len + num_steps + page_size - 1
|
||||
) // page_size
|
||||
prefix_base = seq_len // page_size * page_size
|
||||
chain_base = (
|
||||
prefix_base + topk_id * num_new_pages_per_topk * page_size + last_page_len
|
||||
)
|
||||
chain_pos = chain_base + chain_step
|
||||
extend_data = _load_token_ids(
|
||||
token_pool_ptr,
|
||||
chain_pos,
|
||||
chain_mask,
|
||||
v2p,
|
||||
page_size,
|
||||
ENTRY_PAGE_SIZE,
|
||||
TRANSLATE,
|
||||
)
|
||||
tl.store(out_ptr + prefix_len + extend_offset, extend_data, mask=chain_mask)
|
||||
|
||||
# Update window_kv_indptr: program 0 writes the total at [num_seqs * topk],
|
||||
# every other program its own start; [0] stays the buffer's zero.
|
||||
if zid == 0:
|
||||
zid = num_seqs * topk
|
||||
in_front = bs_offset < zid
|
||||
prev_lens = tl.load(positions + bs_offset, mask=in_front, other=0)
|
||||
prev_window = tl.where(in_front, tl.minimum(prev_lens + iters, window), 0)
|
||||
base = tl.sum(prev_window)
|
||||
tl.store(window_kv_indptr + zid, base)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def align_evict_mask_to_page_size(
|
||||
seq_lens,
|
||||
|
||||
@@ -607,7 +607,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
assert _mc.is_hybrid_swa or mambaish_config(_mc) is not None, (
|
||||
"--enable-unified-memory + EAGLE/EAGLE3 requires a unified "
|
||||
"target (hybrid-SWA or a mamba hybrid): the draft's KV lives "
|
||||
"fused inside the full-attention page envelope."
|
||||
"fused inside the target's sub-pool entries."
|
||||
)
|
||||
# The target verifies on its own pages: the MLA family on an MLA host.
|
||||
eagle_allowed = (
|
||||
|
||||
@@ -998,6 +998,12 @@ class ModelConfig:
|
||||
and self.hf_config.architectures[0] == "InklingForConditionalGeneration"
|
||||
):
|
||||
self.hf_config.architectures[0] = "InklingForConditionalGenerationMTP"
|
||||
if self.hf_text_config.mtp_local_layer_ids:
|
||||
# A local MTP depth keeps the trunk's window unless it declares
|
||||
# its own: the draft backend builds its window rail only if set.
|
||||
self.sliding_window_size = (
|
||||
self.hf_text_config.mtp_local_extent or self.sliding_window_size
|
||||
)
|
||||
if (
|
||||
is_draft_model
|
||||
and self.hf_config.architectures[0] == "GigaChat35ForCausalLM"
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
|
||||
@@ -41,10 +42,12 @@ from sglang.srt.model_executor.cuda_graph_config import (
|
||||
)
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel, get_schedule, get_spec
|
||||
from sglang.srt.speculative.eagle_utils import per_step_draft_out_cache_loc
|
||||
from sglang.srt.speculative.spec_utils import (
|
||||
draft_kv_indices_buffer_width,
|
||||
draft_kv_indices_used_len,
|
||||
generate_draft_decode_kv_indices,
|
||||
generate_draft_decode_window_kv_indices,
|
||||
resolve_draft_decode_window,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
@@ -719,6 +722,22 @@ class TritonAttnBackend(AttentionBackend):
|
||||
kv_indptr = self._fill_kv_indptr_and_indices(
|
||||
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
|
||||
)
|
||||
if self.sliding_window_size is not None and self.sliding_window_size > 0:
|
||||
# A window layer reads the window-clipped prefix, as the eager
|
||||
# draft-extend path builds it.
|
||||
_, _, _, self.cuda_graph_window_kv_offsets[:bs] = (
|
||||
update_sliding_window_buffer(
|
||||
self.window_kv_indptr,
|
||||
self.kv_index_translator,
|
||||
plan,
|
||||
req_pool_indices,
|
||||
self.sliding_window_size,
|
||||
kv_lens,
|
||||
bs,
|
||||
token_to_kv_pool=self.token_to_kv_pool,
|
||||
window_kv_indices=self.cuda_graph_window_kv_indices,
|
||||
)
|
||||
)
|
||||
return qo_indptr, kv_indptr, num_tokens_per_req
|
||||
|
||||
def init_forward_metadata_out_graph(
|
||||
@@ -744,6 +763,19 @@ class TritonAttnBackend(AttentionBackend):
|
||||
# Multi-step spec decode: kv buffers come from spec_info, not the
|
||||
# cuda-graph pool, so replay is not involved.
|
||||
if forward_mode.is_decode_or_idle() and spec_info is not None:
|
||||
# This step's swa write ids, in the buffer the multi-step
|
||||
# backend refills from the step's columns before replay.
|
||||
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
|
||||
forward_batch, in_capture=True
|
||||
)
|
||||
assert not (
|
||||
self.use_sliding_window_kv_pool and swa_out_cache_loc is None
|
||||
), (
|
||||
"multi-step spec decode on a sliding-window KV pool could "
|
||||
"not derive swa_out_cache_loc; the window layers' writes "
|
||||
"have no target. The per-step out_cache_loc must fit "
|
||||
"cuda_graph_swa_out_cache_loc."
|
||||
)
|
||||
self.forward_metadata = ForwardMetadata(
|
||||
attn_logits=self.cuda_graph_attn_logits,
|
||||
attn_lse=self.cuda_graph_attn_lse,
|
||||
@@ -754,11 +786,18 @@ class TritonAttnBackend(AttentionBackend):
|
||||
qo_indptr=None,
|
||||
custom_mask=None,
|
||||
mask_indptr=None,
|
||||
window_kv_indptr=self.window_kv_indptr,
|
||||
window_kv_indices=None,
|
||||
# The per-step window rail lives in the multi-step
|
||||
# backend's persistent buffers, refilled before replay.
|
||||
window_kv_indptr=(
|
||||
spec_info.window_kv_indptr
|
||||
if spec_info.window_kv_indices is not None
|
||||
else self.window_kv_indptr
|
||||
),
|
||||
window_kv_indices=spec_info.window_kv_indices,
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
swa_attn_logits=self.cuda_graph_swa_attn_logits,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
lean_Mp=self.cuda_graph_lean_Mp,
|
||||
lean_Lp=self.cuda_graph_lean_Lp,
|
||||
lean_Op=self.cuda_graph_lean_Op,
|
||||
@@ -800,8 +839,9 @@ class TritonAttnBackend(AttentionBackend):
|
||||
self, forward_batch: ForwardBatch, in_capture: bool = False
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Refill the SWA write-target buffer with the batch's write ids in the
|
||||
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
|
||||
multi-step draft) so the captured store reads fresh slots on replay.
|
||||
sliding-window sub-pool, returning the [:n] view (None for non-SWA) so
|
||||
the captured store reads fresh slots on replay. A multi-step draft step
|
||||
hands in a batch bound to its own columns of the window.
|
||||
"""
|
||||
if not self.use_sliding_window_kv_pool:
|
||||
return None
|
||||
@@ -892,6 +932,18 @@ class TritonAttnBackend(AttentionBackend):
|
||||
else:
|
||||
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
|
||||
bs = kv_indptr.shape[0] - 1
|
||||
if spec_info.window_kv_indices is not None:
|
||||
# The multi-step draft backend built the per-step window
|
||||
# rail (the draft's own chain, window-clipped).
|
||||
window_kv_indptr = spec_info.window_kv_indptr
|
||||
window_kv_indices = spec_info.window_kv_indices
|
||||
window_num_kv_splits = torch.empty(
|
||||
(bs,), dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.get_num_kv_splits(
|
||||
window_num_kv_splits,
|
||||
window_kv_indptr[1 : bs + 1] - window_kv_indptr[:bs],
|
||||
)
|
||||
|
||||
attn_logits = torch.empty(
|
||||
(bs, self.num_head, self.max_kv_splits, self.v_head_dim),
|
||||
@@ -1317,10 +1369,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
qo_indptr=self.qo_indptr[: bs + 1],
|
||||
custom_mask=None,
|
||||
mask_indptr=None,
|
||||
window_kv_indptr=self.window_kv_indptr,
|
||||
window_kv_indices=None,
|
||||
window_kv_indptr=(
|
||||
self.window_kv_indptr[: bs + 1] if swa else self.window_kv_indptr
|
||||
),
|
||||
window_kv_indices=self.cuda_graph_window_kv_indices if swa else None,
|
||||
# Extend reads through qo_indptr, never through kv splits.
|
||||
window_num_kv_splits=None,
|
||||
window_kv_offsets=None,
|
||||
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
|
||||
swa_out_cache_loc=swa_out_cache_loc,
|
||||
)
|
||||
else:
|
||||
@@ -1750,6 +1805,13 @@ class TritonAttnBackend(AttentionBackend):
|
||||
kv_indptr = self.forward_metadata.window_kv_indptr
|
||||
kv_indices = self.forward_metadata.window_kv_indices
|
||||
window_kv_offsets = self.forward_metadata.window_kv_offsets
|
||||
# A null read rail reaches Triton as `None + offsets` and fails as
|
||||
# an unreadable IR-build error; say what is actually missing.
|
||||
assert kv_indices is not None, (
|
||||
f"layer {layer.layer_id} is a sliding-window layer but this "
|
||||
"extend forward built no window_kv_indices; every mode that "
|
||||
"reaches forward_extend must fill the window rail."
|
||||
)
|
||||
else:
|
||||
sliding_window_size = -1
|
||||
kv_indptr = self.forward_metadata.kv_indptr
|
||||
@@ -2234,6 +2296,18 @@ class TritonAttnBackend(AttentionBackend):
|
||||
logits_soft_cap = logit_capping_mod(layer.logit_capping_method, layer.logit_cap)
|
||||
|
||||
if save_kv_cache:
|
||||
swa_loc = self.forward_metadata.swa_out_cache_loc
|
||||
if (
|
||||
self.is_draft_runner
|
||||
and forward_batch.spec_info is not None
|
||||
and swa_loc is not None
|
||||
):
|
||||
# KVWriteLoc keeps a longer swa_loc's prefix, but a draft step's
|
||||
# slice of the window is not a prefix of it.
|
||||
assert swa_loc.shape[0] == forward_batch.out_cache_loc.shape[0], (
|
||||
f"draft step swa_out_cache_loc has {swa_loc.shape[0]} "
|
||||
f"entries for {forward_batch.out_cache_loc.shape[0]} tokens"
|
||||
)
|
||||
if self.use_mla:
|
||||
if layer.k_scale is not None:
|
||||
# MLATokenToKVPool doesn't accept scale parameters; k is unused
|
||||
@@ -2265,6 +2339,14 @@ class TritonAttnBackend(AttentionBackend):
|
||||
if layer.sliding_window_size is not None and layer.sliding_window_size > -1:
|
||||
kv_indptr = self.forward_metadata.window_kv_indptr
|
||||
kv_indices = self.forward_metadata.window_kv_indices
|
||||
# A null read rail reaches Triton as `None + offsets` and fails as
|
||||
# an unreadable IR-build error; say what is actually missing.
|
||||
assert kv_indices is not None, (
|
||||
f"layer {layer.layer_id} is a sliding-window layer but this "
|
||||
"forward built no window_kv_indices; a multi-step draft gets the "
|
||||
"per-step window rail whenever its model has a window "
|
||||
"(TritonMultiStepDraftBackend.window_rail)."
|
||||
)
|
||||
else:
|
||||
kv_indptr = self.forward_metadata.kv_indptr
|
||||
kv_indices = self.forward_metadata.kv_indices
|
||||
@@ -2451,15 +2533,35 @@ class TritonMultiStepDraftBackend:
|
||||
model_runner
|
||||
)
|
||||
self.kv_index_translator = model_runner.kv_index_translator
|
||||
self.token_to_kv_pool = model_runner.token_to_kv_pool
|
||||
# Each draft step's window layers read a window-clipped rail of their
|
||||
# own: the draft's chain is not part of the committed prefix yet.
|
||||
self.sliding_window_size = model_runner.sliding_window_size
|
||||
self.window_rail = (
|
||||
self.sliding_window_size is not None and self.sliding_window_size > 0
|
||||
)
|
||||
self.window_kv_indptr = (
|
||||
torch.zeros(
|
||||
(self.speculative_num_steps, max_bs + 1),
|
||||
dtype=torch.int32,
|
||||
device=model_runner.device,
|
||||
)
|
||||
if self.window_rail
|
||||
else None
|
||||
)
|
||||
self.cuda_graph_window_kv_indices = None
|
||||
|
||||
def common_template(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
kv_indices_buffer: Optional[torch.Tensor],
|
||||
call_fn: int,
|
||||
window_kv_indices_buffer: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if kv_indices_buffer is None:
|
||||
kv_indices_buffer = self.cuda_graph_kv_indices
|
||||
if window_kv_indices_buffer is None:
|
||||
window_kv_indices_buffer = self.cuda_graph_window_kv_indices
|
||||
|
||||
num_seqs = forward_batch.batch_size
|
||||
bs = self.topk * num_seqs
|
||||
@@ -2496,6 +2598,8 @@ class TritonMultiStepDraftBackend:
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
if self.window_rail:
|
||||
self._fill_window_rail(forward_batch, window_kv_indices_buffer, bs)
|
||||
|
||||
if call_fn is None:
|
||||
return
|
||||
@@ -2505,8 +2609,95 @@ class TritonMultiStepDraftBackend:
|
||||
forward_batch.spec_info.kv_indices = kv_indices_buffer[i][
|
||||
: draft_kv_indices_used_len(seq_lens_sum, self.topk, bs, i + 1)
|
||||
]
|
||||
if self.window_rail:
|
||||
forward_batch.spec_info.window_kv_indptr = self.window_kv_indptr[
|
||||
i, : bs + 1
|
||||
]
|
||||
forward_batch.spec_info.window_kv_indices = window_kv_indices_buffer[i][
|
||||
: bs * self.sliding_window_size
|
||||
]
|
||||
call_fn(i, forward_batch)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _step_writes(self, forward_batch, step: int):
|
||||
"""Bind draft step ``step``'s slice of the batch's write ids, as
|
||||
`EagleWorkerV2.draft_forward` slices them, and its columns of the
|
||||
iteration's write window, while a per-step backend derives its write
|
||||
targets (the swa ids included) from the batch."""
|
||||
loc, cols = forward_batch.out_cache_loc, forward_batch.kv_loc_cols
|
||||
if loc is None or not self.attn_backends[step].use_sliding_window_kv_pool:
|
||||
yield
|
||||
return
|
||||
num_steps = self.speculative_num_steps
|
||||
if cols is None:
|
||||
# The draft's own window, each row laid out (topk, num_steps).
|
||||
step_cols = slice(step, None, num_steps)
|
||||
else:
|
||||
# A chain draft writes the leading columns of the verify window.
|
||||
assert self.topk == 1 and isinstance(cols, slice) and cols.step is None
|
||||
step_cols = slice(cols.start + step, cols.start + step + 1)
|
||||
forward_batch.out_cache_loc = per_step_draft_out_cache_loc(
|
||||
loc, forward_batch.batch_size, self.topk, num_steps
|
||||
)[step]
|
||||
forward_batch.kv_loc_cols = step_cols
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
forward_batch.out_cache_loc, forward_batch.kv_loc_cols = loc, cols
|
||||
|
||||
def _fill_window_rail(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
window_kv_indices_buffer: torch.Tensor,
|
||||
bs: int,
|
||||
) -> None:
|
||||
"""Emit every step's window-clipped read ids into the per-step rows
|
||||
of ``window_kv_indices_buffer`` and their indptr into
|
||||
``self.window_kv_indptr``. The reader selects FULL or SWA storage;
|
||||
the layer's window determines which positions to gather."""
|
||||
num_seqs = forward_batch.batch_size
|
||||
width = bs * self.sliding_window_size
|
||||
assert window_kv_indices_buffer.shape[1] >= width, (
|
||||
f"window rail row {window_kv_indices_buffer.shape[1]} < "
|
||||
f"{bs} branch(es) x window {self.sliding_window_size}"
|
||||
)
|
||||
src = self.kv_index_translator.read_source(
|
||||
forward_batch.kv_loc_plan,
|
||||
req_pool_indices=forward_batch.req_pool_indices,
|
||||
bs=num_seqs,
|
||||
kind=self.kv_index_translator.window_kv_kind,
|
||||
)
|
||||
generate_draft_decode_window_kv_indices[
|
||||
(self.speculative_num_steps, num_seqs, self.topk)
|
||||
](
|
||||
src.row_ids,
|
||||
src.ids,
|
||||
forward_batch.seq_lens,
|
||||
window_kv_indices_buffer,
|
||||
self.window_kv_indptr,
|
||||
forward_batch.positions,
|
||||
src.row_stride,
|
||||
window_kv_indices_buffer.shape[1],
|
||||
self.window_kv_indptr.shape[1],
|
||||
next_power_of_2(num_seqs),
|
||||
next_power_of_2(self.speculative_num_steps),
|
||||
next_power_of_2(bs),
|
||||
self.page_size,
|
||||
self.sliding_window_size,
|
||||
ENTRY_PAGE_SIZE=src.entry_page_size,
|
||||
v2p=src.v2p,
|
||||
TRANSLATE=src.v2p is not None,
|
||||
)
|
||||
if (
|
||||
not src.is_translated
|
||||
and self.kv_index_translator.window_kv_kind == IdSpaceKind.SLIDING_WINDOW
|
||||
):
|
||||
# Static pool: the legacy full->swa translate, as
|
||||
# update_sliding_window_buffer applies it to the target's window.
|
||||
for i in range(self.speculative_num_steps):
|
||||
row = window_kv_indices_buffer[i][:width]
|
||||
row.copy_(self.token_to_kv_pool.translate_loc_from_full_to_swa(row))
|
||||
|
||||
def init_forward_metadata(self, forward_batch: ForwardBatch):
|
||||
kv_indices_width = draft_kv_indices_buffer_width(
|
||||
forward_batch.batch_size, self.topk, self.max_context_len
|
||||
@@ -2516,6 +2707,17 @@ class TritonMultiStepDraftBackend:
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
window_kv_indices = None
|
||||
if self.window_rail:
|
||||
# zeros, not empty: a static pool translates the whole row.
|
||||
window_kv_indices = torch.zeros(
|
||||
(
|
||||
self.speculative_num_steps,
|
||||
forward_batch.batch_size * self.topk * self.sliding_window_size,
|
||||
),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def call_fn(i, forward_batch):
|
||||
forward_batch.spec_info.kv_indptr = (
|
||||
@@ -2524,9 +2726,22 @@ class TritonMultiStepDraftBackend:
|
||||
forward_batch.spec_info.kv_indices = (
|
||||
forward_batch.spec_info.kv_indices.clone()
|
||||
)
|
||||
self.attn_backends[i].init_forward_metadata(forward_batch)
|
||||
if self.window_rail:
|
||||
forward_batch.spec_info.window_kv_indptr = (
|
||||
forward_batch.spec_info.window_kv_indptr.clone()
|
||||
)
|
||||
forward_batch.spec_info.window_kv_indices = (
|
||||
forward_batch.spec_info.window_kv_indices.clone()
|
||||
)
|
||||
with self._step_writes(forward_batch, i):
|
||||
self.attn_backends[i].init_forward_metadata(forward_batch)
|
||||
|
||||
self.common_template(forward_batch, kv_indices, call_fn)
|
||||
self.common_template(
|
||||
forward_batch,
|
||||
kv_indices,
|
||||
call_fn,
|
||||
window_kv_indices_buffer=window_kv_indices,
|
||||
)
|
||||
|
||||
def init_cuda_graph_state(self, max_bs: int, max_num_tokens: int):
|
||||
kv_indices_width = draft_kv_indices_buffer_width(
|
||||
@@ -2543,6 +2758,15 @@ class TritonMultiStepDraftBackend:
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
if self.window_rail:
|
||||
self.cuda_graph_window_kv_indices = torch.zeros(
|
||||
(
|
||||
self.speculative_num_steps,
|
||||
max_bs * self.topk * self.sliding_window_size,
|
||||
),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
self.attn_backends[i].init_cuda_graph_state(
|
||||
@@ -2567,9 +2791,10 @@ class TritonMultiStepDraftBackend:
|
||||
)
|
||||
|
||||
def call_fn(i, _forward_batch):
|
||||
self.attn_backends[i].init_forward_metadata_out_graph(
|
||||
inner_fb, in_capture=True
|
||||
)
|
||||
with self._step_writes(inner_fb, i):
|
||||
self.attn_backends[i].init_forward_metadata_out_graph(
|
||||
inner_fb, in_capture=True
|
||||
)
|
||||
|
||||
self.common_template(forward_batch, None, call_fn)
|
||||
else:
|
||||
@@ -2587,6 +2812,14 @@ class TritonMultiStepDraftBackend:
|
||||
self.attn_backends[-1].cuda_graph_num_kv_splits[:num_token],
|
||||
forward_batch.seq_lens[:bs],
|
||||
)
|
||||
if self.attn_backends[0].use_sliding_window_kv_pool:
|
||||
# Each captured step writes its window layers through its own
|
||||
# backend's swa buffer: refill it from that step's columns.
|
||||
for i in range(self.speculative_num_steps - 1):
|
||||
with self._step_writes(forward_batch, i):
|
||||
self.attn_backends[i]._fill_cuda_graph_swa_out_cache_loc(
|
||||
forward_batch
|
||||
)
|
||||
|
||||
def init_forward_metadata_in_graph(self, forward_batch: ForwardBatch) -> None:
|
||||
for attn_backend in self.attn_backends:
|
||||
@@ -2640,12 +2873,7 @@ def update_sliding_window_buffer(
|
||||
indptr=window_kv_indptr,
|
||||
out=window_kv_indices,
|
||||
kv_start_idx=window_kv_start_idx,
|
||||
kind=(
|
||||
IdSpaceKind.SLIDING_WINDOW
|
||||
if translator.reads_are_translated
|
||||
and isinstance(token_to_kv_pool, BaseSWAKVPool)
|
||||
else IdSpaceKind.FULL
|
||||
),
|
||||
kind=translator.window_kv_kind,
|
||||
token_mapping=token_mapping,
|
||||
)
|
||||
if not translated and isinstance(token_to_kv_pool, BaseSWAKVPool):
|
||||
|
||||
@@ -3976,7 +3976,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
def maybe_evict_swa(self):
|
||||
if self.tree_cache.supports_swa():
|
||||
sliding_window_size = self.tree_cache.sliding_window_size
|
||||
retain_window = self.tree_cache.swa_retain_window
|
||||
# Auxiliary windows check their own cursors and prefix locks.
|
||||
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
|
||||
|
||||
@@ -3985,9 +3985,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
)
|
||||
|
||||
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
|
||||
swa_evict_due = self._swa_eviction_trigger(
|
||||
sliding_window_size, eviction_interval
|
||||
)
|
||||
swa_evict_due = self._swa_eviction_trigger(retain_window, eviction_interval)
|
||||
self.token_to_kv_pool_allocator.free_group_begin()
|
||||
for idx, req in enumerate(self.reqs):
|
||||
if self.forward_mode.is_decode():
|
||||
@@ -4009,12 +4007,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
),
|
||||
)
|
||||
|
||||
# Past the window the request no longer reads its prefill's SWA;
|
||||
# release that part of the tree lock so SWA LRU can reclaim it.
|
||||
if (
|
||||
release_leaf_lock
|
||||
and req.decode_batch_idx >= sliding_window_size
|
||||
):
|
||||
# Past the retain window the request no longer reads its prefill's
|
||||
# SWA; release that part of the tree lock so SWA LRU can reclaim it.
|
||||
if release_leaf_lock and req.decode_batch_idx >= retain_window:
|
||||
self.tree_cache.release_swa_prefix_lock(req)
|
||||
elif (
|
||||
self.forward_mode.is_extend()
|
||||
|
||||
@@ -590,6 +590,11 @@ class Scheduler(
|
||||
if self.draft_worker is not None
|
||||
else None
|
||||
),
|
||||
draft_swa_window=(
|
||||
self.draft_worker.draft_swa_window
|
||||
if self.draft_worker is not None
|
||||
else 0
|
||||
),
|
||||
)
|
||||
self.is_hybrid_swa = result.is_hybrid_swa
|
||||
self.is_hybrid_ssm = result.is_hybrid_ssm
|
||||
|
||||
@@ -391,6 +391,9 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
storage_prefetch_retries: Optional[StoragePrefetchRetries] = None
|
||||
# Set by caches that publish KV placement events; None means they don't.
|
||||
kv_events: Optional[KVCacheEventRecorder] = None
|
||||
# Tokens back from the committed length a fused draft reads in the swa
|
||||
# sub-pool (CacheInitParams.draft_swa_window); 0 when it reads none.
|
||||
draft_swa_window: int = 0
|
||||
|
||||
def init_metrics_collector(self):
|
||||
from sglang.srt.layers.dp_attention import is_dp_attention_enabled
|
||||
@@ -663,6 +666,14 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def supports_auxiliary_swa(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def swa_retain_window(self) -> Optional[int]:
|
||||
# What the frees and unlocks of a running request's SWA keep behind it;
|
||||
# every other SWA rule uses sliding_window_size.
|
||||
if self.sliding_window_size is None:
|
||||
return None
|
||||
return max(self.sliding_window_size, self.draft_swa_window)
|
||||
|
||||
def evict_sliding_windows(
|
||||
self, req: Req, pre_len: int, *, eviction_interval: int = 1
|
||||
) -> None:
|
||||
@@ -672,7 +683,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
free_swa_out_of_window_slots(
|
||||
req,
|
||||
pre_len,
|
||||
sliding_window_size=self.sliding_window_size,
|
||||
sliding_window_size=self.swa_retain_window,
|
||||
page_size=self.page_size,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
|
||||
@@ -58,6 +58,9 @@ class CacheInitParams:
|
||||
# paged SWA pool only a draft uses), so it must not gate prefix matching.
|
||||
swa_gates_prefix_match: bool = True
|
||||
decoder_swa_bounded_replay: bool = False
|
||||
# Tokens back from the committed length that a fused draft reads in the
|
||||
# swa sub-pool (BaseSpecWorker.draft_swa_window); 0 when it reads none.
|
||||
draft_swa_window: int = 0
|
||||
|
||||
# Time-to-live for cache entries in seconds. If None, TTL is disabled.
|
||||
cache_ttl_seconds: Optional[float] = None
|
||||
@@ -70,3 +73,12 @@ class CacheInitParams:
|
||||
|
||||
mtp_draft_device_pools: tuple[object, ...] = ()
|
||||
mtp_draft_mamba_pools: tuple[object, ...] = ()
|
||||
|
||||
@property
|
||||
def swa_retain_window(self) -> Optional[int]:
|
||||
# Cache history required by every reader of the SWA sub-pool. Retention,
|
||||
# tree locks, prefix matching and host transfers must use the same span.
|
||||
# The model's attention window remains sliding_window_size.
|
||||
if self.sliding_window_size is None:
|
||||
return None
|
||||
return max(self.sliding_window_size, self.draft_swa_window)
|
||||
|
||||
@@ -300,6 +300,7 @@ def build_kv_cache(
|
||||
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
|
||||
dynamo_kv_event_format: bool = False,
|
||||
kv_event_lora_names: Optional[KvEventLoraNames] = None,
|
||||
draft_swa_window: int = 0,
|
||||
) -> KVCacheBuildResult:
|
||||
parallel = get_parallel()
|
||||
sliding_window_size: Optional[int] = None
|
||||
@@ -324,6 +325,9 @@ def build_kv_cache(
|
||||
full_tokens_per_layer, swa_tokens_per_layer = (
|
||||
tp_worker.get_tokens_per_layer_info()
|
||||
)
|
||||
assert draft_swa_window == 0 or sliding_window_size is not None, (
|
||||
"a fused draft reads an swa sub-pool this target does not build"
|
||||
)
|
||||
|
||||
req_to_token_pool, token_to_kv_pool_allocator = tp_worker.get_memory_pool()
|
||||
mtp_draft_device_pools = tp_worker.model_runner.mtp_draft_device_pools
|
||||
@@ -398,9 +402,17 @@ def build_kv_cache(
|
||||
chunked_prefill_size=effective_chunked_prefill_size,
|
||||
sliding_window_size=sliding_window_size,
|
||||
swa_gates_prefix_match=swa_gates_prefix_match,
|
||||
draft_swa_window=draft_swa_window,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
|
||||
)
|
||||
if params.swa_retain_window != sliding_window_size:
|
||||
logger.info(
|
||||
"SWA eviction keeps %d tokens back: the fused draft reads past the "
|
||||
"%d-token window.",
|
||||
params.swa_retain_window,
|
||||
sliding_window_size,
|
||||
)
|
||||
|
||||
tree_context = TreeCacheBuildContext(
|
||||
server_args=server_args,
|
||||
|
||||
@@ -638,12 +638,8 @@ class KVCacheConfigurator:
|
||||
alloc = token_to_kv_pool_allocator
|
||||
placement = self._fused_draft_from_target_buffer(alloc)
|
||||
if placement is not None:
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
draft_swa_layer_ids,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_draft_pool import (
|
||||
bind_fused_draft,
|
||||
draft_kv_layer_ids,
|
||||
draft_state_layer_classes,
|
||||
)
|
||||
|
||||
@@ -651,27 +647,29 @@ class KVCacheConfigurator:
|
||||
req_to_token_pool = self._build_req_to_token_pool(
|
||||
max_num_reqs=sizes.max_running_requests
|
||||
)
|
||||
runner = self.draft_model_idx or 0
|
||||
# The target placed this draft from its config; the built
|
||||
# model is the ground truth for whether it carries state.
|
||||
state_layers = draft_state_layer_classes(self.model)
|
||||
if state_layers:
|
||||
if state_layers and not placement.lanes_for(runner, "mamba"):
|
||||
raise ValueError(
|
||||
"Fused draft KV: the draft model has recurrent / "
|
||||
f"linear-attention layers ({', '.join(state_layers)}) "
|
||||
"that the fused region gives no state pool."
|
||||
"that the fused placement gives no state lanes."
|
||||
)
|
||||
draft_pool = bind_fused_draft(
|
||||
binding = bind_fused_draft(
|
||||
placement=placement,
|
||||
runner=runner,
|
||||
unified_buffer=alloc.unified_buffer,
|
||||
host_allocator=alloc,
|
||||
placement=placement,
|
||||
runner=self.draft_model_idx or 0,
|
||||
kv_layer_ids=draft_kv_layer_ids(self.model),
|
||||
swa_layer_ids=draft_swa_layer_ids(self.model_config),
|
||||
model=self.model,
|
||||
model_config=self.model_config,
|
||||
req_to_token_pool=req_to_token_pool,
|
||||
page_size=self.page_size,
|
||||
)
|
||||
return _InitializedPools(
|
||||
req_to_token_pool=req_to_token_pool,
|
||||
token_to_kv_pool=draft_pool,
|
||||
req_to_token_pool=binding.req_to_token_pool,
|
||||
token_to_kv_pool=binding.token_to_kv_pool,
|
||||
token_to_kv_pool_allocator=alloc,
|
||||
unified_memory_pool=None,
|
||||
)
|
||||
@@ -823,14 +821,20 @@ class KVCacheConfigurator:
|
||||
# The region holds rows in the target's KV dtype; a draft that resolved
|
||||
# its own would read and write them as something else. Compare the KV
|
||||
# dtypes, not their storage: every fp8 flavor is stored as uint8.
|
||||
region_kv_dtype = placement.region.resolved_kv_dtype()
|
||||
if self.kv_cache_dtype != region_kv_dtype:
|
||||
raise ValueError(
|
||||
f"Fused draft KV: the draft resolved its KV cache dtype to "
|
||||
f"{self.kv_cache_dtype}, but its region inside the target's pages "
|
||||
f"holds {region_kv_dtype}. Set "
|
||||
"--speculative-draft-kv-cache-dtype to the target's KV cache dtype."
|
||||
)
|
||||
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
|
||||
|
||||
for host in placement.hosts():
|
||||
if HOST_KINDS[host].family != "dense":
|
||||
continue
|
||||
region_kv_dtype = placement.region(host).resolved_kv_dtype()
|
||||
if self.kv_cache_dtype != region_kv_dtype:
|
||||
raise ValueError(
|
||||
f"Fused draft KV: the draft resolved its KV cache dtype to "
|
||||
f"{self.kv_cache_dtype}, but its region inside the target's "
|
||||
f"pages holds {region_kv_dtype}. Set "
|
||||
"--speculative-draft-kv-cache-dtype to the target's KV cache "
|
||||
"dtype."
|
||||
)
|
||||
return placement
|
||||
|
||||
def _init_unified_mamba_pools(
|
||||
@@ -1038,6 +1042,15 @@ class KVCacheConfigurator:
|
||||
swa_v_head_dim=mc.swa_v_head_dim,
|
||||
)
|
||||
|
||||
def _unified_host_names(self) -> tuple:
|
||||
"""The sub-pools the unified factory for this model builds, by name."""
|
||||
names = ["full"]
|
||||
if self.is_hybrid_swa:
|
||||
names.append("swa")
|
||||
if self.mambaish_config is not None:
|
||||
names.append("mamba")
|
||||
return tuple(names)
|
||||
|
||||
def _fused_draft_decision(self):
|
||||
"""Whether, and where, the draft's layers fuse into the target's
|
||||
sub-pools. An empty decision means fusion does not apply (unified
|
||||
@@ -1048,6 +1061,7 @@ class KVCacheConfigurator:
|
||||
to the kernel."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
FusedDraftDecision,
|
||||
PlacementContext,
|
||||
draft_kv_profile,
|
||||
place_fused_draft,
|
||||
)
|
||||
@@ -1166,11 +1180,24 @@ class KVCacheConfigurator:
|
||||
return place_fused_draft(
|
||||
profile=profile,
|
||||
num_runners=num_runners,
|
||||
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
|
||||
ctx=PlacementContext(
|
||||
host_names=self._unified_host_names(),
|
||||
target_window=self.model_config.sliding_window_size,
|
||||
asymmetric_rows_ok=self._draft_backends_carry_v_head_dim(),
|
||||
draft_backends=self._draft_attention_backends(),
|
||||
),
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
kv_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
|
||||
def _draft_attention_backends(self) -> tuple:
|
||||
"""The backends the draft worker may run on: its explicit one, else
|
||||
the target's, which it inherits."""
|
||||
explicit = get_spec().speculative_draft_attention_backend
|
||||
if explicit is not None:
|
||||
return (explicit,)
|
||||
return tuple(sorted(b for b in attention_backends() if b is not None))
|
||||
|
||||
def _draft_backends_carry_v_head_dim(self) -> bool:
|
||||
"""Whether every resolved attention backend, the draft's included,
|
||||
carries v_head_dim through to the kernel (ASYMMETRIC_KV_BACKENDS). The
|
||||
@@ -1187,10 +1214,69 @@ class KVCacheConfigurator:
|
||||
pad), built from the same spec as the pool factory's so the priced and
|
||||
allocated entries agree; None when the draft does not fuse into that
|
||||
sub-pool."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
|
||||
|
||||
placement = self._fused_draft_decision().placement
|
||||
if placement is None or sub_pool_name != "full":
|
||||
if placement is None:
|
||||
return None
|
||||
return self._full_host_spec(placement.region).entry_bytes()
|
||||
region = placement.region(sub_pool_name)
|
||||
if region is None:
|
||||
return None
|
||||
builders = {
|
||||
"full": self._full_host_spec,
|
||||
"swa": self._swa_host_spec,
|
||||
"mamba": self._mamba_host_spec,
|
||||
}
|
||||
assert sub_pool_name in HOST_KINDS, (
|
||||
f"sub-pool {sub_pool_name!r} is not a registered host kind"
|
||||
)
|
||||
assert sub_pool_name in builders, (
|
||||
f"no fused host spec builder for sub-pool {sub_pool_name!r}"
|
||||
)
|
||||
return builders[sub_pool_name](region).entry_bytes()
|
||||
|
||||
def _mamba_host_spec(self, region):
|
||||
from sglang.srt.mem_cache.unified_memory_pool import MambaSubPoolSpec
|
||||
|
||||
assert self.mambaish_config is not None, (
|
||||
"only a mamba-ish host builds a mamba sub-pool"
|
||||
)
|
||||
cp = self.mambaish_config.mamba2_cache_params
|
||||
# This runner's OWN state layers, as the mamba factory slices them.
|
||||
layer_ids = [
|
||||
i
|
||||
for i in cp.layers
|
||||
if self.layer_info.start_layer <= i < self.layer_info.end_layer
|
||||
]
|
||||
return MambaSubPoolSpec(
|
||||
name="mamba",
|
||||
layer_num=len(layer_ids),
|
||||
conv_state_shapes=tuple(tuple(int(x) for x in s) for s in cp.shape.conv),
|
||||
conv_dtype=cp.dtype.conv,
|
||||
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
|
||||
temporal_dtype=cp.dtype.temporal,
|
||||
grow_direction="up",
|
||||
draft_region=region,
|
||||
)
|
||||
|
||||
def _swa_host_spec(self, region):
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
MHASubPoolSpec,
|
||||
_store_dtype_for,
|
||||
)
|
||||
|
||||
assert self.is_hybrid_swa, "only a hybrid-SWA host builds an swa sub-pool"
|
||||
geometry = self._unified_swa_head_geometry()
|
||||
return MHASubPoolSpec(
|
||||
name="swa",
|
||||
layer_num=len(self.layer_info.swa_attention_layer_ids),
|
||||
head_num=geometry.swa_head_num,
|
||||
head_dim=geometry.swa_head_dim,
|
||||
v_head_dim=geometry.swa_v_head_dim,
|
||||
store_dtype=_store_dtype_for(self.kv_cache_dtype),
|
||||
grow_direction="up",
|
||||
draft_region=region,
|
||||
)
|
||||
|
||||
def _full_host_spec(self, region):
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
@@ -1292,31 +1378,28 @@ class KVCacheConfigurator:
|
||||
def _fused_draft_for_pool_factory(self, decision):
|
||||
"""The placement a pool factory is handed. It logs the decision it
|
||||
returns, so the boot log shows whether fusion engaged or why not."""
|
||||
from sglang.srt.mem_cache.layout.fused_draft import HOST_KINDS
|
||||
|
||||
if decision.placement is None:
|
||||
if decision.declined is not None:
|
||||
logger.warning("fused draft KV disabled: %s", decision.declined)
|
||||
return None
|
||||
placement = decision.placement
|
||||
region = placement.region
|
||||
kv_dtype = region.resolved_kv_dtype()
|
||||
logger.info(
|
||||
"[unified-memory-pool] fused draft region in 'full': %d lane(s) x %d "
|
||||
"kv head(s) x %d/%d k/v head_dim @ %s = %d B/token; runner lanes %s",
|
||||
region.lane_num,
|
||||
region.head_num,
|
||||
region.head_dim,
|
||||
region.resolved_v_head_dim(),
|
||||
(
|
||||
kv_dtype
|
||||
if kv_dtype == region.store_dtype
|
||||
else f"{kv_dtype} (stored as {region.store_dtype})"
|
||||
),
|
||||
region.entry_bytes(),
|
||||
[
|
||||
tuple(placement.lanes_for(r))
|
||||
for r in range(len(placement.runner_lane_counts))
|
||||
],
|
||||
)
|
||||
if decision.note is not None:
|
||||
logger.info(
|
||||
"[unified-memory-pool] fused draft placement: %s", decision.note
|
||||
)
|
||||
for host in placement.hosts():
|
||||
logger.info(
|
||||
"[unified-memory-pool] %s",
|
||||
HOST_KINDS[host].describe(
|
||||
region=placement.region(host),
|
||||
lanes=[
|
||||
tuple(placement.lanes_for(r, host))
|
||||
for r in range(len(placement.runners))
|
||||
],
|
||||
),
|
||||
)
|
||||
return placement
|
||||
|
||||
def _init_unified_swa_pools(
|
||||
@@ -2956,9 +3039,15 @@ class KVCacheConfigurator:
|
||||
else:
|
||||
max_stage_mamba_layers = len(all_mamba_layers)
|
||||
pp_layer_scale = max_stage_mamba_layers / max(len(all_mamba_layers), 1)
|
||||
stage_per_req = int(
|
||||
config.mamba2_cache_params.mamba_cache_per_req * pp_layer_scale
|
||||
# A fused draft state block widens every state slot: price the entry
|
||||
# the unified factory carves out (host + draft + pad), not the host's.
|
||||
fused_state_entry = self.fused_entry_bytes("mamba")
|
||||
per_req = (
|
||||
config.mamba2_cache_params.mamba_cache_per_req
|
||||
if fused_state_entry is None
|
||||
else fused_state_entry
|
||||
)
|
||||
stage_per_req = int(per_req * pp_layer_scale)
|
||||
|
||||
has_spec_dec = not self.spec_algorithm.is_none()
|
||||
# ReplaySSM drops the per-step intermediate_ssm scratch, so its mamba budget
|
||||
|
||||
@@ -226,6 +226,19 @@ class KVIndexTranslator:
|
||||
None when its pool has no such sub-pool."""
|
||||
return self._spaces.get(kind)
|
||||
|
||||
@property
|
||||
def window_kv_kind(self) -> IdSpaceKind:
|
||||
"""Storage used by window attention, independent of its attention mask.
|
||||
|
||||
A fused window draft may live in FULL when SWA cannot host its window.
|
||||
Such a draft still clips its reads, but uses FULL physical addresses.
|
||||
"""
|
||||
return (
|
||||
IdSpaceKind.SLIDING_WINDOW
|
||||
if IdSpaceKind.SLIDING_WINDOW in self._spaces
|
||||
else IdSpaceKind.FULL
|
||||
)
|
||||
|
||||
def write_ids(
|
||||
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
|
||||
) -> Optional[torch.Tensor]:
|
||||
@@ -406,7 +419,7 @@ class KVIndexTranslator:
|
||||
reads stay virtual (a static pool, or DCP)."""
|
||||
if not self._reads_translated(kind):
|
||||
return self._passthrough_table(req_pool_indices)
|
||||
if plan.has_read_table(kind):
|
||||
if plan.has_read_table(kind, reader=self):
|
||||
return self._plan_table(plan, kind=kind, rows=bs)
|
||||
return KVIndexTable(
|
||||
ids=self.req_to_token,
|
||||
@@ -444,7 +457,7 @@ class KVIndexTranslator:
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
)
|
||||
return plan.read_table(kind=kind, rows=rows, into=into)
|
||||
return plan.read_table(kind=kind, rows=rows, into=into, reader=self)
|
||||
|
||||
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
|
||||
return KVIndexTable(
|
||||
@@ -474,7 +487,7 @@ class KVIndexTranslator:
|
||||
(a static pool, or DCP) for the caller to finish; a static SWA pool can
|
||||
fuse its full->swa table into the gather as ``token_mapping``."""
|
||||
bs = int(seq_lens.numel())
|
||||
if self._reads_translated(kind) and not plan.has_read_table(kind):
|
||||
if self._reads_translated(kind) and not plan.has_read_table(kind, reader=self):
|
||||
assert plan.is_read_by(self), (
|
||||
"a translating reader must read through the plan of its own "
|
||||
"req_to_token rows"
|
||||
@@ -534,7 +547,7 @@ class KVIndexTranslator:
|
||||
"DCP, where the caller selects this rank's share itself)"
|
||||
)
|
||||
rows = out.shape[0]
|
||||
first = not plan.has_read_table(kind)
|
||||
first = not plan.has_read_table(kind, reader=self)
|
||||
table = self._plan_table(
|
||||
plan, kind=kind, rows=rows, into=out if first else None
|
||||
)
|
||||
|
||||
@@ -293,9 +293,14 @@ class KVLocPlan:
|
||||
|
||||
# -- reads -----------------------------------------------------------------
|
||||
|
||||
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
|
||||
def has_read_table(
|
||||
self,
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
*,
|
||||
reader: Optional[KVIndexTranslator] = None,
|
||||
) -> bool:
|
||||
"""Whether a reader has had the ``kind`` table built yet."""
|
||||
return self._source.space(kind).key in self._read_tables
|
||||
return self._read_space(kind, reader).key in self._read_tables
|
||||
|
||||
def read_table(
|
||||
self,
|
||||
@@ -303,14 +308,17 @@ class KVLocPlan:
|
||||
kind: IdSpaceKind = IdSpaceKind.FULL,
|
||||
rows: Optional[int] = None,
|
||||
into: Optional[torch.Tensor] = None,
|
||||
reader: Optional[KVIndexTranslator] = None,
|
||||
) -> KVIndexTable:
|
||||
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
|
||||
when a table reader first asks (in place in ``into``, that reader's
|
||||
capture-stable table, when given) and shared by every later reader.
|
||||
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
|
||||
sink, appended by copying, never by translating again. Where reads stay
|
||||
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
|
||||
space = self._source.space(kind)
|
||||
virtual (a static pool, or DCP) it is the `req_to_token` passthrough.
|
||||
``reader`` names the sub-pool, as in `write_ids`; the plan's own
|
||||
runner by default."""
|
||||
space = self._read_space(kind, reader)
|
||||
table = self._read_tables.get(space.key)
|
||||
if table is None or (
|
||||
table.is_translated and rows is not None and table.ids.shape[0] < rows
|
||||
@@ -329,6 +337,13 @@ class KVLocPlan:
|
||||
)
|
||||
return table
|
||||
|
||||
def _read_space(
|
||||
self, kind: IdSpaceKind, reader: Optional[KVIndexTranslator]
|
||||
) -> IdSpace:
|
||||
# A later depth of a multi-layer draft can route window layers that
|
||||
# the depth whose runner built the plan does not.
|
||||
return (self._source if reader is None else reader).space(kind)
|
||||
|
||||
def is_read_by(self, reader: KVIndexTranslator) -> bool:
|
||||
"""Whether `reader` reads this plan's tables: it indexes the plan's
|
||||
physical ids and gathers the same `req_to_token` rows. A reader with a
|
||||
|
||||
@@ -2,35 +2,53 @@
|
||||
|
||||
The nesting, outermost first:
|
||||
|
||||
sub-pool a named pool inside the unified pool, with its own grow direction
|
||||
and frontier. HOST is the role it plays when its entries also
|
||||
carry draft bytes; only "full" hosts a draft here.
|
||||
sub-pool a named pool inside the unified pool ("full", "swa", "mamba"),
|
||||
with its own grow direction and frontier. HOST is the role it
|
||||
plays when its entries also carry draft bytes.
|
||||
slot one token entry of that sub-pool. The draft's KV is extra parts of
|
||||
every slot, after the host's own parts, so ONE slot id per token
|
||||
covers target and draft: one allocation, one free, one whole-page
|
||||
relocation carry both.
|
||||
region the draft geometry inside the host's entries. A region is not a
|
||||
region the draft geometry inside one host's entries. A region is not a
|
||||
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
|
||||
lane one position in the region's layer dimension -- one draft layer of
|
||||
one runner, present in every slot.
|
||||
range the contiguous lanes one runner owns in one host.
|
||||
|
||||
A lane is not a layer. Runners are the draft's execution copies: a
|
||||
replicated head gives EVERY runner all the draft's layers, on lanes of its own
|
||||
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
|
||||
while a per-depth head (one block per predicted position) gives each runner
|
||||
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
|
||||
contiguous run of lanes, and the runs tile the region in runner order.
|
||||
ONE depth (8 layers across 2 runners = 2 lanes). The runners' ranges tile
|
||||
each region in runner order, with no gap and no overlap.
|
||||
|
||||
A draft whose layers need another sub-pool kind declines fusion as a whole.
|
||||
A draft LAYER kind (`LAYER_*`) is what the draft checkpoint has; a HOST kind
|
||||
(`HostKind`, keyed by sub-pool name) is what a sub-pool can carry in its
|
||||
entries. `place_fused_draft` consults the `HOST_KINDS` registry, and the draft
|
||||
worker binds each host through its family's binder
|
||||
(`unified_draft_pool.DRAFT_BINDERS`). When a kind's host turns it away, its
|
||||
layers FOLD into that host's `fallback_host` if the one-region-one-geometry
|
||||
rule allows.
|
||||
"""
|
||||
|
||||
from typing import List, Optional, Tuple
|
||||
import math
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
|
||||
|
||||
LAYER_FULL = "full"
|
||||
LAYER_WINDOW = "window"
|
||||
LAYER_STATE = "state"
|
||||
|
||||
_LAYER_KIND_NOUN = {
|
||||
LAYER_FULL: "full-attention",
|
||||
LAYER_WINDOW: "sliding-window",
|
||||
LAYER_STATE: "recurrent-state",
|
||||
}
|
||||
|
||||
|
||||
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
|
||||
@@ -94,28 +112,6 @@ class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
)
|
||||
|
||||
|
||||
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Where every draft runner's layers live inside the "full" sub-pool: the
|
||||
region, and each runner's lane count in runner order. Built once on the
|
||||
target, stored on the `UnifiedKVPool`, and read back by each draft
|
||||
runner, so the two sides cannot disagree on a lane.
|
||||
"""
|
||||
|
||||
region: DenseDraftRegion
|
||||
runner_lane_counts: Tuple[int, ...]
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.runner_lane_counts, "a placement needs at least one draft runner"
|
||||
assert sum(self.runner_lane_counts) == self.region.lane_num, (
|
||||
f"runner lane counts {self.runner_lane_counts} must tile "
|
||||
f"range({self.region.lane_num})"
|
||||
)
|
||||
|
||||
def lanes_for(self, runner: int) -> range:
|
||||
start = sum(self.runner_lane_counts[:runner])
|
||||
return range(start, start + self.runner_lane_counts[runner])
|
||||
|
||||
|
||||
class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-GPU K/V row geometry of one kind of draft attention layer."""
|
||||
|
||||
@@ -124,37 +120,345 @@ class DraftKVGeometry(msgspec.Struct, frozen=True, kw_only=True):
|
||||
v_head_dim: int
|
||||
|
||||
|
||||
class DraftStateGeometry(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-GPU shapes of one recurrent-state layer of the draft: one conv
|
||||
tensor per stream plus the temporal state, as `MambaPool.State` lays them
|
||||
out. Every stream is carried, even ones a depth never touches, because
|
||||
the backend indexes ``conv[stream]`` by absolute stream number."""
|
||||
|
||||
conv_state_shapes: Tuple[Tuple[int, ...], ...]
|
||||
conv_dtype: torch.dtype
|
||||
temporal_state_shape: Tuple[int, ...]
|
||||
temporal_dtype: torch.dtype
|
||||
|
||||
def conv_row_bytes(self, idx: int) -> int:
|
||||
return math.prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
|
||||
|
||||
def temporal_row_bytes(self) -> int:
|
||||
return math.prod(self.temporal_state_shape) * self.temporal_dtype.itemsize
|
||||
|
||||
def layer_bytes(self) -> int:
|
||||
conv = sum(self.conv_row_bytes(i) for i in range(len(self.conv_state_shapes)))
|
||||
return conv + self.temporal_row_bytes()
|
||||
|
||||
|
||||
class DraftStateRegion(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""The draft's recurrent state fused into every slot of a host state
|
||||
sub-pool: one block per fused (runner, layer), laid out stream-major like
|
||||
the host's own block, after it. The host's whole-entry clear and copy then
|
||||
carry the draft's state with the target's."""
|
||||
|
||||
lane_num: int
|
||||
state: DraftStateGeometry
|
||||
|
||||
def validate(self) -> None:
|
||||
assert self.lane_num > 0, f"lane_num must be positive; got {self.lane_num}"
|
||||
assert len(self.state.conv_state_shapes) > 0, (
|
||||
"conv_state_shapes must be non-empty"
|
||||
)
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
"""Draft state bytes per slot, before the host entry's alignment."""
|
||||
return self.lane_num * self.state.layer_bytes()
|
||||
|
||||
|
||||
DraftRegion = Union[DenseDraftRegion, DraftStateRegion]
|
||||
|
||||
|
||||
class DraftLayerSet(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""The draft's layers of one kind: layer ids for a replicated head, depth
|
||||
ids for a per-depth head, with the per-GPU geometry they share."""
|
||||
|
||||
layer_ids: Tuple[int, ...]
|
||||
geometry: Any
|
||||
# LAYER_WINDOW only: how many tokens back the layers read.
|
||||
window: Optional[int] = None
|
||||
|
||||
|
||||
class DraftKVProfile(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""What the draft checkpoint asks of the host sub-pools.
|
||||
"""What the draft checkpoint asks of the host sub-pools, by layer kind.
|
||||
|
||||
``num_depths`` > 1 marks a per-depth head (one transformer block per MTP
|
||||
depth, served by one runner each under multi-layer EAGLE); otherwise
|
||||
every runner serves all ``num_layers`` layers.
|
||||
depth, served by one runner each under multi-layer EAGLE), whose layer
|
||||
ids are depth ids; otherwise every runner serves all ``num_layers``
|
||||
layers. A kind is present only when the draft has layers of it.
|
||||
"""
|
||||
|
||||
num_layers: int
|
||||
full: DraftKVGeometry
|
||||
swa_layer_ids: Tuple[int, ...] = ()
|
||||
num_depths: int = 1
|
||||
num_state_layers: int = 0
|
||||
kinds: Dict[str, DraftLayerSet] = {}
|
||||
|
||||
|
||||
def draft_state_layer_num(draft_model_config) -> int:
|
||||
"""Recurrent-state layers the draft HEAD owns -- not the trunk's.
|
||||
class PlacementContext(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Target-side facts a host kind consults when placing a layer set:
|
||||
the sub-pools the host builds, the target's window, whether every
|
||||
resolved backend carries v_head_dim, and the backends the draft worker
|
||||
may run on (its explicit one, else the target's)."""
|
||||
|
||||
A plain NEXTN head of a linear-attention trunk inherits the trunk's
|
||||
config class, so `mamba2_cache_params` lists the TRUNK's state layers
|
||||
while the head itself is a full-attention block. Only a conv-chain MTP
|
||||
head declares `mtp_local_layer_ids`, so that is what separates them.
|
||||
host_names: Tuple[str, ...]
|
||||
target_window: Optional[int] = None
|
||||
asymmetric_rows_ok: bool = False
|
||||
draft_backends: Tuple[str, ...] = ()
|
||||
|
||||
|
||||
class HostKind(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""A unified sub-pool that carries one kind of draft layer in its entries.
|
||||
|
||||
``serves`` is the layer kind this host is the first choice for and
|
||||
``family`` picks the draft worker's binder. ``fallback_host`` carries
|
||||
``serves`` when this host cannot, under the one-region-one-geometry fold
|
||||
rule.
|
||||
"""
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
|
||||
mambaish = mambaish_config(draft_model_config)
|
||||
if mambaish is None:
|
||||
return 0
|
||||
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
|
||||
return 0
|
||||
return len(mambaish.mamba2_cache_params.layers)
|
||||
name: str
|
||||
serves: str
|
||||
family: str
|
||||
fallback_host: Optional[str] = None
|
||||
|
||||
def prerequisite(
|
||||
self, *, layers: DraftLayerSet, ctx: PlacementContext
|
||||
) -> Optional[str]:
|
||||
"""What the runtime must carry for ``layers`` to be served fused at
|
||||
all, whichever host takes them; a reason here declines the draft."""
|
||||
return None
|
||||
|
||||
def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]:
|
||||
"""None when this host's entries can carry ``layers``, else why not."""
|
||||
if self.name not in ctx.host_names:
|
||||
return f"the host has no {self.name!r} sub-pool"
|
||||
return None
|
||||
|
||||
def region_for(
|
||||
self,
|
||||
*,
|
||||
layers: DraftLayerSet,
|
||||
lane_num: int,
|
||||
store_dtype: torch.dtype,
|
||||
kv_dtype: Optional[torch.dtype],
|
||||
) -> DraftRegion:
|
||||
raise NotImplementedError
|
||||
|
||||
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
|
||||
"""The boot-log line for ``region`` and each runner's lanes in it."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class DenseHostKind(HostKind):
|
||||
"""A host whose entries take the draft's K/V rows as two more dense parts."""
|
||||
|
||||
def region_for(
|
||||
self,
|
||||
*,
|
||||
layers: DraftLayerSet,
|
||||
lane_num: int,
|
||||
store_dtype: torch.dtype,
|
||||
kv_dtype: Optional[torch.dtype],
|
||||
) -> DenseDraftRegion:
|
||||
geometry = layers.geometry
|
||||
assert isinstance(geometry, DraftKVGeometry), geometry
|
||||
return DenseDraftRegion(
|
||||
lane_num=lane_num,
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
store_dtype=store_dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
)
|
||||
|
||||
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
|
||||
kv_dtype = region.resolved_kv_dtype()
|
||||
stored = (
|
||||
kv_dtype
|
||||
if kv_dtype == region.store_dtype
|
||||
else f"{kv_dtype} (stored as {region.store_dtype})"
|
||||
)
|
||||
return (
|
||||
f"fused draft region in {self.name!r}: {region.lane_num} lane(s) x "
|
||||
f"{region.head_num} kv head(s) x {region.head_dim}/"
|
||||
f"{region.resolved_v_head_dim()} k/v head_dim @ {stored} "
|
||||
f"= {region.entry_bytes()} B/token; runner lanes {list(lanes)}"
|
||||
)
|
||||
|
||||
|
||||
# Registration order is the order placements report their hosts in.
|
||||
HOST_KINDS: Dict[str, HostKind] = {}
|
||||
|
||||
|
||||
def host_for(layer_kind: str) -> Optional[HostKind]:
|
||||
"""The registered host that serves ``layer_kind`` first, if any."""
|
||||
for kind in HOST_KINDS.values():
|
||||
if kind.serves == layer_kind:
|
||||
return kind
|
||||
return None
|
||||
|
||||
|
||||
def register_host_kind(kind: HostKind) -> HostKind:
|
||||
assert kind.name not in HOST_KINDS, f"host kind {kind.name!r} is already registered"
|
||||
primary = host_for(kind.serves)
|
||||
assert primary is None, (
|
||||
f"layer kind {kind.serves!r} is already served by host {primary.name!r}"
|
||||
)
|
||||
assert kind.fallback_host is None or kind.fallback_host in HOST_KINDS, (
|
||||
f"host kind {kind.name!r} falls back to unregistered host "
|
||||
f"{kind.fallback_host!r}"
|
||||
)
|
||||
HOST_KINDS[kind.name] = kind
|
||||
return kind
|
||||
|
||||
|
||||
FULL_HOST = register_host_kind(
|
||||
DenseHostKind(name="full", serves=LAYER_FULL, family="dense")
|
||||
)
|
||||
|
||||
# Multi-step draft backends that build the per-step sliding-window read and
|
||||
# write rails a fused window layer needs (TritonMultiStepDraftBackend).
|
||||
WINDOW_RAIL_BACKENDS = frozenset({"triton"})
|
||||
|
||||
|
||||
class WindowHostKind(DenseHostKind):
|
||||
"""The swa sub-pool: a draft window layer rides there when its window fits
|
||||
the target's; eviction keeps any further reach the draft worker declares."""
|
||||
|
||||
def prerequisite(
|
||||
self, *, layers: DraftLayerSet, ctx: PlacementContext
|
||||
) -> Optional[str]:
|
||||
unrailed = sorted(set(ctx.draft_backends) - WINDOW_RAIL_BACKENDS)
|
||||
if unrailed:
|
||||
return (
|
||||
f"the draft's attention backend {unrailed} carries no per-step "
|
||||
f"sliding-window rail (only {sorted(WINDOW_RAIL_BACKENDS)} does)"
|
||||
)
|
||||
return None
|
||||
|
||||
def admits(self, *, layers: DraftLayerSet, ctx: PlacementContext) -> Optional[str]:
|
||||
reason = super().admits(layers=layers, ctx=ctx)
|
||||
if reason is not None:
|
||||
return reason
|
||||
if layers.window is None:
|
||||
return "the draft declares no sliding window size"
|
||||
if ctx.target_window is None:
|
||||
return "the target declares no sliding window size"
|
||||
if layers.window > ctx.target_window:
|
||||
return (
|
||||
f"its window {layers.window} exceeds the target's window "
|
||||
f"{ctx.target_window}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
SWA_HOST = register_host_kind(
|
||||
WindowHostKind(
|
||||
name="swa", serves=LAYER_WINDOW, family="dense", fallback_host="full"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class StateHostKind(HostKind):
|
||||
"""The mamba sub-pool: its entries take the draft's recurrent-state block
|
||||
after the host's streams, so the slot that is the request's carries both."""
|
||||
|
||||
def region_for(
|
||||
self,
|
||||
*,
|
||||
layers: DraftLayerSet,
|
||||
lane_num: int,
|
||||
store_dtype: torch.dtype,
|
||||
kv_dtype: Optional[torch.dtype],
|
||||
) -> DraftStateRegion:
|
||||
geometry = layers.geometry
|
||||
assert isinstance(geometry, DraftStateGeometry), geometry
|
||||
return DraftStateRegion(lane_num=lane_num, state=geometry)
|
||||
|
||||
def describe(self, *, region: DraftRegion, lanes: Sequence[Tuple[int, ...]]) -> str:
|
||||
state = region.state
|
||||
return (
|
||||
f"fused draft state in {self.name!r}: {region.lane_num} lane(s) x "
|
||||
f"{len(state.conv_state_shapes)} conv stream(s) @ {state.conv_dtype} + "
|
||||
f"temporal {tuple(state.temporal_state_shape)} @ {state.temporal_dtype} "
|
||||
f"= {region.entry_bytes()} B/slot; runner lanes {list(lanes)}"
|
||||
)
|
||||
|
||||
|
||||
MAMBA_HOST = register_host_kind(
|
||||
StateHostKind(name="mamba", serves=LAYER_STATE, family="state")
|
||||
)
|
||||
|
||||
|
||||
class RunnerLanes(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One draft runner's lanes inside each host region: host -> (start, count)."""
|
||||
|
||||
ranges: Dict[str, Tuple[int, int]] = {}
|
||||
|
||||
def range_for(self, host: str) -> range:
|
||||
start, count = self.ranges.get(host, (0, 0))
|
||||
return range(start, start + count)
|
||||
|
||||
|
||||
class FusedDraftPlacement(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Where every draft runner's layers live inside the host sub-pools.
|
||||
|
||||
One region per host sub-pool that carries draft lanes plus each runner's
|
||||
lane range into it. Built once on the target, stored on the
|
||||
`UnifiedKVPool`, and read back by each draft runner, so the two sides
|
||||
cannot disagree on a lane.
|
||||
"""
|
||||
|
||||
runners: Tuple[RunnerLanes, ...]
|
||||
regions: Dict[str, DraftRegion] = {}
|
||||
|
||||
def __post_init__(self):
|
||||
assert len(self.runners) > 0, "a placement needs at least one draft runner"
|
||||
hosts = set(self.regions).union(*(r.ranges for r in self.runners))
|
||||
for host in sorted(hosts):
|
||||
assert host in HOST_KINDS, (
|
||||
f"no host kind is registered for sub-pool {host!r}"
|
||||
)
|
||||
region = self.regions.get(host)
|
||||
lanes = [s for r in self.runners for s in r.range_for(host)]
|
||||
if region is None:
|
||||
assert not lanes, f"host {host!r} has draft lanes but no region"
|
||||
continue
|
||||
assert lanes == list(range(region.lane_num)), (
|
||||
f"host {host!r}: runner lane ranges {lanes} must tile "
|
||||
f"range({region.lane_num}) in runner order"
|
||||
)
|
||||
|
||||
def region(self, host: str) -> Optional[DraftRegion]:
|
||||
return self.regions.get(host)
|
||||
|
||||
def hosts(self) -> Tuple[str, ...]:
|
||||
"""The hosts carrying draft lanes, in registry order."""
|
||||
return tuple(h for h in HOST_KINDS if h in self.regions)
|
||||
|
||||
def lanes_for(self, runner: int, host: str) -> range:
|
||||
return self.runners[runner].range_for(host)
|
||||
|
||||
@classmethod
|
||||
def from_counts(
|
||||
cls,
|
||||
*,
|
||||
counts: Mapping[str, Sequence[int]],
|
||||
regions: Mapping[str, DraftRegion],
|
||||
) -> "FusedDraftPlacement":
|
||||
"""Tile each host's region with the runners' layer counts, in runner order."""
|
||||
num_runners = {len(per_runner) for per_runner in counts.values()}
|
||||
assert len(num_runners) == 1, (
|
||||
f"every host needs one layer count per runner; got {dict(counts)}"
|
||||
)
|
||||
ranges: List[Dict[str, Tuple[int, int]]] = [
|
||||
{} for _ in range(num_runners.pop())
|
||||
]
|
||||
for host, per_runner in counts.items():
|
||||
assert host in regions, f"host {host!r} has layer counts but no region"
|
||||
start = 0
|
||||
for runner, count in enumerate(per_runner):
|
||||
if count:
|
||||
ranges[runner][host] = (start, count)
|
||||
start += count
|
||||
return cls(
|
||||
runners=tuple(RunnerLanes(ranges=r) for r in ranges),
|
||||
regions=dict(regions),
|
||||
)
|
||||
|
||||
|
||||
def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
|
||||
@@ -167,6 +471,25 @@ def draft_swa_layer_ids(draft_model_config) -> Tuple[int, ...]:
|
||||
return ()
|
||||
|
||||
|
||||
def draft_state_layer_ids(draft_model_config, *, num_depths: int) -> Tuple[int, ...]:
|
||||
"""The draft depth ids that own a recurrent-state block of their own.
|
||||
|
||||
Only a per-depth conv-chain MTP head carries state: the baseline serves
|
||||
it with a state cache cloned beside the target's. A plain NEXTN head of
|
||||
a linear-attention trunk inherits the trunk's mamba-ish config class but
|
||||
is a full-attention block; its config lists the TRUNK's state layers,
|
||||
and the head carries none.
|
||||
"""
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
|
||||
mc = draft_model_config
|
||||
if mambaish_config(mc) is None or mc.num_nextn_predict_layers is None:
|
||||
return ()
|
||||
if getattr(mc.hf_text_config, "mtp_local_layer_ids", None) is None:
|
||||
return ()
|
||||
return tuple(range(num_depths))
|
||||
|
||||
|
||||
def draft_kv_profile(
|
||||
draft_model_config,
|
||||
*,
|
||||
@@ -177,37 +500,76 @@ def draft_kv_profile(
|
||||
"""The profile of a draft `ModelConfig`, heads divided by attn_tp the way
|
||||
the target divides its own (drafts never join the DCP group).
|
||||
``num_depths`` overrides the config's `num_nextn_predict_layers`."""
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
|
||||
mc = draft_model_config
|
||||
if num_depths is None:
|
||||
num_depths = mc.num_nextn_predict_layers
|
||||
return DraftKVProfile(
|
||||
num_layers=int(num_layers),
|
||||
full=DraftKVGeometry(
|
||||
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
|
||||
head_dim=int(mc.head_dim),
|
||||
v_head_dim=int(mc.v_head_dim),
|
||||
),
|
||||
swa_layer_ids=draft_swa_layer_ids(mc),
|
||||
num_depths=1 if num_depths is None else int(num_depths),
|
||||
num_state_layers=draft_state_layer_num(mc),
|
||||
num_depths = 1 if num_depths is None else int(num_depths)
|
||||
num_layers = int(num_layers)
|
||||
window_ids = draft_swa_layer_ids(mc)
|
||||
assert set(window_ids) <= set(range(num_layers)), (
|
||||
f"draft window layer ids {window_ids} outside range({num_layers})"
|
||||
)
|
||||
kinds: Dict[str, DraftLayerSet] = {}
|
||||
full_ids = tuple(i for i in range(num_layers) if i not in window_ids)
|
||||
if full_ids:
|
||||
kinds[LAYER_FULL] = DraftLayerSet(
|
||||
layer_ids=full_ids,
|
||||
geometry=DraftKVGeometry(
|
||||
head_num=int(mc.get_num_kv_heads(attn_tp_size)),
|
||||
head_dim=int(mc.head_dim),
|
||||
v_head_dim=int(mc.v_head_dim),
|
||||
),
|
||||
)
|
||||
if window_ids:
|
||||
kinds[LAYER_WINDOW] = DraftLayerSet(
|
||||
layer_ids=window_ids,
|
||||
geometry=DraftKVGeometry(
|
||||
head_num=int(mc.get_swa_num_kv_heads(attn_tp_size)),
|
||||
head_dim=int(mc.swa_head_dim),
|
||||
v_head_dim=int(mc.swa_v_head_dim),
|
||||
),
|
||||
window=(
|
||||
None if mc.sliding_window_size is None else int(mc.sliding_window_size)
|
||||
),
|
||||
)
|
||||
state_ids = draft_state_layer_ids(mc, num_depths=num_depths)
|
||||
if state_ids:
|
||||
cp = mambaish_config(mc).mamba2_cache_params
|
||||
kinds[LAYER_STATE] = DraftLayerSet(
|
||||
layer_ids=state_ids,
|
||||
geometry=DraftStateGeometry(
|
||||
conv_state_shapes=tuple(
|
||||
tuple(int(x) for x in s) for s in cp.shape.conv
|
||||
),
|
||||
conv_dtype=cp.dtype.conv,
|
||||
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
|
||||
temporal_dtype=cp.dtype.temporal,
|
||||
),
|
||||
)
|
||||
return DraftKVProfile(num_layers=num_layers, num_depths=num_depths, kinds=kinds)
|
||||
|
||||
|
||||
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
|
||||
fuse. Neither means fusion does not apply."""
|
||||
fuse. Neither means fusion does not apply. ``note`` says why a placed
|
||||
layer kind did not get its first-choice host."""
|
||||
|
||||
placement: Optional[FusedDraftPlacement] = None
|
||||
declined: Optional[str] = None
|
||||
note: Optional[str] = None
|
||||
|
||||
|
||||
def _runner_layer_counts(
|
||||
profile: DraftKVProfile, num_runners: int
|
||||
) -> Tuple[Optional[List[Tuple[int, int]]], Optional[str]]:
|
||||
"""Per runner, its (full, swa) layer counts; or why no runner layout exists."""
|
||||
) -> Tuple[Optional[Dict[str, List[int]]], Optional[str]]:
|
||||
"""Per layer kind, each runner's layer count; or why no runner layout exists."""
|
||||
if profile.num_depths <= 1:
|
||||
num_swa = len(profile.swa_layer_ids)
|
||||
return [(profile.num_layers - num_swa, num_swa)] * num_runners, None
|
||||
return {
|
||||
kind: [len(layers.layer_ids)] * num_runners
|
||||
for kind, layers in profile.kinds.items()
|
||||
}, None
|
||||
if num_runners == 1:
|
||||
return None, (
|
||||
f"a per-depth draft head ({profile.num_depths} depths) needs one "
|
||||
@@ -217,69 +579,128 @@ def _runner_layer_counts(
|
||||
return None, (
|
||||
f"{num_runners} draft runners exceed the head's {profile.num_depths} depths"
|
||||
)
|
||||
swa = set(profile.swa_layer_ids)
|
||||
return [(0, 1) if r in swa else (1, 0) for r in range(num_runners)], None
|
||||
return {
|
||||
kind: [1 if r in layers.layer_ids else 0 for r in range(num_runners)]
|
||||
for kind, layers in profile.kinds.items()
|
||||
}, None
|
||||
|
||||
|
||||
def _kv_rows_declined(
|
||||
geometry: Any, *, store_dtype: torch.dtype, asymmetric_rows_ok: bool
|
||||
) -> Optional[str]:
|
||||
"""Why one kind's K/V rows cannot be fused entry parts, or None."""
|
||||
if not isinstance(geometry, DraftKVGeometry):
|
||||
return None
|
||||
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
|
||||
return (
|
||||
"the draft's K/V rows are asymmetric "
|
||||
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
|
||||
"and a resolved attention backend does not carry v_head_dim "
|
||||
"through to the kernel"
|
||||
)
|
||||
for dim in (geometry.head_dim, geometry.v_head_dim):
|
||||
row_bytes = geometry.head_num * dim * store_dtype.itemsize
|
||||
if row_bytes % ROW_ALIGN_BYTES:
|
||||
return (
|
||||
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
|
||||
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _fold_refused(
|
||||
profile: DraftKVProfile, *, kind: str, into: HostKind
|
||||
) -> Optional[str]:
|
||||
"""One region holds one row geometry: ``kind`` folds into ``into`` only
|
||||
when the draft has no layers of the kind ``into`` serves, or both kinds
|
||||
share a geometry."""
|
||||
own = profile.kinds.get(into.serves)
|
||||
if own is None or own.geometry == profile.kinds[kind].geometry:
|
||||
return None
|
||||
return (
|
||||
f"its {_LAYER_KIND_NOUN[kind]} layers' rows differ from its "
|
||||
f"{_LAYER_KIND_NOUN[into.serves]} layers' rows, so they cannot share the "
|
||||
f"{into.name!r} sub-pool"
|
||||
)
|
||||
|
||||
|
||||
def place_fused_draft(
|
||||
*,
|
||||
profile: DraftKVProfile,
|
||||
num_runners: int,
|
||||
ctx: PlacementContext,
|
||||
store_dtype: torch.dtype,
|
||||
asymmetric_rows_ok: bool,
|
||||
kv_dtype: Optional[torch.dtype] = None,
|
||||
) -> FusedDraftDecision:
|
||||
"""Assign every draft layer of every runner to the host sub-pool whose
|
||||
lifetime covers what the layer reads: a full-attention layer rides in
|
||||
``"full"``. A layer kind no host sub-pool serves declines the whole draft,
|
||||
as do asymmetric K/V rows unless the caller vouches that every attention
|
||||
backend carries v_head_dim through to the kernel."""
|
||||
"""Assign every draft layer of every runner to a host sub-pool through the
|
||||
registry: a layer kind goes to the host that serves it, else to that
|
||||
host's fallback under the fold rule. A kind no host takes declines the
|
||||
whole draft, as does a kind whose runtime prerequisite is missing,
|
||||
asymmetric K/V rows unless ``ctx`` vouches that every attention backend
|
||||
carries v_head_dim through to the kernel, and rows off the entry-part
|
||||
alignment."""
|
||||
counts, reason = _runner_layer_counts(profile, num_runners)
|
||||
if counts is None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
if profile.num_state_layers:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft has {profile.num_state_layers} recurrent-state "
|
||||
"layer(s) of its own, which no host state pool carries"
|
||||
)
|
||||
for kind, layers in profile.kinds.items():
|
||||
what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)"
|
||||
host = host_for(kind)
|
||||
if host is None:
|
||||
return FusedDraftDecision(declined=f"{what}: no host kind serves them")
|
||||
reason = host.prerequisite(layers=layers, ctx=ctx)
|
||||
if reason is not None:
|
||||
return FusedDraftDecision(declined=f"{what}: {reason}")
|
||||
reason = _kv_rows_declined(
|
||||
layers.geometry,
|
||||
store_dtype=store_dtype,
|
||||
asymmetric_rows_ok=ctx.asymmetric_rows_ok,
|
||||
)
|
||||
num_swa = sum(swa for _, swa in counts)
|
||||
if num_swa:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft has {num_swa} SWA layer(s) of its own, which the "
|
||||
"fused dense pool cannot serve"
|
||||
if reason is not None:
|
||||
return FusedDraftDecision(declined=reason)
|
||||
|
||||
host_counts: Dict[str, List[int]] = {}
|
||||
host_layers: Dict[str, DraftLayerSet] = {}
|
||||
notes: List[str] = []
|
||||
for kind, layers in profile.kinds.items():
|
||||
host = host_for(kind)
|
||||
what = f"the draft's {len(layers.layer_ids)} {_LAYER_KIND_NOUN[kind]} layer(s)"
|
||||
reason = host.admits(layers=layers, ctx=ctx)
|
||||
if reason is not None:
|
||||
fallback = (
|
||||
None if host.fallback_host is None else HOST_KINDS[host.fallback_host]
|
||||
)
|
||||
)
|
||||
geometry = profile.full
|
||||
if geometry.head_dim != geometry.v_head_dim and not asymmetric_rows_ok:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
"the draft's K/V rows are asymmetric "
|
||||
f"(head_dim={geometry.head_dim}, v_head_dim={geometry.v_head_dim}) "
|
||||
"and a resolved attention backend does not carry v_head_dim "
|
||||
"through to the kernel"
|
||||
)
|
||||
)
|
||||
for dim in (geometry.head_dim, geometry.v_head_dim):
|
||||
row_bytes = geometry.head_num * dim * store_dtype.itemsize
|
||||
if row_bytes % ROW_ALIGN_BYTES:
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"the draft's K/V rows are {row_bytes} B, not a multiple of "
|
||||
f"the {ROW_ALIGN_BYTES}-B row alignment a fused entry part needs"
|
||||
fold = None
|
||||
if fallback is not None:
|
||||
fold = _fold_refused(profile, kind=kind, into=fallback)
|
||||
if fold is None:
|
||||
fold = fallback.admits(layers=layers, ctx=ctx)
|
||||
if fallback is None or fold is not None:
|
||||
if fold is not None:
|
||||
reason = f"{reason}, and {fold}"
|
||||
return FusedDraftDecision(
|
||||
declined=(
|
||||
f"{what} cannot ride in the {host.name!r} sub-pool: {reason}"
|
||||
)
|
||||
)
|
||||
)
|
||||
full_counts = tuple(full for full, _ in counts)
|
||||
region = DenseDraftRegion(
|
||||
lane_num=sum(full_counts),
|
||||
head_num=geometry.head_num,
|
||||
head_dim=geometry.head_dim,
|
||||
v_head_dim=geometry.v_head_dim,
|
||||
store_dtype=store_dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
)
|
||||
notes.append(f"{what} ride in the {fallback.name!r} sub-pool: {reason}")
|
||||
host = fallback
|
||||
per_runner = host_counts.setdefault(host.name, [0] * num_runners)
|
||||
for runner, count in enumerate(counts[kind]):
|
||||
per_runner[runner] += count
|
||||
# The fold rule made every kind sharing a host share its geometry.
|
||||
host_layers.setdefault(host.name, layers)
|
||||
|
||||
placed = {host: c for host, c in host_counts.items() if sum(c) > 0}
|
||||
regions = {
|
||||
host: HOST_KINDS[host].region_for(
|
||||
layers=host_layers[host],
|
||||
lane_num=sum(c),
|
||||
store_dtype=store_dtype,
|
||||
kv_dtype=kv_dtype,
|
||||
)
|
||||
for host, c in placed.items()
|
||||
}
|
||||
return FusedDraftDecision(
|
||||
placement=FusedDraftPlacement(region=region, runner_lane_counts=full_counts)
|
||||
placement=FusedDraftPlacement.from_counts(counts=placed, regions=regions),
|
||||
note="; ".join(notes) or None,
|
||||
)
|
||||
|
||||
@@ -13,7 +13,7 @@ so every per-layer view is ``(num_slots, *row_shape)`` with slot stride
|
||||
is the pool region's offset inside the raw buffer.
|
||||
"""
|
||||
|
||||
from typing import List, Sequence, Tuple
|
||||
from typing import List, Optional, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
@@ -179,21 +179,26 @@ def build_mamba_entry_views(
|
||||
temporal_dtype: torch.dtype,
|
||||
max_slots: int,
|
||||
anchor_bytes: int = 0,
|
||||
entry_bytes: Optional[int] = None,
|
||||
offset_bytes: int = 0,
|
||||
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
||||
"""Per-slot envelope views over ``raw`` for Mamba state.
|
||||
|
||||
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
|
||||
[temporal rows × layers]``. Each returned view has shape
|
||||
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
|
||||
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
|
||||
Layout of one block: ``[conv[0] rows × layers][conv[1] rows × layers]...
|
||||
[temporal rows × layers]``, starting ``offset_bytes`` into a slot's entry
|
||||
of ``entry_bytes`` (the block's own size when omitted, i.e. a standalone
|
||||
envelope). Each returned view has shape ``(num_layers, max_slots,
|
||||
*inner_shape)`` matching ``MambaPool.State.conv[i]`` / ``.temporal``.
|
||||
Mamba state is always token-granular (page_size == 1).
|
||||
"""
|
||||
entry_bytes = mamba_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_state_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_state_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
)
|
||||
if entry_bytes is None:
|
||||
entry_bytes = mamba_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_state_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_state_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
)
|
||||
|
||||
conv_itemsize = conv_dtype.itemsize
|
||||
assert entry_bytes % conv_itemsize == 0, (
|
||||
@@ -207,7 +212,7 @@ def build_mamba_entry_views(
|
||||
as_conv_dtype = raw.view(conv_dtype)
|
||||
conv_slot_stride_elems = entry_bytes // conv_itemsize
|
||||
|
||||
offset_bytes_within_entry = 0
|
||||
offset_bytes_within_entry = offset_bytes
|
||||
conv_views: List[torch.Tensor] = []
|
||||
for shape in conv_state_shapes:
|
||||
inner_shape_bytes = _prod(shape) * conv_itemsize
|
||||
|
||||
@@ -1971,6 +1971,8 @@ class KVWriteLoc:
|
||||
# swa_loc / full_loc are resolved once at metadata-init from the full
|
||||
# (padded) out_cache_loc; piecewise/DP-padded paths later narrow loc per
|
||||
# layer, so slice these pre-resolved locs to match (same per-token order).
|
||||
# A multi-step draft step's slice is NOT a prefix of the all-steps loc;
|
||||
# its backend derives per-step locs and never relies on this.
|
||||
if self.swa_loc is not None and self.swa_loc.shape[0] != self.loc.shape[0]:
|
||||
self.swa_loc = self.swa_loc[: self.loc.shape[0]]
|
||||
if self.full_loc is not None and self.full_loc.shape[0] != self.loc.shape[0]:
|
||||
|
||||
@@ -23,6 +23,7 @@ class PureSWARadixCache(RadixCache):
|
||||
def __init__(self, params: CacheInitParams):
|
||||
super().__init__(params)
|
||||
self.sliding_window_size = params.sliding_window_size
|
||||
self.draft_swa_window = params.draft_swa_window
|
||||
|
||||
def supports_swa(self) -> bool:
|
||||
assert self.sliding_window_size is not None, (
|
||||
|
||||
@@ -468,7 +468,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
enable_hicache=False,
|
||||
write_through_threshold=256,
|
||||
device=str(self.device),
|
||||
swa_sliding_window_size=params.sliding_window_size,
|
||||
# Rust's tree window has the same cache-history contract as
|
||||
# SWAComponent, including the fused draft's front rows.
|
||||
swa_sliding_window_size=params.swa_retain_window,
|
||||
swa_req_ring=not params.swa_gates_prefix_match,
|
||||
enable_kv_cache_events=params.enable_kv_cache_events,
|
||||
dynamo_kv_event_format=params.dynamo_kv_event_format,
|
||||
|
||||
@@ -216,11 +216,12 @@ class SWAKVPool(BaseSWAKVPool):
|
||||
def get_v_head_dim(self):
|
||||
# The FULL side's dim, as HybridLinearKVPool.get_v_head_dim(): a caller
|
||||
# asking a pool for "the" v_head_dim wants the full-attention geometry.
|
||||
# A window-only pool (a sliding-window MTP depth) has no full side;
|
||||
# its swa side is the only geometry there is.
|
||||
# `start_layer`, not 0, so pipeline parallelism (start_layer > 0) works,
|
||||
# and because layer 0 need not be a full-attention layer.
|
||||
return self.full_kv_pool.get_value_buffer(self.full_kv_pool.start_layer).shape[
|
||||
-1
|
||||
]
|
||||
lead = self.full_kv_pool if self.full_layer_nums > 0 else self.swa_kv_pool
|
||||
return lead.get_value_buffer(lead.start_layer).shape[-1]
|
||||
|
||||
def get_kv_buffer(self, layer_id: int):
|
||||
self._wait_for_layer(layer_id)
|
||||
|
||||
@@ -88,7 +88,9 @@ class SWAComponent(TreeComponent):
|
||||
|
||||
super().__init__(cache, params)
|
||||
self._session_leaf_covered_len: dict[str, dict[UnifiedTreeNode, int]] = {}
|
||||
self.sliding_window_size = params.sliding_window_size
|
||||
# This is the tree's required history, including widened draft reads;
|
||||
# it does not change the model's attention window.
|
||||
self.sliding_window_size = params.swa_retain_window
|
||||
self.gates_prefix_match = params.swa_gates_prefix_match
|
||||
self.full_window_pages = (
|
||||
self.sliding_window_size + params.page_size - 1
|
||||
|
||||
@@ -1,14 +1,37 @@
|
||||
"""KV pools a DRAFT runner binds over the draft lanes fused into the target's
|
||||
unified pool: same pages, same slot ids, same v2p table as the target -- one
|
||||
allocation, one free, one relocation."""
|
||||
allocation, one free, one relocation.
|
||||
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence
|
||||
`DRAFT_BINDERS` maps a `HostKind.family` to the binder that builds the
|
||||
runner's pool over every host of that family; registering a binder is how a
|
||||
new host kind reaches the draft worker.
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.layout.fused_draft import FusedDraftPlacement
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
HOST_KINDS,
|
||||
DraftStateRegion,
|
||||
FusedDraftPlacement,
|
||||
draft_state_layer_ids,
|
||||
draft_swa_layer_ids,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
KVWriteLoc,
|
||||
MambaPool,
|
||||
MHATokenToKVPool,
|
||||
unwrap_write_loc,
|
||||
write_loc_is_physical,
|
||||
)
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.mem_cache.unified_memory_pool import (
|
||||
UnifiedHybridReqToTokenPool,
|
||||
UnifiedKVPool,
|
||||
)
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
|
||||
class UnifiedDraftKVPool(MHATokenToKVPool):
|
||||
@@ -102,10 +125,314 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
|
||||
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
|
||||
|
||||
|
||||
class UnifiedDraftSWAKVPool(SWAKVPool):
|
||||
"""A draft's KV pool when it has sliding-window layers: one dense draft
|
||||
pool per host sub-pool it fuses into ("full" and/or "swa"), routed per
|
||||
layer exactly like the target's `UnifiedSWAKVPool`, so the attention
|
||||
backends run their swa rail for it. Inherits `SWAKVPool` for `isinstance`
|
||||
only; never calls its `__init__` (it would allocate static pools)."""
|
||||
|
||||
requires_physical_write_loc = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
unified_buffer: UnifiedKVPool,
|
||||
host_allocator,
|
||||
page_size: int,
|
||||
full_layer_lanes: Mapping[int, int],
|
||||
swa_layer_lanes: Mapping[int, int],
|
||||
):
|
||||
assert swa_layer_lanes, (
|
||||
"UnifiedDraftSWAKVPool binds at least one window layer; a full-only "
|
||||
"draft binds UnifiedDraftKVPool"
|
||||
)
|
||||
self.unified_buffer = unified_buffer
|
||||
self.host_allocator = host_allocator
|
||||
self.page_size = page_size
|
||||
self.device = unified_buffer.device
|
||||
self.layer_transfer_counter = None
|
||||
self.full_layer_nums = len(full_layer_lanes)
|
||||
self.swa_layer_nums = len(swa_layer_lanes)
|
||||
self.layer_num = self.full_layer_nums + self.swa_layer_nums
|
||||
self.start_layer = min([*full_layer_lanes, *swa_layer_lanes])
|
||||
self.size = unified_buffer.max_slots("full") - 1
|
||||
self.size_swa = unified_buffer.max_slots("swa") - 1
|
||||
|
||||
self.full_kv_pool: Optional[UnifiedDraftKVPool] = None
|
||||
if full_layer_lanes:
|
||||
self.full_kv_pool = self._side_pool("full", full_layer_lanes)
|
||||
self.swa_kv_pool = self._side_pool("swa", swa_layer_lanes)
|
||||
lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool
|
||||
self.dtype = lead.dtype
|
||||
self.head_num = lead.head_num
|
||||
self.head_dim = lead.head_dim
|
||||
|
||||
# {layer_id: (per-side index, is_swa_layer)}, sides indexed in layer order.
|
||||
self.layers_mapping: Dict[int, Tuple[int, bool]] = {}
|
||||
for idx, layer_id in enumerate(sorted(full_layer_lanes)):
|
||||
self.layers_mapping[layer_id] = (idx, False)
|
||||
for idx, layer_id in enumerate(sorted(swa_layer_lanes)):
|
||||
self.layers_mapping[layer_id] = (idx, True)
|
||||
# None so dispatch routes through the host's v2p tables, never a
|
||||
# registered mapping.
|
||||
self.full_to_swa_index_mapping: Optional[torch.Tensor] = None
|
||||
self.enable_custom_mem_pool = False
|
||||
self.custom_mem_pool = None
|
||||
self.dsa_kv_cache_store_fp8 = False
|
||||
self.kv_cache_dim = None
|
||||
self.index_head_dim = None
|
||||
self.mem_usage = 0.0 # fused into the host entries
|
||||
|
||||
def _side_pool(
|
||||
self, host_sub_pool_name: str, layer_lanes: Mapping[int, int]
|
||||
) -> UnifiedDraftKVPool:
|
||||
# The side is indexed 0..n-1 in layer order (`layer_id_override`).
|
||||
return UnifiedDraftKVPool(
|
||||
unified_buffer=self.unified_buffer,
|
||||
host_sub_pool_name=host_sub_pool_name,
|
||||
host_allocator=self.host_allocator,
|
||||
layer_lanes={
|
||||
idx: layer_lanes[layer_id]
|
||||
for idx, layer_id in enumerate(sorted(layer_lanes))
|
||||
},
|
||||
page_size=self.page_size,
|
||||
)
|
||||
|
||||
def _side(self, layer_id: int) -> Tuple[UnifiedDraftKVPool, int]:
|
||||
pool_layer_id, is_swa = self.layers_mapping[layer_id]
|
||||
pool = self.swa_kv_pool if is_swa else self.full_kv_pool
|
||||
assert pool is not None, layer_id
|
||||
return pool, pool_layer_id
|
||||
|
||||
# -- KVCache surface a side of None cannot answer --
|
||||
|
||||
@property
|
||||
def post_capture_active(self) -> bool:
|
||||
return False # the host buffer is fully backed at boot
|
||||
|
||||
@property
|
||||
def post_capture_backed_bytes(self) -> int:
|
||||
return 0
|
||||
|
||||
def finalize_backing(self, config) -> None:
|
||||
return
|
||||
|
||||
def register_layer_transfer_counter(self, layer_transfer_counter):
|
||||
self.layer_transfer_counter = layer_transfer_counter
|
||||
|
||||
# -- BaseSWAKVPool ABC surface --
|
||||
|
||||
def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
|
||||
return # the host's swa v2p table IS the mapping
|
||||
|
||||
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
|
||||
"""Virtual token ids -> swa-physical token ids, through the host."""
|
||||
return self.host_allocator.translate_loc_from_full_to_swa(kv_indices)
|
||||
|
||||
def get_state_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"fused draft KV has no per-layer contiguous regions; KV transfer / "
|
||||
"disaggregation is unsupported."
|
||||
)
|
||||
|
||||
# -- size/info --
|
||||
|
||||
def get_kv_size_bytes(self):
|
||||
return 0, 0 # fused into the host entries; UnifiedKVPool logs the total
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"fused draft KV has no per-layer contiguous regions; KV transfer / "
|
||||
"disaggregation is unsupported."
|
||||
)
|
||||
|
||||
def get_v_head_dim(self):
|
||||
lead = self.full_kv_pool if self.full_kv_pool is not None else self.swa_kv_pool
|
||||
return lead.get_value_buffer(lead.start_layer).shape[-1]
|
||||
|
||||
# -- buffer accessors --
|
||||
|
||||
def get_key_buffer(self, layer_id: int):
|
||||
pool, pool_layer_id = self._side(layer_id)
|
||||
return pool.get_key_buffer(pool_layer_id)
|
||||
|
||||
def get_value_buffer(self, layer_id: int):
|
||||
pool, pool_layer_id = self._side(layer_id)
|
||||
return pool.get_value_buffer(pool_layer_id)
|
||||
|
||||
def get_kv_buffer(self, layer_id: int):
|
||||
pool, pool_layer_id = self._side(layer_id)
|
||||
return pool.get_kv_buffer(pool_layer_id)
|
||||
|
||||
# -- kv writing --
|
||||
|
||||
def set_kv_buffer(
|
||||
self,
|
||||
layer,
|
||||
loc_info,
|
||||
cache_k: torch.Tensor,
|
||||
cache_v: torch.Tensor,
|
||||
k_scale: float = 1.0,
|
||||
v_scale: float = 1.0,
|
||||
):
|
||||
"""Route to the right side. Both locs are physical already (the
|
||||
backend derives `swa_loc` once per forward); never translates here.
|
||||
The full side writes through `loc`: the capture-stable `full_loc`
|
||||
alias is sized for the target's writes, not a per-step draft slice."""
|
||||
loc, swa_loc, _ = unwrap_write_loc(loc_info)
|
||||
physical = write_loc_is_physical(loc_info)
|
||||
pool, pool_layer_id = self._side(layer.layer_id)
|
||||
if pool is self.swa_kv_pool:
|
||||
assert swa_loc is not None, (
|
||||
"UnifiedDraftSWAKVPool.set_kv_buffer: window layer received no "
|
||||
"swa_loc; the attention backend must bundle "
|
||||
"forward_metadata.swa_out_cache_loc."
|
||||
)
|
||||
side_loc = swa_loc
|
||||
else:
|
||||
side_loc = loc
|
||||
pool.set_kv_buffer(
|
||||
None,
|
||||
KVWriteLoc(side_loc, physical=physical),
|
||||
cache_k,
|
||||
cache_v,
|
||||
k_scale,
|
||||
v_scale,
|
||||
layer_id_override=pool_layer_id,
|
||||
)
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
raise NotImplementedError(
|
||||
"fused draft KV relocates with the HOST pool's whole-page move; a "
|
||||
"draft-side per-slot move would corrupt the fused layout"
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
|
||||
|
||||
def load_cpu_copy(
|
||||
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
|
||||
):
|
||||
raise NotImplementedError("CPU offloading is unsupported for fused draft KV.")
|
||||
|
||||
|
||||
class UnifiedDraftMambaPool(MambaPool):
|
||||
"""A draft runner's recurrent state as views of the draft block fused into
|
||||
the host's state entries: `mamba_cache.conv[stream][layer]` is laid out
|
||||
exactly as a private `MambaPool`'s. Slot ids are the target's, translated
|
||||
by the shared allocator. Pure view: the host's whole-entry clear and copy
|
||||
already cover the draft block, so the block ops here act on the draft block alone, and it
|
||||
never calls `MambaPool.__init__` (no buffers of its own)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
unified_buffer: UnifiedKVPool,
|
||||
sub_pool_name: str,
|
||||
layer_lanes: Mapping[int, int],
|
||||
):
|
||||
spec = unified_buffer.mamba_spec(sub_pool_name)
|
||||
region = spec.draft_region
|
||||
assert isinstance(region, DraftStateRegion), (
|
||||
f"UnifiedDraftMambaPool: sub-pool {sub_pool_name!r} carries no fused "
|
||||
"draft state region"
|
||||
)
|
||||
layer_ids = sorted(layer_lanes)
|
||||
assert layer_ids, "UnifiedDraftMambaPool binds at least one layer"
|
||||
lanes = [layer_lanes[layer_id] for layer_id in layer_ids]
|
||||
# A runner's lanes are one contiguous range, so its layers are one
|
||||
# strided slice of the region's views (a gather would copy).
|
||||
start = lanes[0]
|
||||
assert lanes == list(range(start, start + len(lanes))) and (
|
||||
0 <= start and start + len(lanes) <= region.lane_num
|
||||
), (
|
||||
f"draft state lanes {lanes} must be a contiguous range within "
|
||||
f"{region.lane_num}"
|
||||
)
|
||||
conv_views, temporal_view = unified_buffer.build_draft_state_views(
|
||||
sub_pool_name
|
||||
)
|
||||
stop = start + len(lanes)
|
||||
|
||||
self._unified_buffer = unified_buffer
|
||||
self._sub_pool_name = sub_pool_name
|
||||
self.layer_lanes: Dict[int, int] = dict(layer_lanes)
|
||||
self.mamba_layer_ids = layer_ids
|
||||
self.num_mamba_layers = len(layer_ids)
|
||||
self._max_size = unified_buffer.max_slots(sub_pool_name) - 1
|
||||
self.size = self._max_size
|
||||
self.device = unified_buffer.device
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(enable=False)
|
||||
self.enable_custom_mem_pool = False
|
||||
self.custom_mem_pool = None
|
||||
# Same disabled-state attributes `UnifiedMambaPool` replicates for the
|
||||
# base class's unconditional reads.
|
||||
self.enable_linear_replayssm = False
|
||||
self.linear_replayssm_cache_len = 16
|
||||
self.replayssm_write_pos = None
|
||||
self.replayssm_is_kda = False
|
||||
self.enable_linear_replayssm_spec = False
|
||||
self.replayssm_spec_fold = False
|
||||
self.replayssm_cache_base = None
|
||||
self.replayssm_is_flush = None
|
||||
self.debug_memory_pool = False
|
||||
self.conv_shard_groups = None
|
||||
self.conv_slice_axis = spec.conv_slice_axis
|
||||
self.mamba_cache = self.State(
|
||||
conv=[view[start:stop] for view in conv_views],
|
||||
temporal=temporal_view[start:stop],
|
||||
)
|
||||
self.mem_usage = 0.0 # fused into the host entries
|
||||
|
||||
def clear_slots(self, indices: torch.Tensor):
|
||||
"""Zero THIS runner's state block in the given slots.
|
||||
|
||||
The block only, never the whole entry: the host's own streams share
|
||||
the entry and must survive, since a new request takes over the
|
||||
draft's block while the target's state for that slot is still live.
|
||||
The host's whole-entry clear covers the block too, but only on a
|
||||
plain EXTEND forward: ModelRunner skips it on decode, target-verify
|
||||
and draft-extend, the modes the draft needs it in.
|
||||
"""
|
||||
for view in self.mamba_cache.conv:
|
||||
view[:, indices] = 0
|
||||
self.mamba_cache.temporal[:, indices] = 0
|
||||
|
||||
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
|
||||
"""Copy-on-write THIS runner's state block, block only (see above)."""
|
||||
for view in self.mamba_cache.conv:
|
||||
view[:, dst_indices] = view[:, src_indices]
|
||||
self.mamba_cache.temporal[:, dst_indices] = self.mamba_cache.temporal[
|
||||
:, src_indices
|
||||
]
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
raise NotImplementedError(
|
||||
"fused draft state relocates with the HOST pool's whole-entry move"
|
||||
)
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
raise NotImplementedError(
|
||||
"fused draft state has no contiguous region of its own; state "
|
||||
"transfer / disaggregation is unsupported."
|
||||
)
|
||||
|
||||
def get_cpu_copy(self, indices):
|
||||
raise NotImplementedError(
|
||||
"CPU offloading is unsupported for fused draft state."
|
||||
)
|
||||
|
||||
def load_cpu_copy(self, mamba_cache_cpu, indices):
|
||||
raise NotImplementedError(
|
||||
"CPU offloading is unsupported for fused draft state."
|
||||
)
|
||||
|
||||
|
||||
def fused_draft_host_allocator(token_to_kv_pool: Any) -> Optional[Any]:
|
||||
"""The host allocator a fused draft pool translates through, or None for
|
||||
any other pool."""
|
||||
if isinstance(token_to_kv_pool, UnifiedDraftKVPool):
|
||||
if isinstance(token_to_kv_pool, (UnifiedDraftKVPool, UnifiedDraftSWAKVPool)):
|
||||
return token_to_kv_pool.host_allocator
|
||||
return None
|
||||
|
||||
@@ -123,8 +450,8 @@ def draft_kv_layer_ids(model) -> List[int]:
|
||||
|
||||
def draft_state_layer_classes(model) -> List[str]:
|
||||
"""Class names of the BUILT draft model's recurrent / linear-attention
|
||||
modules. A fused draft has KV lanes only, so any of these would run with
|
||||
no state pool of its own."""
|
||||
modules. A runner the placement gives no state lanes would run any of
|
||||
these with no state pool of its own."""
|
||||
from sglang.srt.layers.attention.mamba.mamba import MambaMixer2
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
|
||||
@@ -137,37 +464,170 @@ def draft_state_layer_classes(model) -> List[str]:
|
||||
)
|
||||
|
||||
|
||||
class FusedDraftBinding(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""What a draft runner binds: its KV pool over the fused lanes and the
|
||||
request table it reads."""
|
||||
|
||||
token_to_kv_pool: Any = None
|
||||
req_to_token_pool: Any = None
|
||||
|
||||
|
||||
class BindContext(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One runner's inputs to every binder."""
|
||||
|
||||
placement: FusedDraftPlacement
|
||||
runner: int
|
||||
unified_buffer: Any
|
||||
host_allocator: Any
|
||||
model: Any
|
||||
model_config: Any
|
||||
page_size: int
|
||||
|
||||
def hosts_of(self, family: str) -> Tuple[str, ...]:
|
||||
"""This runner's hosts of ``family`` that hold lanes for it."""
|
||||
return tuple(
|
||||
host
|
||||
for host in self.placement.hosts()
|
||||
if HOST_KINDS[host].family == family
|
||||
and len(self.placement.lanes_for(self.runner, host)) > 0
|
||||
)
|
||||
|
||||
|
||||
Binder = Callable[[BindContext, FusedDraftBinding], FusedDraftBinding]
|
||||
|
||||
DRAFT_BINDERS: Dict[str, Binder] = {}
|
||||
|
||||
|
||||
def register_draft_binder(family: str, binder: Binder) -> Binder:
|
||||
assert family not in DRAFT_BINDERS, (
|
||||
f"a draft binder is already registered for host family {family!r}"
|
||||
)
|
||||
DRAFT_BINDERS[family] = binder
|
||||
return binder
|
||||
|
||||
|
||||
def bind_fused_draft(
|
||||
*,
|
||||
unified_buffer: UnifiedKVPool,
|
||||
host_allocator,
|
||||
placement: FusedDraftPlacement,
|
||||
runner: int,
|
||||
kv_layer_ids: Sequence[int],
|
||||
swa_layer_ids: Sequence[int],
|
||||
unified_buffer: UnifiedKVPool,
|
||||
host_allocator,
|
||||
model,
|
||||
model_config,
|
||||
req_to_token_pool,
|
||||
page_size: int,
|
||||
) -> UnifiedDraftKVPool:
|
||||
"""The KV pool draft runner ``runner`` binds over its fused slots.
|
||||
|
||||
The placement sized the lanes from the draft config; the model's real
|
||||
layer ids fill them in layer order, so a count mismatch is a loud boot
|
||||
failure, never a silent alias.
|
||||
"""
|
||||
swa = set(swa_layer_ids)
|
||||
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in swa]
|
||||
assert len(full_ids) == len(kv_layer_ids), (
|
||||
f"draft layers {sorted(swa & set(kv_layer_ids))} are SWA-kind; fused "
|
||||
"SWA KV is not supported yet"
|
||||
)
|
||||
full_lanes = placement.lanes_for(runner)
|
||||
assert len(full_ids) == len(full_lanes), (
|
||||
f"draft runner {runner}: {len(full_ids)} full-attention layer(s) "
|
||||
f"{full_ids} vs {len(full_lanes)} placed lane(s) {list(full_lanes)}"
|
||||
)
|
||||
return UnifiedDraftKVPool(
|
||||
) -> FusedDraftBinding:
|
||||
"""Bind draft runner ``runner`` to its fused lanes: one binder per host
|
||||
family that holds lanes for it, in registry order, each refining the
|
||||
binding the previous one returned."""
|
||||
ctx = BindContext(
|
||||
placement=placement,
|
||||
runner=runner,
|
||||
unified_buffer=unified_buffer,
|
||||
host_sub_pool_name="full",
|
||||
host_allocator=host_allocator,
|
||||
layer_lanes=dict(zip(full_ids, full_lanes)),
|
||||
model=model,
|
||||
model_config=model_config,
|
||||
page_size=page_size,
|
||||
)
|
||||
families: List[str] = []
|
||||
for host in placement.hosts():
|
||||
family = HOST_KINDS[host].family
|
||||
if family not in families and len(placement.lanes_for(runner, host)) > 0:
|
||||
families.append(family)
|
||||
assert families, f"draft runner {runner} holds no lane in any host"
|
||||
binding = FusedDraftBinding(req_to_token_pool=req_to_token_pool)
|
||||
for family in families:
|
||||
assert family in DRAFT_BINDERS, (
|
||||
f"no draft binder is registered for host family {family!r}"
|
||||
)
|
||||
binding = DRAFT_BINDERS[family](ctx, binding)
|
||||
assert binding.token_to_kv_pool is not None, (
|
||||
f"draft runner {runner}: no binder produced a KV pool"
|
||||
)
|
||||
return binding
|
||||
|
||||
|
||||
def _bind_dense(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding:
|
||||
"""This runner's attention layers, in layer order, over its lanes in the
|
||||
dense hosts. The placement sized the lanes from the draft config; the
|
||||
model's real layer ids fill them, so a count mismatch is a loud boot
|
||||
failure, never a silent alias. Window layers follow the placement: they
|
||||
bind the swa sub-pool when it holds a region for them, else the full
|
||||
sub-pool, where a full lifetime covers any window."""
|
||||
hosts = ctx.hosts_of("dense")
|
||||
assert set(hosts) <= {"full", "swa"}, f"unknown dense host(s) in {hosts}"
|
||||
kv_layer_ids = draft_kv_layer_ids(ctx.model)
|
||||
full_lanes = ctx.placement.lanes_for(ctx.runner, "full")
|
||||
swa_lanes = ctx.placement.lanes_for(ctx.runner, "swa")
|
||||
if not swa_lanes:
|
||||
full_ids, swa_ids = list(kv_layer_ids), []
|
||||
elif not full_lanes:
|
||||
full_ids, swa_ids = [], list(kv_layer_ids)
|
||||
else:
|
||||
# Both kinds in one runner: a replicated head, whose config lists every
|
||||
# window layer (a per-depth runner's config is clipped to its block).
|
||||
window = set(draft_swa_layer_ids(ctx.model_config))
|
||||
full_ids = [layer_id for layer_id in kv_layer_ids if layer_id not in window]
|
||||
swa_ids = [layer_id for layer_id in kv_layer_ids if layer_id in window]
|
||||
assert len(full_ids) == len(full_lanes) and len(swa_ids) == len(swa_lanes), (
|
||||
f"draft runner {ctx.runner}: layers full={full_ids} swa={swa_ids} vs placed "
|
||||
f"lanes full={list(full_lanes)} swa={list(swa_lanes)}"
|
||||
)
|
||||
if not swa_ids:
|
||||
pool = UnifiedDraftKVPool(
|
||||
unified_buffer=ctx.unified_buffer,
|
||||
host_sub_pool_name="full",
|
||||
host_allocator=ctx.host_allocator,
|
||||
layer_lanes=dict(zip(full_ids, full_lanes)),
|
||||
page_size=ctx.page_size,
|
||||
)
|
||||
else:
|
||||
pool = UnifiedDraftSWAKVPool(
|
||||
unified_buffer=ctx.unified_buffer,
|
||||
host_allocator=ctx.host_allocator,
|
||||
page_size=ctx.page_size,
|
||||
full_layer_lanes=dict(zip(full_ids, full_lanes)),
|
||||
swa_layer_lanes=dict(zip(swa_ids, swa_lanes)),
|
||||
)
|
||||
return msgspec.structs.replace(binding, token_to_kv_pool=pool)
|
||||
|
||||
|
||||
register_draft_binder("dense", _bind_dense)
|
||||
|
||||
|
||||
def _runner_state_layer_ids(ctx: BindContext) -> Tuple[int, ...]:
|
||||
"""The recurrent-state layer ids this runner serves: its own depth for a
|
||||
per-depth head, every state layer otherwise."""
|
||||
mc = ctx.model_config
|
||||
num_depths = mc.num_nextn_predict_layers
|
||||
num_depths = 1 if num_depths is None else int(num_depths)
|
||||
layer_ids = draft_state_layer_ids(mc, num_depths=num_depths)
|
||||
if num_depths > 1:
|
||||
return (ctx.runner,) if ctx.runner in layer_ids else ()
|
||||
return layer_ids
|
||||
|
||||
|
||||
def _bind_state(ctx: BindContext, binding: FusedDraftBinding) -> FusedDraftBinding:
|
||||
"""This runner's recurrent-state block: its request table is the target's
|
||||
(same request->slot mappings, same slot allocator) with a `mamba_pool`
|
||||
view keyed by the runner's own state layer ids."""
|
||||
hosts = ctx.hosts_of("state")
|
||||
assert hosts == ("mamba",), (
|
||||
f"the state binder serves the 'mamba' host alone; got {hosts}"
|
||||
)
|
||||
assert isinstance(binding.req_to_token_pool, UnifiedHybridReqToTokenPool), (
|
||||
"a fused draft state block needs the target's unified state pool"
|
||||
)
|
||||
layer_ids = _runner_state_layer_ids(ctx)
|
||||
lanes = ctx.placement.lanes_for(ctx.runner, "mamba")
|
||||
assert len(layer_ids) == len(lanes), (
|
||||
f"draft runner {ctx.runner}: state layers {list(layer_ids)} vs placed "
|
||||
f"lanes {list(lanes)}"
|
||||
)
|
||||
req_to_token_pool = binding.req_to_token_pool.clone_for_fused_draft(
|
||||
layer_lanes=dict(zip(layer_ids, lanes))
|
||||
)
|
||||
return msgspec.structs.replace(binding, req_to_token_pool=req_to_token_pool)
|
||||
|
||||
|
||||
register_draft_binder("state", _bind_state)
|
||||
|
||||
@@ -25,6 +25,7 @@ compaction only mutates those (no reference rewriting).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
@@ -40,6 +41,8 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
||||
from sglang.srt.mem_cache.layout.fused_draft import (
|
||||
DenseDraftRegion,
|
||||
DraftRegion,
|
||||
DraftStateRegion,
|
||||
FusedDraftPlacement,
|
||||
)
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
@@ -119,7 +122,9 @@ class SubPoolSpec(ABC):
|
||||
layer_num: int
|
||||
grow_direction: str # "up" | "down" | "float"
|
||||
# Fused draft region riding in this sub-pool's entries; None = unfused.
|
||||
draft_region: Optional[DenseDraftRegion] = None
|
||||
# Each kind accepts the region kind its entry can lay out, after the host
|
||||
# block at `draft_offset_in_entry()`.
|
||||
draft_region: Optional[DraftRegion] = None
|
||||
|
||||
def __post_init__(self):
|
||||
assert self.grow_direction in self._allowed_grow_directions, (
|
||||
@@ -176,6 +181,9 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
f"v_head_dim must be positive; got {self.v_head_dim}"
|
||||
)
|
||||
if self.draft_region is not None:
|
||||
assert isinstance(self.draft_region, DenseDraftRegion), (
|
||||
"dense entries carry dense draft K/V rows, never a state region"
|
||||
)
|
||||
self.draft_region.validate()
|
||||
|
||||
def k_row_bytes(self) -> int:
|
||||
@@ -258,6 +266,9 @@ class MLASubPoolSpec(SubPoolSpec):
|
||||
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
|
||||
)
|
||||
if self.draft_region is not None:
|
||||
assert isinstance(self.draft_region, DenseDraftRegion), (
|
||||
"dense entries carry dense draft K/V rows, never a state region"
|
||||
)
|
||||
self.draft_region.validate()
|
||||
|
||||
@property
|
||||
@@ -305,7 +316,15 @@ class MLASubPoolSpec(SubPoolSpec):
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MambaSubPoolSpec(SubPoolSpec):
|
||||
"""Per-slot layout of one Mamba-shaped sub-pool."""
|
||||
"""Per-slot layout of one Mamba-shaped sub-pool.
|
||||
|
||||
With `draft_region` (a `DraftStateRegion`) set, the draft's state block
|
||||
follows the host's inside every slot's entry:
|
||||
|
||||
[ conv[0] x L | ... | temporal x L | draft conv[0] x Ld | ... | pad ]
|
||||
|
||||
Unfused, the entry keeps its raw (unaligned) size, byte-identical to before.
|
||||
"""
|
||||
|
||||
conv_state_shapes: Tuple[Tuple[int, ...], ...] # one shape per conv tensor
|
||||
conv_dtype: torch.dtype
|
||||
@@ -316,9 +335,11 @@ class MambaSubPoolSpec(SubPoolSpec):
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
|
||||
assert self.draft_region is None, (
|
||||
"mamba state pages carry no fused draft region yet"
|
||||
)
|
||||
if self.draft_region is not None:
|
||||
assert isinstance(self.draft_region, DraftStateRegion), (
|
||||
"mamba state entries carry a draft STATE region, never dense K/V rows"
|
||||
)
|
||||
self.draft_region.validate()
|
||||
|
||||
def conv_row_bytes(self, idx: int) -> int:
|
||||
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
|
||||
@@ -326,13 +347,26 @@ class MambaSubPoolSpec(SubPoolSpec):
|
||||
def temporal_row_bytes(self) -> int:
|
||||
return _prod(self.temporal_state_shape) * self.temporal_dtype.itemsize
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
def host_entry_bytes(self) -> int:
|
||||
"""Host (target-only) bytes for one slot, before any draft block."""
|
||||
total = 0
|
||||
for i in range(len(self.conv_state_shapes)):
|
||||
total += self.layer_num * self.conv_row_bytes(i)
|
||||
total += self.layer_num * self.temporal_row_bytes()
|
||||
return total
|
||||
|
||||
def draft_offset_in_entry(self) -> int:
|
||||
"""Byte offset of the fused draft state block inside one slot's entry."""
|
||||
assert self.draft_region is not None
|
||||
return align_part_offset(self.host_entry_bytes())
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
if self.draft_region is None:
|
||||
return self.host_entry_bytes()
|
||||
return align_entry_bytes(
|
||||
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
|
||||
)
|
||||
|
||||
def get_dtype(self) -> torch.dtype:
|
||||
return self.conv_dtype # representative state dtype; matches MambaPool.dtype
|
||||
|
||||
@@ -432,11 +466,7 @@ class UnifiedKVPool:
|
||||
self._bs1_floor_terms = bs1_floor_terms or []
|
||||
self.fused_draft = fused_draft
|
||||
for spec in sub_pool_specs:
|
||||
expected = (
|
||||
fused_draft.region
|
||||
if fused_draft is not None and spec.name == "full"
|
||||
else None
|
||||
)
|
||||
expected = None if fused_draft is None else fused_draft.region(spec.name)
|
||||
assert spec.draft_region is expected, (
|
||||
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
|
||||
f"not match the fused draft placement's {expected}"
|
||||
@@ -749,6 +779,30 @@ class UnifiedKVPool:
|
||||
temporal_dtype=spec.temporal_dtype,
|
||||
max_slots=max_slots,
|
||||
anchor_bytes=anchor_bytes,
|
||||
entry_bytes=spec.entry_bytes(),
|
||||
)
|
||||
|
||||
def build_draft_state_views(
|
||||
self, sub_pool_name: str
|
||||
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
||||
"""Per-layer conv/temporal views of the DRAFT state block fused into
|
||||
``sub_pool_name``'s entries: same slots, same v2p table as the host."""
|
||||
spec = self.mamba_spec(sub_pool_name)
|
||||
region = spec.draft_region
|
||||
assert isinstance(region, DraftStateRegion), (
|
||||
f"sub-pool {sub_pool_name!r} carries no fused draft state region"
|
||||
)
|
||||
return build_mamba_entry_views(
|
||||
self._raw,
|
||||
layer_num=region.lane_num,
|
||||
conv_state_shapes=region.state.conv_state_shapes,
|
||||
conv_dtype=region.state.conv_dtype,
|
||||
temporal_state_shape=region.state.temporal_state_shape,
|
||||
temporal_dtype=region.state.temporal_dtype,
|
||||
max_slots=self.max_slots(sub_pool_name),
|
||||
anchor_bytes=self._anchor_bytes[sub_pool_name],
|
||||
entry_bytes=spec.entry_bytes(),
|
||||
offset_bytes=spec.draft_offset_in_entry(),
|
||||
)
|
||||
|
||||
|
||||
@@ -1126,8 +1180,29 @@ class UnifiedMambaPool(MambaPool):
|
||||
self.num_mamba_layers,
|
||||
)
|
||||
|
||||
# Inherited MambaPool state ops (copy_from/clear_slots/get_cpu_copy/load_cpu_copy)
|
||||
# take PHYSICAL slot ids; callers translate via the slot allocator first.
|
||||
# State ops take PHYSICAL slot ids and act on whole entries, so a fused
|
||||
# draft state block is cleared, copied and moved with the host streams.
|
||||
|
||||
def _entry_view(self) -> torch.Tensor:
|
||||
spec = self._unified_buffer.mamba_spec(self._sub_pool_name)
|
||||
entry_bytes = spec.entry_bytes()
|
||||
anchor = self._unified_buffer.anchor_bytes(self._sub_pool_name)
|
||||
num_slots = self._max_size + 1
|
||||
raw = self._unified_buffer._raw
|
||||
entries = raw[anchor : anchor + num_slots * entry_bytes]
|
||||
return entries.view(num_slots, entry_bytes)
|
||||
|
||||
def clear_slots(self, indices: torch.Tensor):
|
||||
self._entry_view()[indices] = 0
|
||||
for sibling in self._slot_siblings:
|
||||
sibling.reset_slots(indices)
|
||||
|
||||
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
|
||||
assert self.replayssm_write_pos is None
|
||||
entries = self._entry_view()
|
||||
entries[dst_indices] = entries[src_indices]
|
||||
for sibling in self._slot_siblings:
|
||||
sibling.copy_slots(src_indices, dst_indices)
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
# Compaction moves disjoint physical slots. Copy their whole envelopes
|
||||
@@ -1143,6 +1218,8 @@ class UnifiedMambaPool(MambaPool):
|
||||
self._unified_buffer.max_slots(self._sub_pool_name),
|
||||
self._unified_buffer.mamba_spec(self._sub_pool_name).entry_bytes(),
|
||||
)
|
||||
if self.replayssm_write_pos is not None:
|
||||
self.replayssm_write_pos[tgt_loc] = 0
|
||||
for sibling in self._slot_siblings:
|
||||
sibling.copy_slots(src_loc, tgt_loc)
|
||||
|
||||
@@ -1407,6 +1484,27 @@ class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
|
||||
)
|
||||
)
|
||||
|
||||
def clone_for_fused_draft(
|
||||
self, *, layer_lanes: Dict[int, int]
|
||||
) -> UnifiedHybridReqToTokenPool:
|
||||
"""Shallow copy for a draft runner whose recurrent state is fused into
|
||||
this pool's state entries: it shares the request->slot mappings and
|
||||
the slot allocator, and its `mamba_pool` is a view of the draft block
|
||||
keyed by the runner's own layer ids."""
|
||||
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftMambaPool
|
||||
|
||||
clone = copy.copy(self)
|
||||
clone.mamba_pool = UnifiedDraftMambaPool(
|
||||
unified_buffer=self._unified_buffer,
|
||||
sub_pool_name=self._mamba_sub_pool_name,
|
||||
layer_lanes=layer_lanes,
|
||||
)
|
||||
clone.mamba_map = {
|
||||
layer_id: idx for idx, layer_id in enumerate(sorted(layer_lanes))
|
||||
}
|
||||
clone.mamba_ckpt_pool = None
|
||||
return clone
|
||||
|
||||
@property
|
||||
def mamba_v2p_table(self) -> Optional[torch.Tensor]:
|
||||
"""This pool's ids ARE virtual; page_size is 1, so the translate is the
|
||||
@@ -1552,7 +1650,7 @@ def init_unified_mamba_pools(
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
store_dtype=store_dtype,
|
||||
grow_direction="down",
|
||||
draft_region=None if fused_draft is None else fused_draft.region,
|
||||
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
||||
)
|
||||
else:
|
||||
full_spec = MHASubPoolSpec(
|
||||
@@ -1564,7 +1662,7 @@ def init_unified_mamba_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="down",
|
||||
draft_region=None if fused_draft is None else fused_draft.region,
|
||||
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
||||
)
|
||||
cp = mamba2_cache_params
|
||||
mamba_spec = MambaSubPoolSpec(
|
||||
@@ -1576,6 +1674,7 @@ def init_unified_mamba_pools(
|
||||
temporal_dtype=cp.dtype.temporal,
|
||||
conv_slice_axis=getattr(cp.shape, "conv_slice_axis", 0),
|
||||
grow_direction="up",
|
||||
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
|
||||
)
|
||||
if unified_total_bytes is not None:
|
||||
# PROFILED byte budget for the token side (captured pre-ratio-floor);
|
||||
@@ -2051,10 +2150,10 @@ def init_unified_swa_pools(
|
||||
) -> UnifiedSWAPoolBundle:
|
||||
"""Build the SWA-hybrid unified-memory-pool stack.
|
||||
|
||||
With ``fused_draft``, every entry of the "full" sub-pool carries the
|
||||
draft model's K/V parts after the host parts, and each draft runner
|
||||
binds a `UnifiedDraftKVPool` over its own lanes instead of allocating a
|
||||
pool of its own.
|
||||
With ``fused_draft``, every entry of a sub-pool the placement names
|
||||
carries the draft model's K/V parts after the host parts, and each draft
|
||||
runner binds a `UnifiedDraftKVPool` over its own lanes instead of
|
||||
allocating a pool of its own.
|
||||
"""
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
@@ -2081,7 +2180,7 @@ def init_unified_swa_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="down",
|
||||
draft_region=None if fused_draft is None else fused_draft.region,
|
||||
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
||||
)
|
||||
swa_spec = MHASubPoolSpec(
|
||||
name="swa",
|
||||
@@ -2092,6 +2191,7 @@ def init_unified_swa_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="up",
|
||||
draft_region=None if fused_draft is None else fused_draft.region("swa"),
|
||||
)
|
||||
legacy_allocator_capacities = {}
|
||||
if total_bytes is None:
|
||||
@@ -2251,8 +2351,9 @@ def init_unified_mamba_swa_pools(
|
||||
|
||||
Sizing inputs are the same token counts the 2-pool factories take (ratio-
|
||||
fed until the byte configurator lands); the buffer budget is their byte
|
||||
sum and the runtime split floats. With ``fused_draft``, every entry of the
|
||||
"full" sub-pool carries the draft's parts, as in the 2-pool factories.
|
||||
sum and the runtime split floats. With ``fused_draft``, every entry of a
|
||||
sub-pool the placement names carries the draft's parts, as in the 2-pool
|
||||
factories.
|
||||
"""
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedMambaSWATokenToKVPoolAllocator,
|
||||
@@ -2279,7 +2380,7 @@ def init_unified_mamba_swa_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="down",
|
||||
draft_region=None if fused_draft is None else fused_draft.region,
|
||||
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
||||
)
|
||||
swa_spec = MHASubPoolSpec(
|
||||
name="swa",
|
||||
@@ -2290,6 +2391,7 @@ def init_unified_mamba_swa_pools(
|
||||
store_dtype=store_dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
grow_direction="float",
|
||||
draft_region=None if fused_draft is None else fused_draft.region("swa"),
|
||||
)
|
||||
cp = mamba2_cache_params
|
||||
mamba_spec = MambaSubPoolSpec(
|
||||
@@ -2300,6 +2402,7 @@ def init_unified_mamba_swa_pools(
|
||||
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
|
||||
temporal_dtype=cp.dtype.temporal,
|
||||
grow_direction="up",
|
||||
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
|
||||
)
|
||||
if unified_total_bytes is not None:
|
||||
# PROFILED byte budget for the token side (captured pre-ratio-floor);
|
||||
|
||||
@@ -227,11 +227,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.enable_mamba_extra_buffer = (
|
||||
params.enable_mamba_extra_buffer if self.is_mamba_enabled else False
|
||||
)
|
||||
# SWA window size (None when SWA is not enabled).
|
||||
# Required cache history, shared by retention, budgeting and transfers.
|
||||
self._sliding_window_size = (
|
||||
params.sliding_window_size if self.is_swa_enabled else None
|
||||
params.swa_retain_window if self.is_swa_enabled else None
|
||||
)
|
||||
self.decoder_swa_bounded_replay = params.decoder_swa_bounded_replay
|
||||
self.draft_swa_window = params.draft_swa_window
|
||||
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
|
||||
# evictable leaves) and drives the components' tree-level hooks.
|
||||
self._tree_core_backend = select_tree_core_backend(params)
|
||||
|
||||
@@ -739,7 +739,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
|
||||
self._draft_cell_size = _dflash_draft_cell_size(kvc)
|
||||
self._fused_full_entry = kvc.fused_entry_bytes("full")
|
||||
if self._fused_full_entry is not None:
|
||||
self._fused_swa_entry = kvc.fused_entry_bytes("swa")
|
||||
if self._fused_full_entry is not None or self._fused_swa_entry is not None:
|
||||
self._draft_full_layers_num = 0
|
||||
self._draft_swa_layers_num = 0
|
||||
self._draft_swa_full_layers_num = 0
|
||||
@@ -757,6 +758,8 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
)
|
||||
|
||||
def _swa_cell_bytes(self) -> int:
|
||||
if self._fused_swa_entry is not None:
|
||||
return self._fused_swa_entry
|
||||
return self._swa_per_token * (self._swa_layers_num + self._draft_swa_layers_num)
|
||||
|
||||
def _recompute_cell_size(self) -> None:
|
||||
|
||||
@@ -1841,6 +1841,16 @@ class InklingForConditionalGenerationMTP(nn.Module):
|
||||
hidden_states_before_norm=hidden_states_before_norm,
|
||||
)
|
||||
|
||||
def get_attention_sliding_window_size(self) -> Optional[int]:
|
||||
"""The trunk's convention (`InklingForConditionalGeneration`): the
|
||||
backend window is one less than the extent, so the head reads back
|
||||
the same span the trunk's window layers do. A banded depth runs at
|
||||
its own extent, every other depth at the trunk's window."""
|
||||
text_config = self.text_config
|
||||
if self.draft_model_idx in text_config.mtp_local_layer_ids:
|
||||
return text_config.mtp_local_extent - 1
|
||||
return text_config.sliding_window_size - 1
|
||||
|
||||
def get_embed_and_head(self):
|
||||
return self.model.embed_tokens.weight, self.lm_head.weight
|
||||
|
||||
|
||||
@@ -158,6 +158,12 @@ class BaseSpecWorker(ABC):
|
||||
def hicache_draft_plan(self) -> HiCacheDraftPlan:
|
||||
return self._hicache_draft_plan
|
||||
|
||||
@property
|
||||
def draft_swa_window(self) -> int:
|
||||
"""Tokens back from the committed length that the draft reads in the
|
||||
target's swa sub-pool; 0 when the target's own window covers them."""
|
||||
return 0
|
||||
|
||||
def weight_update_runners(self) -> list[tuple[str, ModelRunner]]:
|
||||
"""(role, runner) pairs weight ops apply to; [] when the draft owns no weights."""
|
||||
# NGRAM and UNO have no draft model
|
||||
|
||||
@@ -177,6 +177,10 @@ class EagleDraftInput(SpecInput):
|
||||
# shape: (b + 1,)
|
||||
kv_indptr: torch.Tensor = None
|
||||
kv_indices: torch.Tensor = None
|
||||
# The window-clipped counterpart of the pair above, for the draft's
|
||||
# sliding-window layers; None on a pool with no window side.
|
||||
window_kv_indptr: torch.Tensor = None
|
||||
window_kv_indices: torch.Tensor = None
|
||||
|
||||
num_tokens_per_req: int = -1
|
||||
num_tokens_for_logprob_per_req: int = -1
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftSWAKVPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import (
|
||||
Backend,
|
||||
Phase,
|
||||
@@ -587,14 +588,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
|
||||
return
|
||||
seen = set()
|
||||
for runner in self.draft_runner_list:
|
||||
pool = runner.req_to_token_pool.mamba_pool
|
||||
req_pool = runner.req_to_token_pool
|
||||
pool = req_pool.mamba_pool
|
||||
if id(pool) in seen:
|
||||
continue
|
||||
seen.add(id(pool))
|
||||
# The pool is a PHYSICAL store; a unified req pool hands out
|
||||
# virtual slot ids (identity on a static pool).
|
||||
if clear is not None and len(clear) > 0:
|
||||
pool.clear_slots(clear)
|
||||
pool.clear_slots(req_pool.translate_mamba_indices(clear))
|
||||
if cow_src is not None and len(cow_src) > 0:
|
||||
pool.copy_from(cow_src, cow_dst)
|
||||
pool.copy_from(
|
||||
req_pool.translate_mamba_indices(cow_src),
|
||||
req_pool.translate_mamba_indices(cow_dst),
|
||||
)
|
||||
forward_batch.mamba_clear_indices = None
|
||||
forward_batch.mamba_cow_src_indices = None
|
||||
forward_batch.mamba_cow_dst_indices = None
|
||||
@@ -1092,6 +1099,20 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def draft_swa_window(self) -> int:
|
||||
# The widened extend starts front rows below the committed length, and
|
||||
# each depth reads its own window back from there.
|
||||
draft = self._draft_worker
|
||||
windows = [
|
||||
runner.sliding_window_size
|
||||
for runner in draft.draft_runner_list
|
||||
if isinstance(runner.token_to_kv_pool, UnifiedDraftSWAKVPool)
|
||||
]
|
||||
if not windows:
|
||||
return 0
|
||||
return max(windows) + draft.draft_extend_num_front_tokens
|
||||
|
||||
def forward_batch_generation(
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
|
||||
@@ -411,6 +411,11 @@ class SpecInput(ABC):
|
||||
future_dsa_topk_indices_available: bool = False
|
||||
dsa_seed_topk_capture: Optional[torch.Tensor] = None
|
||||
|
||||
# A multi-step draft's per-step sliding-window read rail; only the
|
||||
# multi-step draft backend of a sliding-window KV pool builds it.
|
||||
window_kv_indptr: Optional[torch.Tensor] = None
|
||||
window_kv_indices: Optional[torch.Tensor] = None
|
||||
|
||||
def __init__(self, spec_input_type: SpecInputType):
|
||||
self.spec_input_type = spec_input_type
|
||||
|
||||
|
||||
@@ -23,6 +23,9 @@ from sglang.kernels.ops.speculative.cache_locs import (
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_kv_indices as generate_draft_decode_kv_indices,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_window_kv_indices as generate_draft_decode_window_kv_indices,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
get_src_tgt_cache_loc as get_src_tgt_cache_loc,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user