[Refactor] Translate each KV loc once per iteration (#42753)

This commit is contained in:
Cheng Wan
2026-10-08 00:58:28 -07:00
committed by GitHub
parent 508ddc859a
commit b3e6c87be3
97 changed files with 3393 additions and 3289 deletions
@@ -30,10 +30,11 @@ Two delivery forms over that one formula:
which the pool bounds, where a page table's width is bounded
only by `max_context_len`.
PREFIX-ONLY per row: nothing past the row's live prefix is written, so a
caller-owned buffer keeps what it had there -- which is what lets a captured
cuda-graph buffer be refreshed in place. Readers bound themselves by
`cache_seqlens` and never look past the prefix.
PREFIX-ONLY per row by default: nothing past the row's live prefix is written,
so a caller-owned buffer keeps what it had there -- which is what lets a
captured cuda-graph buffer be refreshed in place. Readers bound themselves by
`cache_seqlens` and never look past the prefix. A fresh (`torch.empty`) table
asks for the sink past the prefix instead (`zero_tail`).
A `-1` in `req_to_token` and a freed (`-1`) v2p row both clamp to entry 0, the
reserved padding slot, so a kernel dereferences padding, not a wild address.
@@ -60,6 +61,56 @@ _TARGET_BLOCKS = 1024
@triton.jit
def _gather_translate_items(
row_in, # this row of `req_to_token` -- VIRTUAL token ids
row_out, # where this row's items land
v2p_ptr, # [num_pages + 1] int64 -- virtual->physical page table
live, # scalar: the row reads `req_to_token`; else every item is the sink
kv_start, # first token of the row's window
n_items, # live items in the row
n_written, # items stored: `n_items`, or a wider zeroed row
item_begin, # this program's first item
item_stride, # items one program advances per loop trip
PAGE_SIZE: tl.constexpr,
EMIT_PER_TOKEN: tl.constexpr,
OUT_INT64: tl.constexpr,
ZERO_TAIL: tl.constexpr,
BLOCK: tl.constexpr,
):
for start in range(item_begin, n_written, item_stride):
item = start + tl.arange(0, BLOCK)
mask = item < n_items
load_mask = mask & live
pos = kv_start + item
if EMIT_PER_TOKEN:
page = pos // PAGE_SIZE
else:
page = pos
tok = tl.load(
row_in + page.to(tl.int64) * PAGE_SIZE, mask=load_mask, other=0
).to(tl.int64)
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
# -1 at ps == 1, which would read one element BEFORE `v2p`.
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=load_mask, other=0), 0)
if EMIT_PER_TOKEN:
value = entry * PAGE_SIZE + pos % PAGE_SIZE
else:
value = entry
store_mask = mask
if ZERO_TAIL:
value = tl.where(mask, value, 0)
store_mask = item < n_written
if OUT_INT64:
tl.store(row_out + item, value, mask=store_mask)
else:
tl.store(row_out + item, value.to(tl.int32), mask=store_mask)
# The per-call widths and lengths stay unspecialized: Triton would otherwise
# compile a new variant (`== 1`, `% 16`) mid-serving each time a batch's shape
# hits a combination it has not seen.
@triton.jit(do_not_specialize=["out_stride", "seq_len_delta", "max_row_items"])
def build_kv_read_indices_kernel(
req_to_token_ptr, # in: [max_reqs, max_context] -- VIRTUAL token ids
req_pool_indices_ptr, # in: [bs] -- req_to_token row per batch lane
@@ -76,6 +127,7 @@ def build_kv_read_indices_kernel(
PAGE_SIZE: tl.constexpr,
EMIT_PER_TOKEN: tl.constexpr,
OUT_INT64: tl.constexpr,
ZERO_TAIL: tl.constexpr,
BLOCK: tl.constexpr,
):
bid = tl.program_id(0)
@@ -98,30 +150,24 @@ def build_kv_read_indices_kernel(
# A widened window can outrun a fixed-width row (a verify near the
# context limit); writing past it would land in the next row.
n_items = tl.minimum(n_items, max_row_items)
for start in range(tl.program_id(1) * BLOCK, n_items, item_stride):
item = start + tl.arange(0, BLOCK)
mask = item < n_items
pos = kv_start + item
if EMIT_PER_TOKEN:
page = pos // PAGE_SIZE
else:
page = pos
tok = tl.load(row_in + page.to(tl.int64) * PAGE_SIZE, mask=mask, other=0).to(
tl.int64
)
# Triton's `//` truncates toward zero, so `-1 // ps` is 0 for ps > 1 but
# -1 at ps == 1, which would read one element BEFORE `v2p`.
vpage = tl.where(tok < 0, 0, tok // PAGE_SIZE)
entry = tl.maximum(tl.load(v2p_ptr + vpage, mask=mask, other=0), 0)
if EMIT_PER_TOKEN:
value = entry * PAGE_SIZE + pos % PAGE_SIZE
else:
value = entry
if OUT_INT64:
tl.store(row_out + item, value, mask=mask)
else:
tl.store(row_out + item, value.to(tl.int32), mask=mask)
# A fresh (unzeroed) table also takes the sink past the live prefix.
n_written = max_row_items if ZERO_TAIL else n_items
_gather_translate_items(
row_in,
row_out,
v2p_ptr,
True,
kv_start,
n_items,
n_written,
tl.program_id(1) * BLOCK,
item_stride,
PAGE_SIZE=PAGE_SIZE,
EMIT_PER_TOKEN=EMIT_PER_TOKEN,
OUT_INT64=OUT_INT64,
ZERO_TAIL=ZERO_TAIL,
BLOCK=BLOCK,
)
def _launch(
@@ -138,6 +184,7 @@ def _launch(
kv_start_idx: Optional[torch.Tensor],
emit_per_token: bool,
seq_len_delta: int = 0,
zero_tail: bool = False,
) -> None:
bs = int(req_pool_indices.numel())
item_programs = min(
@@ -159,6 +206,7 @@ def _launch(
PAGE_SIZE=page_size,
EMIT_PER_TOKEN=emit_per_token,
OUT_INT64=out.dtype == torch.int64,
ZERO_TAIL=zero_tail,
BLOCK=_BLOCK_ITEMS,
num_warps=_NUM_WARPS,
)
@@ -188,17 +236,18 @@ def build_kv_read_table(
max_pages: int,
out: torch.Tensor,
seq_len_delta: int = 0,
zero_tail: bool = False,
) -> torch.Tensor:
"""Fill ``out``'s live prefix with PAGE TABLE entries.
``out`` is caller-owned (fresh zeros for the eager path, the module's
capture-stable buffer for replay) and only its ``[:bs, :max_pages]``
region's live prefix is written -- never rebound, never tail-cleared.
``out`` is caller-owned and only its ``[:bs, :max_pages]`` region is
written: each row's live prefix, and with ``zero_tail`` the sink (0) past
it up to ``max_pages`` -- what a fresh ``torch.empty`` table needs, in the
same launch. Never rebound.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). Every row stops at
``max_pages``: a widened window that outruns the row is cut there rather
than spilling into the next row, on CUDA as on CPU.
``seq_len_delta`` widens every row's live prefix (a verify reads its draft
tokens' KV back from the pool); a widened row is cut at ``max_pages``
rather than spilling into the next row.
"""
bs = int(req_pool_indices.numel())
assert out.dtype == torch.int32, (
@@ -228,6 +277,8 @@ def build_kv_read_table(
v2p=v2p,
page_size=page_size,
).to(torch.int32)
if zero_tail:
out[b, live:max_pages] = 0
return out
_launch(
@@ -243,6 +294,7 @@ def build_kv_read_table(
kv_start_idx=None,
emit_per_token=False,
seq_len_delta=seq_len_delta,
zero_tail=zero_tail,
)
return out
@@ -89,6 +89,29 @@ def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
return tl.where(i64 < 0, 0, tl.maximum(phys * page_size + i64 % page_size, 0))
@triton.jit
def _load_token_ids(
row_ptr,
pos,
mask,
v2p,
page_size: tl.constexpr,
ENTRY_PAGE_SIZE: tl.constexpr,
TRANSLATE: tl.constexpr,
):
"""Token ids at positions ``pos`` of one read-source row: the row's own
entries when token-granular, ``entry * ps + pos % ps`` when page-granular;
with ``TRANSLATE``, virtual ids mapped through ``v2p`` on the way out."""
if ENTRY_PAGE_SIZE == 1:
ids = tl.load(row_ptr + pos, mask=mask)
else:
entry = tl.load(row_ptr + pos // ENTRY_PAGE_SIZE, mask=mask, other=0)
ids = entry.to(tl.int64) * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
if TRANSLATE:
ids = _translate_token_ids(ids, v2p, mask, page_size)
return ids
@triton.jit
def generate_draft_decode_kv_indices(
req_pool_indices,
@@ -97,8 +120,9 @@ def generate_draft_decode_kv_indices(
kv_indices,
kv_indptr,
positions,
v2p,
pool_len: tl.constexpr,
# Runtime, not constexpr: the unified pool's read table is as wide as the
# iteration's longest row, so a constexpr stride would recompile per width.
row_stride,
kv_indices_stride: tl.constexpr,
kv_indptr_stride: tl.constexpr,
bs_upper: tl.constexpr,
@@ -108,6 +132,9 @@ def generate_draft_decode_kv_indices(
window_size: tl.constexpr = 0,
sink_size: tl.constexpr = 0,
NUM_STEPS: tl.constexpr = 0,
ENTRY_PAGE_SIZE: tl.constexpr = 1,
# The source's page table (`KVIndexTable.v2p`), applied with TRANSLATE.
v2p=None,
TRANSLATE: tl.constexpr = False,
):
# window_size > 0 restricts the draft (not the target) to sink_size prefix
@@ -151,7 +178,7 @@ def generate_draft_decode_kv_indices(
# Update kv_indices
kv_offset = cum_seq_len * topk + bid * iters * topk + topk_id * (seq_len_w + iters)
kv_ptr = kv_indices + kv_offset
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * pool_len
token_pool_ptr = req_to_token + tl.load(req_pool_indices + bid) * row_stride
num_loop = tl.cdiv(seq_len_w, BLOCK_SIZE)
if NUM_STEPS != 0 and blk >= num_loop and blk > 0:
@@ -165,9 +192,9 @@ def generate_draft_decode_kv_indices(
copy_offset,
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
data = _load_token_ids(
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
)
tl.store(kv_ptr + copy_offset, data, mask=mask)
copy_offset += BLOCK_SIZE
else:
@@ -179,9 +206,9 @@ def generate_draft_decode_kv_indices(
copy_offset,
recent_start + copy_offset - s_eff,
)
data = tl.load(token_pool_ptr + src, mask=mask)
if TRANSLATE:
data = _translate_token_ids(data, v2p, mask, page_size)
data = _load_token_ids(
token_pool_ptr, src, mask, v2p, page_size, ENTRY_PAGE_SIZE, TRANSLATE
)
tl.store(kv_ptr + copy_offset, data, mask=mask)
# Extension entries and kv_indptr belong to token block 0 alone; other
@@ -189,12 +216,14 @@ def generate_draft_decode_kv_indices(
if blk == 0:
extend_offset = tl.arange(0, iter_upper)
if page_size == 1 or topk == 1:
extend_data = tl.load(
token_pool_ptr
+ seq_len
+ topk_id * num_steps
+ tl.arange(0, iter_upper),
mask=extend_offset < iters,
extend_data = _load_token_ids(
token_pool_ptr,
seq_len + topk_id * num_steps + tl.arange(0, iter_upper),
extend_offset < iters,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
else:
prefix_len = seq_len
@@ -208,14 +237,14 @@ def generate_draft_decode_kv_indices(
+ topk_id * num_new_pages_per_topk * page_size
+ last_page_len
)
extend_data = tl.load(
token_pool_ptr + start + extend_offset,
mask=extend_offset < iters,
)
if TRANSLATE:
extend_data = _translate_token_ids(
extend_data, v2p, extend_offset < iters, page_size
extend_data = _load_token_ids(
token_pool_ptr,
start + extend_offset,
extend_offset < iters,
v2p,
page_size,
ENTRY_PAGE_SIZE,
TRANSLATE,
)
tl.store(
@@ -20,6 +20,9 @@ class RaggedVerifyWindow(msgspec.Struct, frozen=True):
positions: torch.Tensor
verify_cache_loc: torch.Tensor
verify_ids: torch.Tensor
# Each packed lane's flat position in the `[bs, verify_num_draft_tokens]`
# verify window, -1 on a padded lane.
window_index: torch.Tensor
class BuildRaggedVerifyWindow:
@@ -129,6 +132,9 @@ def build_ragged_verify_window(
positions=positions,
verify_cache_loc=verify_cache_loc,
verify_ids=verify_ids,
window_index=torch.where(
valid, safe_req.to(torch.int64) * verify_num_draft_tokens + within, -1
),
)
@@ -176,7 +182,7 @@ def build_ragged_verify_window_triton(
verify_lens = layout.verify_lens.to(device=device, dtype=torch.int32)
padded_total = layout.graph_num_tokens
req_id, within, _valid = compact_row_index_triton(
req_id, within, valid = compact_row_index_triton(
verify_lens=verify_lens, padded_total=padded_total, device=device
)
real_cache_loc = assign_extend_cache_locs_func(
@@ -218,6 +224,11 @@ def build_ragged_verify_window_triton(
positions=positions,
verify_cache_loc=verify_cache_loc,
verify_ids=verify_ids,
window_index=torch.where(
valid,
req_id.clamp(max=bs - 1).to(torch.int64) * verify_num_draft_tokens + within,
-1,
),
)
+9 -14
View File
@@ -499,10 +499,9 @@ def _assert_spec_verify_backends(
) -> None:
"""Refuse target backends whose verify id rails are not translation-audited.
Only the target's prefill/decode pair is checked here. A draft that fuses
into the target's pages must run on the translated rails; the fused-draft
decision enforces that by declining fusion otherwise, and a private-pool
draft indexes its own pool by virtual id."""
Only the target's prefill/decode pair is checked here: fusion declines for
a draft off the translated rails, and a private-pool draft indexes its own
pool by virtual id."""
if allowed is None:
allowed = _SPEC_VERIFY_AUDITED_BACKENDS
dcp_note = ""
@@ -591,8 +590,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"DFLASH",
), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK (chain draft), DFLASH (fused block draft), and "
"EAGLE/EAGLE3 (fused draft KV); other speculative algorithms are "
"DSPARK, DFLASH, EAGLE and EAGLE3; other speculative algorithms are "
"not yet audited for the unified pool's virtual-to-physical loc "
"translation. Got "
f"--speculative-algorithm={cfg.speculative_algorithm!r}."
@@ -627,9 +625,7 @@ def handle_unified_memory_pool(server_args: Any) -> None:
f"spec-verify-audited attention backends {sorted(eagle_allowed)} "
f"(got {sorted(eagle_backends, key=str)})."
)
# An unset draft backend inherits the target's pair. A fused draft is
# MHA-shaped on every host, and these rails also serve a draft that
# keeps an MLA pool of its own.
# An unset draft backend inherits the target's pair.
draft_backend = cfg.speculative_draft_attention_backend
draft_backends = {draft_backend} if draft_backend else eagle_backends
assert draft_backends <= TRANSLATED_MHA_RAILS, (
@@ -649,9 +645,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
)
assert not cfg.enable_two_batch_overlap, (
"--enable-unified-memory does not support --enable-two-batch-overlap: "
"TBO's replay split hands each child a view without the pre-translate "
"write loc, so a captured decode replay raises. "
"TODO(ch-wan): carry out_cache_loc_virtual into the child view."
"the iteration's read tables are indexed by the whole batch's rows, and "
"a TBO child's readers do not offset into them."
)
assert not cfg.enable_lmcache, (
"--enable-unified-memory is not yet compatible with --enable-lmcache: "
@@ -679,8 +674,8 @@ def handle_unified_memory_pool(server_args: Any) -> None:
), (
"--enable-unified-memory + EAGLE/EAGLE3 does not support "
"--disaggregation-decode-retraction-backup=host_pool: the backup builds "
"the draft's host pool off a device pool of its own, and an EAGLE draft "
"on the unified pool lives fused in the target's pages, without one."
"the draft's host pool off the draft's own device pool, which a fused "
"EAGLE draft does not have."
)
assert not (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
@@ -698,6 +698,14 @@ class TboForwardBatchPreparer:
start_token_index:end_token_index
]
output_dict["out_cache_loc_is_physical"] = batch.out_cache_loc_is_physical
# Unified memory refuses two-batch overlap, so the plan's reads stay in
# `req_to_token`, which each child indexes at its own rows; each child
# writes its own tokens of the plan's window.
output_dict["kv_loc_plan"] = batch.kv_loc_plan
if batch.kv_loc_plan is not None:
output_dict["kv_loc_cols"] = batch.kv_loc_plan.cols_slice(
batch.kv_loc_cols, slice(start_token_index, end_token_index)
)
attention_tp_size = get_parallel().attn_tp_size
_tbo_padded_len = (
+4 -12
View File
@@ -18,6 +18,7 @@ from sglang.srt.layers.attention.triton_backend import (
ForwardMetadata,
update_sliding_window_buffer,
)
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_utils.capture_mode import (
get_capture_attention_variant,
@@ -202,12 +203,10 @@ class DllmFlashAttention:
swa_out_cache_loc=torch.empty_like(batch.out_cache_loc)
if backend.use_sliding_window_kv_pool
else None,
out_cache_loc_full_physical=torch.empty_like(batch.out_cache_loc)
if backend.kv_index_translator.is_translating
else None,
)
metadata = self._prefill_graph_metadata[key]
backend._fill_kv_indptr_and_indices(
batch.kv_loc_plan,
bs,
batch.extend_prefix_lens,
batch.req_pool_indices,
@@ -220,6 +219,7 @@ class DllmFlashAttention:
_, _, _, offsets = update_sliding_window_buffer(
metadata.window_kv_indptr,
backend.kv_index_translator,
batch.kv_loc_plan,
batch.req_pool_indices,
backend.sliding_window_size,
batch.extend_prefix_lens,
@@ -231,15 +231,7 @@ class DllmFlashAttention:
metadata.window_kv_offsets.copy_(offsets)
if metadata.swa_out_cache_loc is not None:
metadata.swa_out_cache_loc.copy_(
backend.kv_index_translator.sliding_window_write_loc_for(
batch.out_cache_loc
)
)
if metadata.out_cache_loc_full_physical is not None:
backend.kv_index_translator.fill_capture_write_loc(
out=metadata.out_cache_loc_full_physical,
forward_batch=batch,
width=metadata.out_cache_loc_full_physical.numel(),
backend.kv_index_translator.write_ids(batch, IdSpaceKind.SLIDING_WINDOW)
)
backend.forward_metadata = metadata
@@ -40,6 +40,7 @@ from sglang.srt.utils import is_gfx95_supported
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -1951,6 +1952,7 @@ class AiterAttnBackend(AttentionBackend):
paged_kernel_lens=forward_batch.seq_lens,
paged_kernel_lens_sum=forward_batch.seq_lens_sum,
translator=self.kv_index_translator,
plan=forward_batch.kv_loc_plan,
)
)
self.forward_metadata = ForwardMetadata(
@@ -2150,6 +2152,7 @@ class AiterAttnBackend(AttentionBackend):
max(forward_batch.extend_seq_lens_cpu),
forward_batch.seq_lens_cpu.max().item(),
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
max_q_len = self.mla_indices_updater_prefill.max_q_len
@@ -2190,6 +2193,7 @@ class AiterAttnBackend(AttentionBackend):
prefix_lens,
encoder_lens=forward_batch.encoder_lens,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
if self.use_sliding_window_kv_pool:
@@ -4454,6 +4458,8 @@ class AiterIndicesUpdaterPrefill:
prefix_lens: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -4466,6 +4472,8 @@ class AiterIndicesUpdaterPrefill:
prefix_lens: torch.Tensor,
encoder_lens: Optional[torch.Tensor],
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
kv_start_idx = None
kv_indptr = self.kv_indptr
@@ -4515,6 +4523,7 @@ class AiterIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.kv_index_translator,
plan=plan,
)
)
@@ -4547,6 +4556,8 @@ class AiterMlaIndicesUpdaterPrefill:
max_q_len: int,
max_kv_len: int,
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -4560,6 +4571,8 @@ class AiterMlaIndicesUpdaterPrefill:
max_q_len: int,
max_kv_len: int,
spec_info: Optional[SpecInput],
*,
plan: KVLocPlan,
):
bs = len(req_pool_indices)
@@ -4594,6 +4607,7 @@ class AiterMlaIndicesUpdaterPrefill:
paged_kernel_lens=kv_lens,
paged_kernel_lens_sum=kv_lens_sum,
translator=self.kv_index_translator,
plan=plan,
)
)
@@ -4663,18 +4677,21 @@ class AiterMultiStepDraftBackend:
if self.max_context_len >= _KV_INDEX_BLOCKS_MIN_CONTEXT
else 1
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps * num_token_blocks, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
triton.next_power_of_2(num_seqs),
@@ -4684,7 +4701,9 @@ class AiterMultiStepDraftBackend:
# A single token block is the historical launch; NUM_STEPS=0 keeps
# its 128-wide program instead of the token-block specialization.
NUM_STEPS=self.speculative_num_steps if num_token_blocks > 1 else 0,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
for i in range(self.speculative_num_steps - 1):
@@ -34,6 +34,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_flashinfer_available
@@ -176,7 +177,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
query = self._set_kv_and_concat_q_fp8_fused(
layer=layer,
@@ -208,7 +209,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
assert k is not None and k_rope is not None
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
self._kv_write_loc(forward_batch),
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
@@ -33,7 +33,7 @@ from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_ver
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -54,6 +54,7 @@ from sglang.srt.utils import get_compiler_backend
from sglang.srt.utils.common import get_device_capability
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.page_interleave_pool import PageInterleaveKVPoolMixin
from sglang.srt.model_executor.model_runner import ModelRunner
@@ -466,10 +467,9 @@ class FlashAttentionBackend(AttentionBackend):
if not forward_batch.forward_mode.is_draft_extend_v2():
return
if self.kv_index_translator.is_translating:
# The draft-extend runners record this call unconditionally, but a
# translating backend rebuilds these tables out of graph on every
# replay; a captured raw req_to_token gather would then overwrite
# them with virtual ids.
# The draft-extend runners record this call unconditionally; a
# translating backend rebuilds these tables out of graph, and a
# captured req_to_token gather would overwrite them with virtual ids.
return
bs = forward_batch.batch_size
metadata = self.draft_extend_metadata[bs]
@@ -543,7 +543,11 @@ class FlashAttentionBackend(AttentionBackend):
self._init_full_cg_decode_metadata(forward_batch, in_capture)
def _get_swa_write_locations(self, forward_batch: ForwardBatch):
locations = getattr(forward_batch, "out_cache_loc", None)
"""This step's sliding-window write ids: the plan's, sliced per
draft step the way `out_cache_loc` is."""
locations = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
if (
locations is not None
and forward_batch.forward_mode.is_decode_or_idle()
@@ -586,11 +590,7 @@ class FlashAttentionBackend(AttentionBackend):
if in_capture:
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
)
)
self.cuda_graph_swa_out_cache_loc[:n].copy_(swa_out_cache_loc)
if in_capture:
num_tokens = forward_batch.positions.numel()
@@ -627,6 +627,7 @@ class FlashAttentionBackend(AttentionBackend):
return
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -678,6 +679,7 @@ class FlashAttentionBackend(AttentionBackend):
# A stale non-None seq_lens_cpu buffer would under-size max_seq_pages
# (stale page-table rows -> OOB); force None under sync-free.
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -755,22 +757,17 @@ class FlashAttentionBackend(AttentionBackend):
translating = self.kv_index_translator.is_translating
max_seq_len_k = int(forward_batch.seq_lens_cpu[:bs].max().item())
if translating:
# Unified pool: the block table is a TRANSLATED page table, built
# straight into these capture-stable buffers from the LIVE v2p, so
# a page relocated by compaction since capture is picked up. Same
# substitution the eager extend branch makes in its `_unified_read`
# fixup; `build_index_table` emits page-granular physical ids
# directly, so there is no `// page_size` to undo.
self.kv_index_translator.build_index_table(
req_pool_indices=forward_batch.req_pool_indices[:bs],
seq_lens=forward_batch.seq_lens[:bs],
into=KVReadTables(
full=m.page_table,
sliding_window=(
m.swa_page_table if self.use_sliding_window_kv_pool else None
),
),
# Unified pool: the plan's page table, physical page ids from this
# iteration's v2p, copied into these capture-stable buffers.
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=m.page_table
)
if self.use_sliding_window_kv_pool:
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan,
out=m.swa_page_table,
kind=IdSpaceKind.SLIDING_WINDOW,
)
elif max_seq_len_k > 0:
# Build the block table like the eager extend branch: take every
# page_size-th token slot from req_to_token and divide by page_size.
@@ -797,22 +794,15 @@ class FlashAttentionBackend(AttentionBackend):
self.full_cg_prefill_swa_out_cache_loc.shape[0],
"full-CG prefill SWA write-location buffer",
)
# Under the unified pool `out_cache_loc` was rebound to FULL-side
# physical ids at ForwardBatch construction, so the full->swa map
# cannot be re-run on it: a full physical page number is also a
# valid virtual page of the swa v2p, so the map would silently
# resolve it to an unrelated token's swa slot. Phase 2 of the write
# contract derives the swa loc from them instead.
swa_write_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
if in_capture:
# The runner-built capture batch writes the page-0 sink.
self.full_cg_prefill_swa_out_cache_loc[:num_out].zero_()
else:
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
if translating
else self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
)
)
self.full_cg_prefill_swa_out_cache_loc[:num_out].copy_(swa_write_loc)
# Captured kernels read the full bucket. Route its inactive tail to
# SWA's zero dummy slot to prevent stale writes into live slots.
self.full_cg_prefill_swa_out_cache_loc[num_out:].zero_()
@@ -826,17 +816,14 @@ class FlashAttentionBackend(AttentionBackend):
def _spec_read_seq_len_delta(
self, forward_mode: ForwardMode, spec_info: Optional[SpecInput]
) -> int:
"""Columns past ``seq_lens`` this mode's whole-sequence read covers —
the translated table's fill must reach ``cache_seqlens``. Takes the
two fields rather than a ForwardBatch: the captured build slices its
batch and has none. Now read by both builds, so eager and replay
widen identically.
"""Columns past ``seq_lens`` this mode's whole-sequence read covers;
the translated table must reach ``cache_seqlens``.
Zero for the prefix-only shapes: normal decode/extend, the topk>1
split (drafts read via the expand metadata), draft-extend whose lens
already include the window, and draft-extend's idle batch. The ragged
verify layout's per-row lens are bounded by the draft window, so its
upper bound rides the same delta."""
split (drafts read via the expand metadata), draft-extend (whose lens
already include the window) and its idle batch. The ragged verify
layout's per-row lens are bounded by the draft window, so the same
delta bounds it."""
if spec_info is None:
return 0
if forward_mode.is_target_verify() and self.topk <= 1:
@@ -1237,27 +1224,20 @@ class FlashAttentionBackend(AttentionBackend):
else None
)
if _unified_read:
delta = self._spec_read_seq_len_delta(
forward_batch.forward_mode, forward_batch.spec_info
# The plan's table covers this forward's reads, a verify's and a
# draft decode's window included; the kernels bound each row by
# `cache_seqlens`.
kv_view = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
)
if delta:
kv_view = self.kv_index_translator.widened_index_table(
forward_batch, seq_len_delta=delta
)
else:
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
metadata.page_table = kv_view.ids
if self.use_sliding_window_kv_pool:
metadata.swa_page_table = kv_view.sliding_window_ids
if swa_out_cache_loc is not None:
# The swa write loc was computed from the still-VIRTUAL
# loc at ForwardBatch construction; re-running the
# full->swa map on the physical loc would be garbage.
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
swa_out_cache_loc
)
)
metadata.swa_page_table = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan,
kind=IdSpaceKind.SLIDING_WINDOW,
rows=forward_batch.batch_size,
).ids
metadata.swa_out_cache_loc = swa_out_cache_loc
elif self.use_sliding_window_kv_pool:
# FA3 requires an int32 page_table.
metadata.swa_page_table = (
@@ -1265,12 +1245,7 @@ class FlashAttentionBackend(AttentionBackend):
metadata.page_table
).to(torch.int32)
)
if swa_out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.token_to_kv_pool.translate_loc_from_full_to_swa(
swa_out_cache_loc
)
)
metadata.swa_out_cache_loc = swa_out_cache_loc
# Logical-page KV sharding: capture the batch's gather plan and swap the
# page table to scratch rows. During a sharded extend, attention reads
@@ -2864,6 +2839,7 @@ class FlashAttentionBackend(AttentionBackend):
def _set_decode_page_metadata(
self,
plan: KVLocPlan,
metadata,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -2871,9 +2847,8 @@ class FlashAttentionBackend(AttentionBackend):
) -> None:
"""Fill `cache_seqlens_int32`, `cu_seqlens_k` and the page table(s).
Under the unified pool the translator writes the page tables in place,
so the fused kernel is left with the prefix sum alone -- one pass over
the rows instead of a translated build plus a verbatim copy of it.
Under the unified pool the page tables are the plan's, copied in, so
the fused kernel is left with the prefix sum alone.
"""
translated = self.kv_index_translator.reads_are_translated
normal_decode_set_metadata(
@@ -2891,17 +2866,20 @@ class FlashAttentionBackend(AttentionBackend):
skip_page_table=translated,
)
if translated:
# Fill to `cache_seqlens_int32`, which the kernels bound their reads
# by: a draft decode reads `seq_len_delta` past `seq_lens`.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
sliding_window_out=metadata.swa_page_table,
req_pool_indices=req_pool_indices,
seq_lens=metadata.cache_seqlens_int32,
)
# The plan's table reaches `cache_seqlens_int32`, which the kernels
# bound their reads by: a draft decode reads past `seq_lens`.
bs = int(req_pool_indices.numel())
self.kv_index_translator.copy_page_table(plan, out=metadata.page_table[:bs])
if metadata.swa_page_table is not None:
self.kv_index_translator.copy_page_table(
plan,
out=metadata.swa_page_table[:bs],
kind=IdSpaceKind.SLIDING_WINDOW,
)
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -2937,6 +2915,7 @@ class FlashAttentionBackend(AttentionBackend):
# max_seq_len_k left unset -- unread here (scheduler_metadata
# is normal-decode-only).
self._set_decode_page_metadata(
plan,
metadata,
req_pool_indices,
seq_lens,
@@ -3041,6 +3020,7 @@ class FlashAttentionBackend(AttentionBackend):
else self.max_context_len
)
self._set_decode_page_metadata(
plan,
metadata,
req_pool_indices,
seq_lens,
@@ -3092,7 +3072,7 @@ class FlashAttentionBackend(AttentionBackend):
# max_seq_len_k left unset -- unread here (scheduler_metadata is
# normal-decode-only).
self._set_decode_page_metadata(
metadata, req_pool_indices, verify_lens, verify_delta
plan, metadata, req_pool_indices, verify_lens, verify_delta
)
else:
# When topk > 1, we need two specific target verify metadata, and then merge states
@@ -3225,23 +3205,19 @@ class FlashAttentionBackend(AttentionBackend):
metadata.max_seq_len_k + self.page_size - 1
) // self.page_size
if self.kv_index_translator.reads_are_translated:
# The translator writes kernel-facing page ids straight into
# the captured tables; `sliding_window_out` fills the swa twin
# in the same pass.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
sliding_window_out=(
metadata.swa_page_table
if self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
else None
),
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_len_delta=self._spec_read_seq_len_delta(
forward_mode, spec_info
),
rows = int(req_pool_indices.numel())
self.kv_index_translator.copy_page_table(
plan, out=metadata.page_table[:rows]
)
if (
self.use_sliding_window_kv_pool
and metadata.swa_page_table is not None
):
self.kv_index_translator.copy_page_table(
plan,
out=metadata.swa_page_table[:rows],
kind=IdSpaceKind.SLIDING_WINDOW,
)
else:
page_indices = self.req_to_token[
req_pool_indices[:, None],
@@ -35,6 +35,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
@@ -61,6 +62,7 @@ from sglang.srt.utils import (
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -749,6 +751,7 @@ class FlashInferAttnBackend(AttentionBackend):
fixed_split_size=None,
disable_split_kv=self.disable_cuda_graph_kv_split,
req_pool_indices=req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_target_verify():
self.indices_updater_prefill.update(
@@ -761,6 +764,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=spec_info,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_dllm_extend():
self.indices_updater_prefill.update(
@@ -773,6 +777,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=not self.use_paged,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_draft_extend_v2():
self.indices_updater_prefill.update(
@@ -785,6 +790,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=spec_info,
plan=forward_batch.kv_loc_plan,
)
elif forward_mode.is_extend():
# Plain EXTEND under full prefill CUDA graph. plan() runs
@@ -803,6 +809,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=encoder_lens[:bs] if encoder_lens is not None else None,
spec_info=None,
plan=forward_batch.kv_loc_plan,
)
else:
raise ValueError("Invalid forward mode")
@@ -863,8 +870,8 @@ class FlashInferAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
if in_capture:
@@ -955,8 +962,8 @@ class FlashInferAttnBackend(AttentionBackend):
def init_forward_metadata(self, forward_batch: ForwardBatch):
swa_out_cache_loc = None
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
if forward_batch.forward_mode.is_decode_or_idle():
@@ -970,6 +977,7 @@ class FlashInferAttnBackend(AttentionBackend):
fixed_split_size=self.decode_split_tile_size,
disable_split_kv=False,
req_pool_indices=forward_batch.req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = DecodeMetadata(
self.decode_wrappers, swa_out_cache_loc=swa_out_cache_loc
@@ -985,6 +993,7 @@ class FlashInferAttnBackend(AttentionBackend):
use_ragged=False,
encoder_lens=forward_batch.encoder_lens,
spec_info=forward_batch.spec_info,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrappers_verify,
@@ -1038,6 +1047,7 @@ class FlashInferAttnBackend(AttentionBackend):
cross_attention_custom_mask=forward_batch.cross_attention_custom_mask,
extend_prefix_lens_cpu=forward_batch.extend_prefix_lens_cpu,
custom_kv_indices=self.dq_page_table,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrappers_paged,
@@ -1581,6 +1591,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Keep the signature for type checking. It will be assigned during runtime.
@@ -1597,6 +1608,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
decode_wrappers = decode_wrappers or self.decode_wrappers
@@ -1610,6 +1622,7 @@ class FlashInferIndicesUpdaterDecode:
seq_lens_cpu,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1624,6 +1637,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
assert self.sliding_window_size is not None
@@ -1663,6 +1677,7 @@ class FlashInferIndicesUpdaterDecode:
use_sliding_window_kv_pool=use_sliding_window_kv_pool,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1677,6 +1692,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Cache encoder_lens on CPU to avoid GPU→CPU transfer per call
@@ -1703,6 +1719,7 @@ class FlashInferIndicesUpdaterDecode:
seq_lens_cpu=kv_lens_cpu,
fixed_split_size=fixed_split_size,
disable_split_kv=disable_split_kv,
plan=plan,
req_pool_indices=req_pool_indices,
)
@@ -1719,6 +1736,7 @@ class FlashInferIndicesUpdaterDecode:
fixed_split_size: Optional[int] = None,
disable_split_kv: Optional[bool] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
):
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
@@ -1726,6 +1744,7 @@ class FlashInferIndicesUpdaterDecode:
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
if spec_info is None or getattr(spec_info, "kv_indptr", None) is None:
bs = len(paged_kernel_lens)
kv_indptr[1 : bs + 1] = torch.cumsum(paged_kernel_lens, dim=0)
@@ -1739,14 +1758,14 @@ class FlashInferIndicesUpdaterDecode:
paged_kernel_lens_sum, dtype=torch.int32, device="cuda"
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=use_swa_source,
kind=read_kind,
)
else:
kv_indptr, kv_indices = spec_info.kv_indptr, spec_info.kv_indices
@@ -1868,6 +1887,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
# Keep the signature for type checking. It will be assigned during runtime.
raise NotImplementedError()
@@ -1888,6 +1909,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if use_ragged:
assert prefix_lens is not None
@@ -1918,6 +1941,7 @@ class FlashInferIndicesUpdaterPrefill:
multi_item_params=multi_item_params,
seq_lens_cpu=seq_lens_cpu,
custom_kv_indices=custom_kv_indices,
plan=plan,
)
def update_sliding_window(
@@ -1936,6 +1960,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if custom_kv_indices is not None:
raise RuntimeError(
@@ -2031,6 +2057,7 @@ class FlashInferIndicesUpdaterPrefill:
if (wrapper_id == 0 and not use_ragged and spec_info is None)
else -1
),
plan=plan,
)
def _build_swa_prefix_custom_mask(
@@ -2090,6 +2117,8 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask: Optional[torch.Tensor] = None,
extend_prefix_lens_cpu: Optional[List[int]] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
*,
plan: KVLocPlan,
):
if custom_kv_indices is not None:
raise RuntimeError(
@@ -2125,6 +2154,7 @@ class FlashInferIndicesUpdaterPrefill:
cross_attention_custom_mask=(
cross_attention_custom_mask if wrapper_id == 1 else None
),
plan=plan,
)
def call_begin_forward(
@@ -2148,6 +2178,8 @@ class FlashInferIndicesUpdaterPrefill:
seq_lens_cpu: Optional[torch.Tensor] = None,
custom_kv_indices: Optional[torch.Tensor] = None,
window_left: int = -1,
*,
plan: KVLocPlan,
):
bs = len(seq_lens)
# Unified SWA wrapper-0: gather from the swa canonical directly -- its
@@ -2155,6 +2187,7 @@ class FlashInferIndicesUpdaterPrefill:
# full->swa translate below must not run on top of them.
translator = self.attn_backend.kv_index_translator
use_swa_source = use_sliding_window_kv_pool and translator.reads_are_translated
read_kind = IdSpaceKind.SLIDING_WINDOW if use_swa_source else IdSpaceKind.FULL
if spec_info is None:
assert prefix_lens is not None
assert len(seq_lens) == len(req_pool_indices)
@@ -2180,14 +2213,14 @@ class FlashInferIndicesUpdaterPrefill:
dtype=torch.int32,
device=req_pool_indices.device,
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=use_swa_source,
kind=read_kind,
)
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
qo_indptr = qo_indptr[: bs + 1]
@@ -2202,7 +2235,8 @@ class FlashInferIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
plan=plan,
kind=read_kind,
kv_start_idx=kv_start_idx,
)
)
@@ -2213,7 +2247,8 @@ class FlashInferIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=translator,
sliding_window=use_swa_source,
plan=plan,
kind=read_kind,
)
)
@@ -2392,18 +2427,21 @@ class FlashInferMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
@@ -2412,7 +2450,9 @@ class FlashInferMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -57,6 +57,7 @@ if TYPE_CHECKING:
FlashInferMlaAttnBackend,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -367,6 +368,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
init_metadata_replay=False,
spec_info=spec_info,
req_pool_indices=req_pool_indices[:bs],
plan=forward_batch.kv_loc_plan,
)
self.decode_cuda_graph_metadata[bs] = decode_wrapper
self.forward_metadata = DecodeMetadata(decode_wrapper)
@@ -391,6 +393,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
raise ValueError(f"Invalid mode: {forward_mode=}")
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -408,6 +411,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
prefill_wrapper.plan = partial(fast_mla_prefill_plan, prefill_wrapper)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
@@ -425,6 +429,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
decode_wrapper=self.decode_wrapper,
init_metadata_replay=False,
req_pool_indices=forward_batch.req_pool_indices,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = DecodeMetadata(self.decode_wrapper)
elif forward_batch.forward_mode.is_target_verify():
@@ -436,6 +441,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
prefill_wrapper_paged=self.prefill_wrapper_verify,
use_ragged=False,
spec_info=forward_batch.spec_info,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(self.prefill_wrapper_verify, False)
else:
@@ -484,6 +490,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr_cpu=kv_indptr_cpu,
kv_len_arr_cpu=kv_len_arr_cpu,
plan=forward_batch.kv_loc_plan,
)
self.forward_metadata = PrefillMetadata(
self.prefill_wrapper_paged, use_ragged
@@ -531,6 +538,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -566,6 +574,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
init_metadata_replay=True,
spec_info=spec_info,
req_pool_indices=req_pool_indices[:bs],
plan=plan,
**self.fast_decode_kwargs,
)
elif forward_mode.is_target_verify():
@@ -616,6 +625,7 @@ class FlashInferMLAAttnBackend(AttentionBackend):
if use_generic_fast_plan
else None
),
plan=plan,
)
else:
raise ValueError(f"Invalid forward mode: {forward_mode=}")
@@ -859,6 +869,7 @@ class FlashInferMLAIndicesUpdaterDecode:
init_metadata_replay: bool = False,
spec_info: Optional[SpecInput] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
**fast_decode_kwargs,
):
@@ -871,6 +882,7 @@ class FlashInferMLAIndicesUpdaterDecode:
self.kv_indptr,
init_metadata_replay,
spec_info,
plan=plan,
req_pool_indices=req_pool_indices,
**fast_decode_kwargs,
)
@@ -885,6 +897,7 @@ class FlashInferMLAIndicesUpdaterDecode:
init_metadata_replay: bool = False,
spec_info: Optional[SpecInput] = None,
*,
plan: KVLocPlan,
req_pool_indices: torch.Tensor,
**fast_decode_kwargs,
):
@@ -904,11 +917,11 @@ class FlashInferMLAIndicesUpdaterDecode:
else fast_decode_kwargs["kv_indices"]
)
translator = self.attn_backend.kv_index_translator
is_translated = translator.fill_packed_read_stream(
is_translated = translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
)
@@ -1006,6 +1019,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
fast_verify_plan_kwargs: Optional[dict] = None,
*,
plan: KVLocPlan,
qo_indptr_cpu: Optional[torch.Tensor] = None,
kv_indptr_cpu: Optional[torch.Tensor] = None,
kv_len_arr_cpu: Optional[torch.Tensor] = None,
@@ -1031,6 +1045,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
spec_info,
attn_dcp_metadata=attn_dcp_metadata,
fast_verify_plan_kwargs=fast_verify_plan_kwargs,
plan=plan,
qo_indptr_cpu=qo_indptr_cpu,
kv_indptr_cpu=kv_indptr_cpu,
kv_len_arr_cpu=kv_len_arr_cpu,
@@ -1052,6 +1067,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
attn_dcp_metadata: Optional[DecodeContextParallelMetadata] = None,
fast_verify_plan_kwargs: Optional[dict] = None,
*,
plan: KVLocPlan,
qo_indptr_cpu: Optional[torch.Tensor] = None,
kv_indptr_cpu: Optional[torch.Tensor] = None,
kv_len_arr_cpu: Optional[torch.Tensor] = None,
@@ -1068,11 +1084,11 @@ class FlashInferMLAIndicesUpdaterPrefill:
dtype=torch.int32,
device=req_pool_indices.device,
)
self.attn_backend.kv_index_translator.fill_packed_read_stream(
self.attn_backend.kv_index_translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=kv_indptr,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
)
qo_indptr[1 : bs + 1] = torch.cumsum(seq_lens - prefix_lens, dim=0)
@@ -1085,6 +1101,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
plan=plan,
kv_indices_buf=fast_verify_plan_kwargs["kv_indices_buf"],
)
)
@@ -1097,6 +1114,7 @@ class FlashInferMLAIndicesUpdaterPrefill:
paged_kernel_lens=paged_kernel_lens,
paged_kernel_lens_sum=paged_kernel_lens_sum,
translator=self.attn_backend.kv_index_translator,
plan=plan,
)
)
@@ -1241,25 +1259,30 @@ class FlashInferMLAMultiStepDraftBackend:
seq_lens_sum=seq_lens_sum,
)
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
self.generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(self.speculative_num_steps),
next_power_of_2(bs),
self.page_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
assert forward_batch.spec_info is not None
@@ -28,6 +28,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -125,6 +126,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
or forward_mode.is_draft_extend_v2()
):
self._apply_decode_target_verify_metadata(
plan=forward_batch.kv_loc_plan,
bs=forward_batch.batch_size,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
@@ -165,10 +167,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -200,10 +200,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -240,10 +238,8 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
block_kv_indices = self._eager_block_kv_indices(bs, max_seqlen_pad)
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=seq_lens_k,
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=block_kv_indices
)
else:
create_flashmla_kv_indices_triton[
@@ -328,6 +324,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
def _apply_decode_target_verify_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -355,10 +352,9 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
if self.kv_index_translator.reads_are_translated:
assert self.page_size == PAGE_SIZE
self.kv_index_translator.fill_read_table(
out=self.cuda_graph_kv_indices,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
# The plan's table reaches a verify's draft tail.
self.kv_index_translator.copy_page_table(
plan, out=self.cuda_graph_kv_indices[:bs]
)
else:
create_flashmla_kv_indices_triton[
@@ -251,6 +251,7 @@ def _build_tbo_child_replay_fb_view(
child_seq_lens_cpu = fb_view.seq_lens_cpu[seq_slice]
parent_input_ids = getattr(fb_view, "input_ids", None)
parent_out_cache_loc = getattr(fb_view, "out_cache_loc", None)
plan = getattr(fb_view, "kv_loc_plan", None)
return SimpleNamespace(
batch_size=child_bs,
forward_mode=fb_view.forward_mode,
@@ -270,6 +271,15 @@ def _build_tbo_child_replay_fb_view(
if parent_out_cache_loc is not None
else None
),
# Unified memory refuses two-batch overlap, so the plan's reads stay in
# `req_to_token`, which each child indexes at its own rows; each child
# writes its own tokens of the plan's window.
kv_loc_plan=plan,
kv_loc_cols=(
None
if plan is None
else plan.cols_slice(getattr(fb_view, "kv_loc_cols", None), tok_slice)
),
spec_info=child_spec_info,
max_seq_len_override=fb_view.max_seq_len_override,
)
@@ -404,7 +404,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
if save_kv_cache:
self.token_to_kv_pool.set_mla_kv_buffer(
layer,
self._kv_write_loc(forward_batch),
KVWriteLoc.for_batch(forward_batch),
k,
k_rope,
)
@@ -30,6 +30,7 @@ from sglang.srt.layers.dcp import (
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.cuda_graph_config import (
@@ -68,6 +69,7 @@ if _is_cuda:
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.spec_info import SpecInput
@@ -130,9 +132,6 @@ class ForwardMetadata:
swa_attn_logits: Optional[torch.Tensor] = None
# full->SWA translated out_cache_loc (SWA KV-store write target)
swa_out_cache_loc: Optional[torch.Tensor] = None
# PHYSICAL full-attn write target for the unified pool (eager: translated tensor;
# cuda-graph: capture-stable buffer view). None for non-unified pools.
out_cache_loc_full_physical: Optional[torch.Tensor] = None
# Lean decode (persistent-grid partial-result buffers)
lean_Mp: Optional[torch.Tensor] = None
lean_Lp: Optional[torch.Tensor] = None
@@ -503,6 +502,7 @@ class TritonAttnBackend(AttentionBackend):
def _fill_kv_indptr_and_indices(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -512,17 +512,18 @@ class TritonAttnBackend(AttentionBackend):
if kv_indptr is None:
kv_indptr = self.kv_indptr[: bs + 1]
kv_indptr[1:] = torch.cumsum(seq_lens, dim=0)
self.kv_index_translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
self.kv_index_translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens[:bs],
indptr=kv_indptr,
total_tokens=kv_indices.numel(),
out=kv_indices,
)
return kv_indptr
def _update_decode_kv_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -551,7 +552,7 @@ class TritonAttnBackend(AttentionBackend):
num_kv_splits_lens = dcp_seq_lens.clamp_min(1)
else:
kv_indptr = self._fill_kv_indptr_and_indices(
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
)
num_kv_splits_lens = seq_lens
window_kv_indptr = self.window_kv_indptr
@@ -560,6 +561,7 @@ class TritonAttnBackend(AttentionBackend):
window_kv_indptr, _, window_kv_lens, _ = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
seq_lens,
@@ -577,6 +579,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_target_verify_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
spec_info,
@@ -593,7 +596,7 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
kv_indptr = self._fill_kv_indptr_and_indices(
bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, seq_lens, req_pool_indices, self.cuda_graph_kv_indices
)
window_kv_indptr = self.window_kv_indptr
window_kv_indices = None
@@ -607,6 +610,7 @@ class TritonAttnBackend(AttentionBackend):
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
seq_lens[:bs],
@@ -638,6 +642,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_dllm_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
req_pool_indices: torch.Tensor,
@@ -654,12 +659,13 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
self._fill_kv_indptr_and_indices(
bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, prefix_lens, req_pool_indices, self.cuda_graph_kv_indices
)
if self.sliding_window_size is not None and self.sliding_window_size > 0:
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
plan,
req_pool_indices,
self.sliding_window_size,
prefix_lens,
@@ -670,6 +676,7 @@ class TritonAttnBackend(AttentionBackend):
def _update_draft_extend_buffers(
self,
plan: KVLocPlan,
bs: int,
seq_lens: torch.Tensor,
forward_mode: ForwardMode,
@@ -705,7 +712,7 @@ class TritonAttnBackend(AttentionBackend):
extend_seq_lens = torch.zeros(bs, dtype=torch.int32, device=seq_lens.device)
kv_lens = torch.clamp(seq_lens - extend_seq_lens, min=0).to(torch.int32)
kv_indptr = self._fill_kv_indptr_and_indices(
bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
plan, bs, kv_lens, req_pool_indices, self.cuda_graph_kv_indices
)
return qo_indptr, kv_indptr, num_tokens_per_req
@@ -755,15 +762,13 @@ class TritonAttnBackend(AttentionBackend):
return
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
forward_mode=forward_mode,
spec_info=spec_info,
)
out_cache_loc_full_physical = self._fill_cuda_graph_write_locs(
forward_batch, bs
)
swa_out_cache_loc = self._fill_cuda_graph_swa_out_cache_loc(
forward_batch, in_capture=True
)
@@ -772,25 +777,25 @@ class TritonAttnBackend(AttentionBackend):
forward_mode,
spec_info,
swa_out_cache_loc,
out_cache_loc_full_physical,
)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
forward_mode=forward_mode,
spec_info=spec_info,
)
# Metadata view is reused from capture; just refill the buffers.
self._fill_cuda_graph_write_locs(forward_batch, bs)
# Metadata view is reused from capture; the write loc is the
# runner's slot, so only the swa twin needs refilling.
self._fill_cuda_graph_swa_out_cache_loc(forward_batch)
def _fill_cuda_graph_swa_out_cache_loc(
self, forward_batch: ForwardBatch, in_capture: bool = False
) -> Optional[torch.Tensor]:
"""Refill the SWA write-target buffer from the batch's derived
sliding-window write loc, returning the [:n] view (None for non-SWA /
"""Refill the SWA write-target buffer with the batch's write ids in the
sliding-window sub-pool, returning the [:n] view (None for non-SWA /
multi-step draft) so the captured store reads fresh slots on replay.
"""
if not self.use_sliding_window_kv_pool:
@@ -807,24 +812,12 @@ class TritonAttnBackend(AttentionBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(out_cache_loc)
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
return self.cuda_graph_swa_out_cache_loc[:n]
def _fill_cuda_graph_write_locs(
self, forward_batch: ForwardBatch, bs: int
) -> Optional[torch.Tensor]:
"""Runs BEFORE graph.replay(), so it reads the live post-compaction
v2p; no-op for non-unified pools."""
# The buffer exists only for a translating pool; return before naming it.
if not self.kv_index_translator.is_translating:
return None
return self.kv_index_translator.fill_capture_write_loc(
out=self.cuda_graph_out_cache_loc_full_physical,
forward_batch=forward_batch,
width=self.cuda_graph_out_cache_loc_full_physical.numel(),
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Init auxiliary variables for triton attention backend."""
@@ -862,6 +855,7 @@ class TritonAttnBackend(AttentionBackend):
seq_lens_sum, dtype=torch.int64, device=self.device
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.seq_lens,
forward_batch.req_pool_indices,
@@ -875,6 +869,7 @@ class TritonAttnBackend(AttentionBackend):
update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.seq_lens,
@@ -961,6 +956,7 @@ class TritonAttnBackend(AttentionBackend):
seq_lens_sum, dtype=torch.int64, device=self.device
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.seq_lens,
forward_batch.req_pool_indices,
@@ -977,6 +973,7 @@ class TritonAttnBackend(AttentionBackend):
) = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.seq_lens,
@@ -1016,6 +1013,7 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
kv_indptr = self._fill_kv_indptr_and_indices(
forward_batch.kv_loc_plan,
bs,
forward_batch.extend_prefix_lens,
forward_batch.req_pool_indices,
@@ -1030,6 +1028,7 @@ class TritonAttnBackend(AttentionBackend):
) = update_sliding_window_buffer(
self.window_kv_indptr,
self.kv_index_translator,
forward_batch.kv_loc_plan,
forward_batch.req_pool_indices,
self.sliding_window_size,
forward_batch.extend_prefix_lens,
@@ -1054,8 +1053,8 @@ class TritonAttnBackend(AttentionBackend):
swa_out_cache_loc = None
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
swa_out_cache_loc = self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
self.forward_metadata = ForwardMetadata(
@@ -1074,11 +1073,6 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets,
swa_attn_logits=swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=(
forward_batch.out_cache_loc
if self.kv_index_translator.is_translating
else None
),
lean_Mp=lean_Mp,
lean_Lp=lean_Lp,
lean_Op=lean_Op,
@@ -1201,22 +1195,12 @@ class TritonAttnBackend(AttentionBackend):
device=self.device,
)
if self.kv_index_translator.is_translating:
# Unified pool full-attention write-target buffer, refilled at replay
# (-> KVWriteLoc.full_loc). Capture-stable, mirrors cuda_graph_swa_out_cache_loc.
self.cuda_graph_out_cache_loc_full_physical = torch.zeros(
(max_num_tokens,),
dtype=torch.int64,
device=self.device,
)
def _build_cuda_graph_forward_metadata(
self,
bs: int,
forward_mode: ForwardMode,
spec_info: Optional[SpecInput],
swa_out_cache_loc: Optional[torch.Tensor] = None,
out_cache_loc_full_physical: Optional[torch.Tensor] = None,
) -> ForwardMetadata:
"""Construct ForwardMetadata from the current cuda-graph buffer state.
@@ -1247,7 +1231,6 @@ class TritonAttnBackend(AttentionBackend):
window_kv_offsets=None,
swa_attn_logits=self.cuda_graph_swa_attn_logits,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
lean_Mp=self.cuda_graph_lean_Mp,
lean_Lp=self.cuda_graph_lean_Lp,
lean_Op=self.cuda_graph_lean_Op,
@@ -1279,7 +1262,6 @@ class TritonAttnBackend(AttentionBackend):
),
window_kv_offsets=self.cuda_graph_window_kv_offsets if swa else None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_dllm_extend():
return ForwardMetadata(
@@ -1297,7 +1279,6 @@ class TritonAttnBackend(AttentionBackend):
window_num_kv_splits=None,
window_kv_offsets=None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
elif forward_mode.is_draft_extend_v2():
return ForwardMetadata(
@@ -1323,13 +1304,13 @@ class TritonAttnBackend(AttentionBackend):
window_num_kv_splits=None,
window_kv_offsets=None,
swa_out_cache_loc=swa_out_cache_loc,
out_cache_loc_full_physical=out_cache_loc_full_physical,
)
else:
raise ValueError(f"Invalid forward mode: {forward_mode=} for CUDA Graph.")
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -1344,7 +1325,7 @@ class TritonAttnBackend(AttentionBackend):
if forward_mode.is_decode_or_idle():
assert spec_info is None, "Multi-step cuda graph init is not done here."
_, _, window_kv_lens, num_kv_splits_lens = self._update_decode_kv_buffers(
bs, seq_lens, req_pool_indices
plan, bs, seq_lens, req_pool_indices
)
self.get_num_kv_splits(
self.cuda_graph_num_kv_splits[:bs], num_kv_splits_lens[:bs]
@@ -1356,13 +1337,13 @@ class TritonAttnBackend(AttentionBackend):
elif forward_mode.is_target_verify():
bs = len(req_pool_indices)
self._update_target_verify_buffers(
bs, seq_lens, spec_info, req_pool_indices
plan, bs, seq_lens, spec_info, req_pool_indices
)
elif forward_mode.is_dllm_extend():
self._update_dllm_buffers(bs, seq_lens, req_pool_indices)
self._update_dllm_buffers(plan, bs, seq_lens, req_pool_indices)
elif forward_mode.is_draft_extend_v2():
self._update_draft_extend_buffers(
bs, seq_lens, forward_mode, spec_info, req_pool_indices
plan, bs, seq_lens, forward_mode, spec_info, req_pool_indices
)
else:
raise ValueError(
@@ -1649,7 +1630,6 @@ class TritonAttnBackend(AttentionBackend):
loc_info = KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
)
if layer.k_scale is None:
self._set_kv_buffer(forward_batch, layer, loc_info, k, v)
@@ -2123,8 +2103,6 @@ class TritonAttnBackend(AttentionBackend):
"window-layer extend before the metadata carried a "
"sliding-window write loc"
)
elif self.forward_metadata.out_cache_loc_full_physical is not None:
extend_kv_indices = self.forward_metadata.out_cache_loc_full_physical
# Capture batches may not have a spec_info, so use the attention
# metadata's resolved uniform verify width when extend lengths are absent.
@@ -2234,14 +2212,9 @@ class TritonAttnBackend(AttentionBackend):
k.div_(layer.k_scale)
self.token_to_kv_pool.set_kv_buffer(
layer,
# `full_loc` carries the pre-translated loc under the unified
# pool, refreshed into a capture-stable buffer before replay —
# translating inside set_kv_buffer would be captured and replay
# a stale v2p. None (-> raw loc) for static pools.
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
v,
@@ -2253,7 +2226,6 @@ class TritonAttnBackend(AttentionBackend):
KVWriteLoc.for_batch(
forward_batch,
swa_loc=self.forward_metadata.swa_out_cache_loc,
full_loc=self.forward_metadata.out_cache_loc_full_physical,
),
k,
v,
@@ -2468,18 +2440,21 @@ class TritonMultiStepDraftBackend:
# over-estimate is safe. Use a static UB to skip the per-iter .sum().item() D2H.
seq_lens_sum = num_seqs * self.max_context_len
v2p = self.kv_index_translator.full_flat_v2p()
src = self.kv_index_translator.read_source(
forward_batch.kv_loc_plan,
req_pool_indices=forward_batch.req_pool_indices,
bs=num_seqs,
)
generate_draft_decode_kv_indices[
(self.speculative_num_steps, num_seqs, self.topk)
](
forward_batch.req_pool_indices,
self.req_to_token_pool.req_to_token,
src.row_ids,
src.ids,
forward_batch.seq_lens,
kv_indices_buffer,
self.kv_indptr,
forward_batch.positions,
v2p,
self.pool_len,
src.row_stride,
kv_indices_buffer.shape[1],
self.kv_indptr.shape[1],
next_power_of_2(num_seqs),
@@ -2488,7 +2463,9 @@ class TritonMultiStepDraftBackend:
self.page_size,
self.draft_window_size,
self.draft_sink_size,
TRANSLATE=v2p is not None,
ENTRY_PAGE_SIZE=src.entry_page_size,
v2p=src.v2p,
TRANSLATE=src.v2p is not None,
)
if call_fn is None:
@@ -2590,6 +2567,7 @@ class TritonMultiStepDraftBackend:
def update_sliding_window_buffer(
window_kv_indptr,
translator,
plan,
req_pool_indices,
sliding_window_size,
seq_lens,
@@ -2626,16 +2604,18 @@ def update_sliding_window_buffer(
if isinstance(token_to_kv_pool, SWAKVPool)
else None
)
translated = translator.fill_packed_read_stream(
req_pool_indices=req_pool_indices[:bs],
translated = translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=window_kv_lens,
indptr=window_kv_indptr,
total_tokens=window_kv_indices.numel(),
out=window_kv_indices,
kv_start_idx=window_kv_start_idx,
sliding_window=(
translator.reads_are_translated
kind=(
IdSpaceKind.SLIDING_WINDOW
if translator.reads_are_translated
and isinstance(token_to_kv_pool, BaseSWAKVPool)
else IdSpaceKind.FULL
),
token_mapping=token_mapping,
)
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
KVCacheAttentionAccessKind,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
@@ -1088,16 +1089,18 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# fused kernel skips its page-table writes so the graph reads the
# refreshed content through pointers baked at capture).
metadata = self.forward_metadata
# `cache_seqlens_int32` is what the attention kernels bound their
# page-table reads by, and the fused metadata call above wrote it.
# A target verify reads `draft_token_num` further than `seq_lens`
# goes, so filling to `seq_lens` leaves those columns untranslated.
self.kv_index_translator.fill_read_table(
out=metadata.page_table,
req_pool_indices=forward_batch.req_pool_indices[:bs],
seq_lens=metadata.cache_seqlens_int32,
sliding_window_out=metadata.swa_page_table,
# The kernels bound their page-table reads by `cache_seqlens_int32`
# (the fused metadata call above wrote it); the plan's table
# reaches that far, a target verify's draft tail included.
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan, out=metadata.page_table[:bs]
)
if metadata.swa_page_table is not None:
self.kv_index_translator.copy_page_table(
forward_batch.kv_loc_plan,
out=metadata.swa_page_table[:bs],
kind=IdSpaceKind.SLIDING_WINDOW,
)
# A capture batch carries no prepared write loc; zeros are the
# page-0 sink.
if (
@@ -1110,8 +1113,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
@@ -1153,7 +1156,7 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
metadata, req_pool_indices, metadata.cache_seqlens_int32
)
# The fused in-graph kernel also skips ragged batches, so refill the
# SWA write-target buffer here (out_cache_loc -> SWA locs).
# SWA write-target buffer here, from the plan.
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
n = forward_batch.out_cache_loc.shape[0]
self.cuda_graph_swa_out_cache_loc[n:].zero_()
@@ -1161,8 +1164,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
self.cuda_graph_swa_out_cache_loc[:n].zero_()
else:
self.cuda_graph_swa_out_cache_loc[:n].copy_(
self.token_to_kv_pool.translate_loc_from_full_to_swa(
forward_batch.out_cache_loc
self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
)
@@ -1266,12 +1269,23 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
else:
metadata.cu_seqlens_q = metadata.cu_seqlens_k
kv_view = self.kv_index_translator.index_table_for_batch(forward_batch)
kv_view = self.kv_index_translator.read_table(
forward_batch.kv_loc_plan, rows=forward_batch.batch_size
)
if kv_view.is_translated:
# No fill kernel: the kernels take the tensor's own width/stride
# and bound their reads by cache_seqlens.
metadata.page_table = kv_view.ids
metadata.swa_page_table = kv_view.sliding_window_ids
metadata.swa_page_table = (
self.kv_index_translator.read_table(
forward_batch.kv_loc_plan,
kind=IdSpaceKind.SLIDING_WINDOW,
rows=forward_batch.batch_size,
).ids
if self.kv_index_translator.space(IdSpaceKind.SLIDING_WINDOW)
is not None
else None
)
else:
has_swa = self._swa_kv_pool is not None
metadata.page_table = torch.empty(
@@ -1300,10 +1314,8 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
# int64 scatter index (unlike the int32 read page table above).
if self.use_sliding_window_kv_pool and forward_batch.out_cache_loc is not None:
metadata.swa_out_cache_loc = (
self.kv_index_translator.sliding_window_write_loc_for(
forward_batch.out_cache_loc
)
metadata.swa_out_cache_loc = self.kv_index_translator.write_ids(
forward_batch, IdSpaceKind.SLIDING_WINDOW
)
self.forward_metadata = metadata
@@ -79,6 +79,7 @@ if is_flashinfer_available():
if TYPE_CHECKING:
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
logger = logging.getLogger(__name__)
@@ -299,10 +300,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
# Tree-mask scratch is fetched from the target backend only.
self.is_draft_runner = model_runner.is_draft_worker
# [:n] view of a capture-stable buffer on the cuda-graph path; None on
# the eager path, which passes forward_batch.out_cache_loc through.
self._decode_kernel_loc: Optional[torch.Tensor] = None
self.cuda_graph_out_cache_loc_kernel: Optional[torch.Tensor] = None
# Fused KV-scatter + q-concat on the decode dense-loc path (one launch
# instead of set_mla_kv_buffer + concat_mla_absorb_q). Disabled under
# async asserts: the fused path writes the pool directly and would
@@ -411,6 +408,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
def _create_block_kv_indices(
self,
plan: KVLocPlan,
batch_size: int,
max_blocks: int,
req_pool_indices: torch.Tensor,
@@ -430,11 +428,19 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
Returns:
Block KV indices tensor
"""
dcp_enabled = get_parallel().dcp_enabled
translated = self.kv_index_translator.is_translating and not dcp_enabled
# A translated table holds the plan's page ids and reads the sink past
# them: a draft extend pads every row's key length to its widest row,
# and the padded queries (whose outputs are dropped) read that far.
block_kv_indices = torch.full(
(batch_size, max_blocks), -1, dtype=torch.int32, device=device
(batch_size, max_blocks),
0 if translated else -1,
dtype=torch.int32,
device=device,
)
if get_parallel().dcp_enabled:
if dcp_enabled:
self._fill_dcp_block_kv_indices(
block_kv_indices,
req_pool_indices,
@@ -442,12 +448,8 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
return block_kv_indices
if self.kv_index_translator.is_translating:
self.kv_index_translator.fill_read_table(
out=block_kv_indices,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
)
if translated:
self.kv_index_translator.copy_page_table(plan, out=block_kv_indices)
else:
create_flashmla_kv_indices_triton[
(
@@ -480,13 +482,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
self.decode_cuda_graph_kv_indices = torch.full(
(max_bs, max_blocks_per_seq), -1, dtype=torch.int32, device=self.device
)
# Unified pool: capture-stable buffer for the physical KV write loc, filled
# out-of-graph in init_forward_metadata_out_graph so the in-graph
# set_mla_kv_buffer captures no translate.
if self.kv_index_translator.is_translating:
self.cuda_graph_out_cache_loc_kernel = torch.zeros(
max_num_tokens, dtype=torch.int64, device=self.device
)
num_tokens_per_req = max_num_tokens // max_bs
if is_float4_e2m1fn_x2(self.data_type):
@@ -603,6 +598,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
def _apply_cuda_graph_metadata(
self,
plan: KVLocPlan,
bs: int,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
@@ -642,10 +638,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
# Update block indices for new sequences.
if self.kv_index_translator.is_translating:
self.kv_index_translator.fill_read_table(
out=metadata.block_kv_indices,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
# The plan's table reaches a verify's draft tail.
self.kv_index_translator.copy_page_table(
plan, out=metadata.block_kv_indices[:bs]
)
else:
create_flashmla_kv_indices_triton[
@@ -755,6 +750,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
forward_batch.seq_lens.device,
)
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
@@ -762,56 +758,13 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
)
else:
self._apply_cuda_graph_metadata(
plan=forward_batch.kv_loc_plan,
bs=bs,
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
forward_mode=forward_mode,
)
# Out-of-graph on capture AND every replay-prep, so the in-graph
# set_mla_kv_buffer captures no translate.
if self.kv_index_translator.is_translating and (
forward_mode.is_decode_or_idle() or forward_mode.is_target_verify()
):
# The captured kernel consumes the whole buffer, so the tail a
# shorter replay leaves must go to slot 0 rather than live pages.
self._decode_kernel_loc = self.kv_index_translator.fill_capture_write_loc(
out=self.cuda_graph_out_cache_loc_kernel,
forward_batch=forward_batch,
width=self.cuda_graph_out_cache_loc_kernel.numel(),
)
else:
self._decode_kernel_loc = None
def _kv_write_loc(self, forward_batch: ForwardBatch) -> KVWriteLoc:
"""The loc an unfused KV scatter must write at: the capture-stable
buffer under a captured unified-pool decode, since the translate
rebinds `out_cache_loc` to a fresh tensor the graph never recorded;
the batch's own loc everywhere else.
"""
if self._decode_kernel_loc is not None:
# Filled by `fill_capture_write_loc`, which translates it.
return KVWriteLoc(self._decode_kernel_loc, physical=True)
return KVWriteLoc.for_batch(forward_batch)
def _resolve_fused_write_loc(
self, forward_batch: ForwardBatch
) -> Optional[torch.Tensor]:
"""Write loc for the fused fp8 KV scatter, or None when this batch is
not covered by it.
Captured decode refills `_decode_kernel_loc` out of the graph, and the
captured kernel must read that buffer. Eager decode on a unified pool
has no such buffer, and the caller falls back to the unfused path.
"""
if self._decode_kernel_loc is not None:
return self._decode_kernel_loc
return (
None
if self.kv_index_translator.is_translating
else forward_batch.out_cache_loc
)
def init_forward_metadata(self, forward_batch: ForwardBatch):
"""Initialize the metadata for a forward pass."""
if self._kv_shard_pool is not None:
@@ -821,7 +774,6 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
forward_batch,
)
self._decode_kernel_loc = None
# Delegate to parent for non-decode modes.
if (
forward_batch.forward_mode.is_extend()
@@ -912,6 +864,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
max_seqlen_pad = self._calc_padded_blocks(max_seq)
block_kv_indices = self._create_block_kv_indices(
forward_batch.kv_loc_plan,
bs,
max_seqlen_pad,
forward_batch.req_pool_indices,
@@ -1287,7 +1240,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
# Fused: bf16->fp8 quantize + KV scatter + q concat
# in one launch; None when not covered.
@@ -1323,42 +1276,23 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert k is not None and k_rope is not None, (
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
)
if self._decode_kernel_loc is not None:
if merge_query and self._fused_set_kv_concat_q:
# Fused: KV scatter + [q_nope | q_rope] concat in one
# launch; None when the inputs are not covered.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=self._decode_kernel_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
)
else:
# eager (or static pool): out_cache_loc is physical.
if (
merge_query
and self._fused_set_kv_concat_q
and not self.kv_index_translator.is_translating
):
# Static pool only, conservatively.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=forward_batch.out_cache_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# `out_cache_loc` is physical on every pool, eager or captured
# (the runner's slot).
if merge_query and self._fused_set_kv_concat_q:
# Fused: KV scatter + [q_nope | q_rope] concat in one launch;
# None when the inputs are not covered.
query = self._set_kv_and_concat_q_fused(
layer=layer,
loc=forward_batch.out_cache_loc,
k=k,
k_rope=k_rope,
q=q,
q_rope=q_rope,
)
if query is None:
self.token_to_kv_pool.set_mla_kv_buffer(
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# Prepare query tensor inline (already built when the fused save-KV
# path ran)
@@ -1515,7 +1449,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
assert q_rope is not None and k_rope is not None
if cos_sin_cache is None:
if save_kv_cache and self._fused_set_kv_concat_q_fp8:
loc = self._resolve_fused_write_loc(forward_batch)
loc = forward_batch.out_cache_loc
if loc is not None:
# Fused: bf16->fp8 quantize + KV scatter + q concat
# in one launch; None when not covered.
@@ -1551,7 +1485,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
"For populating trtllm_mla kv cache, both k_nope and k_rope should be not None."
)
self.token_to_kv_pool.set_mla_kv_buffer(
layer, self._kv_write_loc(forward_batch), k, k_rope
layer, KVWriteLoc.for_batch(forward_batch), k, k_rope
)
# TODO refactor to avoid code duplication
+2
View File
@@ -604,6 +604,7 @@ class TpModelWorker(BaseTpWorker):
skip_attn_backend_init: Optional[bool] = None, # deprecated
*,
capture_hidden_mode: Optional[CaptureHiddenMode] = None,
return_kv_loc_plan: bool = False,
) -> GenerationBatchResult:
# Get forward batch from schedule batch
if batch is not None:
@@ -649,6 +650,7 @@ class TpModelWorker(BaseTpWorker):
expert_distribution_metrics=out.expert_distribution_metrics,
routed_experts_output=out.routed_experts_output,
indexer_topk_output=out.indexer_topk_output,
kv_loc_plan=forward_batch.kv_loc_plan if return_kv_loc_plan else None,
)
capture_pre_sample_logits(batch, forward_batch, logits_output)
+6
View File
@@ -26,6 +26,7 @@ from sglang.srt.utils.common import async_d2h as _async_d2h
if TYPE_CHECKING:
from sglang.srt.managers.auxiliary_output import HostAuxiliaryOutput
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.speculative.spec_info import SpecInput
@@ -118,6 +119,11 @@ class GenerationBatchResult:
# until the accepted path comes back over the relay.
spec_verify_out_cache_loc: Optional[torch.Tensor] = None
# The plan the forward wrote its KV through, when the caller asked for it
# (`return_kv_loc_plan`): a speculative worker's own writes into the same
# slots take their ids from it.
kv_loc_plan: Optional[KVLocPlan] = None
# PP+spec: [bs, spec_steps + 1] global node indices of the accepted path.
# Every stage holds the KV for its own layers, so every stage has to compact
# that path into its committed prefix; only the last stage can compute it.
@@ -510,10 +510,9 @@ class KVCacheConfigurator:
token_to_kv_pool = None
# Unified-pool fast path: build req_to_token + token_to_kv pool + allocator
# from one byte buffer, then return. Gated to the target worker by ROLE:
# a compact-window DFLASH draft also passes req_to_token_pool=None, and
# would allocate a SECOND unified byte buffer here. Supports hybrid
# Mamba and hybrid SWA (not DSV4).
# from one byte buffer, then return. Target worker only: a compact-window
# DFLASH draft also passes req_to_token_pool=None. Supports hybrid Mamba
# and hybrid SWA (not DSV4).
if (
get_memory().enable_unified_memory
and req_to_token_pool is None
@@ -581,7 +580,6 @@ class KVCacheConfigurator:
alloc = token_to_kv_pool_allocator
placement = self._fused_draft_from_target_buffer(alloc)
if placement is not None:
# FUSED arm
from sglang.srt.mem_cache.layout.fused_draft import (
draft_swa_layer_ids,
)
@@ -619,7 +617,6 @@ class KVCacheConfigurator:
token_to_kv_pool_allocator=alloc,
unified_memory_pool=None,
)
# PRIVATE arm
draft_virtual_id_space = alloc.draft_virtual_id_space
assert draft_virtual_id_space >= sizes.max_total_num_tokens, (
"unified allocator virtual space smaller than the token "
@@ -627,7 +624,7 @@ class KVCacheConfigurator:
f"max_total_num_tokens={sizes.max_total_num_tokens}"
)
# Round UP to page alignment (paged draft backends view the
# pool as (-1, page_size, H, D); the space is not aligned).
# pool as (-1, page_size, H, D); the virtual space is not aligned).
page = max(int(self.pool_page_size or 1), 1)
draft_virtual_id_space = (
(draft_virtual_id_space + page - 1) // page * page
@@ -733,11 +730,8 @@ class KVCacheConfigurator:
)
def _fused_draft_from_target_buffer(self, alloc):
"""The placement this draft binds to, or None for the private-pool
fallback (the draft then owns a raw-virtual-indexed pool of its own).
Target boot declines a placement whenever the fused region cannot
serve the draft (`_fused_draft_decision`), so the private arm is the
rollback lever, not a boot-order bug."""
"""The placement this draft binds to, or None when the target placed
none and the draft builds a private pool over the virtual id space."""
if not (
self.spec_algorithm.is_eagle() or self.spec_algorithm.is_dflash_family()
):
@@ -961,14 +955,10 @@ class KVCacheConfigurator:
def _fused_draft_decision(self):
"""Whether, and where, the draft's layers fuse into the target's
sub-pools. Fusion applies only for: unified memory ON, a unified target
(hybrid-SWA or mamba hybrid, either full-pool kind), an EAGLE-family
or DFLASH/DSPARK algorithm whose draft config was loaded at target
boot. It declines, keeping the draft's private pool, for a non-MHA
draft, a draft backend off the translated MHA rails, DCP, HiCache,
host-pool decode retraction, or an explicit draft KV dtype unlike the
host's; `place_fused_draft` then admits or declines the draft's layer
kinds."""
sub-pools. An empty decision means fusion does not apply (unified
memory off, no hybrid host, or no EAGLE- or DFLASH-family draft config
loaded at target boot); a declined one says why the draft cannot fuse.
The pool factories decide whether a decline refuses the boot."""
from sglang.srt.mem_cache.layout.fused_draft import (
FusedDraftDecision,
draft_kv_profile,
@@ -998,10 +988,9 @@ class KVCacheConfigurator:
"the fused region holds dense MHA K/V rows"
)
)
# The backend(s) the draft runner will run. A DFLASH-family draft
# worker resolves one as `build_draft_tp_worker` does; an EAGLE draft
# runs the published draft backend, else the target's prefill/decode
# pair. A published draft backend carries model-hook declarations.
# The draft runner's backend, resolved as its worker will: an EAGLE
# draft with no backend of its own runs the target's prefill/decode
# pair. The published draft backend carries model-hook declarations.
if self.spec_algorithm.is_dflash_family():
from sglang.srt.speculative.draft_worker_common import (
resolve_draft_worker_attention_backend,
@@ -1066,10 +1055,9 @@ class KVCacheConfigurator:
else 1
)
# A fused draft stores its rows in the host's KV dtype. An explicit
# draft dtype resolves without the draft model, as the draft runner
# resolves it, so a mismatch is declined here; `auto` follows the
# draft's quant config, which only the draft runner knows, and is
# checked when it binds.
# draft dtype resolves here as the draft runner will; `auto` follows
# the draft's quant config, which only the draft runner knows, so it
# is checked when the draft binds.
draft_kv_cache_dtype = get_spec().speculative_draft_kv_cache_dtype
if draft_kv_cache_dtype not in (None, "auto"):
from sglang.srt.mem_cache.kv_cache_dtype import configure_kv_cache_dtype
@@ -1098,11 +1086,10 @@ class KVCacheConfigurator:
)
def fused_entry_bytes(self, sub_pool_name: str) -> Optional[int]:
"""Per-token bytes of ``sub_pool_name``'s FUSED entry (host + draft +
pad) for the boot solve's cell model. Single source of truth: assembled
through the same spec the pool factory builds, so the priced entry and
the allocated entry cannot drift. None when the draft does not fuse
into that sub-pool."""
"""Per-token bytes of ``sub_pool_name``'s fused entry (host + draft +
pad), built from the same spec as the pool factory's so the priced and
allocated entries agree; None when the draft does not fuse into that
sub-pool."""
placement = self._fused_draft_decision().placement
if placement is None or sub_pool_name != "full":
return None
@@ -1150,14 +1137,11 @@ class KVCacheConfigurator:
def _fused_draft_for_mamba_factory(self):
"""`_fused_draft_for_pool_factory` for a mamba-hybrid host. Its unified
buffer takes the WHOLE profiled KV budget (`unified_total_bytes`), so a
private draft pool -- sized to the full virtual id span -- would be
allocated on top of it, unbudgeted. Until the mamba solve prices that
pool, an EAGLE-family or DFLASH draft that does not fuse is refused here
instead of overcommitting GPU memory. A DSPARK draft that does not fuse
keeps its private pool, unpriced: refusing it would refuse DSPARK on
these hosts whenever its draft declines fusion, as the Kimi-Linear
default (a trtllm_mha draft) does."""
buffer takes the whole profiled KV budget, so a private draft pool
would sit on top of it unbudgeted: an EAGLE-family or DFLASH draft that
does not fuse is refused. A DSPARK draft that does not fuse keeps its
private pool, unpriced, so DSPARK still boots where its draft declines
fusion (as the Kimi-Linear default, a trtllm_mha draft, does)."""
decision = self._fused_draft_decision()
placement = self._fused_draft_for_pool_factory(decision)
if (
@@ -1169,18 +1153,16 @@ class KVCacheConfigurator:
raise ValueError(
"--enable-unified-memory + EAGLE/EAGLE3/DFLASH on a mamba-hybrid "
"target needs the draft's KV fused into the target's pages, but "
f"this draft would keep a private pool ({reason}). A mamba host's "
"unified buffer takes the whole KV budget, so a private draft "
"pool would overcommit GPU memory."
f"this draft cannot fuse ({reason}). A private draft pool would "
"overcommit the KV budget the unified buffer already takes."
)
return placement
def _fused_draft_for_swa_factory(self):
"""`_fused_draft_for_pool_factory` for a hybrid-SWA host. Its boot solve
prices a private EAGLE draft pool at the target's per-token size, not
at the draft's own heads, head_dim and KV dtype, so a draft that does
not fuse is refused here, as on a mamba host, instead of overcommitting
GPU memory."""
the draft's own geometry and KV dtype, so an EAGLE draft that does not
fuse is refused rather than overcommit GPU memory."""
decision = self._fused_draft_decision()
placement = self._fused_draft_for_pool_factory(decision)
if (
@@ -1192,15 +1174,14 @@ class KVCacheConfigurator:
raise ValueError(
"--enable-unified-memory + EAGLE/EAGLE3 on a hybrid-SWA target "
"needs the draft's KV fused into the target's pages, but this "
f"draft would keep a private pool ({reason}), which the unified "
"pool's sizing does not price."
f"draft cannot fuse ({reason}), and the unified pool's sizing "
"does not price a private draft pool."
)
return placement
def _fused_draft_for_pool_factory(self, decision):
"""The placement a pool factory is handed, from ONE resolved decision,
so the boot log reports exactly that placement: a declined fusion and
an engaged one otherwise look identical from outside."""
"""The placement a pool factory is handed. It logs the decision it
returns, so the boot log shows whether fusion engaged or why not."""
if decision.placement is None:
if decision.declined is not None:
logger.warning("fused draft KV disabled: %s", decision.declined)
+418 -397
View File
@@ -23,18 +23,29 @@ A KV slot can be named in two id spaces:
The two coincide on a plain pool, so nothing here does any work there.
Backends get a `KVIndexTable`, which answers "what do I gather from, and
which row is mine?":
A pool can hold several sub-pools (full attention, sliding window), each with
its own physical ids and named by an `IdSpace` (`space`). Writes are translated
once per sub-pool, by the iteration's `KVLocPlan`. A read is translated exactly
once, by the readers below and nowhere else; a backend never translates. The
plan's read table exists only once a table reader asks for it (`read_table`,
`copy_page_table`), built into that reader's own capture-stable table when it
is the first; later readers take it, earlier ones translate in their own
gather. The readers answer "what do I gather from, and which row is mine?"
with a `KVIndexTable`:
ids[row_ids[b], pos]
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
objects - no copy, no kernel)
unified : ids = a built array of physical ids,
row_ids = arange(batch_size)
plain pool : ids = req_to_token, row_ids = req_pool_indices (those very
objects - no copy, no kernel)
shared table : ids = the plan's array of physical page ids,
row_ids = arange(batch_size)
no table : ids = req_to_token, row_ids = req_pool_indices, and
v2p = the sub-pool's page table, applied by the gather
Backends call their own copy a *page table* (fa3) or a *block table*
(trtllm); here it is the **index table**.
(trtllm); here it is the **index table**. `read_source` serves a gather
kernel, `pack_read_stream` a CSR stream, and `read_table` / `copy_page_table`
a page-table kernel.
Converting only ever rewrites the page number and keeps the in-page offset, so
one page-granular table serves both kinds of consumer: a block-table backend
@@ -42,16 +53,15 @@ uses its rows as-is, and one that wants flat per-token ids rebuilds them as
token_id = entry * entry_page_size + pos % entry_page_size
WRITES, IN TWO PHASES. The full-side write loc is rebound to physical
ids at ForwardBatch construction - the earliest consumer can snapshot it
right after. The sliding-window write loc is derived at the same moment as
read table, into the same index table.
The one translation outside a plan is DCP's read production
(`translate_dcp_read_ids`): which ids a rank reads is only known where the
DCP index kernels select them.
"""
from __future__ import annotations
import weakref
from typing import Optional, Tuple
import functools
from typing import Dict, Optional
import msgspec
import torch
@@ -63,6 +73,7 @@ from sglang.kernels.ops.kvcache.kv_read_table import (
build_kv_read_table,
build_kv_read_table_packed,
)
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
@@ -71,30 +82,36 @@ from sglang.srt.mem_cache.allocator.unified_mamba import (
)
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.kv_loc_plan import (
IdSpace,
IdSpaceKind,
KVLocPlan,
pad_with_sink,
window_read_extent,
)
from sglang.srt.mem_cache.unified_draft_pool import fused_draft_host_allocator
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_npu
class KVReadTables(msgspec.Struct, frozen=True):
"""One capture-stable destination, however many id spaces the pool has.
A backend holds this and hands it back to `build_index_table(into=...)`;
it never has to know whether there is a sliding-window space behind it.
"""
full: torch.Tensor
sliding_window: Optional[torch.Tensor]
_is_npu = is_npu()
class KVIndexTable(msgspec.Struct, frozen=True):
"""Collection of what one batch gathers from."""
"""What one batch gathers from, and how its entries become the ids a
kernel reads: lane ``b``'s entry ``j`` is ``ids[row_ids[b] * row_stride +
j // entry_page_size]``, expanded to tokens when an entry is a page, then
mapped through ``v2p`` (page-granular, at the translator's page size) when
one is given. A gather kernel executes exactly this; it never decides
whether or how to translate."""
ids: torch.Tensor # 2-D array of KV ids to gather from
row_ids: torch.Tensor # which row belongs to batch lane b
row_stride: int # stride between rows of `ids`, in elements
entry_page_size: int # what one entry covers: 1 = a token, N = a page of N
is_translated: bool # entries are already physical ids
sliding_window_ids: Optional[torch.Tensor] # SWA models: the parallel swa array
is_translated: bool # the ids a gather yields are physical
# Virtual rows a gather translates on the way out (no shared table this
# iteration); None when the entries are already what the kernel reads.
v2p: Optional[torch.Tensor] = None
class KVIndexTranslator:
@@ -108,6 +125,7 @@ class KVIndexTranslator:
token_to_kv_pool,
page_size: int,
device: str,
is_draft_worker: bool = False,
):
self.req_to_token = req_to_token
self.page_size = page_size
@@ -125,55 +143,104 @@ class KVIndexTranslator:
host_allocator is not None and host_allocator is token_to_kv_pool_allocator
)
self.is_translating = is_unified_target or is_fused_draft
self._spaces: Dict[IdSpaceKind, IdSpace] = {}
if self.is_translating:
alloc = token_to_kv_pool_allocator
self._capture_page_size = alloc.page_size
self._full_v2p_table = alloc.full_v2p_page_table
self._full_p2v_table = alloc.full_p2v_page_table
self._translate_full = alloc.translate_kv_loc
# The WRITE loc is the one id that arrives DCP-WIDENED: read indices
# are collapsed by the DCP index kernels, `out_cache_loc` still
# carries the owner rule in `loc % dcp_size`. Identity with the read
# translate when dcp_size == 1.
self._translate_write_full = alloc.translate_write_loc
# DCP read ids stay WIDENED to the consumer: selecting this rank's
# share changes the length, so only the production site can do it.
self.defer_read_translate = get_parallel().attn_dcp_size > 1
# Keyed by the allocator, which owns the tables: a fused draft's
# runner shares the target's full sub-pool, and so its key.
self._spaces[IdSpaceKind.FULL] = IdSpace(
key=(IdSpaceKind.FULL, id(alloc)),
# The WRITE loc is the one id that arrives DCP-WIDENED: read
# indices are collapsed by the DCP index kernels, `out_cache_loc`
# still carries the owner rule in `loc % dcp_size`.
write=alloc.translate_write_loc,
read_v2p=None if self.defer_read_translate else self._full_v2p_table,
)
routes_window_layers = is_unified_target or isinstance(
token_to_kv_pool, BaseSWAKVPool
)
if isinstance(alloc, UnifiedSWAAllocatorBase) and routes_window_layers:
self._swa_v2p_table = alloc.swa_v2p_page_table
self._swa_write_loc_from_full = self._swa_write_loc_unified
else:
self._swa_v2p_table = None
self._swa_write_loc_from_full = None
# Straight from the virtual ids through the sliding-window
# sub-pool's own table.
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
key=(IdSpaceKind.SLIDING_WINDOW, id(alloc)),
write=alloc.translate_loc_from_full_to_swa,
read_v2p=(
None if self.defer_read_translate else alloc.swa_v2p_page_table
),
)
else:
self._capture_page_size = page_size
self._full_v2p_table = None
self._full_p2v_table = None
self._translate_full = None
self._translate_write_full = None
self.defer_read_translate = False
self._swa_v2p_table = None
parallel = get_parallel()
if _is_npu and parallel.dcp_enabled and not is_draft_worker:
# An NPU DCP target writes its rank's share of the window in
# rank-local slots (-1: another rank's); the scheduler,
# `req_to_token` and the replicated draft pool keep the
# allocator-global ones the window is planned in.
localize = functools.partial(
localize_dcp_indices,
dcp_size=parallel.dcp_size,
dcp_rank=parallel.dcp_rank,
interleave_size=page_size,
)
self._spaces[IdSpaceKind.FULL] = IdSpace(
key=(IdSpaceKind.FULL, "dcp-rank-local"), write=localize
)
else:
localize = None
self._spaces[IdSpaceKind.FULL] = IdSpace(key=(IdSpaceKind.FULL, None))
# `translate_loc_from_full_to_swa` is abstract on `BaseSWAKVPool`,
# which is also what the backends' `_resolve_swa_kv_pool` keys on.
self._swa_write_loc_from_full = (
token_to_kv_pool.translate_loc_from_full_to_swa
if isinstance(token_to_kv_pool, BaseSWAKVPool)
and (
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
or token_to_kv_pool.request_window is None
# Reads stay in `req_to_token`; the backends map them themselves.
if isinstance(token_to_kv_pool, BaseSWAKVPool) and (
not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
or token_to_kv_pool.request_window is None
):
to_swa = token_to_kv_pool.translate_loc_from_full_to_swa
self._spaces[IdSpaceKind.SLIDING_WINDOW] = IdSpace(
key=(IdSpaceKind.SLIDING_WINDOW, id(token_to_kv_pool)),
write=(
to_swa
if localize is None
else lambda ids: to_swa(localize(ids))
),
)
else None
)
self._rows: Optional[torch.Tensor] = (
torch.arange(req_to_token.shape[0], dtype=torch.int64, device=device)
if self.is_translating
else None
)
self._index_table_memo: Optional[Tuple[weakref.ref, KVIndexTable]] = None
def space(self, kind: IdSpaceKind) -> Optional[IdSpace]:
"""How this runner's ``kind`` sub-pool names the iteration's slots, or
None when its pool has no such sub-pool."""
return self._spaces.get(kind)
def write_ids(
self, forward_batch, kind: IdSpaceKind = IdSpaceKind.FULL
) -> Optional[torch.Tensor]:
"""The ids ``forward_batch`` writes in this runner's ``kind`` sub-pool,
taken from its plan and padded with the sink to its `out_cache_loc`'s
length (a padded batch, a captured graph's buffer). None when there is
no write loc or no such sub-pool."""
plan = getattr(forward_batch, "kv_loc_plan", None)
out_cache_loc = getattr(forward_batch, "out_cache_loc", None)
if plan is None or out_cache_loc is None:
return None
ids = plan.write_ids(
self, kind=kind, cols=getattr(forward_batch, "kv_loc_cols", None)
)
return pad_with_sink(ids, out_cache_loc.shape[0])
def capture_token_capacity(self, max_token_pool_size: int) -> int:
"""Host capture rows are indexed by request-token IDs, not kernel IDs.
@@ -185,240 +252,305 @@ class KVIndexTranslator:
return self._full_v2p_table.numel() * self._capture_page_size
return max_token_pool_size + self.page_size
def full_flat_v2p(self) -> Optional[torch.Tensor]:
"""The full-side v2p page table for a kernel that translates flat ids
itself, or ``None`` when this runner does not translate (pass-through
and static pools)."""
if not self.is_translating:
return None
return self._full_v2p_table
# -- the iteration's plan --------------------------------------------------
# -- per-batch view --------------------------------------------------------
def plan(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
) -> KVLocPlan:
"""This iteration's ids, translated once. ``write_virtual`` is the
iteration's write window (``[batch, window]`` flattened when its
forwards write columns of it); ``read_extent`` is how far past
``seq_lens`` the iteration's reads reach. ``write_slots`` replaces
``write_virtual`` for a runner's own buffer (`bind_runner_slots`)."""
return KVLocPlan(
source=self,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=write_virtual,
read_extent=read_extent,
write_slots=write_slots,
)
def own_plan(self, forward_batch, *, runner_slots: bool = False) -> KVLocPlan:
"""The plan of a forward that is its own iteration, from its own
fields: its ``out_cache_loc`` is the write window (virtual, or with
``runner_slots`` a runner's own buffer), and its reads reach
`window_read_extent` past its lengths."""
write_ids = forward_batch.out_cache_loc
seq_lens = getattr(forward_batch, "seq_lens", None)
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
encoder_lens = getattr(forward_batch, "encoder_lens", None)
if (
encoder_lens is not None
and seq_lens is not None
and self.reads_are_translated
):
# An encoder-decoder row holds its encoder tokens ahead of the
# decoder's, and its self-attention reads past `seq_lens`.
seq_lens = seq_lens + encoder_lens
encoder_lens_cpu = getattr(forward_batch, "encoder_lens_cpu", None)
seq_lens_cpu = (
None
if seq_lens_cpu is None or encoder_lens_cpu is None
else seq_lens_cpu + torch.tensor(encoder_lens_cpu).to(seq_lens_cpu)
)
return self.plan(
req_pool_indices=getattr(forward_batch, "req_pool_indices", None),
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=None if runner_slots else write_ids,
write_slots=write_ids if runner_slots else None,
read_extent=window_read_extent(
getattr(forward_batch, "forward_mode", None),
getattr(forward_batch, "spec_info", None),
write_ids,
getattr(forward_batch, "batch_size", 0),
),
)
def bind_runner_slots(self, forward_batch) -> None:
"""Bind a batch a runner builds over its own buffers, to capture a
graph or to warm up with. Its write ids are the runner's buffer, used
as it is; its reads plan over the runner's own rows and lengths."""
self.own_plan(forward_batch, runner_slots=True).bind(forward_batch, self)
def build_iteration_table(
self,
plan: KVLocPlan,
space: IdSpace,
*,
rows: Optional[int],
previous: Optional[KVIndexTable],
into: Optional[torch.Tensor] = None,
) -> KVIndexTable:
"""Build ``plan``'s page table in ``space`` for `KVLocPlan.read_table`:
the passthrough where reads stay virtual, else one build over ``[0,
seq_lens + read_extent)``. Rows past the plan's batch read the sink; a
request wider than ``previous`` copies its rows rather than translating
again. ``into``, the asking reader's capture-stable table, is built in
place when wide enough and then serves as the plan's table."""
if space.read_v2p is None:
return self._passthrough_table(plan.req_pool_indices)
bs = int(plan.req_pool_indices.numel())
rows = max(bs, rows or 0)
if previous is not None:
out = torch.zeros(
(rows, previous.ids.shape[1]), dtype=torch.int32, device=self.device
)
out[: previous.ids.shape[0]].copy_(previous.ids)
else:
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
slc = plan.seq_lens_cpu
if slc is not None and slc.numel() > 0:
max_seq = int(slc.max()) + plan.read_extent
width = min(max(-(-max_seq // self.page_size), 1), row_pages)
elif into is not None:
# No host lengths: a row's speculative headroom outruns the
# captured table, which is as wide as its graph reads.
width = min(into.shape[1], row_pages)
else:
width = row_pages
# `zero_tail` writes the sink past each live prefix, so neither a
# reused `into` nor an `empty` table keeps stale columns.
if into is not None and into.shape[0] >= rows and into.shape[1] >= width:
out = into[:rows, :width]
else:
out = torch.empty((rows, width), dtype=torch.int32, device=self.device)
if rows > bs:
out[bs:].zero_()
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=plan.req_pool_indices,
seq_lens=plan.seq_lens,
v2p=space.read_v2p,
page_size=self.page_size,
max_pages=width,
out=out,
zero_tail=True,
seq_len_delta=plan.read_extent,
)
return KVIndexTable(
ids=out,
row_ids=self._rows[:rows],
row_stride=out.stride(0),
entry_page_size=self.page_size,
is_translated=True,
)
# -- readers ---------------------------------------------------------------
def _reads_translated(self, kind: IdSpaceKind) -> bool:
space = self._spaces.get(kind)
return space is not None and space.read_v2p is not None
def read_source(
self,
plan: KVLocPlan,
*,
req_pool_indices: torch.Tensor,
bs: int,
kind: IdSpaceKind = IdSpaceKind.FULL,
) -> KVIndexTable:
"""The row source a gather kernel reads lane ``b``'s ``kind`` ids from:
the plan's table once a table reader has had it built, else the virtual
`req_to_token` rows with ``v2p`` for the kernel's gather to apply (no
table is built for it). The plain `req_to_token` passthrough where
reads stay virtual (a static pool, or DCP)."""
if not self._reads_translated(kind):
return self._passthrough_table(req_pool_indices)
if plan.has_read_table(kind):
return self._plan_table(plan, kind=kind, rows=bs)
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=True,
v2p=self.space(kind).read_v2p,
)
def read_table(
self,
plan: KVLocPlan,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
rows: Optional[int] = None,
) -> KVIndexTable:
"""The plan's ``kind`` page table, exactly ``rows`` rows when given
(lanes past the plan's batch read the sink); the ``req_to_token``
passthrough where reads stay virtual (a static or private pool, or DCP,
where the producing kernel selects this rank's share)."""
if self._reads_translated(kind):
return self._plan_table(plan, kind=kind, rows=rows)
return self._passthrough_table(plan.req_pool_indices)
def _plan_table(
self,
plan: KVLocPlan,
*,
kind: IdSpaceKind,
rows: Optional[int] = None,
into: Optional[torch.Tensor] = None,
):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
"req_to_token rows"
)
return plan.read_table(kind=kind, rows=rows, into=into)
def _passthrough_table(self, req_pool_indices: torch.Tensor) -> KVIndexTable:
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=False,
)
def pack_read_stream(
self,
plan: KVLocPlan,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
indptr: torch.Tensor,
out: torch.Tensor,
kv_start_idx: Optional[torch.Tensor] = None,
kind: IdSpaceKind = IdSpaceKind.FULL,
token_mapping: Optional[torch.Tensor] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ``kind`` ids a paged wrapper plans
over; return whether they are physical. Packed from the plan's table
once a table reader has had it built, else gathered and translated
straight into ``out``. ``False`` means still-VIRTUAL full-attention ids
(a static pool, or DCP) for the caller to finish; a static SWA pool can
fuse its full->swa table into the gather as ``token_mapping``."""
bs = int(seq_lens.numel())
if self._reads_translated(kind) and not plan.has_read_table(kind):
assert plan.is_read_by(self), (
"a translating reader must read through the plan of its own "
"req_to_token rows"
)
# `seq_lens` sizes the batch: a caller may hold a wider
# req_pool_indices (a padded graph buffer), whose extra lanes have
# no length to bound.
assert req_pool_indices.numel() >= bs, (
f"pack_read_stream: {bs} stream lanes, "
f"{req_pool_indices.numel()} request rows"
)
build_kv_read_table_packed(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices[:bs],
seq_lens=seq_lens,
v2p=self.space(kind).read_v2p,
indptr=indptr,
page_size=self.page_size,
max_tokens=out.numel(),
out=out,
kv_start_idx=kv_start_idx,
)
return True
# The plan's table or a passthrough: the entries are the stream's ids.
src = self.read_source(
plan, req_pool_indices=req_pool_indices, bs=bs, kind=kind
)
assert src.v2p is None, "pack_read_stream: a table-less translating read"
assert token_mapping is None or not src.is_translated
create_flashinfer_kv_indices_triton[(bs,)](
src.ids,
src.row_ids[:bs],
seq_lens,
indptr,
kv_start_idx,
out,
src.row_stride,
ENTRY_PAGE_SIZE=src.entry_page_size,
token_mapping=token_mapping,
)
return src.is_translated or token_mapping is not None
def copy_page_table(
self,
plan: KVLocPlan,
*,
out: torch.Tensor,
kind: IdSpaceKind = IdSpaceKind.FULL,
) -> None:
"""Fill ``out``, a captured graph's page table (padded lanes read the
sink), with the plan's ``kind`` table. The plan's first table reader
builds straight into ``out``, which then serves as the plan's table; a
later one copies it. Columns past the plan's width keep stale values,
which kernels never read past their own lengths."""
assert self._reads_translated(kind), (
"copy_page_table: reads stay virtual here (a non-unified pool, or "
"DCP, where the caller selects this rank's share itself)"
)
rows = out.shape[0]
first = not plan.has_read_table(kind)
table = self._plan_table(
plan, kind=kind, rows=rows, into=out if first else None
)
if table.ids.data_ptr() == out.data_ptr():
return # built in place, or this very buffer is the plan's table
width = min(out.shape[1], table.ids.shape[1])
out[:, :width].copy_(table.ids[:rows, :width])
# -- dispositions ----------------------------------------------------------
@property
def reads_are_translated(self) -> bool:
"""Whether a read this translator fills comes out physical. False
on a non-unified pool, and under DCP, where the ids stay VIRTUAL for
``translate_dcp_read_ids`` to finish."""
return self.is_translating and not self.defer_read_translate
def fill_packed_read_stream(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
indptr: torch.Tensor,
total_tokens: int,
out: torch.Tensor,
kv_start_idx: Optional[torch.Tensor] = None,
sliding_window: bool = False,
token_mapping: Optional[torch.Tensor] = None,
) -> bool:
"""Fill ``out``'s CSR rows with the ids a paged wrapper plans over, and
report whether they came out translated.
Non-unified: gather from ``req_to_token``, optionally through ``token_mapping``.
Unified: one fused gather-and-translate, so no caller needs a
``[bs, max_pages]`` rectangle to repack from -- ``out`` holds one id per
resident token, a length the pool bounds.
``sliding_window`` selects the swa sub-pool's own id space, built from
VIRTUAL ids and never chained through full-physical. A ``False`` return
means the ids are still VIRTUAL: the DCP path defers translation to
``translate_dcp_read_ids``. A static SWA pool can pass its full->swa
table as ``token_mapping`` to fuse that translation into the gather.
"""
# `seq_lens` sizes the batch: a caller may hold a wider req_pool_indices
# (the padded graph buffer), and the extra lanes have no length to bound.
bs = int(seq_lens.numel())
assert req_pool_indices.numel() >= bs, (
f"fill_packed_read_stream: {req_pool_indices.numel()} req rows for "
f"{bs} lengths"
)
req_pool_indices = req_pool_indices[:bs]
if not self.reads_are_translated:
create_flashinfer_kv_indices_triton[(bs,)](
self.req_to_token,
req_pool_indices,
seq_lens,
indptr,
kv_start_idx,
out,
self.req_to_token.stride(0),
ENTRY_PAGE_SIZE=1,
token_mapping=token_mapping,
)
return token_mapping is not None
if sliding_window:
assert self._swa_v2p_table is not None, (
"fill_packed_read_stream: sliding_window on a pool with no swa sub-pool"
)
build_kv_read_table_packed(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._swa_v2p_table if sliding_window else self._full_v2p_table,
indptr=indptr,
page_size=self.page_size,
max_tokens=total_tokens,
out=out,
kv_start_idx=kv_start_idx,
)
return True
def build_index_table(
self,
*,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
max_pages: Optional[int] = None,
into: Optional[KVReadTables] = None,
seq_len_delta: int = 0,
) -> KVIndexTable:
"""The one per-batch entry point.
Non-unified: the raw ``(req_to_token, req_pool_indices)`` passthrough,
no tensor ops and no copies. Unified: fills each row's live prefix and
returns the table WHOLE, so a caller needing a stable pointer (a
captured graph bakes it) passes its own tables in ``into``;
``into=None`` allocates of width ``max_pages`` instead.
``seq_len_delta`` widens every row's live prefix -- the whole-sequence
verify contract (draft KV read back from the pool). An eager caller's
``max_pages`` must already cover the delta.
"""
if not self.is_translating or self.defer_read_translate:
return KVIndexTable(
ids=self.req_to_token,
row_ids=req_pool_indices,
row_stride=self.req_to_token.stride(0),
entry_page_size=1,
is_translated=False,
sliding_window_ids=None,
)
bs = int(req_pool_indices.numel())
if into is not None:
out_full = into.full
out_swa = into.sliding_window
# A caller-owned table may be padded wider than req_to_token's span
# (trtllm_mla / flashmla pad to a page-count bound); the columns
# past it have no source to read, so stop there.
width = min(
out_full.shape[1] if max_pages is None else max_pages,
-(-self.req_to_token.shape[1] // self.page_size),
)
else:
assert max_pages is not None, (
"KVIndexTranslator.build_index_table: allocating needs max_pages "
"(from the batch's seq_lens_cpu max)"
)
width = max_pages
out_full = torch.zeros((bs, width), dtype=torch.int32, device=self.device)
out_swa = (
torch.zeros((bs, width), dtype=torch.int32, device=self.device)
if self._swa_v2p_table is not None
else None
)
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._full_v2p_table,
page_size=self.page_size,
max_pages=width,
out=out_full,
seq_len_delta=seq_len_delta,
)
if out_swa is not None:
build_kv_read_table(
req_to_token=self.req_to_token,
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
v2p=self._swa_v2p_table,
page_size=self.page_size,
max_pages=width,
out=out_swa,
seq_len_delta=seq_len_delta,
)
return KVIndexTable(
ids=out_full,
row_ids=self._rows[:bs],
row_stride=out_full.stride(0),
entry_page_size=self.page_size,
is_translated=True,
sliding_window_ids=out_swa,
)
def fill_read_table(
self,
*,
out: torch.Tensor,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
sliding_window_out: Optional[torch.Tensor] = None,
seq_len_delta: int = 0,
) -> None:
"""`build_index_table(into=...)` for a caller that owns a bare block
table rather than a KVReadTables: the page-table consumers read that
table directly, its rows already being the index table's rows.
`sliding_window_out` fills the swa twin in the same pass, for a hybrid
model whose kernels take two block tables.
"""
# `reads_are_translated`, not `is_translating`: under DCP the builder
# returns the passthrough view and writes nothing, so `is_translating`
# would let a caller keep a stale table and never hear about it.
assert self.reads_are_translated, (
"KVIndexTranslator.fill_read_table cannot fill a page table when "
"reads stay virtual (a non-unified pool, or DCP, where the caller "
"must select this rank's share itself)"
)
assert sliding_window_out is None or self._swa_v2p_table is not None, (
"KVIndexTranslator.fill_read_table: asked for a sliding-window "
"table on a pool with no swa sub-pool"
)
self.build_index_table(
req_pool_indices=req_pool_indices,
seq_lens=seq_lens,
into=KVReadTables(full=out, sliding_window=sliding_window_out),
seq_len_delta=seq_len_delta,
)
def index_table_for_batch(self, forward_batch) -> KVIndexTable:
"""Eager per-batch view, memoized in one slot keyed by batch identity
so multi-consumer metadata builds share a build. The next batch
replaces the slot; consumers only read during their own build. The
captured path does not memoize -- it refreshes its buffers per
replay."""
memo = self._index_table_memo
if memo is not None and memo[0]() is forward_batch:
return memo[1]
max_pages = None
if self.is_translating:
# `seq_lens_cpu` is a non-None but STALE slice on a gpu_only
# batch; `seq_lens_sum` is the signal that it is live. A stale max
# under-sizes the table and the tail then reads as the sink.
slc = forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is not None
and slc is not None
and slc.numel() > 0
):
max_seq = int(slc.max())
else:
max_seq = self.req_to_token.shape[1]
max_pages = max(-(-max_seq // self.page_size), 1)
view = self.build_index_table(
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
max_pages=max_pages,
)
self._index_table_memo = (weakref.ref(forward_batch), view)
return view
return self._reads_translated(IdSpaceKind.FULL)
@property
def full_v2p_table(self) -> Optional[torch.Tensor]:
@@ -426,43 +558,11 @@ class KVIndexTranslator:
pool needs no translation.
For the DCP page-table builders, whose gather is over a rank's cyclic
slice rather than a row prefix, so `build_index_table` cannot serve
them.
slice rather than a row prefix, so a plan's row-prefix table cannot
serve them.
"""
return self._full_v2p_table
def widened_index_table(self, forward_batch, *, seq_len_delta: int) -> KVIndexTable:
"""Whole-sequence-verify view: every row's live prefix widened by
`seq_len_delta` columns (the drafts are read back from the pool).
Not memoized -- verify is one consumer, and the batch's memoized
prefix table stays valid for the others.
The width comes from the LIVE prefix: a DSPARK/DFLASH verify batch
carries `seq_lens_cpu` already expanded by the window and keeps the
live lens on `spec_info.live_seq_lens_cpu`, so widening the expanded
lens would count the window twice. The width never exceeds the
`req_to_token` row it is read from."""
max_pages = None
if self.is_translating:
row_pages = -(-self.req_to_token.shape[1] // self.page_size)
live = getattr(forward_batch.spec_info, "live_seq_lens_cpu", None)
slc = live if live is not None else forward_batch.seq_lens_cpu
if (
forward_batch.seq_lens_sum is not None
and slc is not None
and slc.numel() > 0
):
max_seq = int(slc.max()) + seq_len_delta
max_pages = min(max(-(-max_seq // self.page_size), 1), row_pages)
else:
max_pages = row_pages
return self.build_index_table(
req_pool_indices=forward_batch.req_pool_indices,
seq_lens=forward_batch.seq_lens,
max_pages=max_pages,
seq_len_delta=seq_len_delta,
)
def bind_and_verify_backends(self, backends) -> None:
"""Boot: make every reachable backend carry THIS translator.
@@ -481,84 +581,13 @@ class KVIndexTranslator:
"copy, not build its own."
)
# -- write loc (phase 1; phase 2 lives in build_index_table) ----------------
def bind_own_plan(self, forward_batch) -> None:
"""Bind a batch built outside `ForwardBatch.init_new` that is its own
iteration to a plan of its own fields (`own_plan`). A forward that
shares its iteration's window takes that plan instead."""
self.own_plan(forward_batch).bind(forward_batch, self)
def rebind_write_loc(self, forward_batch) -> None:
"""Phase 1 of the WRITE contract: translate the batch's write loc to
FULL-side physical ids exactly once, at ForwardBatch construction, and
mark it physical. On non-unified pools the allocation is already
physical, so only the mark is set.
REBIND, never mutate: the translate returns a FRESH tensor, so the
ScheduleBatch's aliased tensor stays VIRTUAL for the radix / accept /
in-flight machinery that reads it. The pre-translate tensor stays on
the batch for `fill_capture_write_loc`.
"""
self._index_table_memo = None
if forward_batch.out_cache_loc is None:
return
if self.is_translating:
forward_batch.out_cache_loc_virtual = forward_batch.out_cache_loc
forward_batch.out_cache_loc = self._translate_write_full(
forward_batch.out_cache_loc
)
forward_batch.out_cache_loc_is_physical = True
def fill_capture_write_loc(
self,
*,
out: torch.Tensor,
forward_batch,
width: Optional[int] = None,
) -> Optional[torch.Tensor]:
"""Translate this batch's WRITE loc straight into ``out``, a backend's
capture-stable buffer, and return the live ``[:n]`` view. One launch
fills the live prefix and clears the tail a shorter replay leaves;
None when this pool needs no translation.
Must run at metadata-init time: `out` is reused every step, so filling
it sooner would race a still-pending previous step under overlap
scheduling.
"""
if not self.is_translating:
return None
virtual = forward_batch.out_cache_loc_virtual
if virtual is None:
loc = forward_batch.out_cache_loc
if loc is None:
return None
# The runner builds the capture batch outside `init_new`, so no
# rebind marked its virtual source; bake it holding sink ids.
width = int(loc.numel()) if width is None else int(width)
out[:width].zero_()
return out[: int(loc.numel())]
n = int(virtual.numel())
width = n if width is None else int(width)
buf = out[:width]
self._translate_write_full(virtual, out=buf, out_width=width)
return buf[:n]
def sliding_window_write_loc_for(
self, out_cache_loc: Optional[torch.Tensor]
) -> Optional[torch.Tensor]:
"""This batch's sliding-window write loc, or None when there is no loc
this forward or the pool has no sliding-window id space."""
if out_cache_loc is None or self._swa_write_loc_from_full is None:
return None
return self._swa_write_loc_from_full(out_cache_loc)
def _swa_write_loc_unified(self, full_loc: torch.Tensor) -> torch.Tensor:
"""Sliding-window write loc, derived pointwise from FULL-side physical
ids (phase 2 of the write contract).
"""
ps = self.page_size
offset = full_loc % ps # == virtual_token % page_size
# An unmapped physical page reads back as -1; clamp it rather than let
# the gather wrap onto the v2p table's last element.
virt_page = self._full_p2v_table[full_loc // ps].clamp_(min=0)
return (self._swa_v2p_table[virt_page] * ps + offset).clamp_(min=0)
# -- token-level translate surface (the mixin / local-attn consumers) ------
# -- DCP read production ---------------------------------------------------
@property
def needs_read_translate(self) -> bool:
@@ -575,14 +604,6 @@ class KVIndexTranslator:
dcp_size = get_parallel().attn_dcp_size
if dcp_size > 1:
widened_ids = widened_ids // dcp_size
return self.translate_full_attn_ids(widened_ids)
def translate_full_attn_ids(
self, kv_indices: torch.Tensor, *, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
"""Virtual token ids -> physical full-attention ids (the identity
when no translation is needed, so callers never branch)."""
if not self.is_translating:
assert out is None, "passthrough translate takes no out="
return kv_indices
return self._translate_full(kv_indices, out=out)
return widened_ids
return self._translate_full(widened_ids)
+348
View File
@@ -0,0 +1,348 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""One scheduler iteration's KV slot ids, per sub-pool.
Under the unified pool a KV slot has a virtual id (what the scheduler allocates
and `req_to_token` stores) and, in each sub-pool, a physical id (what that
sub-pool's kernels index). Every forward of one iteration -- the target's, and
a fused draft's, which lives in the target's pages -- writes the same slots and
reads the same rows, and compaction does not move a page while the iteration's
forwards are in flight. So:
* The write window is translated once per sub-pool, by the plan: the
full-attention ids when the plan is built, another sub-pool's on first
use (`write_ids`). Every writer binds them.
* A read is translated exactly once, only by the translator's read
primitives (`KVIndexTranslator`); a consumer never translates.
* The read table (`read_table`) exists only once a table reader asks for
it, built into that reader's own capture-stable table when it is the
first. Later readers take it; earlier ones translate in their own gather.
* A sub-pool is named by its `IdSpace`; one that indexes virtual ids as they
are (a static pool's, a private draft pool's) translates nothing.
One plan per iteration: `ForwardBatch.init_new` builds it for a lone forward, a
speculative worker as soon as the iteration's slots are allocated (shared by
its draft and target forwards and direct KV writers), and a graph runner for a
batch over its own buffers (`write_slots`).
"""
from __future__ import annotations
import enum
from typing import TYPE_CHECKING, Callable, Dict, Hashable, Optional, Union
import msgspec
import torch
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_index_translator import (
KVIndexTable,
KVIndexTranslator,
)
class IdSpaceKind(enum.Enum):
"""A sub-pool of a runner's KV pool, as the consumers that index it name
it."""
# Full-attention layers: the pool's own token ids.
FULL = enum.auto()
# The sliding-window layers of a hybrid pool: the same token ids through
# the sliding-window sub-pool's table.
SLIDING_WINDOW = enum.auto()
class IdSpace(msgspec.Struct, frozen=True):
"""How one sub-pool names the iteration's slots.
``write`` maps the iteration's virtual token ids to this sub-pool's ids
(None: they index it as they are). ``read_v2p`` is the page table its reads
go through (None: reads stay in `req_to_token`, for the consumer to
finish). Runners whose pools share a sub-pool share its ``key``, and with
it a plan's ids for it."""
key: Hashable
write: Optional[Callable[[torch.Tensor], torch.Tensor]] = None
read_v2p: Optional[torch.Tensor] = None
def window_read_extent(forward_mode, spec_info, write_ids, batch_size: int) -> int:
"""How far past ``seq_lens`` a forward reads: the window it writes, when
its ``seq_lens`` do not count that window yet. A verify reads at most
``draft_token_num`` past them (a ragged verify writes fewer for some rows);
a draft decode reads back its own earlier steps; a draft extend's lengths
take its window only after its batch is built. Zero otherwise."""
if spec_info is None or write_ids is None or not batch_size:
return 0
if forward_mode.is_target_verify():
return spec_info.draft_token_num
if forward_mode.is_decode() or forward_mode.is_draft_extend_v2():
return write_ids.numel() // batch_size
return 0
class TokenSpan(msgspec.Struct, frozen=True):
"""A contiguous run ``[start, stop)`` of a write window's flat tokens (one
half of a two-batch-overlap split), which a reader takes as a view; a
``slice`` in `Cols` names columns of every row instead."""
start: int
stop: int
# Part of a `[batch, window]` write window: a slice of its columns, a
# contiguous run of its flat tokens, or a flat index into it (-1: the sink).
Cols = Union[slice, TokenSpan, torch.Tensor]
def pad_with_sink(ids: Optional[torch.Tensor], n: int) -> Optional[torch.Tensor]:
"""``ids`` padded to ``n`` entries with the sink (id 0 in every space), for
a batch whose write ids cover more lanes than its plan's (a padded batch,
a captured graph's buffer)."""
if ids is None or ids.shape[0] >= n:
return ids
padded = ids.new_zeros(n)
padded[: ids.shape[0]].copy_(ids)
return padded
class KVLocPlan:
"""The ids of one iteration, in every sub-pool, each computed once."""
def __init__(
self,
*,
source: KVIndexTranslator,
req_pool_indices: torch.Tensor,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
write_virtual: Optional[torch.Tensor],
read_extent: int = 0,
write_slots: Optional[torch.Tensor] = None,
):
self._source = source
self.req_pool_indices = req_pool_indices
# Committed lengths when the plan is built; every read this iteration
# stays within `seq_lens + read_extent` (the window it writes).
self.seq_lens = seq_lens
self.seq_lens_cpu = seq_lens_cpu
self.read_extent = read_extent
full = source.space(IdSpaceKind.FULL)
if write_slots is not None:
# A runner's own write buffer (graph capture, warmup): already in
# its pool's ids, naming the sink until a replay fills it, and kept
# by address by a captured graph.
assert write_virtual is None
self.write_virtual = None if full.write is not None else write_slots
self.write_physical = write_slots
else:
# Aliases the ScheduleBatch's tensor, which stays virtual for the
# radix tree, the accept path and lazy compaction's in-flight write
# set.
self.write_virtual = write_virtual
# FIXME: a speculative iteration builds its plan before its draft
# runs, so this launch sits on the host path ahead of the draft's
# graph, idle GPU time at low concurrency. Translate the window
# inside the iteration's first captured forward instead, into a
# capture-stable buffer the plan owns, which every later forward
# and direct writer reads -- once per sub-pool, the sliding-window
# ids (derived in `write_ids`) included.
self.write_physical = (
full.write(write_virtual)
if full.write is not None and write_virtual is not None
else write_virtual
)
self._full_key = full.key
self._write_ids: Dict[Hashable, Optional[torch.Tensor]] = {
full.key: self.write_physical
}
self._read_tables: Dict[Hashable, KVIndexTable] = {}
# -- writes ----------------------------------------------------------------
def bind(self, batch, reader: KVIndexTranslator, *, cols: Optional[Cols] = None):
"""Give ``batch`` (a ForwardBatch or a stand-in) this plan, the part of
the window it writes (``cols``), and its full-attention write ids in
`reader`'s pool. The one way a forward gets its write ids; another
sub-pool's come from `KVIndexTranslator.write_ids`."""
batch.kv_loc_plan = self
batch.kv_loc_cols = cols
batch.out_cache_loc = self.write_ids(reader, cols=cols)
batch.out_cache_loc_virtual = (
self.virtual_write_ids(cols=cols)
if self.is_translated_for(reader)
else None
)
batch.out_cache_loc_is_physical = batch.out_cache_loc is not None
def bind_replay(self, batch, reader: KVIndexTranslator, *, slots: torch.Tensor):
"""`bind`, for the batch a captured graph replays this iteration's
forward with. Its write ids are ``slots``, the runner's capture-stable
buffer that already holds this plan's write ids and names the sink past
them. The virtual mirror stays with the live batch."""
self.bind(batch, reader)
batch.out_cache_loc = slots
batch.out_cache_loc_virtual = None
def write_ids(
self,
reader: KVIndexTranslator,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
cols: Optional[Cols] = None,
) -> Optional[torch.Tensor]:
"""The write window as `reader`'s ``kind`` sub-pool indexes it, or None
when its pool has no such sub-pool. ``cols`` selects part of a
``[batch, window]`` window, for a forward that writes only part of it:
a slice of columns (a draft step ahead of the verify window it opens),
or a flat index into the window, -1 naming the sink (a ragged verify's
packed rows)."""
space = reader.space(kind)
if space is None:
return None
if space.key not in self._write_ids:
self._write_ids[space.key] = self._window_in(space)
return self._cols(self._write_ids[space.key], cols)
def _window_in(self, space: IdSpace) -> Optional[torch.Tensor]:
if self.write_virtual is None:
# No window this iteration, or a runner's own slots, which name
# the sink in every other sub-pool too.
if self.write_physical is None:
return None
return torch.zeros_like(self.write_physical)
if space.write is None:
return self.write_virtual
return space.write(self.write_virtual)
def virtual_write_ids(self, *, cols: Optional[Cols] = None):
"""The same columns in the virtual space: the ids the scheduler's
bookkeeping (lazy compaction's in-flight write set, the radix tree)
names."""
return self._cols(self.write_virtual, cols)
def _cols(self, ids: Optional[torch.Tensor], cols: Optional[Cols]):
if ids is None or cols is None:
return ids
if isinstance(cols, TokenSpan):
return ids[cols.start : cols.stop]
if isinstance(cols, torch.Tensor):
return torch.where(cols >= 0, ids[cols.clamp(min=0)], 0)
bs = int(self.req_pool_indices.numel())
if bs == 1:
return ids[cols]
return ids.view(bs, -1)[:, cols].reshape(-1)
def cols_slice(self, cols: Optional[Cols], tokens: slice) -> Cols:
"""The ``tokens`` of a forward that writes ``cols``: the columns of
one half of a batch split in two (two-batch overlap). A contiguous
half stays a `TokenSpan`; only columns spread over the rows become a
flat index into the window. Tokens past the window are left out; the
consumer pads them with the sink."""
if isinstance(cols, torch.Tensor):
return cols[tokens]
window = (
self.write_virtual
if self.write_virtual is not None
else self.write_physical
)
if cols is None or isinstance(cols, TokenSpan):
base = 0 if cols is None else cols.start
end = window.numel() if cols is None else cols.stop
start, stop, step = tokens.indices(end - base)
if step == 1:
return TokenSpan(base + start, base + max(start, stop))
index = torch.arange(window.numel(), device=window.device)
return self._cols(index, cols)[tokens]
def reads_from(
self,
source: KVIndexTranslator,
*,
seq_lens: torch.Tensor,
seq_lens_cpu: Optional[torch.Tensor],
read_extent: int,
) -> KVLocPlan:
"""This plan's write ids, with reads planned over ``source``'s own
`req_to_token` rows and lengths -- a draft that reads a compact table
of its own while writing the iteration's window. The write ids are
shared, not translated again."""
plan = KVLocPlan(
source=source,
req_pool_indices=self.req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=None,
read_extent=read_extent,
)
plan.write_virtual = self.write_virtual
plan.write_physical = self.write_physical
plan._write_ids = self._write_ids
return plan
# -- reads -----------------------------------------------------------------
def has_read_table(self, kind: IdSpaceKind = IdSpaceKind.FULL) -> bool:
"""Whether a reader has had the ``kind`` table built yet."""
return self._source.space(kind).key in self._read_tables
def read_table(
self,
*,
kind: IdSpaceKind = IdSpaceKind.FULL,
rows: Optional[int] = None,
into: Optional[torch.Tensor] = None,
) -> KVIndexTable:
"""The ``kind`` page table over ``[0, seq_lens + read_extent)``, built
when a table reader first asks (in place in ``into``, that reader's
capture-stable table, when given) and shared by every later reader.
``rows=n`` returns exactly n rows; lanes past the plan's batch read the
sink, appended by copying, never by translating again. Where reads stay
virtual (a static pool, or DCP) it is the `req_to_token` passthrough."""
space = self._source.space(kind)
table = self._read_tables.get(space.key)
if table is None or (
table.is_translated and rows is not None and table.ids.shape[0] < rows
):
table = self._source.build_iteration_table(
self, space, rows=rows, previous=table, into=into
)
self._read_tables[space.key] = table
else:
assert into is None, "a destination goes to the table's first build"
if rows is not None and table.ids.shape[0] > rows:
# Built wider, for a captured graph's padded lanes: this reader's
# kernels size their batch by the table's rows.
table = msgspec.structs.replace(
table, ids=table.ids[:rows], row_ids=table.row_ids[:rows]
)
return table
def is_read_by(self, reader: KVIndexTranslator) -> bool:
"""Whether `reader` reads this plan's tables: it indexes the plan's
physical ids and gathers the same `req_to_token` rows. A reader with a
table of its own (a draft's compact `req_to_token`) plans its own
reads."""
return (
self.is_translated_for(reader)
and reader.req_to_token is self._source.req_to_token
)
def is_translated_for(self, reader: KVIndexTranslator) -> bool:
"""Whether `reader`'s pool indexes this plan's physical ids: the
target's, and a fused draft's, which lives in the target's full
sub-pool. A pass-through reader (a static pool, a private draft pool)
indexes the virtual ids."""
full = reader.space(IdSpaceKind.FULL)
return full.write is not None and full.key == self._full_key
@@ -5,45 +5,23 @@ The nesting, outermost first:
sub-pool a named pool inside the unified pool, with its own grow direction
and frontier. HOST is the role it plays when its entries also
carry draft bytes; only "full" hosts a draft here.
slot one token entry of that sub-pool. The draft's KV becomes extra
parts of every slot, after the host's own parts at
`draft_offset_in_entry()`, so ONE slot id per token covers target
and draft: one allocation, one free, one whole-page relocation
carry both.
slot one token entry of that sub-pool. The draft's KV is extra parts of
every slot, after the host's own parts, so ONE slot id per token
covers target and draft: one allocation, one free, one whole-page
relocation carry both.
region the draft geometry inside the host's entries. A region is not a
`SubPoolSpec`: no grow direction, no frontier logic, only geometry.
lane one position in the region's layer dimension -- one draft layer of
one runner, present in every slot.
Runners are the draft's execution copies, and the two head shapes place them
oppositely. A replicated head gives EVERY runner all the draft's layers, so
each needs its own lanes or the runners clobber each other's KV (3 runners x
2 layers = 6 lanes). A per-depth head (depth 0 predicts the next token,
depth 1 the one after, and so on) gives each runner ONE depth (8 layers
across 2 runners = 2 lanes). Lane count is therefore not layer count, in
either direction. Each runner owns a contiguous run of lanes, and the runs
tile the region in runner order.
A lane is not a layer. Runners are the draft's execution copies: a
replicated head gives EVERY runner all the draft's layers, on lanes of its own
so runners do not clobber each other's KV (3 runners x 2 layers = 6 lanes),
while a per-depth head (one block per predicted position) gives each runner
ONE depth (8 layers across 2 runners = 2 lanes). Each runner owns a
contiguous run of lanes, and the runs tile the region in runner order.
Three phases, three moments in boot, and what carries each:
PLAN on the TARGET, before any memory exists
KVCacheConfigurator._fused_draft_decision()
draft_kv_profile(draft_model_config) what the checkpoint asks for
place_fused_draft(profile, runners) admit, or decline with a reason
KVCacheConfigurator.fused_entry_bytes() prices it for the boot solve
CARVE as the byte buffer is cut
KVCacheConfigurator._fused_draft_for_pool_factory() resolve once
init_unified_*_pools(fused_draft=...)
MHASubPoolSpec.draft_offset_in_entry() / .entry_bytes()
UnifiedKVPool.build_dense_draft_views()
BIND on each DRAFT runner, built after the target
KVCacheConfigurator._fused_draft_from_target_buffer(alloc)
bind_fused_draft(...) -> UnifiedDraftKVPool
KVIndexTranslator: fused_draft_host_allocator(pool) is alloc -> translate
A draft whose layers need another sub-pool kind declines to a private pool.
A draft whose layers need another sub-pool kind declines fusion as a whole.
"""
from typing import List, Optional, Tuple
@@ -57,8 +35,8 @@ from sglang.srt.mem_cache.layout.token_major import ROW_ALIGN_BYTES, DensePart
class DenseDraftRegion(msgspec.Struct, frozen=True, kw_only=True):
"""Geometry of the DRAFT model's K/V rows fused into every slot of a host
sub-pool. The draft is a separate checkpoint, so its head geometry and
layer count differ from the host's; its K and V rows are two more parts of
the host entry, indexed by the host's physical token id."""
layer count can differ from the host's; its K and V rows are two more parts
of the host entry, indexed by the host's physical token id."""
lane_num: int
head_num: int
@@ -174,7 +152,6 @@ def draft_state_layer_num(draft_model_config) -> int:
mambaish = mambaish_config(draft_model_config)
if mambaish is None:
return 0
# Optional HF attribute: only conv-chain MTP configs declare it.
if getattr(draft_model_config.hf_text_config, "mtp_local_layer_ids", None) is None:
return 0
return len(mambaish.mamba2_cache_params.layers)
@@ -217,8 +194,8 @@ def draft_kv_profile(
class FusedDraftDecision(msgspec.Struct, frozen=True, kw_only=True):
"""`place_fused_draft`'s answer: the placement, or why the draft keeps a
private pool. Neither means fusion simply does not apply."""
"""`place_fused_draft`'s answer: the placement, or why the draft cannot
fuse. Neither means fusion does not apply."""
placement: Optional[FusedDraftPlacement] = None
declined: Optional[str] = None
@@ -253,8 +230,8 @@ def place_fused_draft(
) -> FusedDraftDecision:
"""Assign every draft layer of every runner to the host sub-pool whose
lifetime covers what the layer reads: a full-attention layer rides in
``"full"``. A layer kind no host arm serves declines the whole draft to
its private pool."""
``"full"``. A layer kind no host sub-pool serves declines the whole
draft."""
counts, reason = _runner_layer_counts(profile, num_runners)
if counts is None:
return FusedDraftDecision(declined=reason)
+15 -20
View File
@@ -1871,26 +1871,21 @@ class KVWriteLoc:
All location info lives here (in the attention metadata), NOT in the pool:
- ``loc``: the generic per-token write location (``out_cache_loc``).
PHYSICAL on every pool: by allocation on non-unified pools,
rebound at ForwardBatch construction (``rebind_write_loc``) on
the unified pool.
PHYSICAL on every pool: by allocation on non-unified pools, the
iteration's plan's write ids on the unified pool (``KVLocPlan.bind``).
- ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None``
otherwise); under the unified pool the translator derives it from the
same rebound loc (``sliding_window_write_loc_for``).
- ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the
construction-time rebind it is the SAME id space as ``loc``, so pools
fall back to ``loc`` when it is ``None`` -- only triton's captured path
still passes its capture-stable
``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a
same-space alias slated for collapse).
otherwise): the same plan's write ids in the sliding-window sub-pool
(``KVIndexTranslator.write_ids``).
- ``full_loc``: OPTIONAL full-attention-sub-pool location, in the same id
space as ``loc``; pools fall back to ``loc`` when it is ``None``.
``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved
loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``);
``loc`` is the generic fallback. Bundling them lets a backend issue one
``set_kv_buffer`` call regardless of pool type.
``physical`` marks the locs as physical token ids: the batch's write loc
after ``rebind_write_loc``, or ids a backend translated itself. A unified
``physical`` marks the locs as physical token ids: the write ids a plan
bound to the batch, or ids produced separately and marked so. A unified
pool's write door refuses a loc not marked physical; other pools do not
check.
"""
@@ -1910,9 +1905,9 @@ class KVWriteLoc:
) -> KVWriteLoc:
"""The batch's ``out_cache_loc`` as a write loc, carrying the batch's
physical mark. A ``swa_loc`` or ``full_loc`` passed here travels under
the same mark, so it must be derived from that rebound loc (as
``sliding_window_write_loc_for`` does); a loc produced separately
states its own mark with ``KVWriteLoc(loc, physical=...)``."""
the same mark, so it must come from the same plan (as
``KVIndexTranslator.write_ids`` does); a loc produced separately states
its own mark with ``KVWriteLoc(loc, physical=...)``."""
return cls(
forward_batch.out_cache_loc,
swa_loc,
@@ -2095,9 +2090,9 @@ class KVCache(abc.ABC):
if self.requires_physical_write_loc and not write_loc_is_physical(loc_info):
raise ValueError(
f"{where}: write loc is not marked physical. Hand the pool "
"KVWriteLoc.for_batch(forward_batch) after "
"KVIndexTranslator.rebind_write_loc, or KVWriteLoc(loc, "
"physical=True) for ids translated separately."
"KVWriteLoc.for_batch(forward_batch) for a batch bound to a "
"plan (KVLocPlan.bind), or KVWriteLoc(loc, physical=True) for "
"ids translated separately."
)
@abc.abstractmethod
@@ -4734,7 +4729,7 @@ class MLATokenToKVPool(KVCache):
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
# False: this pool takes a WIDENED loc. The unified pool resolves it in
# `KVIndexTranslator.rebind_write_loc` and flips this.
# the plan's write translation (`translate_write_loc`) and flips this.
write_loc_is_dcp_resolved = False
@property
@@ -15,15 +15,11 @@ class UnifiedDraftKVPool(MHATokenToKVPool):
"""Dense draft KV over the draft parts of one host sub-pool's entries.
Per-layer `k_buffer` / `v_buffer` are views of the draft parts inside
every slot of the host sub-pool (`UnifiedKVPool.build_dense_draft_views`);
``layer_lanes`` maps each of this runner's layer ids to its region lane.
Locs arriving through the KVCache API are the target's PHYSICAL token ids,
produced by the allocator's translate (the id-space choke point binds it,
see KVIndexTranslator); the pool exposes `host_allocator` for that
binding. Relocation needs no method here: compaction moves whole page
envelopes on the HOST pool, which carries the draft bytes; `move_kv_cache`
raises so a stray per-slot move fails loudly instead of corrupting the
fused layout.
every slot of the host sub-pool; ``layer_lanes`` maps each of this
runner's layer ids to its region lane. Locs are the target's PHYSICAL
token ids, translated through `host_allocator`. Compaction moves whole
page envelopes on the HOST pool, draft bytes included, so `move_kv_cache`
raises rather than let a per-slot move corrupt the fused layout.
"""
requires_physical_write_loc = True
@@ -149,8 +149,7 @@ class MHASubPoolSpec(SubPoolSpec):
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
The slot stride stays the one entry, so host and draft views are indexed
by the same physical token id; only their offsets differ. `draft_region
is None` keeps the layout byte-identical to the unfused one.
by the same physical token id; only their offsets differ.
"""
head_num: int
@@ -930,8 +929,9 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
# Lifetime owned by UnifiedKVPool; do not delete the views.
pass
# `rebind_write_loc` already collapsed the widened id and sent the rows this
# rank does not own to the padding sink.
# The plan's write translation (`translate_write_loc`) already collapsed
# the widened id and sent the rows this rank does not own to the padding
# sink.
write_loc_is_dcp_resolved = True
def get_kv_size_bytes(self):
@@ -814,7 +814,6 @@ class CPUGraphRunner:
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -832,6 +831,7 @@ class CPUGraphRunner:
num_token_non_padded=self.num_token_non_padded,
global_forward_mode=self.capture_forward_mode,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
# Wrap all forward calls with capture_with_skip_cross_attention so that
# mllama (and any other encoder-decoder model) sees the correct compile-
# time constant for skip_cross_attention during tracing.
@@ -83,6 +83,7 @@ class ForwardBatchDeepSeekMHAMixin:
):
self.prefix_chunk_kv_indices = []
req_to_token = get_req_to_token_pool().req_to_token
translator = get_attn_backend().kv_index_translator
for idx in range(self.num_prefix_chunks):
chunk_starts = self.prefix_chunk_starts[idx]
chunk_seq_lens = self.prefix_chunk_seq_lens[idx]
@@ -93,6 +94,18 @@ class ForwardBatchDeepSeekMHAMixin:
num_chunk_tokens, dtype=torch.int32, device=device
)
if translator.reads_are_translated:
# The plan's table already holds the chunk's physical ids.
translator.pack_read_stream(
self.kv_loc_plan,
req_pool_indices=self.req_pool_indices,
seq_lens=chunk_seq_lens,
indptr=chunk_cu_seq_lens,
out=chunk_kv_indices,
kv_start_idx=chunk_starts,
)
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
continue
create_chunked_prefix_cache_kv_indices[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
@@ -107,7 +120,6 @@ class ForwardBatchDeepSeekMHAMixin:
self.prefix_chunk_starts_cpu[idx],
self.prefix_chunk_seq_lens_cpu[idx],
)
translator = get_attn_backend().kv_index_translator
chunk_kv_indices = translator.translate_dcp_read_ids(chunk_kv_indices)
self.prefix_chunk_kv_indices.append(chunk_kv_indices)
@@ -266,19 +278,28 @@ class ForwardBatchDeepSeekMHAMixin:
device=self.req_pool_indices.device,
)
kv_indptr[1:] = torch.cumsum(self.seq_lens, dim=0)
req_to_token = get_req_to_token_pool().req_to_token
create_flashinfer_kv_indices_triton[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
self.seq_lens,
kv_indptr,
None,
kv_indices,
req_to_token.shape[1],
)
# None on a backend that never bound a translator.
# None on a backend that never bound a translator, which reads virtual
# ids; otherwise one gather from the iteration's plan (DCP takes its
# own branch upstream).
src = get_attn_backend().kv_index_translator
if src is not None:
kv_indices = src.translate_full_attn_ids(kv_indices)
if src is None:
req_to_token = get_req_to_token_pool().req_to_token
create_flashinfer_kv_indices_triton[(self.batch_size,)](
req_to_token,
self.req_pool_indices,
self.seq_lens,
kv_indptr,
None,
kv_indices,
req_to_token.shape[1],
)
else:
src.pack_read_stream(
self.kv_loc_plan,
req_pool_indices=self.req_pool_indices,
seq_lens=self.seq_lens,
indptr=kv_indptr,
out=kv_indices,
)
self.mha_one_shot_kv_indices = kv_indices
return kv_indices
@@ -43,7 +43,6 @@ from sglang.srt.environ import envs
from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
compute_req_all_ids_info,
)
from sglang.srt.layers.dcp.layout import localize_dcp_indices
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_slot_in,
@@ -77,6 +76,7 @@ if TYPE_CHECKING:
from sglang.srt.layers.layer_boundary.residual.stream import ResidualStream
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.managers.schedule_batch import MultimodalInputs, ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm
@@ -192,21 +192,6 @@ def _elastic_should_preserve_local_token_counts(
return uneven_token_count
def _localize_npu_dcp_out_cache_loc(
out_cache_loc: torch.Tensor,
*,
interleave_size: int,
) -> torch.Tensor:
"""Map allocator-global NPU DCP slots to this target rank."""
parallel = get_parallel()
return localize_dcp_indices(
out_cache_loc,
parallel.dcp_size,
parallel.dcp_rank,
interleave_size,
)
class ForwardMode(IntEnum):
# Extend a sequence. The KV cache of the beginning part of the sequence is already computed (e.g., system prompt).
# It is also called "prefill" in common terminology.
@@ -532,14 +517,23 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
# The original sequence length without being chunked. Qwen-1M related.
orig_seq_lens: Optional[torch.Tensor] = None
# The write loc before `rebind_write_loc` replaced it with physical
# ids; a backend re-derives from it into its capture-stable buffer.
# This iteration's KV ids, translated once per sub-pool (`KVLocPlan`).
# Every forward of the iteration holds the same plan; `out_cache_loc` is
# its write window in the full-attention ids this runner's pool indexes,
# and a backend that writes another sub-pool asks the plan for its ids
# (`KVIndexTranslator.write_ids`).
kv_loc_plan: Optional[KVLocPlan] = None
# The part of the plan's window this forward writes (None: all of it).
kv_loc_cols: Optional[Cols] = None
# The same write ids in the virtual space when `out_cache_loc` holds
# translated ones (None on a pool that indexes virtual ids), for the
# scheduler's bookkeeping: lazy compaction's in-flight write set, TBO's
# split, state capture.
out_cache_loc_virtual: Optional[torch.Tensor] = None
# DSV4-NPU only: per-pool slot bundle from DSV4NPUTokenToKVPoolAllocator,
# consumed by the Ascend backend for PA_ND block tables. None elsewhere.
out_cache_loc_dsv4: Optional[DSV4OutCacheLoc] = None
# Whether `out_cache_loc` holds physical ids: set by
# KVIndexTranslator.rebind_write_loc; capture-time batches declare it.
# Whether `out_cache_loc` holds physical ids: set by `KVLocPlan.bind`.
out_cache_loc_is_physical: bool = False
# The indices to track mamba state with
mamba_track_indices: Optional[torch.Tensor] = None # shape: [b], int64
@@ -937,9 +931,16 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
return_hidden_states_before_norm: bool,
extend_position_info=None,
spec_mrope_positions: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
write_cols: Optional[Cols] = None,
):
# init_new must not mutate the input ScheduleBatch; per-forward
# overrides go through explicit keyword arguments.
#
# `kv_loc_plan`: the iteration's plan when this forward is one of
# several over the same slots (a speculative iteration), `write_cols`
# the columns of its window this forward writes. Without one, this
# forward is its own iteration and builds the plan itself.
# capture_hidden_mode=None means no override: capture the server's
# configured maximum so lower-mode requests can share one graph.
@@ -1051,15 +1052,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
spec_info=batch.spec_info,
)
# ScheduleBatch and req_to_token keep allocator-global slot identities.
# Preserve that view before exposing rank-local NPU DCP write slots.
# ScheduleBatch and req_to_token keep allocator-global slot identities,
# and so does the plan's window. An NPU DCP target writes rank-local
# slots, which its translator maps the window to at `bind` below; keep
# the global view for the consumers that read it.
if _is_npu and get_parallel().dcp_enabled and not model_runner.is_draft_worker:
ret.origin_out_cache_loc = ret.out_cache_loc
if ret.out_cache_loc is not None:
ret.out_cache_loc = _localize_npu_dcp_out_cache_loc(
ret.out_cache_loc,
interleave_size=model_runner.page_size,
)
ret._maybe_init_non_generation_fields(batch)
device = model_runner.device
@@ -1073,7 +1071,10 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
device,
)
model_runner.kv_index_translator.rebind_write_loc(ret)
translator = model_runner.kv_index_translator
if kv_loc_plan is None:
kv_loc_plan = translator.own_plan(ret)
kv_loc_plan.bind(ret, translator, cols=write_cols)
if envs.SGLANG_KV_CANARY_ENABLE_TOKEN_ORACLE.get():
hashed = _hash_rids_to_tensor(
@@ -2143,6 +2144,8 @@ def build_inner_fb_view(
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
# A caller may hand in another view that does not carry this field.
out_cache_loc_virtual=getattr(forward_batch, "out_cache_loc_virtual", None),
kv_loc_plan=getattr(forward_batch, "kv_loc_plan", None),
kv_loc_cols=getattr(forward_batch, "kv_loc_cols", None),
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
spec_info=forward_batch.spec_info,
@@ -898,6 +898,7 @@ class ModelRunner:
token_to_kv_pool=self.token_to_kv_pool,
page_size=self.page_size or 1,
device=self.device,
is_draft_worker=self.is_draft_worker,
)
def max_shared_logits_buffer_rows(self) -> int:
@@ -662,7 +662,6 @@ class BaseRunner(ABC):
forward_batch = ForwardBatch(
forward_mode=capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=batch_size,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -698,6 +697,7 @@ class BaseRunner(ABC):
global_forward_mode=capture_forward_mode,
lora_ids=lora_ids,
)
mr.kv_index_translator.bind_runner_slots(forward_batch)
if buffers.ngram_embedding_info is not None:
forward_batch.ngram_embedding_info = buffers.ngram_embedding_info.slice(
@@ -189,6 +189,8 @@ def build_replay_fb_view(
encoder_lens=buffers.encoder_lens[:bs] if is_encoder_decoder else None,
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
origin_out_cache_loc=getattr(forward_batch, "origin_out_cache_loc", None),
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
max_seq_len_override=forward_batch.max_seq_len_override,
@@ -1005,7 +1007,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.capture_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -1043,6 +1044,7 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
rids_int=rids_int,
bootstrap_room_ids_int=bootstrap_room_ids_int,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
# Trip the coordinator so the hisparse code path is captured into the
# graph; backends read it from self.model_runner.hisparse_coordinator.
@@ -1118,9 +1120,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
self._post_process_after_profile(prof)
self._profiler = None
# No pool-side pin to clear: the captured full-physical write loc rides the
# backend's `ForwardMetadata.out_cache_loc_full_physical` (-> KVWriteLoc.full_loc).
def _capture_one_stream(self, stream_idx: Optional[int] = None) -> None:
avail_mem = get_available_gpu_memory(
self.model_runner.device,
@@ -1271,9 +1270,6 @@ class DecodeCudaGraphRunner(BaseCudaGraphRunner):
if (c := self.model_runner.canary_manager) is not None
else contextlib.nullcontext()
)
# Full-physical write loc lives in the attention metadata (the backend's
# `out_cache_loc_full_physical` -> KVWriteLoc.full_loc), so the runner
# wires no buffer here. (SWA write loc rides the `swa_out_cache_loc` rail.)
with canary_ctx:
shape_key = self._make_graph_key(
@@ -1496,7 +1496,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
with torch.device(self.device):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.EXTEND,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=_slot("input_ids"),
input_embeds=(
@@ -1562,6 +1561,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
return_pooled_hidden_states=self.capture_return_pooled_hidden_states,
max_seq_len_override=self.max_context_size,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
ngram_manager = self.model_runner.ngram_embedding_manager
if ngram_manager.enabled:
forward_batch.ngram_embedding_info = NgramEmbeddingInfo.create(
@@ -1861,7 +1861,6 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
static_forward_batch = ForwardBatch(
forward_mode=pcg_forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
input_embeds=input_embeds,
@@ -1926,6 +1925,11 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
),
max_seq_len_override=self.max_context_size,
)
forward_batch.kv_loc_plan.bind_replay(
static_forward_batch,
self.model_runner.kv_index_translator,
slots=out_cache_loc,
)
# The n-gram hasher runs outside the graph and reads this at replay.
static_forward_batch.ngram_embedding_info = forward_batch.ngram_embedding_info
static_forward_batch.engram_history = forward_batch.engram_history
+2 -2
View File
@@ -803,8 +803,8 @@ class DSparkDraftMixin:
attn.attn.v_scale,
)
else:
# Translated by the injector (physical by allocation on a plain
# pool).
# `cache_loc` comes from `TargetHiddenKvInjector.ids_for`,
# already in the ids the draft pool indexes.
pool.set_kv_buffer(
attn.attn,
KVWriteLoc(cache_loc, physical=True),
@@ -481,12 +481,8 @@ class InklingAttention(nn.Module):
do_store = do_bf16_store or do_mxfp8_store
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
# SWA sub-pool + the backend's full->SWA translated write location.
# SWA sub-pool + the plan's sliding-window write ids (backend buffer).
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
es = q.element_size()
@@ -584,10 +580,6 @@ class InklingAttention(nn.Module):
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
is_v2 = forward_batch.forward_mode.is_draft_extend_v2()
@@ -691,10 +683,6 @@ class InklingAttention(nn.Module):
metadata = get_attn_backend().forward_metadata
if do_store and self.is_local:
loc = metadata.swa_out_cache_loc
elif do_store:
loc = getattr(metadata, "out_cache_loc_full_physical", None)
if loc is None:
loc = forward_batch.out_cache_loc
else:
loc = forward_batch.out_cache_loc
es = q.element_size()
+15 -7
View File
@@ -7,6 +7,7 @@ import torch
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import Cols, IdSpaceKind, KVLocPlan
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
ForwardBatch,
@@ -51,6 +52,10 @@ class DFlashVerifyInput(SpecInput):
live_seq_lens_cpu: Optional[torch.Tensor] = None
# Conservative request-lifetime bound for candidate graph dispatch.
candidate_max_seq_len_upper_bound: Optional[int] = None
# The iteration's plan when the caller planned the verify window, and the
# part of it this verify writes (`KVLocPlan.write_ids`).
kv_loc_plan: Optional[KVLocPlan] = None
kv_loc_cols: Optional[Cols] = None
def __post_init__(self):
super().__init__(spec_input_type=SpecInputType.DFLASH_VERIFY)
@@ -101,6 +106,8 @@ class DFlashVerifyInput(SpecInput):
target_worker.model_runner,
capture_hidden_mode=self.capture_hidden_mode,
return_hidden_states_before_norm=False,
kv_loc_plan=self.kv_loc_plan,
write_cols=self.kv_loc_cols,
)
can_run_cuda_graph = bool(
@@ -133,13 +140,14 @@ class DFlashVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
plan: KVLocPlan,
kv_start_idx: Optional[torch.Tensor] = None,
kv_indices_buf: Optional[torch.Tensor] = None,
sliding_window: bool = False,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
device = req_pool_indices.device
bs = req_pool_indices.numel()
@@ -179,14 +187,14 @@ class DFlashVerifyInput(SpecInput):
dtype=torch.int32,
device=device,
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum + kv_indices_extra,
out=kv_indices,
kv_start_idx=kv_start_idx,
sliding_window=sliding_window,
kind=kind,
)
mask = self.custom_mask
if mask is not None:
@@ -26,6 +26,7 @@ from sglang.srt.lora.layers import unwrap_lora_layer
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
@@ -1744,6 +1745,11 @@ class DFlashWorkerV2(BaseSpecWorker):
return out_tokens
def _draft_write_ids(self, plan: KVLocPlan) -> torch.Tensor:
"""The plan's write ids as the draft pool indexes them: the target's
physical ids for a fused draft, the virtual ids for a private one."""
return plan.write_ids(self.draft_model_runner.kv_index_translator)
def _append_target_hidden_to_draft_kv_by_loc(
self,
*,
@@ -1845,17 +1851,9 @@ class DFlashWorkerV2(BaseSpecWorker):
if commit_lens.dtype != torch.int32:
commit_lens = commit_lens.to(torch.int32)
# On a translating pool the translate writes a new tensor, so the
# callers' locs stay virtual: the post-verify 2-D buffer is re-read as
# virtual ids by the compact req_to_token rebuild. On a plain pool it
# returns `cache_loc` itself.
translator = self.draft_model_runner.kv_index_translator
cache_loc = translator.translate_full_attn_ids(cache_loc)
if cache_loc_2d is not None:
# Same ids, so reshape the translated copy rather than translate
# them a second time.
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
# `cache_loc` arrives in the draft pool's id space (`_draft_write_ids`);
# the callers' window buffers stay virtual for the compact
# req_to_token rebuild.
with (
torch.inference_mode(),
draft_tp_context(self.draft_owns_attention),
@@ -2265,7 +2263,9 @@ class DFlashWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=CaptureHiddenMode.FULL,
return_kv_loc_plan=True,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
logits_output, next_token_ids = (
batch_output.logits_output,
@@ -2316,9 +2316,10 @@ class DFlashWorkerV2(BaseSpecWorker):
ctx_lens,
int(sum(batch.extend_lens)),
)
# The draft KV goes to the slots the target prefill just wrote.
self._append_target_hidden_to_draft_kv_by_loc(
target_hidden=logits_output.hidden_states,
cache_loc=batch.out_cache_loc,
cache_loc=self._draft_write_ids(kv_loc_plan),
positions=positions,
extend_lens=ctx_lens,
)
@@ -2494,6 +2495,19 @@ class DFlashWorkerV2(BaseSpecWorker):
positions = positions_2d.reshape(-1)
verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1)
# The iteration's window -- the draft block, the verify and the
# target-hidden KV writes all write it -- planned once.
kv_loc_plan = self.target_worker.model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=(
batch.seq_lens_cpu
if batch.seq_lens_cpu is not None
else draft_input.nxt_kv_lens_cpu
),
write_virtual=verify_out_cache_loc,
read_extent=block_size,
)
seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs]
if self.use_compact_draft_cache:
@@ -2555,10 +2569,20 @@ class DFlashWorkerV2(BaseSpecWorker):
),
global_num_token_non_padded_cpu=bs * block_size,
)
# Hand-built draft batch bypasses ForwardBatch.init_new: under the
# unified pool the write loc must be rebound to the draft's
# kernel-facing ids here (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(forward_batch)
# The draft block writes the whole window. A compact draft reads its
# own `req_to_token` rows over its own lengths.
draft_translator = self.draft_model_runner.kv_index_translator
draft_plan = (
kv_loc_plan.reads_from(
draft_translator,
seq_lens=draft_seq_lens,
seq_lens_cpu=seq_lens_cpu,
read_extent=block_size,
)
if self.use_compact_draft_cache
else kv_loc_plan
)
draft_plan.bind(forward_batch, draft_translator)
if self.selector is not None or self.lilicorr is not None:
self._selector_sample = None
@@ -2713,6 +2737,7 @@ class DFlashWorkerV2(BaseSpecWorker):
draft_token_num=int(self.block_size),
custom_mask=custom_mask,
capture_hidden_mode=CaptureHiddenMode.FULL,
kv_loc_plan=kv_loc_plan,
)
batch.out_cache_loc = verify_out_cache_loc
@@ -2877,10 +2902,11 @@ class DFlashWorkerV2(BaseSpecWorker):
# Consume in this step: every decode graph size shares one aux output,
# which the next target forward overwrites (resolve_aux_hidden_states_width).
cache_loc = self._draft_write_ids(kv_loc_plan)
self._append_target_hidden_to_draft_kv_by_loc(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
cache_loc=verify_out_cache_loc,
cache_loc_2d=verify_out_cache_loc_2d,
cache_loc=cache_loc,
cache_loc_2d=cache_loc.view(verify_out_cache_loc_2d.shape),
positions=positions,
commit_lens=commit_lens,
)
@@ -360,7 +360,7 @@ class DraftBlockProposer:
)
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
# loc to the draft's kernel-facing ids (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(idle_batch)
self.draft_model_runner.kv_index_translator.bind_own_plan(idle_batch)
self._fill_dp_moe_sync_metadata(idle_batch, batch)
with torch.inference_mode():
self.draft_model_runner.forward(idle_batch)
@@ -435,10 +435,10 @@ class DraftBlockProposer:
),
global_num_token_non_padded_cpu=draft_num_tokens,
)
# Hand-built batch bypasses ForwardBatch.init_new: rebind the write
# loc to the draft's kernel-facing ids (no-op on plain pools).
self.draft_model_runner.kv_index_translator.rebind_write_loc(
draft_forward_batch
verify_window.kv_loc_plan.bind(
draft_forward_batch,
self.draft_model_runner.kv_index_translator,
cols=slice(0, query_token_num),
)
self._fill_dp_moe_sync_metadata(draft_forward_batch, batch)
graph_runner = self.draft_model_runner.decode_cuda_graph_runner
@@ -5,12 +5,12 @@ import torch
from sglang.kernels.ops.attention.dsv4.unified_kv_kernels.env_gate import (
is_unified_kv_triton,
)
from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func
from sglang.kernels.ops.speculative.dspark.dspark_verify_window import (
BuildCommitInjectLayout,
build_unified_commit_inject_layout,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.speculative.ragged_verify import RaggedVerifyLayout
@@ -32,6 +32,11 @@ class TargetHiddenKvInjector:
self.verify_num_draft_tokens = verify_num_draft_tokens
self._block_pos_offsets = block_pos_offsets
def ids_for(self, plan: KVLocPlan, *, cols: Optional[Cols] = None) -> torch.Tensor:
"""The plan's write ids as the draft pool indexes them: the target's
physical ids for a fused draft, the virtual ids for a private one."""
return plan.write_ids(self.draft_model_runner.kv_index_translator, cols=cols)
def inject_target_hidden(
self,
*,
@@ -70,16 +75,7 @@ class TargetHiddenKvInjector:
device=device, dtype=torch.int64, non_blocking=True
)
# On a translating pool the translate writes a new tensor, so the
# callers' locs stay virtual; on a plain pool it returns `cache_loc`
# itself.
translator = self.draft_model_runner.kv_index_translator
cache_loc = translator.translate_full_attn_ids(cache_loc)
if cache_loc_2d is not None:
# Callers pass `cache_loc` viewed [bs, verify_len]: reshape the
# translated copy rather than translate the same ids twice.
cache_loc_2d = cache_loc.reshape(cache_loc_2d.shape)
# `cache_loc` arrives in the draft pool's id space (`ids_for`).
pool = self.draft_model_runner.token_to_kv_pool
if hasattr(pool, "set_swa_key_buffer_radix_fused_norm_rope"):
if target_hidden_is_projected:
@@ -214,6 +210,7 @@ class TargetHiddenKvInjector:
hidden_strided: torch.Tensor,
commit_lens: torch.Tensor,
bs: int,
kv_loc_plan: KVLocPlan,
) -> None:
stride = self.verify_num_draft_tokens
prefix_lens = batch.seq_lens
@@ -252,15 +249,7 @@ class TargetHiddenKvInjector:
return
positions_2d = prefix_lens.unsqueeze(1) + self._block_pos_offsets
verify_cache_loc = assign_extend_cache_locs_func(
req_pool_indices=batch.req_pool_indices,
req_to_token=self.model_runner.req_to_token_pool.req_to_token,
start_offset=prefix_lens,
end_offset=prefix_lens + stride,
batch_size=bs,
draft_token_num=stride,
device=self.device,
)
verify_cache_loc = self.ids_for(kv_loc_plan)
verify_cache_loc_2d = verify_cache_loc.view(bs, stride)
self.inject_target_hidden(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
@@ -20,6 +20,7 @@ from sglang.srt.managers.overlap_utils import (
ResolvedConfidence,
)
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments
@@ -65,6 +66,9 @@ class VerifyWindow(msgspec.Struct, frozen=True):
positions_2d: torch.Tensor
verify_cache_loc: torch.Tensor
verify_cache_loc_2d: torch.Tensor
# The iteration's plan of `verify_cache_loc`: the draft block, the verify
# and the target-hidden KV writes all take their ids from it.
kv_loc_plan: KVLocPlan
class DSparkVerifyPlanner:
@@ -827,7 +831,10 @@ def alloc_verify_window(
verify_num_draft_tokens: int,
block_pos_offsets: torch.Tensor,
model_runner,
seq_lens_cpu: Optional[torch.Tensor],
) -> VerifyWindow:
"""``seq_lens_cpu``: a host bound on ``batch.seq_lens``, which sizes the
plan's read table."""
prefix_lens = batch.seq_lens
verify_w = verify_num_draft_tokens
positions_2d = prefix_lens.unsqueeze(1) + block_pos_offsets
@@ -845,6 +852,13 @@ def alloc_verify_window(
positions_2d=positions_2d,
verify_cache_loc=verify_cache_loc,
verify_cache_loc_2d=verify_cache_loc_2d,
kv_loc_plan=model_runner.kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=prefix_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=verify_cache_loc,
read_extent=verify_w,
),
)
@@ -312,6 +312,7 @@ class TargetVerifyExecutor:
custom_mask=None,
capture_hidden_mode=CaptureHiddenMode.FULL,
live_seq_lens_cpu=batch.seq_lens_cpu,
kv_loc_plan=verify_window.kv_loc_plan,
)
batch.out_cache_loc = verify_cache_loc
seq_lens_cpu_backup = batch.seq_lens_cpu
@@ -388,6 +389,7 @@ class TargetVerifyExecutor:
hidden_strided=hidden_strided,
commit_lens=commit_lens,
bs=bs,
kv_loc_plan=verify_window.kv_loc_plan,
)
return
hidden = logits_output.hidden_states
@@ -404,10 +406,11 @@ class TargetVerifyExecutor:
state_slot = (
batch.req_pool_indices[:bs].view(-1, 1).expand(bs, vlen).reshape(-1)
)
cache_loc = self.kv_injector.ids_for(verify_window.kv_loc_plan)
self.kv_injector.inject_target_hidden(
target_hidden=hidden.reshape(-1, hidden.shape[-1]),
cache_loc=verify_window.verify_cache_loc,
cache_loc_2d=verify_window.verify_cache_loc_2d,
cache_loc=cache_loc,
cache_loc_2d=cache_loc.view(verify_window.verify_cache_loc_2d.shape),
positions=verify_window.positions_2d.reshape(-1),
commit_lens=commit_lens,
state_slot=state_slot,
@@ -420,6 +423,7 @@ class TargetVerifyExecutor:
layout: RaggedVerifyLayout,
ragged_window: RaggedVerifyWindow,
sampling_info,
kv_loc_plan,
) -> TargetVerifyResult:
verify_input = DFlashVerifyInput(
draft_token=ragged_window.verify_ids,
@@ -429,6 +433,9 @@ class TargetVerifyExecutor:
capture_hidden_mode=CaptureHiddenMode.FULL,
ragged_verify_layout=layout,
live_seq_lens_cpu=batch.seq_lens_cpu,
# The packed rows are a selection of the planned verify window.
kv_loc_plan=kv_loc_plan,
kv_loc_cols=ragged_window.window_index,
)
batch.out_cache_loc = ragged_window.verify_cache_loc
seq_lens_cpu_backup = batch.seq_lens_cpu
@@ -461,6 +468,7 @@ class TargetVerifyExecutor:
bs: int,
device: str,
sampling_info,
verify_window: VerifyWindow,
inject_gate: bool = False,
) -> tuple[TargetVerifyResult, torch.Tensor]:
ragged_window = BuildRaggedVerifyWindow.execute(
@@ -480,6 +488,7 @@ class TargetVerifyExecutor:
layout=layout,
ragged_window=ragged_window,
sampling_info=sampling_info,
kv_loc_plan=verify_window.kv_loc_plan,
)
logits_output = target_verify.logits_output
@@ -642,6 +642,7 @@ class DSparkWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=CaptureHiddenMode.FULL,
return_kv_loc_plan=True,
)
# BCG replay skips model-side Python, so re-evaluate the same pure predicate.
target_hidden_is_projected = (
@@ -705,7 +706,9 @@ class DSparkWorkerV2(BaseSpecWorker):
repeats,
output_size=num_tokens,
)
cache_loc = batch.out_cache_loc
# The draft KV goes to the slots the target prefill just wrote.
cache_loc = self._kv_injector.ids_for(batch_output.kv_loc_plan)
batch_output.kv_loc_plan = None
token_indices = logits_output.hidden_states_token_indices
if token_indices is not None:
cache_loc = cache_loc[token_indices]
@@ -842,6 +845,11 @@ class DSparkWorkerV2(BaseSpecWorker):
verify_num_draft_tokens=self.verify_num_draft_tokens,
block_pos_offsets=self._block_pos_offsets,
model_runner=self.model_runner,
seq_lens_cpu=(
batch.seq_lens_cpu
if batch.seq_lens_cpu is not None
else draft_input.nxt_kv_lens_cpu
),
)
sampling_info = batch.sampling_info
@@ -936,6 +944,7 @@ class DSparkWorkerV2(BaseSpecWorker):
bs=bs,
device=device,
sampling_info=sampling_info,
verify_window=verify_window,
inject_gate=fold_eligible,
)
else:
@@ -459,7 +459,6 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=num_seqs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -485,6 +484,7 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
spec_info.capture_hidden_mode if spec_info else CaptureHiddenMode.NULL
),
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
def run_once():
self.draft_attn_backend.init_forward_metadata_in_graph(forward_batch)
@@ -424,7 +424,6 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -448,6 +447,7 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
if self.buffers.dsa_seed_topk_capture is not None:
spec_info.dsa_seed_topk_capture = self.buffers.dsa_seed_topk_capture[
@@ -649,6 +649,8 @@ class EAGLEDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
encoder_lens=None,
out_cache_loc=buffers.out_cache_loc[:num_tokens],
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
out_cache_loc_dsv4=getattr(forward_batch, "out_cache_loc_dsv4", None),
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
+19 -13
View File
@@ -5,6 +5,7 @@ from typing import Callable, List, Optional
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -34,6 +35,9 @@ class EagleVerifyInput(SpecInput):
draft_probs: torch.Tensor = None
prepared_out_cache_loc: Optional[torch.Tensor] = None
prepared_mrope_positions: Optional[torch.Tensor] = None
# The iteration's plan when the draft planned the verify window
# (`prepared_out_cache_loc`); verify and draft extend take it.
kv_loc_plan: Optional[KVLocPlan] = None
# Shape info for padding
num_tokens_per_req: int = -1 # -1 auto-fills from draft_token_num.
@@ -90,11 +94,12 @@ class EagleVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
device = req_pool_indices.device
batch_size = req_pool_indices.numel()
qo_indptr = torch.arange(
@@ -113,13 +118,13 @@ class EagleVerifyInput(SpecInput):
total_tokens = paged_kernel_lens_sum + self.draft_token_num * batch_size
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=device)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
mask_numel = (
paged_kernel_lens_sum * self.draft_token_num
@@ -402,10 +407,11 @@ class EagleDraftExtendInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: Optional[int],
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""Draft-extend CSR args. The lens already include the window, so
unlike verify nothing is widened here."""
"""Draft-extend CSR args. ``paged_kernel_lens`` already includes the
draft-extend window."""
device = req_pool_indices.device
bs = self.num_correct_drafts.numel()
# Constant num_tokens_per_req qo layout (required for cuda-graph capture).
@@ -426,12 +432,12 @@ class EagleDraftExtendInput(SpecInput):
paged_kernel_lens_sum, dtype=torch.int32, device=device
)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=paged_kernel_lens_sum,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
return kv_indices, cum_kv_seq_len, qo_indptr, None
@@ -607,6 +607,7 @@ def eagle_prepare_for_verify(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=verify_input.prepared_mrope_positions,
kv_loc_plan=verify_input.kv_loc_plan,
)
# Run attention backend plan and cuda graph preparation
@@ -55,6 +55,8 @@ if _is_cpu:
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.speculative.eagle_draft_cuda_graph_runner import (
@@ -121,7 +123,10 @@ def prepare_for_draft_extend(
return_hidden_states_before_norm: bool,
widened_out_cache_loc: Optional[torch.Tensor] = None,
widened_positions: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""``kv_loc_plan``: the iteration's plan, whose window the draft extend
writes again (the verify window); a widened window plans its own."""
bs = len(batch.seq_lens)
# Optional window widening (num_front_tokens=0 -> off): prepend that many
# rows below the boundary. Locs/positions arrive precomputed; token/hidden
@@ -204,6 +209,7 @@ def prepare_for_draft_extend(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=return_hidden_states_before_norm,
extend_position_info=extend_position_info,
kv_loc_plan=None if widen else kv_loc_plan,
)
# Forward sees post-write length (draft extend writes num_draft_tokens
# slots); mutation stays on forward_batch to preserve SB.seq_lens.
@@ -243,18 +249,35 @@ def prepare_for_draft(
draft_model_runner: ModelRunner,
topk: int,
num_steps: int,
):
*,
target_translator: Optional[KVIndexTranslator] = None,
num_draft_tokens: Optional[int] = None,
) -> tuple[ForwardBatch, bool, Optional[KVLocPlan]]:
"""The draft-decode batch. With ``target_translator``, a chain draft plans
the iteration's whole write window (the ``num_draft_tokens`` slots past
``seq_lens`` that verify and draft extend also write) and writes its first
``num_steps`` columns; that plan is returned, else None."""
prepared_positions = None
prepared_mrope_positions = None
kv_loc_plan = None
if not batch.forward_mode.is_idle():
bs = len(batch.seq_lens)
# Assign cache locations (draft-write targets).
page_size = batch.token_to_kv_pool_allocator.page_size
plans_window = (
target_translator is not None
and topk == 1
and num_draft_tokens is not None
and num_draft_tokens > num_steps
and not _is_cpu
)
# A chain draft's slots lead the iteration's window: gather the whole
# window in the same launch.
window_cols = num_draft_tokens if plans_window else num_steps
if page_size == 1 or topk == 1:
batch.out_cache_loc = torch.empty(
(bs * topk * num_steps,),
(bs * topk * window_cols,),
dtype=torch.int64,
device=batch.device,
)
@@ -285,13 +308,21 @@ def prepare_for_draft(
batch.out_cache_loc,
req_to_token_pool.req_to_token.shape[1],
topk,
num_steps,
window_cols,
positions=prepared_positions,
mrope=prepared_mrope_positions,
BS=bs,
WRITE_POSITIONS=prepared_positions is not None,
WRITE_MROPE=prepared_mrope_positions is not None,
)
if plans_window:
kv_loc_plan = target_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=batch.seq_lens,
seq_lens_cpu=batch.seq_lens_cpu,
write_virtual=batch.out_cache_loc,
read_extent=num_draft_tokens,
)
else:
# page_size > 1 + topk > 1: per-branch page-aligned draft pages.
# Reduce out_cache_loc from the page-aligned tree region down to the
@@ -352,11 +383,13 @@ def prepare_for_draft(
capture_hidden_mode=capture_mode,
return_hidden_states_before_norm=False,
spec_mrope_positions=prepared_mrope_positions,
kv_loc_plan=kv_loc_plan,
write_cols=None if kv_loc_plan is None else slice(0, num_steps),
)
can_run_decode_cuda_graph = cuda_graph_runner and cuda_graph_runner.can_run_graph(
forward_batch
)
return forward_batch, can_run_decode_cuda_graph
return forward_batch, can_run_decode_cuda_graph, kv_loc_plan
def build_eagle_verify_input(
@@ -41,6 +41,7 @@ from sglang.srt.layers.moe.utils import (
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
from sglang.srt.managers.tp_worker import TpModelWorker
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
Phase,
@@ -606,7 +607,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
def draft(self, batch: ScheduleBatch, *, with_topology: bool = False):
draft_input: EagleDraftInput = batch.spec_info
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
draft_input,
self.req_to_token_pool,
batch,
@@ -614,6 +615,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
self.topk,
self.speculative_num_steps,
target_translator=self.target_worker.model_runner.kv_index_translator,
num_draft_tokens=self.speculative_num_draft_tokens,
)
if (
can_run_decode_cuda_graph
@@ -673,6 +676,10 @@ class EagleDraftWorker(EagleDraftWorkerBase):
tree_mask_mode=self.tree_mask_mode,
device=self.device,
)
if kv_loc_plan is not None:
# Verify and draft extend write the window the draft planned.
verify_input.kv_loc_plan = kv_loc_plan
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
if with_topology:
# PP+spec relays the tree so every stage rebuilds the same verify
# input; the mask build needs the topology this one was built from.
@@ -900,6 +907,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
target_hidden_states: torch.Tensor,
next_token_ids: torch.Tensor,
mm_input_embeds: Optional[torch.Tensor] = None,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""
Run draft model extend to correctly fill the KV cache.
@@ -908,6 +916,8 @@ class EagleDraftWorker(EagleDraftWorkerBase):
batch: The batch to run.
target_hidden_states: Hidden states from the target model forward
next_token_ids: Next token ids generated from the target forward.
kv_loc_plan: The target prefill's plan; the draft writes the same
slots.
"""
# Construct input_ids
if not batch.forward_mode.is_idle():
@@ -959,6 +969,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
capture_hidden_mode=capture_hidden_mode,
return_hidden_states_before_norm=False,
kv_loc_plan=kv_loc_plan,
)
forward_batch.return_logprob = False
if mm_input_embeds is not None:
@@ -1035,7 +1046,11 @@ class EagleDraftWorker(EagleDraftWorkerBase):
return buf[:num_tokens]
def _draft_extend_for_decode(
self, batch: ScheduleBatch, batch_result: GenerationBatchResult
self,
batch: ScheduleBatch,
batch_result: GenerationBatchResult,
*,
kv_loc_plan: Optional[KVLocPlan] = None,
):
# Cast to int64 before entering plan stream to avoid cross-stream
# synchronization issues with .to() inside the plan stream context.
@@ -1093,6 +1108,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner,
self.cuda_graph_runner_for_draft_extend,
return_hidden_states_before_norm=False,
kv_loc_plan=kv_loc_plan,
)
if self.plan_stream:
@@ -1412,7 +1428,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
speculative_moe_a2a_backend_context(),
spec_stage_span("draft_extend"),
):
self.draft_worker._draft_extend_for_decode(batch, batch_output)
self.draft_worker._draft_extend_for_decode(
batch, batch_output, kv_loc_plan=verify_input.kv_loc_plan
)
if (
get_parallel().pp_size > 1
@@ -1468,7 +1486,9 @@ class EAGLEWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
return_kv_loc_plan=self._draft_worker is not None,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
@@ -1505,6 +1525,7 @@ class EAGLEWorkerV2(BaseSpecWorker):
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
batch_output.logits_output.mm_input_embeds,
kv_loc_plan=kv_loc_plan,
)
return batch_output
@@ -302,7 +302,6 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
forward_batch = ForwardBatch(
forward_mode=ForwardMode.DECODE,
out_cache_loc_is_physical=True,
batch_size=expanded_bs,
input_ids=None,
req_pool_indices=req_pool_indices,
@@ -321,6 +320,7 @@ class FrozenKVMTPCudaGraphRunner(DecodeCudaGraphRunner):
spec_info=spec_info,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
def run_once():
# Record the metadata rebuild against the committed target-prefix
@@ -365,6 +365,8 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker):
encoder_lens=None,
out_cache_loc=getattr(forward_batch, "out_cache_loc", None),
out_cache_loc_virtual=forward_batch.out_cache_loc_virtual,
kv_loc_plan=forward_batch.kv_loc_plan,
kv_loc_cols=forward_batch.kv_loc_cols,
spec_info=None,
)
with self._frozen_kv_target_view(forward_batch):
@@ -99,6 +99,7 @@ if is_npu():
fill_draft_extend_prepare_buffers = fill_draft_extend_prepare_buffers_native
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import Cols, KVLocPlan
from sglang.srt.speculative.multi_layer_eagle_worker_v2 import (
MultiLayerEagleDraftWorker,
)
@@ -333,7 +334,6 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Forward batch
forward_batch = ForwardBatch(
forward_mode=self.forward_mode,
out_cache_loc_is_physical=True,
batch_size=bs,
input_ids=input_ids,
req_pool_indices=req_pool_indices,
@@ -361,6 +361,7 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
global_num_token_non_padded_cpu=self.captured_req_width * bs,
return_hidden_states_before_norm=True,
)
self.model_runner.kv_index_translator.bind_runner_slots(forward_batch)
return forward_batch
def _postprocess_forward_batch(self, forward_batch: ForwardBatch, bs: int):
@@ -476,6 +477,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
seq_lens_cpu: Optional[torch.Tensor],
*,
out_cache_loc_virtual: Optional[torch.Tensor],
kv_loc_plan: Optional[KVLocPlan],
kv_loc_cols: Optional[Cols],
):
"""Init this step's attention metadata for the prepared bucket and
replay its graph. Buffers must already be populated by the composite
@@ -501,6 +504,8 @@ class MultiLayerEagleDraftExtendCudaGraphRunner(DecodeCudaGraphRunner):
# Virtual input stays separate from the backend's physical buffer.
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
out_cache_loc_virtual=out_cache_loc_virtual,
kv_loc_plan=kv_loc_plan,
kv_loc_cols=kv_loc_cols,
spec_info=spec_info,
)
if (
@@ -550,6 +555,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
self.num_front_tokens = 0
self.prune_draft_extend_logits = False
self._out_cache_loc_virtual = None
self._kv_loc_plan = None
self._kv_loc_cols = None
self._init_and_capture()
@@ -740,11 +747,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
req_pool_indices,
out_cache_loc,
positions=None,
out_cache_loc_virtual=None,
out_cache_loc_virtual,
kv_loc_plan,
):
# Staging runs before `prepare`, so the rail arrives here rather than
# off a ForwardBatch; `prepare` overwrites it with the batch's own.
# Staging runs before `prepare` has a ForwardBatch, so the virtual write
# ids and the plan come in here; `prepare` replaces them with the batch's.
self._out_cache_loc_virtual = out_cache_loc_virtual
self._kv_loc_plan = kv_loc_plan
self._kv_loc_cols = None
raw_bs = req_pool_indices.shape[0]
bs = self.get_runner(0)._pad_to_bucket(raw_bs, self.capture_bs)
buffers = self.buffers
@@ -780,6 +790,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
out_cache_loc=buffers.out_cache_loc[: bs * self.captured_req_width],
mamba_track_indices=track_indices(buffers.mamba_track_indices, bs),
out_cache_loc_virtual=self._out_cache_loc_virtual,
kv_loc_plan=self._kv_loc_plan,
kv_loc_cols=self._kv_loc_cols,
)
for backend in backends:
backend.init_forward_metadata_out_graph(batch)
@@ -792,6 +804,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
num_tokens = raw_bs * self.captured_req_width
self._out_cache_loc_virtual = forward_batch.out_cache_loc_virtual
self._kv_loc_plan = forward_batch.kv_loc_plan
self._kv_loc_cols = forward_batch.kv_loc_cols
# Bucketize to a captured batch size (padding the tail).
if self.require_mlp_tp_gather:
@@ -900,6 +914,8 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
self._replay_spec_info,
self.seq_lens_cpu,
out_cache_loc_virtual=self._out_cache_loc_virtual,
kv_loc_plan=self._kv_loc_plan,
kv_loc_cols=self._kv_loc_cols,
)
raw_bs = self.raw_bs
raw_num_tokens = self.raw_num_tokens
@@ -17,7 +17,7 @@ from __future__ import annotations
import logging
import time
from dataclasses import replace
from typing import TYPE_CHECKING, List, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
@@ -107,6 +107,7 @@ _is_cpu = is_cpu()
if TYPE_CHECKING:
from sglang.srt.mem_cache.kv_loc_plan import KVLocPlan
from sglang.srt.model_executor.model_runner import ModelRunner, ModelRunnerOutput
@@ -460,7 +461,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
def draft(self, batch: ScheduleBatch):
draft_input: EagleDraftInput = batch.spec_info
forward_batch, can_run_decode_cuda_graph = prepare_for_draft(
forward_batch, can_run_decode_cuda_graph, kv_loc_plan = prepare_for_draft(
draft_input,
self.req_to_token_pool,
batch,
@@ -468,12 +469,14 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[0],
self.topk,
self.speculative_num_steps,
target_translator=self.target_worker.model_runner.kv_index_translator,
num_draft_tokens=self.speculative_num_draft_tokens,
)
# Run draft
parent_list, top_scores_index, draft_tokens = self.draft_forward(forward_batch)
return build_eagle_verify_input(
verify_input = build_eagle_verify_input(
batch,
draft_input,
parent_list,
@@ -487,6 +490,11 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
tree_mask_mode=self.tree_mask_mode,
device=self.device,
)
if kv_loc_plan is not None:
# Verify and draft extend write the window the draft planned.
verify_input.kv_loc_plan = kv_loc_plan
verify_input.prepared_out_cache_loc = kv_loc_plan.write_virtual
return verify_input
def draft_forward(self, forward_batch: ForwardBatch):
# Parse args
@@ -596,6 +604,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch: ScheduleBatch,
target_hidden_states: torch.Tensor,
next_token_ids: torch.Tensor,
kv_loc_plan: Optional[KVLocPlan] = None,
):
"""
Run draft model extend to correctly fill the KV cache.
@@ -604,6 +613,8 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch: The batch to run.
target_hidden_states: Hidden states from the target model forward
next_token_ids: Next token ids generated from the target forward.
kv_loc_plan: The target prefill's plan; the draft writes the same
slots.
"""
# The draft embed clamps unconditionally (to tolerate multimodal pad
# sentinels), so probe next_token_ids here first -- otherwise a corrupted id
@@ -641,6 +652,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
self.draft_runner_list[0],
capture_hidden_mode=draft_capture_hidden_mode,
return_hidden_states_before_norm=True,
kv_loc_plan=kv_loc_plan,
)
self._apply_deferred_mamba_init_to_draft_pools(forward_batch)
@@ -766,14 +778,20 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
draft_token_num=self.speculative_num_draft_tokens,
device=batch.device,
)
seq_lens = batch.seq_lens + self.speculative_num_draft_tokens
runner.stage_shared_reads(
seq_lens=batch.seq_lens + self.speculative_num_draft_tokens,
seq_lens=seq_lens,
req_pool_indices=batch.req_pool_indices,
out_cache_loc=locs,
positions=positions,
# Both loc sources read req_to_token untranslated, so `locs` is the
# PRE-translate write loc the capture rail needs.
# Both loc sources read req_to_token untranslated: `locs` is virtual.
out_cache_loc_virtual=locs,
kv_loc_plan=self.draft_runner_list[0].kv_index_translator.plan(
req_pool_indices=batch.req_pool_indices,
seq_lens=seq_lens,
seq_lens_cpu=None,
write_virtual=locs,
),
)
return True
@@ -783,6 +801,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
batch_result: GenerationBatchResult,
*,
staged: bool = False,
kv_loc_plan: Optional[KVLocPlan] = None,
):
# Batch 2: Draft extend
draft_extend_input = EagleDraftExtendInput(
@@ -826,6 +845,7 @@ class MultiLayerEagleDraftWorker(EagleDraftWorkerBase):
return_hidden_states_before_norm=True,
widened_out_cache_loc=boundary_kv_locs,
widened_positions=boundary_kv_positions,
kv_loc_plan=kv_loc_plan,
)
if self.plan_stream:
@@ -1091,7 +1111,9 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
batch,
pp_proxy_tensors=pp_proxy_tensors,
capture_hidden_mode=target_capture_mode,
return_kv_loc_plan=True,
)
kv_loc_plan, batch_output.kv_loc_plan = batch_output.kv_loc_plan, None
# Spec_v2 convention: batch.seq_lens = length BEFORE this iter's tokens.
# Extend processed L prompt tokens; next verify iter expects same L.
@@ -1106,6 +1128,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
batch,
batch_output.logits_output.hidden_states,
batch_output.next_token_ids,
kv_loc_plan=kv_loc_plan,
)
return batch_output
else:
@@ -1134,7 +1157,10 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
if on_publish is not None:
on_publish(batch_output.new_seq_lens)
self.draft_worker._draft_extend_for_decode(
batch, batch_output, staged=staged
batch,
batch_output,
staged=staged,
kv_loc_plan=verify_input.kv_loc_plan,
)
return batch_output
+9 -7
View File
@@ -5,6 +5,7 @@ from typing import List, Optional
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, KVLocPlan
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
@@ -63,11 +64,12 @@ class NgramVerifyInput(SpecInput):
paged_kernel_lens: torch.Tensor,
paged_kernel_lens_sum: int,
translator: KVIndexTranslator,
sliding_window: bool = False,
plan: KVLocPlan,
kind: IdSpaceKind = IdSpaceKind.FULL,
):
"""CSR verify args, gathered straight into the packed stream. The lens
are widened here and handed to the translator, so nothing materializes
a ``[bs, max_pages]`` rectangle to repack from."""
"""CSR verify args. ``paged_kernel_lens`` excludes the verify tokens and
is widened here; the translator packs the read ids straight into
``kv_indices``."""
bs = req_pool_indices.numel()
cum_kv_seq_len = torch.zeros((bs + 1,), dtype=torch.int32, device=self.device)
@@ -83,13 +85,13 @@ class NgramVerifyInput(SpecInput):
total_tokens = paged_kernel_lens_sum + self.draft_token_num * bs
kv_indices = torch.empty(total_tokens, dtype=torch.int32, device=self.device)
translator.fill_packed_read_stream(
translator.pack_read_stream(
plan,
req_pool_indices=req_pool_indices,
seq_lens=paged_kernel_lens,
indptr=cum_kv_seq_len,
total_tokens=total_tokens,
out=kv_indices,
sliding_window=sliding_window,
kind=kind,
)
# Pad custom_mask when CUDA graph pads batch size beyond the actual number of requests.
@@ -297,7 +297,7 @@ class UnoWorkerV2(BaseSpecWorker):
draft_token_num=self.forward_width,
)
return ForwardBatch(
forward_batch = ForwardBatch(
forward_mode=ForwardMode.TARGET_VERIFY,
batch_size=len(prefix_lens),
input_ids=input_ids,
@@ -312,6 +312,8 @@ class UnoWorkerV2(BaseSpecWorker):
capture_hidden_mode=CaptureHiddenMode.NULL,
return_hidden_states_before_norm=False,
)
self.model_runner.kv_index_translator.bind_own_plan(forward_batch)
return forward_batch
def _run_target_block(
self,
@@ -761,6 +761,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
batch.extend_prefix_lens = torch.tensor(
@@ -26,6 +26,7 @@ from sglang.srt.environ import envs
from sglang.srt.layers.attention.attention_registry import ATTENTION_BACKENDS
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.model_executor.cuda_graph_config import (
Backend,
@@ -419,6 +420,13 @@ class MockDSV4ModelRunner:
identity = torch.arange(swa_size, dtype=torch.int64, device=device)
self.token_to_kv_pool.register_mapping(identity)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.kv_index_translator = KVIndexTranslator(
req_to_token=self.req_to_token_pool.req_to_token,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
token_to_kv_pool=self.token_to_kv_pool,
page_size=case.page_size,
device=device,
)
self.attention_chunk_size = None
self.hisparse_coordinator = None
self.init_new_workspace = False
@@ -661,6 +669,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
# extend_* fields are only populated for extend-shaped modes. DECODE leaves
# them at their defaults; the flash_mla path reads metadata directly from
# DSV4AttnMetadata so the extend fields are unused for the compress_ratio=0
@@ -322,6 +322,7 @@ class MockGDNModelRunner(ModelRunner):
page_size=case.page_size,
get_kvcache=lambda: self.token_to_kv_pool,
)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -329,7 +330,6 @@ class MockGDNModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -562,6 +562,8 @@ def _make_forward_batch(
else None
),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -318,6 +318,7 @@ class MockKDAModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -325,7 +326,6 @@ class MockKDAModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -552,6 +552,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -327,6 +327,7 @@ class MockLightningModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -334,7 +335,6 @@ class MockLightningModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -523,6 +523,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -453,6 +453,7 @@ class MockMamba2ModelRunner(ModelRunner):
enable_alt_stream=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -460,7 +461,6 @@ class MockMamba2ModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = False
self.is_draft_worker = False
self._kernel_warmed_up = True
@property
@@ -653,6 +653,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -307,6 +307,7 @@ class MockMLAModelRunner(ModelRunner):
enable_memory_saver=False,
)
self.token_to_kv_pool_allocator = SimpleNamespace(page_size=case.page_size)
self.is_draft_worker = False
self.init_kv_index_translator()
self.attention_chunk_size = None
self.hisparse_coordinator = None
@@ -314,7 +315,6 @@ class MockMLAModelRunner(ModelRunner):
self.is_hybrid_swa = False
self.sliding_window_size = None
self.use_mla_backend = True
self.is_draft_worker = False
self._kernel_warmed_up = True
# Runner-mode helpers mutate speculative graph sizes after construction.
self.graph_shared_output = GraphSharedOutput(
@@ -693,6 +693,8 @@ def _make_forward_batch(
seq_lens_sum=sum(seq_lens),
positions=torch.tensor(positions, dtype=torch.int64, device=device),
)
# Production batches take their KV ids from a plan (`init_new`).
runner.kv_index_translator.bind_own_plan(batch)
if case.forward_mode.is_extend(include_draft_extend_v2=True):
extend_seq_lens = torch.tensor(input_lens, dtype=torch.int32, device=device)
@@ -297,6 +297,8 @@ def _init_cuda_graph_replay_metadata(backend, capture_batch_size: int, batch):
encoder_lens=batch.encoder_lens,
extend_seq_lens=getattr(batch, "extend_seq_lens", None),
out_cache_loc=getattr(batch, "out_cache_loc", None),
kv_loc_plan=batch.kv_loc_plan,
kv_loc_cols=batch.kv_loc_cols,
spec_info=batch.spec_info,
)
backend.init_forward_metadata_out_graph(fb_view)
@@ -510,6 +510,7 @@ def run_eagle_draft_cuda_graph_runner_case(
)
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
expected = _run_eagle_draft_eager(
eager_worker,
eager_batch,
@@ -526,6 +527,7 @@ def run_eagle_draft_cuda_graph_runner_case(
)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
graph_runner = _capture_eagle_draft_graph_runner(
graph_worker,
graph_backend,
@@ -565,6 +567,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
)
adapter.prepare_replay_state(eager_fixture, case, draft_inputs, settings)
eager_batch = adapter.make_forward_batch(case, draft_inputs, settings)
eager_fixture.runner.kv_index_translator.bind_own_plan(eager_batch)
expected = _run_frozen_kv_mtp_eager(eager_worker, eager_batch)
graph_fixture, graph_worker, _ = _build_frozen_kv_mtp_fixture(
@@ -576,6 +579,7 @@ def run_frozen_kv_mtp_cuda_graph_runner_case(
)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
graph_batch = adapter.make_forward_batch(case, draft_inputs, settings)
graph_fixture.runner.kv_index_translator.bind_own_plan(graph_batch)
graph_runner = _capture_frozen_kv_mtp_graph_runner(graph_worker)
adapter.prepare_replay_state(graph_fixture, case, draft_inputs, settings)
@@ -19,8 +19,8 @@ FULL side's ids while it used to be sized by its own page count. A model with
few full-attention layers and many sliding ones gives the id owner more pages,
and the top of that id space then lands off the end of the swa table. On GPU
the write is unchecked (`alloc_bind_inplace`'s `tl.store`), so the symptom
surfaces later and elsewhere: a device-side index assert on the read in
`_swa_write_loc_unified`, and a dead scheduler.
surfaces later and elsewhere: a device-side index assert on a later read of
the swa table, and a dead scheduler.
The reachability of it is a KV-budget property, not just a model property: ids
come off the front of the owner's free list and freed ids return to the back, so
@@ -55,6 +55,7 @@ def _apply(backend, *, bs: int, seq_lens: torch.Tensor, forward_mode):
patch.object(backend, "_fill_dcp_block_kv_indices") as fill,
):
backend._apply_cuda_graph_metadata(
plan=None, # DCP builds its own block table
bs=bs,
req_pool_indices=torch.arange(bs, dtype=torch.int32, device="cuda"),
seq_lens=seq_lens,
@@ -108,7 +109,6 @@ class _DCPMetadataTests:
heads, v_head_dim, n = 4, 512, 3 * NUM_DRAFT_TOKENS
backend = object.__new__(self.backend_cls)
backend.data_type = backend.q_data_type = torch.bfloat16
backend._decode_kernel_loc = None
def real_path(*args, **kwargs):
raise _RealVerifyPath
@@ -24,7 +24,7 @@ class TestKimiLinearUnifiedMemoryFlashMLA(
GSM8KMixin, PrefixCacheBranchingMixin, DefaultServerBase
):
"""flashmla at its ps=64 snap: the block-table route
(KVIndexTranslator.fill_read_table into flashmla's padded tables) plus the
(the plan's page table copied into flashmla's padded tables) plus the
ps=64 sub-pool sizing (64-token sink floor, per-layer-view tail pad).
Hopper-only, like the rest of this nightly suite."""
@@ -12,6 +12,7 @@ from sglang.srt.layers.attention.linear.inkling_sconv_backend import (
InklingShortConvHybridAttnBackend,
)
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool
from sglang.srt.mem_cache.unified_memory_pool import (
MambaSubPoolSpec,
@@ -461,6 +462,9 @@ class TestInklingCheckpointIndices(CustomTestCase):
composite.buffers = buffers
composite.captured_req_width = 4
composite.draft_extend_attn_backend_list = [wrapper]
composite._out_cache_loc_virtual = None
composite._kv_loc_plan = None
composite._kv_loc_cols = None
for step, tracking in enumerate((True, False, True)):
with self.subTest(tracking=tracking, step=step):
ids = slots[-4:].roll(step).clone()
@@ -814,6 +818,15 @@ class TestInklingCheckpointIndices(CustomTestCase):
spec_algorithm=SimpleNamespace(is_standalone=lambda: False),
device_timer=None,
canary_manager=None,
# The KV side is not under test: a static pool's translator binds
# the captured batch to its own write ids.
kv_index_translator=KVIndexTranslator(
req_to_token=torch.zeros((1, 8), dtype=torch.int32, device="cuda"),
token_to_kv_pool_allocator=None,
token_to_kv_pool=object(),
page_size=1,
device="cuda",
),
)
return runner, backend, allocator, pool, slots, buffers
@@ -926,6 +939,9 @@ class TestInklingCheckpointIndices(CustomTestCase):
input_ids=buffers.input_ids[: live * 4],
positions=buffers.positions[: live * 4],
out_cache_loc=buffers.out_cache_loc[: live * 4],
out_cache_loc_virtual=None,
kv_loc_plan=None,
kv_loc_cols=None,
req_pool_indices=torch.arange(1, live + 1, device="cuda"),
seq_lens=torch.full((live,), 6, dtype=torch.int64, device="cuda"),
seq_lens_sum=6 * live,
@@ -973,7 +989,13 @@ class TestInklingCheckpointIndices(CustomTestCase):
composite._stage_metadata = Mock()
composite.prepare(fresh)
runner.replay(
4, composite.seq_lens_sum, composite._replay_spec_info, None
4,
composite.seq_lens_sum,
composite._replay_spec_info,
None,
out_cache_loc_virtual=composite._out_cache_loc_virtual,
kv_loc_plan=composite._kv_loc_plan,
kv_loc_cols=composite._kv_loc_cols,
)
else:
runner.execute(fresh, torch.arange(live, device="cuda") * 4 + 1)
@@ -0,0 +1,126 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`generate_draft_decode_kv_indices` over the unified pool's read tables.
Over the plan's physical page table (token ids rebuilt as
`entry * ps + pos % ps`), and over virtual rows translated in the kernel's own
gather, the kernel must emit exactly the translated virtual ids and the same
kv_indptr. The static `req_to_token` path is covered by
test_spec_kv_indices_grid.py.
python -m pytest test/registered/kernels/ops/speculative/test_draft_decode_kv_indices_read_table.py -v
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=20, stage="base-b-kernel-unit", runner_config="1-gpu-large")
_SENTINEL = -7
def _run_kernel(
*, table, entry_page_size, seq_lens, positions, num_steps, topk, page_size, v2p=None
):
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_kv_indices,
)
from sglang.srt.utils import next_power_of_2
num_seqs = seq_lens.numel()
bs = num_seqs * topk
width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64)
kv_indices = torch.full(
(num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda"
)
kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda")
generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)](
torch.arange(num_seqs, dtype=torch.int64, device="cuda"),
table,
seq_lens,
kv_indices,
kv_indptr,
positions,
table.stride(0),
width,
kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(num_steps),
next_power_of_2(bs),
page_size,
ENTRY_PAGE_SIZE=entry_page_size,
v2p=v2p,
TRANSLATE=v2p is not None,
)
return kv_indices, kv_indptr
class TestDraftDecodeKVIndicesReadTable(CustomTestCase):
def _check(self, *, page_size, num_steps=3, topk=1):
torch.manual_seed(7)
num_seqs, max_context = 3, 64
num_pages = max_context // page_size
# Each row holds whole virtual pages in a shuffled order, and the
# virtual->physical table is a shuffle too, so nothing is identity.
v2p = torch.randperm(num_pages, device="cuda")
rows = []
for _ in range(num_seqs):
vpages = torch.randperm(num_pages, device="cuda")
offsets = torch.arange(page_size, device="cuda")
rows.append((vpages[:, None] * page_size + offsets).reshape(-1))
req_to_token = torch.stack(rows)
# The plan's table: one physical page per virtual page of the row.
table = v2p[req_to_token[:, ::page_size] // page_size].to(torch.int32)
seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")
positions = seq_lens.repeat_interleave(topk)
common = dict(
seq_lens=seq_lens,
positions=positions,
num_steps=num_steps,
topk=topk,
page_size=page_size,
)
raw, raw_indptr = _run_kernel(table=req_to_token, entry_page_size=1, **common)
out, out_indptr = _run_kernel(table=table, entry_page_size=page_size, **common)
# No shared table: the virtual rows, translated in the kernel's gather.
gathered, gathered_indptr = _run_kernel(
table=req_to_token, entry_page_size=1, v2p=v2p, **common
)
torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0)
written = raw != _SENTINEL
self.assertTrue(bool((out[~written] == _SENTINEL).all()))
virt = raw[written]
expected = v2p[virt // page_size] * page_size + virt % page_size
torch.testing.assert_close(out[written], expected, rtol=0, atol=0)
torch.testing.assert_close(gathered_indptr, out_indptr, rtol=0, atol=0)
torch.testing.assert_close(gathered, out, rtol=0, atol=0)
def test_page_size_one(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=1)
def test_page_size_two(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=2)
if __name__ == "__main__":
unittest.main()
@@ -117,7 +117,6 @@ def _run(io, window_size=0, sink_size=0):
kv_indices,
kv_indptr,
io["positions"],
None, # v2p: no unified-pool translation
io["pool_len"],
kv_indices.shape[1],
kv_indptr.shape[1],
@@ -1,172 +0,0 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`generate_draft_decode_kv_indices` under the unified pool's translation.
BUG REGRESSION. The multi-step draft-decode kernel copied req_to_token values
verbatim into the per-step kv_indices. Under `--enable-unified-memory` those
values are VIRTUAL ids while the fused draft pool is indexed by physical
ids, so with `speculative_num_steps > 1` every draft decode step read the
wrong KV rows -- silently: bad drafts only depress acceptance, verify stays
correct. Pinned here: TRANSLATE=True must emit exactly `translate_kv_loc`'s
physical ids, and TRANSLATE=False must stay byte-identical to the historical
raw copy.
python -m pytest test/registered/kernels/test_generate_draft_decode_kv_indices_translate.py -v
"""
import unittest
import torch
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
_SENTINEL = -7
def _run_kernel(
*,
req_to_token,
seq_lens,
positions,
num_steps,
topk,
page_size,
v2p,
translate,
):
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_kv_indices,
)
from sglang.srt.utils import next_power_of_2
num_seqs = seq_lens.numel()
bs = num_seqs * topk
width = int((seq_lens.sum().item() + num_steps * bs) * 2 + 64)
kv_indices = torch.full(
(num_steps, width), _SENTINEL, dtype=torch.int64, device="cuda"
)
kv_indptr = torch.zeros((num_steps, bs + 1), dtype=torch.int32, device="cuda")
generate_draft_decode_kv_indices[(num_steps, num_seqs, topk)](
torch.arange(num_seqs, dtype=torch.int64, device="cuda"),
req_to_token,
seq_lens,
kv_indices,
kv_indptr,
positions,
v2p if translate else None,
req_to_token.shape[1],
width,
kv_indptr.shape[1],
next_power_of_2(num_seqs),
next_power_of_2(num_steps),
next_power_of_2(bs),
page_size,
TRANSLATE=translate,
)
return kv_indices, kv_indptr
class TestGenerateDraftDecodeKVIndicesTranslate(CustomTestCase):
def _check(self, *, page_size, num_steps=3, topk=1, unused_slots=False):
torch.manual_seed(7)
num_seqs, max_context = 3, 64
num_pages = max_context // page_size + 1
# Virtual ids: a shuffled page layout so translation is not identity.
perm = torch.randperm(num_pages, device="cuda")
v2p = torch.empty(num_pages, dtype=torch.int64, device="cuda")
v2p[perm] = torch.arange(num_pages, device="cuda")
req_to_token = (
torch.arange(max_context, dtype=torch.int64, device="cuda")
.flip(0)
.repeat(num_seqs, 1)
)
if unused_slots:
# Negative entries inside the read window must land on the sink.
req_to_token[:, ::3] = -1
seq_lens = torch.tensor([5, 1, 9], dtype=torch.int64, device="cuda")[:num_seqs]
positions = seq_lens.repeat_interleave(topk)
common = dict(
req_to_token=req_to_token,
seq_lens=seq_lens,
positions=positions,
num_steps=num_steps,
topk=topk,
page_size=page_size,
v2p=v2p,
)
raw, raw_indptr = _run_kernel(translate=False, **common)
out, out_indptr = _run_kernel(translate=True, **common)
torch.testing.assert_close(raw_indptr, out_indptr, rtol=0, atol=0)
written = raw != _SENTINEL
# Unwritten lanes stay untouched in both compilations.
self.assertTrue(bool((out[~written] == _SENTINEL).all()))
# Written lanes: translate_kv_loc's formula over the raw ids.
virt = raw[written]
page = torch.where(virt < 0, 0, virt // page_size)
expected = torch.where(
virt < 0,
0,
torch.clamp_min(v2p[page] * page_size + virt % page_size, 0),
)
torch.testing.assert_close(out[written], expected, rtol=0, atol=0)
def test_page_size_one(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=1)
def test_page_size_two(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
self._check(page_size=2)
def test_negative_ids_map_to_the_sink(self):
if not torch.cuda.is_available():
self.skipTest("CUDA required")
for page_size in (1, 2):
self._check(page_size=page_size, unused_slots=True)
def test_translate_false_is_a_raw_copy(self):
"""The static-pool compilation must stay byte-identical to the
historical kernel: the emitted indices ARE the req_to_token values."""
if not torch.cuda.is_available():
self.skipTest("CUDA required")
torch.manual_seed(11)
num_seqs, max_context, page_size = 2, 32, 1
req_to_token = torch.randint(
0, 1 << 20, (num_seqs, max_context), dtype=torch.int64, device="cuda"
)
seq_lens = torch.tensor([4, 7], dtype=torch.int64, device="cuda")
raw, _ = _run_kernel(
req_to_token=req_to_token,
seq_lens=seq_lens,
positions=seq_lens,
num_steps=2,
topk=1,
page_size=page_size,
v2p=None,
translate=False,
)
written = raw != _SENTINEL
emitted = raw[written]
self.assertTrue(bool(torch.isin(emitted, req_to_token.flatten()).all()))
if __name__ == "__main__":
unittest.main()
@@ -60,7 +60,6 @@ def _run_draft(kern_inputs, topk, steps, page_size, nb, kw):
kv_i,
kv_p,
positions,
None, # v2p: no unified-pool translation
POOL_LEN,
kv_i.shape[1],
kv_p.shape[1],
@@ -9,6 +9,7 @@ import torch
from sglang.srt.dllm.algorithm.gemma4_renoise import Gemma4Renoise
from sglang.srt.layers.attention.triton_backend import TritonAttnBackend
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import (
DecodeCudaGraphRunner,
@@ -26,16 +27,16 @@ class _ReadStream:
def __init__(self):
self.tokens = torch.arange(64).view(4, 16)
def fill_packed_read_stream(
def pack_read_stream(
self,
plan,
*,
req_pool_indices,
seq_lens,
indptr,
total_tokens,
out,
kv_start_idx=None,
sliding_window=False,
kind=IdSpaceKind.FULL,
token_mapping=None,
):
for i, (req, length) in enumerate(zip(req_pool_indices, seq_lens)):
@@ -91,6 +92,7 @@ class TestGemma4GraphMetadata(unittest.TestCase):
requests = torch.tensor([1, 2, 0])
# Prefixes 3, 11, and an empty padded request; each has a four-token canvas.
backend._apply_cuda_graph_metadata(
plan=None,
bs=3,
req_pool_indices=requests,
seq_lens=torch.tensor([7, 15, 4]),
@@ -126,6 +128,7 @@ class TestGemma4GraphMetadata(unittest.TestCase):
metadata.window_kv_indices.data_ptr(),
)
backend._apply_cuda_graph_metadata(
plan=None,
bs=3,
req_pool_indices=torch.tensor([3, 1, 0]),
seq_lens=torch.tensor([4, 9, 4]),
@@ -89,10 +89,8 @@ class TestFlashAttentionGraphMetadata(CustomTestCase):
class TestSpecReadSeqLenDelta(CustomTestCase):
"""The mode -> seq_len_delta map IS the widening the whole-sequence spec
reads apply to cache_seqlens; a drift silently truncates the verify or
draft tail of the translated read table (the metadata builders widen
cache_seqlens while the source fill stays prefix-only)."""
"""How far past seq_lens each spec mode's translated read table reaches;
too short silently leaves the verify or draft tail stale."""
def _backend(self, *, topk=1, num_steps=3, step_id=1, num_draft=4):
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
@@ -127,11 +125,8 @@ class TestSpecReadSeqLenDelta(CustomTestCase):
class TestDraftExtendInGraph(CustomTestCase):
"""The draft-extend cuda-graph runners record
`init_forward_metadata_in_graph` unconditionally. Under translation fa3
rebuilds the draft-extend tables out of graph on every replay, so the
in-graph gather -- raw req_to_token, i.e. virtual ids -- must not be
recorded, or each replay overwrites the translated tables."""
"""A translating fa3 must not record the in-graph draft-extend gather: it
reads raw (virtual) req_to_token ids over the tables rebuilt out of graph."""
def _run(self, *, translating):
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
@@ -140,10 +135,8 @@ class TestDraftExtendInGraph(CustomTestCase):
fb = SimpleNamespace(forward_mode=ForwardMode.DRAFT_EXTEND_V2, batch_size=2)
b.init_forward_metadata_in_graph(fb)
def test_translating_backend_records_nothing(self):
def test_only_a_static_backend_builds_in_graph(self):
self._run(translating=True)
def test_static_backend_still_builds_in_graph(self):
with self.assertRaises(KeyError):
self._run(translating=False)
@@ -1,9 +1,10 @@
"""Nothing under layers/attention may translate KV ids for itself.
Ownership is exactly two places: `KVIndexTranslator` for READS and the
ForwardBatch rebind (`rebind_write_loc`) for WRITES. Virtual and physical ids
share a value range, so a backend that forgets a translate -- or does one
twice -- reads the wrong rows and nothing crashes.
Ownership is exactly one place: the iteration's `KVLocPlan`, built through
`KVIndexTranslator`, translates the WRITE window and the READ table once, and
backends read the result. Virtual and physical ids share a value range, so a
backend that forgets a translate -- or does one twice -- reads the wrong rows
and nothing crashes.
Deliberately out of scope: the allocator-internal implementations
(`allocator/unified_*`, `unified_memory_pool`), which ARE the mechanism the
@@ -9,6 +9,7 @@ from sglang.srt.layers.attention.triton_backend import (
TritonAttnBackend,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
from sglang.test.ci.ci_register import register_cpu_ci
@@ -26,16 +27,19 @@ class _BonusTokenVerifyInput(SpecInput):
class _KVIndexTranslator:
is_translating = False
def fill_packed_read_stream(
def pack_read_stream(
self,
plan,
*,
req_pool_indices,
seq_lens,
indptr,
total_tokens,
out,
kv_start_idx=None,
kind=IdSpaceKind.FULL,
):
out.zero_()
return False
class _RecordingTritonBackend(TritonAttnBackend):
@@ -92,6 +96,8 @@ def _make_forward_batch(batch_size, spec_info):
spec_info=spec_info,
forward_mode=ForwardMode.TARGET_VERIFY,
out_cache_loc=torch.zeros(batch_size * 7, dtype=torch.int64),
# A static pool's reads never touch the plan.
kv_loc_plan=None,
)
@@ -15,14 +15,10 @@
A whole-sequence verify reads `[committed prefix | drafts]` back out of the
pool, so under the unified pool the translated page table a verify kernel
reads has to cover `seq_lens + num_draft_tokens` columns per row. A table
filled only to `seq_lens` leaves the draft tail stale: invisible at
page_size 256, where the un-widened `ceil(seq/ps)` pages still cover a few
draft tokens, and at page_size 1 observed as accept length collapsing
(6.20 -> 1.37) or wrong tokens (gsm8k 0.905 -> 0.730).
reads has to cover `seq_lens + num_draft_tokens` columns per row; a table
filled only to `seq_lens` silently leaves the draft tail stale.
These tests drive fa3's two verify builds -- the eager one and the captured
one a cuda-graph replay runs -- through a translating KV-index translator
These tests drive fa3's eager and captured verify builds from a verify plan
over a non-identity virtual->physical page map, and check the page table the
kernel reads: every row translated, and filled through the draft tail.
@@ -39,6 +35,7 @@ from sglang.srt.layers.attention.flashattention_backend import (
FlashAttentionMetadata,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -67,9 +64,16 @@ def _translator(req_to_token, page_size, n_pages):
t.defer_read_translate = False
t._capture_page_size = page_size
t._full_v2p_table = v2p.to(_DEV)
t._swa_v2p_table = None
t._spaces = {
IdSpaceKind.FULL: IdSpace(
key=(IdSpaceKind.FULL, "test"),
write=lambda ids: (
t._full_v2p_table[ids // page_size] * page_size + ids % page_size
),
read_v2p=t._full_v2p_table,
)
}
t._rows = torch.arange(req_to_token.shape[0], dtype=torch.int64, device=_DEV)
t._index_table_memo = None
return t
@@ -99,6 +103,25 @@ def _expected(req_to_token, live, v2p, page_size):
return rows
def _verify_plan(translator, req_to_token, live, host_lens=None):
"""The plan a verify forward builds for itself: it writes the draft window
past `live`, so it reads that far."""
bs = live.numel()
rows = torch.arange(bs, device=_DEV)
cols = live.to(_DEV)[:, None] + torch.arange(_NUM_DRAFT, device=_DEV)[None, :]
return translator.own_plan(
SimpleNamespace(
forward_mode=ForwardMode.TARGET_VERIFY,
spec_info=SimpleNamespace(draft_token_num=_NUM_DRAFT),
batch_size=bs,
req_pool_indices=rows,
seq_lens=live.to(_DEV),
seq_lens_cpu=live if host_lens is None else host_lens,
out_cache_loc=req_to_token[rows[:, None], cols].long().reshape(-1),
)
)
def _backend(translator, req_to_token, page_size):
b = FlashAttentionBackend.__new__(FlashAttentionBackend)
b.kv_index_translator = translator
@@ -153,6 +176,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase):
metadata.swa_page_table = None
b.target_verify_metadata = {bs: metadata}
b._apply_cuda_graph_metadata(
_verify_plan(translator, req_to_token, live),
bs,
torch.arange(bs, device=_DEV),
live.to(_DEV),
@@ -193,6 +217,7 @@ class TestFa3VerifyReadsTheDraftTail(CustomTestCase):
spec_info=spec_info,
encoder_lens=None,
out_cache_loc=None,
kv_loc_plan=_verify_plan(translator, req_to_token, live, host_lens),
)
b.init_forward_metadata(fb)
self._check(
@@ -225,7 +225,12 @@ class TestMambaPrefillTrackMetadata(unittest.TestCase):
model_config=SimpleNamespace(
requires_mm_token_modalities=False, model_is_mrope=False
),
kv_index_translator=SimpleNamespace(rebind_write_loc=lambda forward: None),
# A pass-through runner: its plan hands the batch its ids untouched.
kv_index_translator=SimpleNamespace(
own_plan=lambda batch: SimpleNamespace(
bind=lambda batch, reader, cols=None: None
)
),
prefill_attention_backend_str="torch_native",
ngram_embedding_manager=SimpleNamespace(enabled=False),
lora_manager=None,
@@ -1,17 +1,12 @@
"""Dense fused-draft geometry: the draft model's KV as two more parts of every
host slot's entry.
Derived-property pins for the fused entry layout
host slot's entry,
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
the draft parts start at the 16-B-aligned end of the host parts and the entry
rounds up to the 32-B entry alignment (never to an lcm of row widths), so the
draft's row width is free to differ from the host's; host and draft views
share the slot stride and are indexed by the same physical token id; and
the draft parts starting at the aligned end of the host parts. Host and draft
views share the slot stride and are indexed by the same physical token id;
writes through either family's views land inside their own part of their own
slot (compaction moves whole page envelopes, so confinement IS the correctness
of the fused move).
slot, and a host page move carries the draft bytes with it.
python -m pytest test/registered/unit/mem_cache/test_dense_fused_draft_geometry.py -v
"""
@@ -27,7 +22,6 @@ from sglang.srt.mem_cache.layout.fused_draft import (
)
from sglang.srt.mem_cache.layout.token_major import (
ENTRY_ALIGN_BYTES,
ROW_ALIGN_BYTES,
align_entry_bytes,
align_part_offset,
)
@@ -43,11 +37,10 @@ from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
register_cpu_ci(est_time=7, suite="base-a-test-cpu")
_DEV = "cpu"
_DTYPE = torch.bfloat16
_ITEM = _DTYPE.itemsize
def _host_spec(draft_region=None, layer_num=2, head_num=2, head_dim=4):
@@ -75,12 +68,6 @@ def _placement(region):
class TestFusedSpecMath(unittest.TestCase):
def test_unfused_spec_is_byte_identical_to_before(self):
s = _host_spec()
self.assertEqual(s.entry_bytes(), 2 * (16 + 16))
self.assertEqual(s.host_entry_bytes(), s.entry_bytes())
self.assertEqual([p.name for p in s.layout().parts], ["k", "v"])
def test_fused_entry_appends_the_draft_parts(self):
f = _host_spec(_draft_region())
r = f.draft_region
@@ -101,46 +88,6 @@ class TestFusedSpecMath(unittest.TestCase):
self.assertEqual(layout.part("draft_v").offset_bytes, 64 + 48)
self.assertEqual(layout.part("draft_k").layer_stride_bytes, 64)
def test_entry_alignment_pads_only_to_the_entry_quantum(self):
# host 32 B + draft 48 B = 80 B of rows -> one 96 B entry (16 B pad),
# not an lcm of the two row widths.
f = _host_spec(
DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, v_head_dim=16, store_dtype=_DTYPE
),
layer_num=1,
head_num=1,
head_dim=8,
)
self.assertEqual(f.host_entry_bytes(), 32)
self.assertEqual(f.draft_region.entry_bytes(), 48)
self.assertEqual(f.entry_bytes(), 96)
self.assertEqual(f.draft_offset_in_entry() % ROW_ALIGN_BYTES, 0)
f.layout() # parts fit and do not overlap
def test_draft_rows_must_be_row_aligned(self):
with self.assertRaises(AssertionError):
_host_spec(
DenseDraftRegion(lane_num=1, head_num=1, head_dim=3, store_dtype=_DTYPE)
).layout()
def test_only_page_envelope_kinds_accept_a_draft_region(self):
"""`draft_region` is a universal spec field (None = unfused), but a
kind without the fused entry layout must refuse one at construction;
a silently-carried region would never reach the layout. MHA and MLA
entries carry the draft parts; mamba state pages carry none yet."""
with self.assertRaises(AssertionError):
MambaSubPoolSpec(
name="mamba",
layer_num=1,
conv_state_shapes=((2, 2),),
conv_dtype=torch.float32,
temporal_state_shape=(2,),
temporal_dtype=torch.float32,
grow_direction="up",
draft_region=_draft_region(),
)
class TestFusedRegionConfinement(unittest.TestCase):
"""Writes through host/draft views must land in their own part of their
@@ -199,20 +146,6 @@ class TestFusedRegionConfinement(unittest.TestCase):
self.assertGreaterEqual(lo, page * self.PS * entry)
self.assertLessEqual(hi, (page + 1) * self.PS * entry)
def test_host_and_draft_views_share_the_slot_stride(self):
spec, _, (hk, hv), (dk, dv) = self._build()
stride = spec.entry_bytes()
for view in (*hk, *hv, *dk, *dv):
self.assertEqual(view.stride(0) * view.element_size(), stride)
self.assertEqual(view.shape[0], hk[0].shape[0])
self.assertEqual(tuple(dk[0].shape[1:]), (1, 24))
self.assertEqual(tuple(dv[0].shape[1:]), (1, 8))
def test_pool_helper_refuses_an_unfused_sub_pool(self):
_, pool, _, _ = self._build()
with self.assertRaises(AssertionError):
pool.build_dense_draft_views("swa") # no fused region there
class TestUnifiedDraftKVPool(unittest.TestCase):
PS = 2
@@ -268,9 +201,9 @@ class TestUnifiedDraftKVPool(unittest.TestCase):
)
def test_host_page_move_carries_the_draft_bytes(self):
# THE fused-layout property: compaction relocates whole page envelopes
# on the HOST pool; a draft marker written in page A must arrive at
# page B after host.move_kv_cache(B, A), with zero draft-side moves.
# Compaction relocates whole page envelopes on the HOST pool: a draft
# marker written in page A arrives at page B after
# host.move_kv_cache(B, A), with no draft-side move.
pool = self._pool()
dp, _ = self._draft_pool(pool)
host = UnifiedMHATokenToKVPool(
@@ -336,17 +269,6 @@ class TestUnifiedDraftKVPool(unittest.TestCase):
self.assertEqual(dp.get_key_buffer(0).dtype, fp8)
torch.testing.assert_close(dp.get_key_buffer(0)[loc].float(), k)
def test_draft_side_moves_and_transfers_fail_loudly(self):
pool = self._pool()
dp, _ = self._draft_pool(pool)
one = torch.zeros(self.PS, dtype=torch.int64)
with self.assertRaises(NotImplementedError):
dp.move_kv_cache(one, one)
with self.assertRaises(NotImplementedError):
dp.get_contiguous_buf_infos()
with self.assertRaises(NotImplementedError):
dp.get_cpu_copy(one)
def _mla_host_spec(draft_region=None):
# 32 B latent rows, two layers: a 64 B host entry before the draft parts.
@@ -376,8 +298,7 @@ def _mamba_spec():
class TestFusedMLAHost(unittest.TestCase):
"""MLA entries carry the fused draft parts exactly like MHA entries: the
same aligned entry, one slot stride for both families, and byte-disjoint
host/draft parts within every slot. The unfused spec stays byte-identical;
that identity guards every existing MLA deploy."""
host/draft parts within every slot."""
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
@@ -389,13 +310,6 @@ class TestFusedMLAHost(unittest.TestCase):
PS = 2
PAGES = 8
def test_unfused_spec_is_byte_identical_to_before(self):
s = _mla_host_spec()
self.assertEqual(s.row_bytes(), 32)
self.assertEqual(s.entry_bytes(), 64)
self.assertEqual(s.host_entry_bytes(), s.entry_bytes())
self.assertEqual([p.name for p in s.layout().parts], ["kv"])
def test_fused_entry_appends_the_draft_parts(self):
f = _mla_host_spec(_draft_region())
self.assertEqual(f.host_entry_bytes(), 64)
@@ -448,10 +362,9 @@ class TestFusedMLAHost(unittest.TestCase):
self.assertTrue(bool((nz >= split).all() and (nz < slot_hi).all()))
def test_translator_takes_the_fused_disposition_on_the_mla_host(self):
"""A draft runner bound over the MLA host's entries must translate
through the mamba allocator's full-side v2p, exactly as on the SWA
host. A kind-specific probe regression here silently reverts the
draft to passthrough (raw virtual ids into the views)."""
"""A draft runner bound over the MLA host's entries translates through
the mamba allocator's full-side v2p, as on the SWA host; a passthrough
here would write raw virtual ids into the views."""
from sglang.srt.mem_cache.allocator.unified_mamba import (
UnifiedMambaTokenToKVPoolAllocator,
)
@@ -489,7 +402,7 @@ class TestFusedMLAHost(unittest.TestCase):
)
self.assertTrue(translator.is_translating)
self.assertIs(
translator.full_flat_v2p(), alloc.full_attn_allocator.virtual_to_physical
translator.full_v2p_table, alloc.full_attn_allocator.virtual_to_physical
)
def test_draft_pool_binds_over_the_mla_host(self):
@@ -280,9 +280,9 @@ class TestHybridLinearMLARouting(unittest.TestCase):
return pool
def test_set_mla_kv_buffer_door_never_translates(self):
"""The translate happens exactly once, at ForwardBatch construction
(`rebind_write_loc`); a door that translated again would
double-translate every unified MLA write."""
"""The translate happens exactly once, in the iteration's plan
(`KVLocPlan`); a door that translated again would double-translate
every unified MLA write."""
pool = self._make_bare_pool()
loc = torch.tensor([107, 108, 109], dtype=torch.int64)
layer = types.SimpleNamespace(layer_id=0)
@@ -11,29 +11,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`place_fused_draft`: which host/draft pairs fuse, and where each runner's
layers land.
"""`place_fused_draft` and the target's fused-draft decision: which drafts
fuse into the host's entries, and where each runner's layers land.
A decline keeps a DFLASH or DSPARK draft on its private pool (a DFLASH draft
on a mamba host is refused instead), and refuses an EAGLE draft at boot. A
wrong answer for a draft that keeps its pool silently changes what it binds
and what the boot solve prices. Pinned:
A decline keeps a draft on a private pool (or refuses it at boot); admitting
one that cannot read the fused rows correctly gives silently wrong drafts.
Pinned:
- a replicated head under multi-layer EAGLE gets one lane RANGE per runner
(one shared region would let the runners clobber each other's KV);
- a per-depth head serves one depth per runner and needs one runner per
depth;
- a draft with SWA or recurrent-state layers of its own, or asymmetric K/V
rows, declines: the fused arm binds one dense pool over the host's full
slots;
- a draft whose attention backend is off the translated MHA rails
declines: it would read the fused rows without the KV-index translator;
- so does a draft under --dcp-size > 1, where each rank's host rows hold
only its share of the tokens the replicated draft reads, and a draft
under HiCache, which builds its host pool off a device pool of its own;
- an explicit draft KV dtype unlike the host's declines;
- the profile divides the draft's heads by attn_tp, as the target does;
- a placement whose runner lane counts do not fill its region is refused;
- the priced entry counts the layers THIS runner owns, not the whole model's.
rows, declines;
- so does a draft whose attention backend is off the translated MHA rails
(it would read the fused rows with virtual ids), a draft under
--dcp-size > 1 (each rank's rows hold only its share of the tokens), an
explicit draft KV dtype unlike the host's, and a draft under HiCache or
host-pool decode retraction (they build the draft's host pool off a
device pool of its own).
python -m pytest test/registered/unit/mem_cache/test_fused_draft_placement.py -v
"""
@@ -48,9 +43,6 @@ from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
DraftKVGeometry,
DraftKVProfile,
FusedDraftDecision,
FusedDraftPlacement,
draft_kv_profile,
place_fused_draft,
)
from sglang.srt.runtime_context import get_parallel, override_platform
@@ -130,118 +122,6 @@ class TestPlaceFusedDraft(CustomTestCase):
)
class TestDraftKVProfile(CustomTestCase):
def test_heads_are_divided_by_attn_tp(self):
mc = SimpleNamespace(
is_hybrid_swa=True,
is_deepseek_v4_arch=False,
swa_attention_layer_ids=[0],
num_nextn_predict_layers=None,
get_num_kv_heads=lambda tp: max(1, 8 // tp),
head_dim=64,
v_head_dim=32,
)
with patch("sglang.srt.configs.hybrid_arch.mambaish_config", return_value=None):
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=2)
self.assertEqual(
profile.full, DraftKVGeometry(head_num=4, head_dim=64, v_head_dim=32)
)
self.assertEqual(profile.swa_layer_ids, (0,))
self.assertEqual(profile.num_depths, 1)
self.assertEqual(profile.num_state_layers, 0)
def _linear_trunk_mc(self, hf_text_config):
return SimpleNamespace(
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
num_nextn_predict_layers=1,
get_num_kv_heads=lambda tp: 8,
head_dim=64,
v_head_dim=64,
hf_text_config=hf_text_config,
)
def test_a_nextn_head_of_a_linear_trunk_owns_no_state(self):
"""BUG REGRESSION. A NEXTN head ships inside the trunk checkpoint and
inherits its config CLASS, so `mamba2_cache_params` lists the TRUNK's
state layers while the head is a full-attention block. Counting them
declined a fusable draft to its private pool."""
mc = self._linear_trunk_mc(SimpleNamespace())
trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2]))
with patch(
"sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk
):
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1)
self.assertEqual(profile.num_state_layers, 0)
def test_a_conv_chain_head_still_counts_its_state(self):
"""The discriminator must not silence a head that really owns state:
only a conv-chain MTP config declares `mtp_local_layer_ids`."""
mc = self._linear_trunk_mc(SimpleNamespace(mtp_local_layer_ids=[0]))
trunk = SimpleNamespace(mamba2_cache_params=SimpleNamespace(layers=[0, 1, 2]))
with patch(
"sglang.srt.configs.hybrid_arch.mambaish_config", return_value=trunk
):
profile = draft_kv_profile(mc, num_layers=1, attn_tp_size=1)
self.assertEqual(profile.num_state_layers, 3)
class TestFusedDraftPlacement(CustomTestCase):
def test_runner_lane_counts_must_fill_the_region(self):
region = DenseDraftRegion(
lane_num=2, head_num=1, head_dim=8, store_dtype=_DTYPE
)
with self.assertRaises(AssertionError):
FusedDraftPlacement(region=region, runner_lane_counts=(1,))
with self.assertRaises(AssertionError):
FusedDraftPlacement(region=region, runner_lane_counts=())
class TestFusedEntryPricing(CustomTestCase):
"""The boot solve prices a fused entry through the same spec the pool
factory builds, so the two cannot drift. The factory builds the full
sub-pool from this runner's OWN layer slice; pricing from the whole-model
split over-counts every layer another pipeline rank holds, and the solve
then hands out fewer tokens than fit."""
def _configurator(self, *, whole, hybrid_swa, owned=(), span=(0, 0)):
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
cfg = KVCacheConfigurator.__new__(KVCacheConfigurator)
cfg.is_hybrid_swa = hybrid_swa
cfg.layer_info = SimpleNamespace(
full_attention_layer_ids=list(owned),
start_layer=span[0],
end_layer=span[1],
)
cfg.mambaish_config = SimpleNamespace(full_attention_layer_ids=whole)
cfg.model_config = SimpleNamespace(
full_attention_layer_ids=whole,
head_dim=8,
get_num_kv_heads=lambda *_: 1,
)
cfg.use_mla_backend = False
cfg.kv_cache_dtype = _DTYPE
return cfg
def _priced(self, cfg):
region = DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, store_dtype=_DTYPE
)
with get_parallel().override(attn_tp_size=1, attn_dcp_size=1):
return cfg._full_host_spec(region).layer_num
def test_an_swa_host_prices_this_runners_slice(self):
cfg = self._configurator(
whole=[0, 1, 2, 3], hybrid_swa=True, owned=[0, 1], span=(0, 2)
)
self.assertEqual(self._priced(cfg), 2)
def test_a_mamba_host_prices_only_the_layers_in_its_span(self):
cfg = self._configurator(whole=[0, 1, 2, 3], hybrid_swa=False, span=(2, 4))
self.assertEqual(self._priced(cfg), 2)
class TestFusedDraftDecision(CustomTestCase):
"""The target's boot decision over a host whose full sub-pool fuses."""
@@ -252,7 +132,6 @@ class TestFusedDraftDecision(CustomTestCase):
draft_kv_dtype=None,
host_kv_dtype=_DTYPE,
kv_cache_dtype_flag="auto",
attention_arch=None,
draft_backend=None,
target_backends=("triton", "triton"),
dcp_size=1,
@@ -286,7 +165,7 @@ class TestFusedDraftDecision(CustomTestCase):
head_dim=64,
v_head_dim=64,
dtype=torch.bfloat16,
attention_arch=attention_arch or AttentionArch.MHA,
attention_arch=AttentionArch.MHA,
),
)
memory = SimpleNamespace(
@@ -326,15 +205,6 @@ class TestFusedDraftDecision(CustomTestCase):
def test_a_stateless_draft_is_placed(self):
self.assertIsNotNone(self._decide().placement)
def test_a_non_mha_draft_keeps_the_private_pool(self):
"""The region holds dense MHA K/V rows. An MLA draft is kept out by its
own architecture, not by the accident of asymmetric head dims."""
from sglang.srt.configs.model_config import AttentionArch
declined = self._decide(attention_arch=AttentionArch.MLA)
self.assertIsNone(declined.placement)
self.assertIn("MLA", declined.declined)
def test_a_draft_kv_dtype_unlike_the_host_keeps_the_private_pool(self):
"""A fused draft stores its rows in the host's KV dtype; an explicit
draft dtype that differs declines instead of mis-typing those rows,
@@ -400,90 +270,5 @@ class TestFusedDraftDecision(CustomTestCase):
)
class TestMambaHostPrivateDraftRefused(CustomTestCase):
"""A mamba host's unified buffer takes the whole KV budget, so a private
draft pool on top of it would overcommit; the fused arm is the only
EAGLE-family or DFLASH arm such a host builds. A DSPARK draft that does
not fuse keeps its private pool."""
_DECLINED = "the draft's K/V rows are asymmetric"
def _resolve(self, *, decision, algorithm="EAGLE"):
from sglang.srt.mem_cache import kv_cache_configurator as kvc
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator)
cfg.is_draft_worker = False
cfg.spec_algorithm = SpeculativeAlgorithm[algorithm]
with patch.object(
kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision
):
return cfg._fused_draft_for_mamba_factory()
def test_a_declined_eagle_draft_is_refused(self):
for algorithm in ("EAGLE", "EAGLE3"):
with self.subTest(algorithm=algorithm):
with self.assertRaisesRegex(ValueError, "rows are asymmetric"):
self._resolve(
decision=FusedDraftDecision(declined=self._DECLINED),
algorithm=algorithm,
)
def test_a_declined_dflash_draft_is_refused_and_dspark_keeps_its_pool(self):
"""A DFLASH decliner is refused like EAGLE's; a DSPARK decliner keeps
its private pool instead of failing the boot."""
declined = FusedDraftDecision(declined=self._DECLINED)
with self.assertRaisesRegex(ValueError, "rows are asymmetric"):
self._resolve(decision=declined, algorithm="DFLASH")
self.assertIsNone(self._resolve(decision=declined, algorithm="DSPARK"))
def test_a_placed_draft_and_other_algorithms_pass(self):
placement = _place(_profile()).placement
for algorithm in ("EAGLE", "DFLASH", "DSPARK"):
with self.subTest(algorithm=algorithm):
self.assertIs(
self._resolve(
decision=FusedDraftDecision(placement=placement),
algorithm=algorithm,
),
placement,
)
self.assertIsNone(
self._resolve(decision=FusedDraftDecision(), algorithm="NONE")
)
class TestSWAHostPrivateDraftRefused(CustomTestCase):
"""A hybrid-SWA host's boot solve prices a private EAGLE draft at the
target's per-token size, so a draft that does not fuse is refused there
too; the fused arm is the only EAGLE arm such a host builds."""
def _resolve(self, *, decision, eagle=True):
from sglang.srt.mem_cache import kv_cache_configurator as kvc
cfg = kvc.KVCacheConfigurator.__new__(kvc.KVCacheConfigurator)
cfg.is_draft_worker = False
cfg.spec_algorithm = SimpleNamespace(is_eagle=lambda: eagle)
with patch.object(
kvc.KVCacheConfigurator, "_fused_draft_decision", return_value=decision
):
return cfg._fused_draft_for_swa_factory()
def test_a_declined_eagle_draft_is_refused(self):
with self.assertRaisesRegex(ValueError, "KV cache dtype"):
self._resolve(
decision=FusedDraftDecision(
declined="the draft's KV cache dtype (bf16) differs from the host's"
)
)
def test_a_placed_draft_and_other_algorithms_pass(self):
placement = _place(_profile()).placement
self.assertIs(
self._resolve(decision=FusedDraftDecision(placement=placement)), placement
)
self.assertIsNone(self._resolve(decision=FusedDraftDecision(), eagle=False))
if __name__ == "__main__":
unittest.main()
@@ -11,20 +11,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Draft-shaped KVCacheConfigurator dispatch under the unified pool.
"""How a draft worker's KVCacheConfigurator binds its KV under the unified pool.
BUG REGRESSION (fast path). The fast path was gated on `req_to_token_pool is
None` alone, but a compact-window DFLASH draft (`--speculative-draft-window-size`)
also passes None — it builds a private req_to_token of its own — so the draft
worker would allocate a SECOND unified byte buffer at boot. A draft-shaped
configurator must fall through to the normal pool build instead.
DERIVED PROPERTY (binding dispatch). The fused-vs-private draft binding
dispatches on the spec algorithm, not the allocator kind: a non-EAGLE draft on
a unified SWA target takes the private arm, sized by the full sub-allocator's
VIRTUAL id space (`max_slots - 1`). The SWA allocator's `size_full` reports
the static token budget — smaller than the id space — so sizing by it would
put verify-window writes at high virtual ids out of bounds.
With a fused region the draft binds `UnifiedDraftKVPool` over the target's
allocator (a compact-window DFLASH draft with a private `req_to_token` of its
own), refusing a KV dtype unlike the region's. Without one it takes the private
arm, sized by the full sub-allocator's VIRTUAL id space (`max_slots - 1`), not
`size_full`, the smaller static token budget: sizing by that would put
verify-window writes at high virtual ids out of bounds.
python -m pytest test/registered/unit/mem_cache/test_kv_cache_configurator_draft_gate.py -v
"""
@@ -56,82 +50,7 @@ from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
class _ReachedNormalBuild(Exception):
"""Sentinel: control flow fell past the unified fast path."""
class TestUnifiedFastPathDraftGate(CustomTestCase):
def _run(self, *, is_draft_worker: bool):
cfg = kcc.KVCacheConfigurator.__new__(kcc.KVCacheConfigurator)
cfg.is_draft_worker = is_draft_worker
cfg.mambaish_config = object() # would select the mamba unified arm
cfg.is_hybrid_swa = False # not the mamba+SWA tri-pool arm
cfg.model_config = SimpleNamespace(hf_config={}) # not DSV4
# Every field the fast path reads: a missing one raises while the
# arm's arguments are evaluated, which reads here as "took the fast
# path but called nothing".
sizes = SimpleNamespace(
max_running_requests=8,
max_total_num_tokens=64,
full_max_total_num_tokens=64,
swa_max_total_num_tokens=32,
unified_total_bytes=1 << 20,
)
taken = []
with (
patch.object(
kcc,
"get_memory",
return_value=SimpleNamespace(enable_unified_memory=True),
),
patch.object(
kcc,
"get_disagg",
return_value=SimpleNamespace(disaggregation_mode="null"),
),
patch.object(
kcc.KVCacheConfigurator,
"_init_unified_mamba_pools",
lambda self, **kw: taken.append("mamba") or None,
),
patch.object(
kcc.KVCacheConfigurator,
"_init_unified_swa_pools",
lambda self, **kw: taken.append("swa") or None,
),
patch.object(
kcc.KVCacheConfigurator,
"_build_req_to_token_pool",
side_effect=_ReachedNormalBuild,
),
):
try:
cfg._init_pools(
sizes=sizes,
req_to_token_pool=None,
token_to_kv_pool_allocator=None,
)
except _ReachedNormalBuild:
return "normal", taken
except (AttributeError, TypeError):
# The stubbed unified arm returned None; anything past the
# fast-path dispatch counts as having taken it.
return "unified", taken
return "unified", taken
def test_draft_worker_falls_through_to_the_normal_build(self):
path, taken = self._run(is_draft_worker=True)
self.assertEqual(path, "normal")
self.assertEqual(taken, [])
def test_target_worker_still_takes_the_fast_path(self):
"""The guard must narrow to drafts only — a target regression here
silently turns --enable-unified-memory into a no-op."""
path, taken = self._run(is_draft_worker=False)
self.assertEqual(taken, ["mamba"])
register_cpu_ci(est_time=5, stage="weekly", runner_config="cpu")
class _FakeKVCache:
@@ -240,7 +159,6 @@ class TestDraftBindingDispatch(CustomTestCase):
alloc,
max_total_num_tokens,
kv_dtype=torch.bfloat16,
state_layers=(),
req_to_token_pool="shared",
):
if req_to_token_pool == "shared":
@@ -295,9 +213,7 @@ class TestDraftBindingDispatch(CustomTestCase):
# The real scan walks the draft nn.Module for RadixAttention layers.
patch.object(unified_draft_pool, "draft_kv_layer_ids", return_value=[0]),
patch.object(
unified_draft_pool,
"draft_state_layer_classes",
return_value=list(state_layers),
unified_draft_pool, "draft_state_layer_classes", return_value=[]
),
patch.object(
kcc.KVCacheConfigurator,
@@ -325,34 +241,33 @@ class TestDraftBindingDispatch(CustomTestCase):
sized = caught.exception.sizes.max_total_num_tokens
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
def test_eagle_draft_still_binds_the_fused_pool(self):
def test_a_draft_with_a_region_binds_the_fused_pool(self):
"""An EAGLE or DSPARK draft binds the fused pool over the target's
allocator when the target resolved a region; the region-less private
arm above is the fallback."""
alloc = self._swa_allocator(with_draft_region=True)
pools = self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
self.assertIs(pools.token_to_kv_pool_allocator, alloc)
for algorithm in (SpeculativeAlgorithm.EAGLE3, SpeculativeAlgorithm.DSPARK):
pools = self._run(
algorithm=algorithm,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
self.assertIs(pools.token_to_kv_pool_allocator, alloc)
def test_a_draft_kv_dtype_unlike_the_region_refuses_to_bind(self):
"""The region stores the target's KV dtype. A draft that resolved its
own -- `auto` picking up an fp8 quant config the target lacks, say --
must fail loudly rather than read those rows as another dtype."""
alloc = self._swa_allocator(with_draft_region=True)
"""The region stores the target's KV dtype, every fp8 flavor as uint8:
a draft resolving the target's fp8 binds a pool that casts to it, and
one resolving another dtype refuses instead of reading the rows as
that dtype."""
bf16 = self._swa_allocator(with_draft_region=True)
with self.assertRaisesRegex(ValueError, "speculative-draft-kv-cache-dtype"):
self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
alloc=bf16,
max_total_num_tokens=bf16.size_full,
kv_dtype=torch.float8_e4m3fn,
)
def test_an_fp8_region_binds_an_fp8_draft_and_refuses_another_fp8(self):
"""Every fp8 flavor is stored as uint8, so the bind compares KV dtypes:
a draft resolving the target's fp8 binds a pool that casts to that fp8,
and one resolving a different fp8 refuses instead of reading e4m3 bytes
as e5m2."""
alloc = self._swa_allocator(
with_draft_region=True, kv_dtype=torch.float8_e4m3fn
)
@@ -372,30 +287,6 @@ class TestDraftBindingDispatch(CustomTestCase):
kv_dtype=torch.float8_e5m2,
)
def test_a_draft_with_state_layers_refuses_to_bind(self):
"""The target places a draft from its config; the built model is the
ground truth. Recurrent layers would run with no state pool."""
alloc = self._swa_allocator(with_draft_region=True)
with self.assertRaisesRegex(ValueError, "MambaMixer2"):
self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
state_layers=["MambaMixer2"],
)
def test_dspark_draft_with_a_region_binds_the_fused_pool(self):
"""DSPARK's draft KV fuses when the target resolved a region (its
block rows are indexed by the same token->page identity); the
region-less private arm above remains the automatic fallback."""
alloc = self._swa_allocator(with_draft_region=True)
pools = self._run(
algorithm=SpeculativeAlgorithm.DSPARK,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
def test_compact_dflash_draft_fuses_with_a_private_req_table(self):
"""Compact-window DFLASH passes req_to_token_pool=None (it keeps a
private table narrowing WHICH pages the draft reads) while the KV
@@ -411,22 +302,6 @@ class TestDraftBindingDispatch(CustomTestCase):
self.assertIsInstance(pools.token_to_kv_pool, UnifiedDraftKVPool)
self.assertEqual(pools.req_to_token_pool.kind, "private-compact")
def test_eagle_draft_without_a_placement_falls_back_to_the_private_arm(self):
"""Target boot declines a placement for legitimate configurations (a
draft with SWA layers of its own, asymmetric rows), so a
placement-less EAGLE draft binds the private pool -- sized by the id
space like any other private draft -- instead of failing the boot."""
alloc = self._swa_allocator(with_draft_region=False)
id_space = alloc.full_attn_allocator.max_slots - 1
with self.assertRaises(_CapturedSizes) as caught:
self._run(
algorithm=SpeculativeAlgorithm.EAGLE3,
alloc=alloc,
max_total_num_tokens=alloc.size_full,
)
sized = caught.exception.sizes.max_total_num_tokens
self.assertEqual(sized, (id_space + _PS - 1) // _PS * _PS)
if __name__ == "__main__":
unittest.main()
@@ -11,7 +11,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""KVIndexTranslator -- the read-path id translator.
"""KVIndexTranslator -- what a runner reads and writes an iteration's plan
through: the plan's read table and write ids per disposition, the readers
(`read_source`, `pack_read_stream`, `copy_page_table`), and the boot-time
dispositions.
CPU-only: these exercise the builder's pure-torch reference path, not the
Triton kernel.
@@ -32,7 +35,8 @@ from test_multi_ended_allocator import _FakeKVCache, _FakeUnifiedSWAKVPool
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator, KVReadTables
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
from sglang.srt.state_capturer.base import BaseTopkCapturer
@@ -113,12 +117,26 @@ def _reference_table(req_to_token, req_pool_indices, seq_lens, v2p, ps, width):
return out
def _publish():
# The code under test reads its config from the bags.
reset_context()
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _plan(src, rows, seq_lens, *, write_virtual=None, read_extent=0):
return src.plan(
req_pool_indices=rows,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens.clone(),
write_virtual=write_virtual,
read_extent=read_extent,
)
class TestPassthrough(unittest.TestCase):
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
_publish()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_non_unified_returns_same_objects(self):
"""Strict passthrough: no copy, no branch. Any tensor op on the
@@ -134,18 +152,15 @@ class TestPassthrough(unittest.TestCase):
self.assertFalse(src.is_translating)
self.assertEqual(src.capture_token_capacity(17), 18)
rows = torch.tensor([2, 0])
view = src.build_index_table(
req_pool_indices=rows, seq_lens=torch.tensor([5, 3])
)
loc = torch.tensor([1, 2, 3])
plan = _plan(src, rows, torch.tensor([5, 3]), write_virtual=loc)
self.assertIs(plan.write_ids(src), loc)
view = src.read_source(plan, req_pool_indices=rows, bs=2)
self.assertIs(view.ids, req_to_token)
self.assertIs(view.row_ids, rows)
self.assertEqual(view.row_stride, req_to_token.stride(0))
self.assertEqual(view.entry_page_size, 1)
self.assertFalse(view.is_translated)
self.assertIsNone(view.sliding_window_ids)
# And the translate surface is the identity, not a wrapped copy.
t = torch.tensor([1, 2, 3])
self.assertIs(src.translate_full_attn_ids(t), t)
def _alloc_and_fill(allocator, ps, lens):
@@ -167,10 +182,8 @@ def _alloc_and_fill(allocator, ps, lens):
class TestReadTableBuild(unittest.TestCase):
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
_publish()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_read_table_matches_reference(self):
"""Both read tables must equal the independent per-element derivation
@@ -183,71 +196,27 @@ class TestReadTableBuild(unittest.TestCase):
)
src = _make_source(allocator, req_to_token, ps)
self.assertTrue(src.is_translating)
width = 6
view = src.build_index_table(
req_pool_indices=rows, seq_lens=seq_lens, max_pages=width
)
plan = _plan(src, rows, seq_lens)
view = src.read_table(plan)
width = view.ids.shape[1]
self.assertEqual(width, 5) # the widest row's pages
self.assertTrue(view.is_translated)
self.assertEqual(view.entry_page_size, ps)
self.assertTrue(
torch.equal(view.row_ids, torch.arange(3, dtype=torch.int64))
)
want_full = _reference_table(
req_to_token,
rows,
seq_lens,
allocator.full_v2p_page_table,
ps,
width,
)
want_swa = _reference_table(
req_to_token,
rows,
seq_lens,
allocator.swa_v2p_page_table,
ps,
width,
)
self.assertTrue(
torch.equal(view.ids, want_full),
f"full read table off-formula (ps={ps})",
)
self.assertTrue(
torch.equal(view.sliding_window_ids, want_swa),
f"swa read table off-formula (ps={ps})",
)
def test_packed_stream_equals_the_rectangle_it_replaces(self):
"""The two builders must agree element for element:
packed[indptr[b] + p] == ids[b, p // ps] * ps + p % ps."""
for ps in (1, 4):
allocator = _build_composite(ps)
req_to_token, rows, seq_lens = _alloc_and_fill(
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
)
src = _make_source(allocator, req_to_token, ps)
view = src.build_index_table(
req_pool_indices=rows, seq_lens=seq_lens, max_pages=6
)
indptr = torch.zeros(len(seq_lens) + 1, dtype=torch.int32)
indptr[1:] = torch.cumsum(seq_lens, dim=0)
total = int(indptr[-1])
packed = torch.zeros(total, dtype=torch.int32)
translated = src.fill_packed_read_stream(
req_pool_indices=rows,
seq_lens=seq_lens,
indptr=indptr,
total_tokens=total,
out=packed,
)
self.assertTrue(translated)
for b, n in enumerate(seq_lens.tolist()):
for pos in range(n):
self.assertEqual(
int(packed[int(indptr[b]) + pos]),
int(view.ids[b, pos // ps]) * ps + pos % ps,
f"packed stream off the page table (ps={ps}, b={b}, {pos=})",
)
for table, v2p, side in (
(view.ids, allocator.full_v2p_page_table, "full"),
(
src.read_table(plan, kind=IdSpaceKind.SLIDING_WINDOW).ids,
allocator.swa_v2p_page_table,
"swa",
),
):
want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width)
self.assertTrue(
torch.equal(table, want), f"{side} read table off-formula (ps={ps})"
)
def test_sink_routing(self):
"""Dead lanes, -1 slots inside the live prefix, and tombstoned v2p
@@ -265,64 +234,112 @@ class TestReadTableBuild(unittest.TestCase):
allocator.full_v2p_page_table[tomb_page] = -1
allocator.swa_v2p_page_table[tomb_page] = -1
src = _make_source(allocator, req_to_token, ps)
view = src.build_index_table(
req_pool_indices=rows, seq_lens=seq_lens, max_pages=4
)
for table in (view.ids, view.sliding_window_ids):
plan = _plan(src, rows, seq_lens)
for kind in (IdSpaceKind.FULL, IdSpaceKind.SLIDING_WINDOW):
table = src.read_table(plan, kind=kind).ids
self.assertTrue(bool((table >= 0).all()))
self.assertTrue(bool((table[1] == 0).all()), "dead lane not sunk")
self.assertEqual(int(table[0, 1]), 0, "-1 slot not sunk")
self.assertEqual(int(table[2, 0]), 0, "tombstone not sunk")
class TestBuildInto(unittest.TestCase):
"""fill_read_table fills a backend-owned padded block table's live prefix with
FULL-side read-table entries -- the trtllm_mla / flashmla consumption route
(their rows ARE the read table's rows)."""
class TestReadExtent(unittest.TestCase):
"""A forward that reads past its lengths (a verify's draft tail) plans its
table that far: `seq_lens + read_extent`, capped at the request row."""
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
_publish()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_prefix_filled_tail_sentinel_preserved_width_capped(self):
"""The -1 tail sentinel belongs to the backend, and a table padded
WIDER than the req_to_token page span (trtllm's LCM alignment) must be
capped rather than trip the builder's width assert."""
def test_table_covers_the_extent(self):
window = 3
for ps in (1, 4):
allocator = _build_composite(ps)
req_to_token, rows, live = _alloc_and_fill(
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
)
src = _make_source(allocator, req_to_token, ps)
view = src.read_table(_plan(src, rows, live, read_extent=window))
width = -(-(int(live.max()) + window) // ps)
self.assertEqual(tuple(view.ids.shape), (rows.numel(), width))
want = _reference_table(
req_to_token,
rows,
live + window,
allocator.full_v2p_page_table,
ps,
width,
)
self.assertTrue(torch.equal(view.ids, want), f"ps={ps}")
def test_width_never_exceeds_the_request_row(self):
"""Near the context limit the host bound plus the extent can pass the
end of the `req_to_token` row; the table stops at the row instead of
tripping the read-table width assert."""
window = 6
allocator = _build_composite(1)
req_to_token, rows, live = _alloc_and_fill(allocator, 1, lens=[6, 1])
row_pages = req_to_token.shape[1]
src = _make_source(allocator, req_to_token, 1)
plan = src.plan(
req_pool_indices=rows,
seq_lens=live,
seq_lens_cpu=live + 2 * window,
write_virtual=None,
read_extent=window,
)
self.assertGreater(int(live.max()) + 3 * window, row_pages)
self.assertEqual(src.read_table(plan).ids.shape[1], row_pages)
class TestCopyPageTable(unittest.TestCase):
"""`copy_page_table`: a captured graph's own table takes the plan's rows,
padded lanes reading the sink; columns past the plan's width keep their
values (the kernels bound their reads by their own lengths)."""
def setUp(self):
_publish()
self.addCleanup(reset_context)
def test_rows_copied_tail_kept_padded_lanes_sunk(self):
ps = 4
allocator = _build_composite(ps)
lens = [5, 2 * ps + 1, 1]
lens = [5, 2 * ps + 1]
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens)
src = _make_source(allocator, req_to_token, ps)
self.assertTrue(src.is_translating)
plan = _plan(src, rows, seq_lens)
width = src.read_table(plan).ids.shape[1]
# Wider than the req_to_token span and one lane more than the batch.
out = torch.full((3, req_to_token.shape[1] // ps + 3), 7, dtype=torch.int32)
out_swa = torch.full_like(out, 7)
src.copy_page_table(plan, out=out)
src.copy_page_table(plan, out=out_swa, kind=IdSpaceKind.SLIDING_WINDOW)
for v2p, got in (
(allocator.full_v2p_page_table, out),
(allocator.swa_v2p_page_table, out_swa),
):
want = _reference_table(req_to_token, rows, seq_lens, v2p, ps, width)
self.assertTrue(torch.equal(got[:2, :width], want))
self.assertTrue(bool((got[2, :width] == 0).all()), "padded lane")
self.assertTrue(bool((got[:, width:] == 7).all()), "tail clobbered")
width_pages = req_to_token.shape[1] // ps + 3 # wider than the span
out = torch.full((len(lens), width_pages), -1, dtype=torch.int32)
src.fill_read_table(out=out, req_pool_indices=rows, seq_lens=seq_lens)
want = _reference_table(
req_to_token,
rows,
seq_lens,
allocator.full_v2p_page_table,
ps,
width_pages,
)
for b, n in enumerate(lens):
n_pages = -(-n // ps)
self.assertTrue(
torch.equal(out[b, :n_pages], want[b, :n_pages]),
f"row {b} live prefix off-formula",
)
self.assertTrue(
bool((out[b, n_pages:] == -1).all()),
f"row {b} tail sentinel clobbered",
)
def test_row_ids_not_reallocated_across_plans(self):
"""`row_ids` is one arange sized from the request pool and sliced per
table; a per-table `torch.arange` would cost an allocation and a launch
on every replay prep."""
allocator = _build_composite(1)
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3])
src = _make_source(allocator, req_to_token, 1)
first = src.read_table(_plan(src, rows[:2], seq_lens[:2]))
second = src.read_table(_plan(src, rows, seq_lens))
self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr())
self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV)))
self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV)))
self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0])
def test_passthrough_source_refuses(self):
"""Callers dispatch on `reads_are_translated`; a passthrough source has
no v2p to build from and must fail loud, not fill garbage."""
no plan table to copy and must fail loud, not fill garbage."""
src = KVIndexTranslator(
req_to_token=torch.zeros((2, 4), dtype=torch.int64),
token_to_kv_pool_allocator=SimpleNamespace(),
@@ -330,12 +347,9 @@ class TestBuildInto(unittest.TestCase):
page_size=1,
device=_DEV,
)
plan = _plan(src, torch.tensor([0]), torch.tensor([1]))
with self.assertRaises(AssertionError):
src.fill_read_table(
out=torch.zeros((1, 4), dtype=torch.int32),
req_pool_indices=torch.tensor([0]),
seq_lens=torch.tensor([1]),
)
src.copy_page_table(plan, out=torch.zeros((1, 4), dtype=torch.int32))
class TestPoolOwnership(unittest.TestCase):
@@ -422,81 +436,26 @@ class TestPoolOwnership(unittest.TestCase):
device=_DEV,
)
rows = torch.tensor([1, 0], dtype=torch.int32, device=_DEV)
view = src.build_index_table(
req_pool_indices=rows,
seq_lens=torch.tensor([3, 2], dtype=torch.int32, device=_DEV),
loc = torch.tensor([5, 6], dtype=torch.int64, device=_DEV)
plan = _plan(
src,
rows,
torch.tensor([3, 2], dtype=torch.int32, device=_DEV),
write_virtual=loc,
)
# Read table: the EXACT objects a static-pool backend reads today.
# Writes and reads: the EXACT objects a static-pool backend uses.
self.assertIs(plan.write_ids(src), loc)
view = src.read_source(plan, req_pool_indices=rows, bs=2)
self.assertIs(view.ids, req_to_token)
self.assertIs(view.row_ids, rows)
self.assertFalse(view.is_translated)
# And the token-level surface is the identity, same guard.
t = torch.tensor([5, 6], dtype=torch.int64, device=_DEV)
self.assertIs(src.translate_full_attn_ids(t), t)
class TestCaptureContract(unittest.TestCase):
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_caller_owned_table_is_returned_whole_and_filled_prefix_only(self):
ps = 4
allocator = _build_composite(ps)
req_to_token = torch.full((4, 16 * ps), -1, dtype=torch.int64)
v = allocator.alloc(2 * ps)
req_to_token[1, : 2 * ps] = v
src = _make_source(allocator, req_to_token, ps)
# A page-table consumer owns its buffers; entry 0 is the reserved sink
# in every id space, so zeros are what an unfilled column must read as.
cap = torch.zeros((4, 8), dtype=torch.int32, device=_DEV)
cap_swa = torch.zeros((4, 8), dtype=torch.int32, device=_DEV)
tables = KVReadTables(full=cap, sliding_window=cap_swa)
# Poison everything, then refresh a 1-row batch: ONLY its live prefix
# may change -- stale tails and other rows are the fa3 contract.
cap.fill_(7)
cap_swa.fill_(7)
view = src.build_index_table(
req_pool_indices=torch.tensor([1]),
seq_lens=torch.tensor([2 * ps]),
into=tables,
)
self.assertIs(view.ids, cap, "the caller's table comes back WHOLE")
want = allocator.full_v2p_page_table[req_to_token[1, ::ps][:2] // ps]
self.assertTrue(torch.equal(cap[0, :2], want.to(torch.int32)))
self.assertTrue(bool((cap[0, 2:] == 7).all()), "stale tail was cleared")
self.assertTrue(bool((cap[1:] == 7).all()), "rows beyond bs were touched")
def test_row_ids_not_reallocated_across_builds(self):
"""`row_ids` is one arange sized from the request pool and sliced per
build; a per-build `torch.arange` would be correct but costs an
allocation and a launch on every replay prep."""
allocator = _build_composite(1)
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, 1, lens=[4, 2, 3])
src = _make_source(allocator, req_to_token, 1)
self.assertTrue(src.is_translating)
first = src.build_index_table(
req_pool_indices=rows[:2], seq_lens=seq_lens[:2], max_pages=4
)
second = src.build_index_table(
req_pool_indices=rows, seq_lens=seq_lens, max_pages=4
)
self.assertEqual(first.row_ids.data_ptr(), second.row_ids.data_ptr())
self.assertTrue(torch.equal(first.row_ids, torch.arange(2, device=_DEV)))
self.assertTrue(torch.equal(second.row_ids, torch.arange(3, device=_DEV)))
# Sized to bound any batch the request pool can hold.
self.assertGreaterEqual(src._rows.numel(), req_to_token.shape[0])
class _FakeForwardBatch:
"""Weakref-able stand-in (SimpleNamespace is not) carrying the fields
`index_table_for_batch` and `rebind_write_loc` read. `seq_lens_sum`
defaults to the real sum: it is the signal that the CPU mirror is live,
and a real ForwardBatch always carries it (None only when gpu_only)."""
"""Stand-in carrying the fields a plan of its own (`own_plan`) and the
capturers read. `seq_lens_sum` defaults to the real sum: it is the signal
that the CPU mirror is live, and a real ForwardBatch always carries it
(None only when gpu_only)."""
def __init__(
self,
@@ -520,138 +479,14 @@ class _FakeForwardBatch:
)
class TestWidenedTable(unittest.TestCase):
"""`widened_index_table`: the whole-sequence verify view, `seq_len_delta`
columns past every row's live prefix."""
class TestWriteIds(CustomTestCase):
"""A plan translates its write window once, into a fresh tensor, and
derives the sliding-window ids from the virtual window -- pads and
tombstones landing on the sink."""
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
_publish()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _check(self, view, req_to_token, rows, lens, allocator, ps, width):
self.assertEqual(tuple(view.ids.shape), (rows.numel(), width))
want = _reference_table(
req_to_token, rows, lens, allocator.full_v2p_page_table, ps, width
)
self.assertTrue(torch.equal(view.ids, want), f"ps={ps}")
def test_width_comes_from_the_live_prefix(self):
"""A DSPARK verify batch carries `seq_lens_cpu` already expanded by the
window and the live lens on `spec_info.live_seq_lens_cpu`; widening the
expanded lens would count the window twice."""
window = 3
for ps in (1, 4):
allocator = _build_composite(ps)
req_to_token, rows, live = _alloc_and_fill(
allocator, ps, lens=[5 * ps, 2 * ps, 3 * ps - 1]
)
src = _make_source(allocator, req_to_token, ps)
fb = _FakeForwardBatch(
req_pool_indices=rows,
seq_lens=live,
seq_lens_cpu=live + window,
spec_info=SimpleNamespace(live_seq_lens_cpu=live),
)
view = src.widened_index_table(fb, seq_len_delta=window)
width = -(-(int(live.max()) + window) // ps)
self._check(view, req_to_token, rows, live + window, allocator, ps, width)
def test_width_never_exceeds_the_request_row(self):
"""Near the context limit the widened host bound can pass the end of
the `req_to_token` row; the table stops at the row instead of tripping
the read-table width assert."""
window = 6
ps = 1
allocator = _build_composite(ps)
req_to_token, rows, live = _alloc_and_fill(allocator, ps, lens=[6, 1])
row_pages = req_to_token.shape[1] // ps
src = _make_source(allocator, req_to_token, ps)
# No live lens on the spec info: the expanded mirror is all there is.
fb = _FakeForwardBatch(
req_pool_indices=rows, seq_lens=live, seq_lens_cpu=live + window
)
self.assertGreater(int(live.max()) + 2 * window, row_pages)
view = src.widened_index_table(fb, seq_len_delta=window)
self._check(view, req_to_token, rows, live + window, allocator, ps, row_pages)
class TestViewMemo(unittest.TestCase):
"""The eager view is memoized ON THE SOURCE in a single slot keyed by batch
identity, so per-batch state stays out of the ForwardBatch while one
metadata build's many consumers still share one table build."""
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _fb(self, allocator, ps, lens):
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=lens)
fb = _FakeForwardBatch(
req_pool_indices=rows,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
)
return fb, req_to_token
def test_same_batch_returns_the_memoized_view(self):
ps = 1
allocator = _build_composite(ps)
fb, req_to_token = self._fb(allocator, ps, lens=[3, 2])
src = _make_source(allocator, req_to_token, ps)
v1 = src.index_table_for_batch(fb)
v2 = src.index_table_for_batch(fb)
self.assertIs(v1, v2)
def test_next_batch_replaces_the_single_slot(self):
ps = 1
allocator = _build_composite(ps)
fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2])
src = _make_source(allocator, req_to_token, ps)
v1 = src.index_table_for_batch(fb1)
fb2 = _FakeForwardBatch(
req_pool_indices=fb1.req_pool_indices,
seq_lens=fb1.seq_lens,
seq_lens_cpu=fb1.seq_lens_cpu,
)
v2 = src.index_table_for_batch(fb2)
self.assertIsNot(v1, v2)
# Single slot: fb1 no longer matches and rebuilds.
v1_again = src.index_table_for_batch(fb1)
self.assertIsNot(v1_again, v1)
def test_dead_batch_never_matches(self):
"""A garbage-collected batch's slot must not serve a later batch: the
weakref key goes dead and the build runs fresh."""
import gc
ps = 1
allocator = _build_composite(ps)
fb1, req_to_token = self._fb(allocator, ps, lens=[3, 2])
src = _make_source(allocator, req_to_token, ps)
v1 = src.index_table_for_batch(fb1)
del fb1
gc.collect()
fb2, _ = self._fb(allocator, ps, lens=[2])
v2 = src.index_table_for_batch(fb2)
self.assertIsNot(v2, v1)
self.assertEqual(v2.ids.shape[0], 1)
class TestWriteLoc(CustomTestCase):
"""The two-phase write contract: `rebind_write_loc` rebinds the full side
once at ForwardBatch construction, and the sliding-window write loc derives
POINTWISE from the full-side values -- pads, slices, and fresh copies
included -- with no handover and no stored per-forward state."""
def setUp(self):
# The code under test reads its config from the bags.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _built(self, ps=1, n=4):
allocator = _build_composite(ps)
@@ -662,20 +497,18 @@ class TestWriteLoc(CustomTestCase):
want_swa = allocator.translate_loc_from_full_to_swa(virt)
return src, allocator, rows, seq_lens, virt, want_full, want_swa
def _field(self, src, rows, seq_lens, kernel_loc):
return src.sliding_window_write_loc_for(kernel_loc)
def test_rebind_translates_full_side_only(self):
def test_own_plan_translates_full_side_only(self):
for ps in (1, 4):
src, _, _, _, virt, want_full, _ = self._built(ps=ps, n=3 * ps)
keep = virt.clone()
fb = _FakeForwardBatch(out_cache_loc=virt)
src.rebind_write_loc(fb)
# Full side: rebound to a FRESH physical tensor; the
# ScheduleBatch's aliased virtual tensor is untouched.
src.bind_own_plan(fb)
# Full side: a FRESH physical tensor; the ScheduleBatch's aliased
# virtual tensor is untouched and mirrored for its bookkeeping.
self.assertIsNot(fb.out_cache_loc, virt)
self.assertTrue(torch.equal(fb.out_cache_loc, want_full))
self.assertTrue(torch.equal(virt, keep))
self.assertIs(fb.out_cache_loc_virtual, virt)
def test_capture_capacity_covers_dcp_widened_mamba_ids(self):
"""A capturer must store the highest virtual IDs issued under DCP."""
@@ -755,7 +588,7 @@ class TestWriteLoc(CustomTestCase):
req_pool = SimpleNamespace(req_to_token=virt.clone()[None, :])
fb = _FakeForwardBatch(out_cache_loc=virt.clone())
if translating:
src.rebind_write_loc(fb)
src.bind_own_plan(fb)
fb.out_cache_loc = torch.cat(
[fb.out_cache_loc, virt.new_zeros(2)]
)
@@ -795,35 +628,36 @@ class TestWriteLoc(CustomTestCase):
)
)
def test_swa_write_loc_round_trips_from_full_side(self):
"""Derived property: `field(full(t)) == swa(t)` for any virtual run t,
across page sizes."""
def test_swa_ids_come_from_the_virtual_ids(self):
for ps in (1, 4, 64):
src, _, rows, seq_lens, _, want_full, want_swa = self._built(
ps=ps, n=3 * ps
src, _, rows, seq_lens, virt, _, want_swa = self._built(ps=ps, n=3 * ps)
plan = _plan(src, rows, seq_lens, write_virtual=virt)
self.assertTrue(
torch.equal(
plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW), want_swa
)
)
got = self._field(src, rows, seq_lens, want_full)
self.assertTrue(torch.equal(got, want_swa))
def test_pad_lanes_derive_to_sink(self):
"""The DP pad appends zeros, and physical 0 is the reserved
padding slot in every id space, so pad lanes derive to swa slot 0 with
no `num_live` bookkeeping."""
src, _, rows, seq_lens, _, want_full, want_swa = self._built(n=3)
padded = torch.cat([want_full, want_full.new_zeros(2)])
got = self._field(src, rows, seq_lens, padded)
def test_pad_lanes_read_the_sink(self):
"""The DP pad appends zeros, and id 0 is the reserved padding slot in
every id space, so pad lanes land on swa slot 0."""
src, _, rows, seq_lens, virt, _, want_swa = self._built(n=3)
padded = torch.cat([virt, virt.new_zeros(2)])
plan = _plan(src, rows, seq_lens, write_virtual=padded)
got = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
self.assertTrue(torch.equal(got[:3], want_swa))
self.assertTrue(bool((got[3:] == 0).all()), "pad lanes must land on slot 0")
def test_tombstoned_swa_page_clamps_to_sink(self):
src, allocator, rows, seq_lens, virt, want_full, _ = self._built(ps=1, n=2)
def test_tombstoned_swa_page_reads_the_sink(self):
src, allocator, rows, seq_lens, virt, _, _ = self._built(ps=1, n=2)
allocator.swa_v2p_page_table[int(virt[0])] = -1
got = self._field(src, rows, seq_lens, want_full[:1])
self.assertEqual(int(got[0]), 0)
plan = _plan(src, rows, seq_lens, write_virtual=virt[:1])
swa_ids = plan.write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
self.assertEqual(int(swa_ids[0]), 0)
def test_static_swa_pool_derives_via_pool_translate(self):
"""Static SWA pools: the field is the pool's own legacy full->swa
translate, computed at the same build; the rebind stays a no-op."""
"""Static SWA pools: the swa ids are the pool's own full->swa
translate; the write ids stay the batch's own."""
pool = SWAKVPool.__new__(SWAKVPool)
pool.full_to_swa_index_mapping = torch.arange(10, dtype=torch.int64)
pool.translate_loc_from_full_to_swa = lambda t: t + 100
@@ -836,14 +670,18 @@ class TestWriteLoc(CustomTestCase):
)
loc = torch.tensor([5, 6], dtype=torch.int64)
fb = _FakeForwardBatch(out_cache_loc=loc)
src.rebind_write_loc(fb)
self.assertIs(fb.out_cache_loc, loc, "disabled rebind must be a no-op")
self.assertTrue(torch.equal(src.sliding_window_write_loc_for(loc), loc + 100))
src.bind_own_plan(fb)
self.assertIs(fb.out_cache_loc, loc, "a static pool's ids are its own")
self.assertTrue(
torch.equal(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), loc + 100)
)
def test_no_loc_or_no_swa_side_yields_none(self):
# Unified swa composite, but there is no write loc this forward.
src, _, rows, seq_lens, _, _, _ = self._built(n=2)
self.assertIsNone(src.sliding_window_write_loc_for(None))
self.assertIsNone(
_plan(src, rows, seq_lens).write_ids(src, kind=IdSpaceKind.SLIDING_WINDOW)
)
# Passthrough on a non-SWA pool: a loc is given, but there is no swa
# id space to derive into.
plain = KVIndexTranslator(
@@ -853,23 +691,13 @@ class TestWriteLoc(CustomTestCase):
page_size=1,
device=_DEV,
)
loc = torch.tensor([3], dtype=torch.int64)
self.assertIsNone(
plain.sliding_window_write_loc_for(torch.tensor([3], dtype=torch.int64))
_plan(
plain, torch.tensor([0]), torch.tensor([1]), write_virtual=loc
).write_ids(plain, kind=IdSpaceKind.SLIDING_WINDOW)
)
def test_rebind_retires_the_view_memo(self):
ps = 1
allocator = _build_composite(ps)
req_to_token, rows, seq_lens = _alloc_and_fill(allocator, ps, lens=[3, 2])
src = _make_source(allocator, req_to_token, ps)
fb = _FakeForwardBatch(
req_pool_indices=rows, seq_lens=seq_lens, seq_lens_cpu=seq_lens
)
v1 = src.index_table_for_batch(fb)
src.rebind_write_loc(_FakeForwardBatch(out_cache_loc=None))
v2 = src.index_table_for_batch(fb)
self.assertIsNot(v2, v1, "rebind starts the next forward: stale views die")
if __name__ == "__main__":
unittest.main()
@@ -11,36 +11,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""KVIndexTranslator's three id-space dispositions with a fused draft region.
"""KVIndexTranslator with a fused draft region.
1. TARGET runner (allocator's own kvcache): translates to physical ids.
2. FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to this allocator): the
SAME translate -- same pages, same v2p table, same physical ids (the draft
parts sit inside the host entry). A dense draft pool routes no window
layers, so it has no swa id space: the window index table falls back to
the one dense table and `sliding_window_write_loc_for()` answers None (a
separate-swa assumption here crashed the read side and left the write side
with nothing to derive).
3. PRIVATE-POOL draft (DSPARK/DFLASH shape: target's allocator, own
virtual-indexed buffer): strict passthrough — translating such a runner
would address a slot-count buffer with dense ids (OOB both directions),
so the no-op is load-bearing, not a default.
A FUSED-DRAFT runner (`UnifiedDraftKVPool` bound to the target's allocator)
translates exactly as the target does -- same v2p table, same physical ids,
the draft parts sitting inside the host entries -- and, routing no window
layers, has no sliding-window write ids. A PRIVATE-POOL draft (own
virtual-indexed buffer) stays a strict passthrough. The draft writers outside
the draft's forward (DFLASH/DSPARK's target-hidden writes, the multi-step
draft decode's kv indices) read the same ids.
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
"""
import ast
import pathlib
import unittest
from types import SimpleNamespace
import torch
from sglang.srt.mem_cache import kv_index_translator
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
@@ -51,7 +44,7 @@ from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
register_cpu_ci(est_time=6, suite="base-a-test-cpu")
_DEV = "cpu"
_PS = 2
@@ -150,163 +143,32 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_target_runner_translates_to_physical_ids(self):
_, allocator, kvcache, _ = _build()
src = _source(allocator, kvcache)
self.assertTrue(src.is_translating)
v = allocator.alloc(2 * _PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
def test_fused_draft_runner_translates_to_the_same_physical_ids(self):
_, allocator, _, draft_pool = _build()
src = _source(allocator, draft_pool)
self.assertTrue(src.is_translating)
self.assertIsNone(src._swa_v2p_table) # single space: no swa id space
# A dense draft pool: no sliding-window sub-pool.
self.assertIsNone(src.space(IdSpaceKind.SLIDING_WINDOW))
# rebind_write_loc translates to the HOST's physical ids: the draft
# parts live inside the same slots.
# Its own plan translates to the HOST's physical ids: the draft parts
# live inside the same slots.
v = allocator.alloc(2 * _PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
src.bind_own_plan(fb)
self.assertIsNot(fb.out_cache_loc, v)
fa = allocator.full_attn_allocator
expected = torch.clamp_min(fa.virtual_to_physical[v // _PS] * _PS + v % _PS, 0)
torch.testing.assert_close(fb.out_cache_loc, expected, rtol=0, atol=0)
self.assertTrue(torch.equal(fb.out_cache_loc, allocator.translate_kv_loc(v)))
# A dense draft pool routes no window layers: no swa write loc.
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
def test_dense_fused_draft_has_no_window_write_loc(self):
"""`sliding_window_write_loc_for()` on a dense fused-draft batch answers
None: the pool routes no window layers, so there is nothing to
derive. The hybrid-SWA TARGET on the same allocator keeps a genuinely
derived swa loc, so the None can never paper over a real swa id
space."""
_, allocator, kvcache, draft_pool = _build()
src = _source(allocator, draft_pool)
v = allocator.alloc(_PS)
self.assertIsNotNone(v)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertIsNone(src.sliding_window_write_loc_for(fb.out_cache_loc))
# Target contrast: a real swa side derives a DIFFERENT loc.
tgt = _source(allocator, kvcache)
tv = allocator.alloc(_PS)
self.assertIsNotNone(tv)
tfb = SimpleNamespace(out_cache_loc=tv)
tgt.rebind_write_loc(tfb)
tswa = tgt.sliding_window_write_loc_for(tfb.out_cache_loc)
self.assertIsNotNone(tswa)
self.assertIsNot(tswa, tfb.out_cache_loc)
self.assertTrue(torch.equal(tswa, allocator.translate_loc_from_full_to_swa(tv)))
def test_private_pool_draft_stays_a_strict_passthrough(self):
_, allocator, _, _ = _build()
private_draft_pool = _FakeKVCache(64) # own buffer, not the allocator's
src = _source(allocator, private_draft_pool)
self.assertFalse(src.is_translating)
v = torch.arange(2 * _PS, dtype=torch.int64)
fb = SimpleNamespace(out_cache_loc=v)
src.rebind_write_loc(fb)
self.assertIs(fb.out_cache_loc, v) # untouched, not even a copy
def test_foreign_allocator_draft_pool_is_not_enabled(self):
# A UnifiedDraftKVPool bound to a DIFFERENT allocator must not enable
# against this one (identity, not type, decides).
pool, allocator, kvcache, _ = _build()
_, other_alloc, _, other_draft = _build()
src = _source(allocator, other_draft)
self.assertFalse(src.is_translating)
def test_seq_len_delta_matches_widened_lens(self):
"""``seq_len_delta=k`` must be byte-identical to building with
``seq_lens + k`` -- the two spellings of the whole-sequence verify
widening. A kernel applying the delta to the page count but not the
loads (or vice versa) silently truncates the verify tail."""
_, allocator, kvcache, _ = _build()
v = allocator.alloc(6 * _PS)
self.assertIsNotNone(v)
rt = torch.zeros((2, 16), dtype=torch.int32)
rt[0, : v.numel()] = v.to(torch.int32)
src = KVIndexTranslator(
req_to_token=rt,
token_to_kv_pool_allocator=allocator,
token_to_kv_pool=kvcache,
page_size=_PS,
device=_DEV,
)
rpi = torch.tensor([0], dtype=torch.int64)
seq = torch.tensor([3], dtype=torch.int64)
delta = 2 * _PS + 1
max_pages = -(-(3 + delta) // _PS)
widened = src.build_index_table(
req_pool_indices=rpi,
seq_lens=seq,
max_pages=max_pages,
seq_len_delta=delta,
)
by_lens = src.build_index_table(
req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages
)
torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0)
# The delta genuinely widened: entries exist past the unwidened prefix.
plain_pages = -(-3 // _PS)
self.assertTrue(bool((widened.ids[0, plain_pages:] > 0).any()))
def test_widened_index_table_matches_widened_lens(self):
"""`widened_index_table` (the verify entry point) must equal the
widened-lens build over the SAME batch: derive max_pages from the
widened max and forward the delta. Deriving max_pages from the
un-widened lens silently truncates the verify tail's pages."""
_, allocator, kvcache, _ = _build()
v = allocator.alloc(6 * _PS)
self.assertIsNotNone(v)
rt = torch.zeros((2, 16), dtype=torch.int32)
rt[0, : v.numel()] = v.to(torch.int32)
src = KVIndexTranslator(
req_to_token=rt,
token_to_kv_pool_allocator=allocator,
token_to_kv_pool=kvcache,
page_size=_PS,
device=_DEV,
)
rpi = torch.tensor([0], dtype=torch.int64)
seq = torch.tensor([3], dtype=torch.int64)
delta = 2 * _PS + 1
fb = SimpleNamespace(
req_pool_indices=rpi,
seq_lens=seq,
seq_lens_cpu=seq.cpu(),
# `seq_lens_sum` is the liveness signal for seq_lens_cpu: it is a
# non-None but STALE slice on a gpu_only batch, so the build only
# trusts it when the sum is present. Without the field the stand-in
# batch falls back to the full req_to_token width.
seq_lens_sum=int(seq.sum()),
out_cache_loc=None,
spec_info=None,
)
widened = src.widened_index_table(fb, seq_len_delta=delta)
max_pages = -(-(3 + delta) // _PS)
by_lens = src.build_index_table(
req_pool_indices=rpi, seq_lens=seq + delta, max_pages=max_pages
)
self.assertEqual(widened.ids.shape, by_lens.ids.shape)
torch.testing.assert_close(widened.ids, by_lens.ids, rtol=0, atol=0)
self.assertIsNone(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW))
def test_target_hidden_writers_write_physical_ids(self):
"""BUG REGRESSION. DFLASH and DSPARK do not compute their draft KV from
the draft's own forward -- they PROJECT the target's hidden states and
write them straight into the draft pool, at locs read off the target's
req_to_token (VIRTUAL). Under fusion the draft pool takes the target's
physical ids, so an untranslated write lands at the wrong row: the
draft attends over the wrong KV and accept length collapses to 1.0
with no crash. Each writer must hand the pool translated ids and leave
the caller's locs virtual (the compact req_to_token rebuild re-reads
them)."""
"""DFLASH and DSPARK project the target's hidden states straight into
the draft pool. Under fusion that pool takes the target's physical ids:
both writers take them from the iteration's plan in the draft pool's
space, never the virtual ids (a wrong row, with no crash)."""
from sglang.srt.speculative.dflash_worker_v2 import DFlashWorkerV2
from sglang.srt.speculative.dspark_components.dspark_kv_inject import (
TargetHiddenKvInjector,
@@ -317,11 +179,15 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
is_translating = True
def translate_full_attn_ids(self, ids, *, out=None):
return ids + 100
virtual = torch.tensor([4, 5, 6, 9], dtype=torch.int64)
physical = virtual + 100
draft_translator = _Shift()
# The iteration's plan: physical ids for the fused draft's translator.
plan = SimpleNamespace(
write_ids=lambda reader, cols=None: (
physical.clone() if reader is draft_translator else virtual.clone()
)
)
hidden = torch.randn(4, 8)
positions = torch.arange(4)
commit_lens = torch.tensor([2, 1], dtype=torch.int32)
@@ -348,7 +214,7 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
worker = SimpleNamespace(
model_runner=SimpleNamespace(device=torch.device(_DEV)),
draft_model_runner=SimpleNamespace(
kv_index_translator=_Shift(), token_to_kv_pool=_DflashPool()
kv_index_translator=draft_translator, token_to_kv_pool=_DflashPool()
),
draft_model=SimpleNamespace(
project_target_hidden=lambda h: h,
@@ -365,30 +231,29 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
)
# Per-token writes (prefill).
cache_loc = virtual.clone()
cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan)
DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc(
worker, target_hidden=hidden, cache_loc=cache_loc, positions=positions
)
self.assertTrue(writes["loc"].physical)
torch.testing.assert_close(writes["loc"].loc, physical, rtol=0, atol=0)
torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0)
# Prefix-valid writes (post-verify), from the 2-D view of the same ids.
cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2)
cache_loc = DFlashWorkerV2._draft_write_ids(worker, plan)
DFlashWorkerV2._append_target_hidden_to_draft_kv_by_loc(
worker,
target_hidden=hidden,
cache_loc=cache_loc,
positions=positions,
cache_loc_2d=cache_loc_2d,
cache_loc_2d=cache_loc.view(2, 2),
commit_lens=commit_lens,
)
torch.testing.assert_close(
writes["loc_2d"], physical.view(2, 2), rtol=0, atol=0
)
torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0)
# DSPARK's injector, on an MHA draft pool.
# DSPARK's injector, on an MHA draft pool: the plan's ids for the
# draft's translator, written as they are.
injected = {}
def write_target_hidden_kv(**kwargs):
@@ -397,50 +262,47 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
injector = TargetHiddenKvInjector(
draft_model=SimpleNamespace(write_target_hidden_kv=write_target_hidden_kv),
draft_model_runner=SimpleNamespace(
kv_index_translator=_Shift(), token_to_kv_pool=SimpleNamespace()
kv_index_translator=draft_translator, token_to_kv_pool=SimpleNamespace()
),
model_runner=SimpleNamespace(device=torch.device(_DEV)),
device=torch.device(_DEV),
verify_num_draft_tokens=2,
block_pos_offsets=torch.arange(2),
)
cache_loc, cache_loc_2d = virtual.clone(), virtual.clone().view(2, 2)
cache_loc = injector.ids_for(plan)
injector.inject_target_hidden(
target_hidden=hidden,
cache_loc=cache_loc,
positions=positions,
cache_loc_2d=cache_loc_2d,
cache_loc_2d=cache_loc.view(2, 2),
commit_lens=commit_lens,
)
torch.testing.assert_close(injected["cache_loc"], physical, rtol=0, atol=0)
torch.testing.assert_close(
injected["cache_loc_2d"], physical.view(2, 2), rtol=0, atol=0
)
torch.testing.assert_close(cache_loc, virtual, rtol=0, atol=0)
torch.testing.assert_close(cache_loc_2d, virtual.view(2, 2), rtol=0, atol=0)
def test_full_flat_v2p_per_disposition(self):
"""The flat-translate accessor must hand a kernel exactly what
def test_full_v2p_table_per_disposition(self):
"""The v2p accessor must hand a DCP table builder exactly what
`translate_kv_loc` would use: the shared v2p table on the target and
on the fused draft, and None on a pass-through runner (a translated
pass-through would address a private buffer with the target's ids)."""
_, allocator, kvcache, draft_pool = _build()
target = _source(allocator, kvcache)
self.assertIs(target.full_flat_v2p(), allocator.full_v2p_page_table)
self.assertIs(target.full_v2p_table, allocator.full_v2p_page_table)
draft = _source(allocator, draft_pool)
self.assertIs(draft.full_flat_v2p(), allocator.full_v2p_page_table)
self.assertIs(draft.full_v2p_table, allocator.full_v2p_page_table)
passthrough = _source(allocator, _FakeKVCache(64))
self.assertIsNone(passthrough.full_flat_v2p())
self.assertIsNone(passthrough.full_v2p_table)
def test_multi_step_containers_pass_the_v2p_table_to_the_kernel(self):
"""Every `generate_draft_decode_kv_indices` launch must thread the
runner's v2p table and a matching TRANSLATE flag. A launch
without them emits raw req_to_token values, which under the unified
pool are VIRTUAL ids the fused draft pool cannot address — the exact
silent-garbage bug this series fixed."""
def test_multi_step_containers_read_the_plans_table(self):
"""Every `generate_draft_decode_kv_indices` launch must gather from
`read_source` with its entry granularity. A launch over raw
req_to_token emits VIRTUAL ids, which the fused draft pool cannot
address -- silent garbage drafts."""
import pathlib
import re
@@ -454,112 +316,20 @@ class TestKVIndexTranslatorDraftDisposition(unittest.TestCase):
if launches:
launching[path.name] = (
launches,
len(re.findall(r"TRANSLATE=v2p is not None", text)),
"full_flat_v2p" in text,
len(re.findall(r"ENTRY_PAGE_SIZE=src\.entry_page_size", text)),
"kv_index_translator.read_source(" in text,
)
self.assertGreaterEqual(
len(launching), 4, f"launch sites disappeared: {sorted(launching)}"
)
for name, (launches, translated, has_accessor) in launching.items():
for name, (launches, granular, has_source) in launching.items():
self.assertEqual(
launches,
translated,
f"{name}: {launches} kernel launch(es) but only {translated} "
"carry TRANSLATE=v2p is not None",
granular,
f"{name}: {launches} kernel launch(es) but only {granular} "
"pass the read table's entry granularity",
)
self.assertTrue(
has_accessor,
f"{name} launches the kernel without full_flat_v2p",
)
class TestDispositionBranchesAgree(unittest.TestCase):
"""Every `__init__` disposition must assign the SAME attribute set.
BUG REGRESSION. `KVIndexTranslator.__init__` picks a disposition --
translating (unified target or fused draft) or passthrough -- and later
methods read attributes off `self` unconditionally. When upstream adds an
attribute it naturally adds it to the branches IT knows about; a branch
added here is silently left short, and nothing fails until the missing
attribute is read at RUNTIME. That is exactly how the fused-draft branch
lost `_translate_write_full` and `defer_read_translate` across a rebase:
py_compile passes (the attribute is only ever read, never declared), the
undefined-NAME check passes (it is an attribute, not a bare name), and the
target and passthrough paths both work -- only a fused-draft forward raises
`AttributeError: 'KVIndexTranslator' object has no attribute
'_translate_write_full'`.
Comparing the branches against EACH OTHER needs no list to maintain: a new
attribute is covered the moment any one branch sets it.
python -m pytest test/registered/unit/mem_cache/test_kv_index_translator_draft_disposition.py -v
"""
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def _init_branch_assignments(self):
"""{branch index: {attr names it assigns}} for __init__'s if/elif/else."""
src = pathlib.Path(kv_index_translator.__file__).read_text()
tree = ast.parse(src)
cls = next(
n
for n in ast.walk(tree)
if isinstance(n, ast.ClassDef) and n.name == "KVIndexTranslator"
)
init = next(
n
for n in cls.body
if isinstance(n, ast.FunctionDef) and n.name == "__init__"
)
# The disposition chain is the `if` whose body assigns the id-space
# fields; find it by the attribute every disposition must set.
def assigns(stmts):
out = set()
for st in stmts:
for node in ast.walk(st):
if isinstance(node, ast.Assign):
for t in node.targets:
if (
isinstance(t, ast.Attribute)
and isinstance(t.value, ast.Name)
and t.value.id == "self"
):
out.add(t.attr)
return out
for node in init.body:
if not isinstance(node, ast.If):
continue
branches, cur = [], node
while True:
branches.append(assigns(cur.body))
if len(cur.orelse) == 1 and isinstance(cur.orelse[0], ast.If):
cur = cur.orelse[0]
continue
if cur.orelse:
branches.append(assigns(cur.orelse))
break
if len(branches) >= 2 and all("_full_v2p_table" in b for b in branches):
return branches
self.fail("could not locate the disposition if/elif/else in __init__")
def test_every_disposition_assigns_the_same_attributes(self):
branches = self._init_branch_assignments()
union = set().union(*branches)
missing = {i: sorted(union - b) for i, b in enumerate(branches) if union - b}
self.assertEqual(
missing,
{},
"a KVIndexTranslator.__init__ disposition does not assign every "
"attribute its siblings do; the branch raises AttributeError only "
"when that path is taken at runtime: " + repr(missing),
)
self.assertTrue(has_source, f"{name} launches without read_source")
if __name__ == "__main__":
@@ -0,0 +1,745 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`KVLocPlan`: one iteration's KV ids, translated once.
- the target and a fused draft get the same physical write ids, a
pass-through reader (a private draft pool, a static pool) the virtual
window, which is never mutated; `bind` and a split batch's `cols_slice`
pick the same tokens in both spaces;
- a forward whose `seq_lens` do not yet count the window it writes (a
verify, a speculative draft decode, a draft extend) reads that window past
them;
- one iteration's forwards translate the window once per sub-pool and build
a sub-pool's read table at most once, only when a table reader asks; the
table covers `seq_lens + read_extent`, padded lanes reading the sink;
- only readers of the plan's own rows read its table; a pass-through reader
gathers `req_to_token`, and a draft with its own `req_to_token` plans its
own reads over the shared write ids (`reads_from`);
- a runner's own write buffer and a replayed graph's padded buffer are used
as they are, the sliding-window ids naming the sink past the plan's.
python -m pytest test/registered/unit/mem_cache/test_kv_loc_plan.py -v
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import msgspec
import torch
from sglang.srt.mem_cache import kv_index_translator
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind, TokenSpan
from sglang.srt.mem_cache.layout.fused_draft import (
DenseDraftRegion,
FusedDraftPlacement,
)
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftKVPool
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
_DEV = "cpu"
_PS = 2
_FULL = IdSpaceKind.FULL
_SWA = IdSpaceKind.SLIDING_WINDOW
def _reference(req_to_token, rows, lens, v2p, width):
"""Independent derivation of a read table: each row's pages over `lens`,
through `v2p`, the sink past them."""
out = torch.zeros((rows.numel(), width), dtype=torch.int32)
for b, row in enumerate(rows.tolist()):
for c in range(min(-(-int(lens[b]) // _PS), width)):
out[b, c] = max(int(v2p[int(req_to_token[row, c * _PS]) // _PS]), 0)
return out
class _HostStreamGather:
"""`create_flashinfer_kv_indices_triton` on the host, for these CPU tests:
lane ``b``'s ids from row ``row_ids[b]`` of the source, packed at
``indptr[b]``."""
def __getitem__(self, grid):
return self
def __call__(
self,
ids,
row_ids,
seq_lens,
indptr,
kv_start_idx,
out,
row_stride,
ENTRY_PAGE_SIZE=1,
token_mapping=None,
):
assert row_stride == ids.stride(0)
for b in range(int(seq_lens.numel())):
start = 0 if kv_start_idx is None else int(kv_start_idx[b])
pos = torch.arange(start, start + int(seq_lens[b]))
entry = ids[int(row_ids[b]), pos // ENTRY_PAGE_SIZE].to(torch.int64)
tok = entry * ENTRY_PAGE_SIZE + pos % ENTRY_PAGE_SIZE
if token_mapping is not None:
tok = token_mapping[tok]
lo = int(indptr[b])
out[lo : lo + int(seq_lens[b])] = tok.to(out.dtype)
class _FakeKVCache:
def __init__(self, max_slots):
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
self.allocator = None
def attach_allocator(self, allocator):
self.allocator = allocator
class _FakeUnifiedSWAKVPool:
def __init__(self, shared_pool):
self.full_kv_pool = _FakeKVCache(shared_pool.max_slots("full"))
self.swa_kv_pool = _FakeKVCache(shared_pool.max_slots("swa"))
self.full_to_swa_index_mapping = None
def attach_allocators(self, *, full_allocator, swa_allocator):
self._full_allocator = full_allocator
self._swa_allocator = swa_allocator
class TestKVLocPlan(unittest.TestCase):
def setUp(self):
# `KVIndexTranslator.__init__` reads `attn_dcp_size`, a derived
# parallel width that only exists once a config is published.
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
full_spec = MHASubPoolSpec(
name="full",
layer_num=2,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="down",
draft_region=DenseDraftRegion(
lane_num=1, head_num=1, head_dim=8, store_dtype=torch.bfloat16
),
)
swa_spec = MHASubPoolSpec(
name="swa",
layer_num=1,
head_num=2,
head_dim=4,
store_dtype=torch.bfloat16,
grow_direction="up",
)
n_full, n_swa = 32, 16
pool = UnifiedKVPool(
total_bytes=n_full * full_spec.entry_bytes()
+ n_swa * swa_spec.entry_bytes(),
sub_pool_specs=[full_spec, swa_spec],
device=_DEV,
enable_memory_saver=False,
page_size=_PS,
fused_draft=FusedDraftPlacement(
region=full_spec.draft_region, runner_lane_counts=(1,)
),
)
kvcache = _FakeUnifiedSWAKVPool(pool)
self.allocator = UnifiedSWATokenToKVPoolAllocator(
unified_buffer=pool,
kvcache=kvcache,
device=_DEV,
full_max_total_num_tokens=n_full,
swa_max_total_num_tokens=n_swa,
page_size=_PS,
need_sort=False,
forward_stream=None,
)
draft_pool = UnifiedDraftKVPool(
unified_buffer=pool,
host_sub_pool_name="full",
host_allocator=self.allocator,
layer_lanes={0: 0},
page_size=_PS,
)
# Every runner of one server shares the target's req_to_token.
self.req_to_token = torch.zeros((4, 16), dtype=torch.int32)
self.draft_pool = draft_pool
self.target = self._translator(kvcache)
self.fused_draft = self._translator(draft_pool)
self.private_draft = self._translator(_FakeKVCache(64))
# Two requests, two pages each; the second page of each is this
# iteration's write window (one token per request).
self.rpi = torch.tensor([1, 2], dtype=torch.int64)
for row in (1, 2):
ids = self.allocator.alloc(2 * _PS)
self.assertIsNotNone(ids)
self.req_to_token[row, : 2 * _PS] = ids.to(torch.int32)
self.seq_lens = torch.tensor([3, 3], dtype=torch.int64)
self.window = self.req_to_token[self.rpi, 2:4].to(torch.int64).reshape(-1)
def _translator(self, pool):
return KVIndexTranslator(
req_to_token=self.req_to_token,
token_to_kv_pool_allocator=self.allocator,
token_to_kv_pool=pool,
page_size=_PS,
device=_DEV,
)
def _plan(self, source=None, read_extent=1):
return (source or self.target).plan(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=self.seq_lens.clone(),
write_virtual=self.window,
read_extent=read_extent,
)
def test_readers_get_ids_in_their_own_space(self):
window = self.window.clone()
plan = self._plan()
physical = self.allocator.translate_write_loc(window)
self.assertTrue(torch.equal(plan.write_ids(self.target), physical))
# A fused draft shares the target's pages: the same tensor, untranslated.
self.assertIs(plan.write_ids(self.fused_draft), plan.write_physical)
# A private draft pool indexes virtual ids.
self.assertIs(plan.write_ids(self.private_draft), self.window)
self.assertTrue(torch.equal(self.window, window)) # never mutated
def test_cols_slice_names_a_split_batch_s_tokens(self):
"""Two-batch overlap splits a forward's tokens; each half writes its
own tokens of the window, whatever the forward's own columns were."""
plan = self._plan()
bs = int(self.rpi.numel())
everything = plan.cols_slice(None, slice(1, None))
self.assertEqual(everything, TokenSpan(1, self.window.numel()))
self.assertEqual(plan.cols_slice(everything, slice(1, 3)), TokenSpan(2, 4))
# A static plan's halves, and a translating one's, both read views.
static = self._plan(self.private_draft)
for p, reader in ((plan, self.target), (static, self.private_draft)):
half = p.cols_slice(None, slice(0, 2))
self.assertIsInstance(half, TokenSpan)
whole = p.write_ids(reader)
ids = p.write_ids(reader, cols=half)
self.assertTrue(torch.equal(ids, whole[:2]))
self.assertEqual(
ids.untyped_storage().data_ptr(), whole.untyped_storage().data_ptr()
)
first = plan.cols_slice(slice(0, 1), slice(None))
width = self.window.numel() // bs
self.assertTrue(torch.equal(first, torch.arange(bs) * width))
ragged = torch.tensor([3, -1, 0])
self.assertTrue(torch.equal(plan.cols_slice(ragged, slice(1, 3)), ragged[1:]))
for cols, tokens in (
(None, slice(1, None)),
(everything, slice(1, 3)),
(slice(0, 1), slice(None)),
):
self.assertTrue(
torch.equal(
plan.write_ids(self.target, cols=plan.cols_slice(cols, tokens)),
plan.write_ids(self.target, cols=cols)[tokens],
)
)
def test_one_iteration_translates_once(self):
writes, builds = [], []
real_write = self.allocator.translate_write_loc
real_build = kv_index_translator.build_kv_read_table
swa_writes = []
real_swa_write = self.allocator.translate_loc_from_full_to_swa
def counting_write(ids, *a, **kw):
writes.append(ids)
return real_write(ids, *a, **kw)
def counting_swa_write(ids):
swa_writes.append(ids)
return real_swa_write(ids)
def counting_build(**kwargs):
builds.append(kwargs["v2p"])
return real_build(**kwargs)
with patch.object(kv_index_translator, "build_kv_read_table", counting_build):
# The spaces hold the allocator's translates from when the
# translator was built; count through them. The fused draft's full
# space is the target's, so it counts there too.
for reader, kind, write in (
(self.target, _FULL, counting_write),
(self.fused_draft, _FULL, counting_write),
(self.target, _SWA, counting_swa_write),
):
reader._spaces[kind] = msgspec.structs.replace(
reader.space(kind), write=write
)
plan = self._plan(read_extent=2)
draft, verify, extend = (SimpleNamespace() for _ in range(3))
plan.bind(draft, self.fused_draft, cols=slice(0, 1))
plan.bind(verify, self.target)
plan.bind(extend, self.fused_draft)
bs = int(self.rpi.numel())
for reader in (self.target, self.fused_draft):
# What a packed stream gathers from, and a captured table.
reader.read_source(plan, req_pool_indices=self.rpi, bs=bs + 1)
reader.read_table(plan)
reader.copy_page_table(
plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32)
)
# The target's sliding-window layers, written and read twice.
for _ in range(2):
self.target.write_ids(verify, _SWA)
self.target.read_table(plan, kind=_SWA)
self.target.copy_page_table(
plan, out=torch.zeros((bs + 2, 4), dtype=torch.int32), kind=_SWA
)
self.assertEqual(len(writes), 1)
self.assertEqual(len(swa_writes), 1)
# The full and the sliding-window id space, one build each.
self.assertEqual(len(builds), 2)
self.assertTrue(torch.equal(verify.out_cache_loc, plan.write_physical))
self.assertIs(extend.out_cache_loc, verify.out_cache_loc)
def test_a_verify_reads_the_window_it_writes(self):
def own_plan(mode, spec_info):
return self.target.own_plan(
SimpleNamespace(
forward_mode=mode,
spec_info=spec_info,
batch_size=int(self.rpi.numel()),
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=self.seq_lens.clone(),
out_cache_loc=self.window,
)
)
width = self.window.numel() // self.rpi.numel()
# A verify's rows read up to `draft_token_num` past their lengths, more
# than the window when the verify is ragged.
spec = SimpleNamespace(draft_token_num=width + 1)
self.assertEqual(
own_plan(ForwardMode.TARGET_VERIFY, spec).read_extent, width + 1
)
self.assertEqual(own_plan(ForwardMode.DECODE, spec).read_extent, width)
# A draft extend's lengths take its window after its batch is built.
self.assertEqual(own_plan(ForwardMode.DRAFT_EXTEND_V2, spec).read_extent, width)
# Their lengths already count what an extend or a plain decode writes.
self.assertEqual(own_plan(ForwardMode.EXTEND, spec).read_extent, 0)
self.assertEqual(own_plan(ForwardMode.DECODE, None).read_extent, 0)
def test_a_stream_only_iteration_builds_no_table(self):
"""A plan builds a table only when a table reader asks for it: stream
readers -- one runner, or a target and a fused draft over the same
rows -- each translate in their own gather. Once the table is built,
a stream packs the same ids from it."""
calls = {"packed": 0, "table": 0}
def counting(name, real):
def wrapped(**kwargs):
calls[name] += 1
return real(**kwargs)
return wrapped
# A runner of rows no other translator here reads.
self.req_to_token = self.req_to_token.clone()
solo = self._translator(self.allocator.get_kvcache())
bs = int(self.rpi.numel())
lens = torch.tensor([3, 2], dtype=torch.int64)
indptr = torch.tensor([0, 3, 5], dtype=torch.int32)
def pack(translator, plan, out, kind=_FULL):
return translator.pack_read_stream(
plan,
req_pool_indices=self.rpi,
seq_lens=lens,
indptr=indptr,
out=out,
kind=kind,
)
with (
patch.object(
kv_index_translator,
"build_kv_read_table_packed",
counting("packed", kv_index_translator.build_kv_read_table_packed),
),
patch.object(
kv_index_translator,
"build_kv_read_table",
counting("table", kv_index_translator.build_kv_read_table),
),
patch.object(
kv_index_translator,
"create_flashinfer_kv_indices_triton",
_HostStreamGather(),
),
):
plan = self._plan(solo, read_extent=1)
out = torch.full((6,), -7, dtype=torch.int32)
self.assertTrue(pack(solo, plan, out))
self.assertEqual(calls, {"packed": 1, "table": 0})
self.assertFalse(plan.has_read_table())
for b in range(bs):
row = self.req_to_token[int(self.rpi[b]), : int(lens[b])]
want = self.allocator.translate_write_loc(row.to(torch.int64))
got = out[int(indptr[b]) : int(indptr[b + 1])].to(torch.int64)
self.assertTrue(torch.equal(got, want))
self.assertEqual(int(out[5]), -7) # past the stream: untouched
# A caller holding a padded graph buffer of request rows: the
# lengths size the batch, the extra lanes are left out.
padded = torch.cat([self.rpi, torch.zeros(5, dtype=torch.int64)])
again = torch.full((6,), -7, dtype=torch.int32)
solo.pack_read_stream(
self._plan(solo, read_extent=1),
req_pool_indices=padded,
seq_lens=lens,
indptr=indptr,
out=again,
)
self.assertTrue(torch.equal(again, out))
self.assertEqual(calls, {"packed": 2, "table": 0})
# Once a table reader has the table built, a stream packs from it:
# nothing gathered again, the same ids.
plan.read_table()
after = torch.full((6,), -7, dtype=torch.int32)
pack(solo, plan, after)
self.assertEqual(calls, {"packed": 2, "table": 1})
self.assertTrue(torch.equal(after, out))
# Nothing asked for the sliding-window table: its stream comes
# alone.
pack(solo, plan, torch.empty(6, dtype=torch.int32), kind=_SWA)
self.assertEqual(calls, {"packed": 3, "table": 1})
self.assertFalse(plan.has_read_table(_SWA))
# Another stream reader of the same rows (a fused draft): still no
# table -- each translates its own gather.
peer = self._translator(self.draft_pool)
for seq_lens_cpu in (self.seq_lens.clone(), None):
both = solo.plan(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=seq_lens_cpu,
write_virtual=self.window,
read_extent=1,
)
pack(solo, both, torch.empty(6, dtype=torch.int32))
pack(peer, both, torch.empty(6, dtype=torch.int32))
# The draft's row gather gets the virtual rows and the page
# table, translated in its own kernel.
src = peer.read_source(both, req_pool_indices=self.rpi, bs=bs)
self.assertTrue(src.is_translated)
self.assertIs(src.ids, self.req_to_token)
self.assertIs(src.v2p, solo.space(_FULL).read_v2p)
self.assertFalse(both.has_read_table())
self.assertEqual(calls, {"packed": 7, "table": 1})
def test_a_captured_first_reader_holds_the_table(self):
"""When the plan's first reader is a captured table, the table is built
in that buffer -- no table of the plan's own, no copy -- and serves the
plan's other readers; the same buffer again is left alone, and another
captured reader copies it rather than building again; an eager reader
of the batch takes the batch's rows of it."""
builds = []
real = kv_index_translator.build_kv_read_table
def counting(**kwargs):
builds.append(kwargs["out"])
return real(**kwargs)
bs = int(self.rpi.numel())
captured = torch.full((bs + 1, 8), 7, dtype=torch.int32)
with patch.object(kv_index_translator, "build_kv_read_table", counting):
plan = self._plan(read_extent=1)
self.target.copy_page_table(plan, out=captured)
self.assertEqual(len(builds), 1)
self.assertEqual(builds[0].data_ptr(), captured.data_ptr())
table = plan.read_table(rows=bs + 1)
self.assertEqual(table.ids.data_ptr(), captured.data_ptr())
self.assertEqual(table.row_stride, captured.stride(0))
# An eager reader of the batch (fa3, trtllm_mha) gets the batch's
# rows of that table: its kernels size the batch by them.
eager = self.target.read_table(plan, rows=bs)
self.assertEqual(eager.ids.shape[0], bs)
self.assertEqual(eager.row_ids.numel(), bs)
self.assertEqual(eager.ids.data_ptr(), captured.data_ptr())
width = table.ids.shape[1]
reference = _reference(
self.req_to_token,
self.rpi,
self.seq_lens + 1,
self.allocator.full_v2p_page_table,
width,
)
self.assertTrue(torch.equal(captured[:bs, :width], reference))
self.assertEqual(int(captured[bs, :width].abs().sum()), 0) # padded lane
self.assertTrue(bool((captured[:, width:] == 7).all())) # past the width
self.target.copy_page_table(plan, out=captured)
other = torch.full_like(captured, 5)
self.target.copy_page_table(plan, out=other)
self.assertEqual(len(builds), 1)
self.assertTrue(torch.equal(other[:, :width], captured[:, :width]))
def test_a_captured_table_bounds_a_build_without_host_lengths(self):
"""Without host lengths, a captured first reader narrower than a row
has the table built in its own buffer, to its width."""
builds = []
real = kv_index_translator.build_kv_read_table
def counting(**kwargs):
builds.append(kwargs["out"])
return real(**kwargs)
bs = int(self.rpi.numel())
row_pages = -(-self.req_to_token.shape[1] // _PS)
captured = torch.full((bs + 1, row_pages // 2), 7, dtype=torch.int32)
with patch.object(kv_index_translator, "build_kv_read_table", counting):
plan = self.target.plan(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=None,
write_virtual=self.window,
read_extent=1,
)
self.target.copy_page_table(plan, out=captured)
self.assertEqual(len(builds), 1)
self.assertEqual(builds[0].data_ptr(), captured.data_ptr())
self.assertEqual(builds[0].shape[1], captured.shape[1])
reference = _reference(
self.req_to_token,
self.rpi,
self.seq_lens + 1,
self.allocator.full_v2p_page_table,
captured.shape[1],
)
self.assertTrue(torch.equal(captured[:bs], reference))
self.assertEqual(int(captured[bs].abs().sum()), 0) # padded lane
def test_read_table_is_built_once_and_padded_by_copy(self):
builds = []
real = kv_index_translator.build_kv_read_table
def counting(**kwargs):
builds.append(kwargs["v2p"])
return real(**kwargs)
with patch.object(kv_index_translator, "build_kv_read_table", counting):
plan = self._plan(read_extent=1)
table = plan.read_table()
self.assertIs(plan.read_table(), table)
# Only the space that was read.
self.assertEqual(len(builds), 1)
swa = plan.read_table(kind=_SWA)
self.assertEqual(len(builds), 2)
padded = plan.read_table(rows=4)
padded_swa = plan.read_table(kind=_SWA, rows=4)
self.assertEqual(len(builds), 2) # copied, not built again
self.assertTrue(torch.equal(padded.ids[:2], table.ids))
self.assertEqual(int(padded.ids[2:].abs().sum()), 0) # the sink
self.assertTrue(torch.equal(padded_swa.ids[:2], swa.ids))
self.assertEqual(int(padded_swa.ids[2:].abs().sum()), 0)
# The rows cover `seq_lens + read_extent`.
reference = _reference(
self.req_to_token,
self.rpi,
self.seq_lens + 1,
self.allocator.full_v2p_page_table,
table.ids.shape[1],
)
self.assertTrue(torch.equal(table.ids, reference))
def test_only_readers_of_its_rows_read_its_table(self):
plan = self._plan()
self.assertTrue(plan.is_read_by(self.target))
self.assertTrue(plan.is_read_by(self.fused_draft))
self.assertFalse(plan.is_read_by(self.private_draft))
compact = KVIndexTranslator(
req_to_token=self.req_to_token.clone(),
token_to_kv_pool_allocator=self.allocator,
token_to_kv_pool=self.draft_pool,
page_size=_PS,
device=_DEV,
)
self.assertFalse(plan.is_read_by(compact))
# Before a table reader asks, the fused draft's row gather translates
# in its own kernel; after, it reads the plan's table.
src = self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2)
self.assertIs(src.v2p, self.target.space(_FULL).read_v2p)
self.assertFalse(plan.has_read_table())
table = plan.read_table(rows=2)
self.assertIs(
self.fused_draft.read_source(plan, req_pool_indices=self.rpi, bs=2), table
)
passthrough = self.private_draft.read_source(
plan, req_pool_indices=self.rpi, bs=2
)
self.assertFalse(passthrough.is_translated)
self.assertIs(passthrough.ids, self.req_to_token)
def test_a_compact_draft_reads_its_own_rows_over_the_shared_writes(self):
plan = self._plan()
compact_rows = self.req_to_token.clone()
compact = KVIndexTranslator(
req_to_token=compact_rows,
token_to_kv_pool_allocator=self.allocator,
token_to_kv_pool=self.draft_pool,
page_size=_PS,
device=_DEV,
)
lens = torch.tensor([2, 1], dtype=torch.int64)
derived = plan.reads_from(
compact, seq_lens=lens, seq_lens_cpu=lens.clone(), read_extent=1
)
# The writes are the plan's, not translated again.
self.assertIs(derived.write_physical, plan.write_physical)
self.assertIs(derived.write_ids(compact), plan.write_physical)
self.assertTrue(derived.is_read_by(compact))
compact_rows[self.rpi, :2] = compact_rows[self.rpi, 2:4]
reference = _reference(
compact_rows,
self.rpi,
lens + 1,
self.allocator.full_v2p_page_table,
derived.read_table().ids.shape[1],
)
self.assertTrue(torch.equal(derived.read_table().ids, reference))
def test_bind_sets_the_write_fields(self):
"""`bind` gives a batch the columns it writes, the same ones in both
spaces, with the virtual mirror only when its ids were translated."""
plan = self._plan()
bs = int(self.rpi.numel())
first = slice(0, 1)
fused = SimpleNamespace()
plan.bind(fused, self.fused_draft, cols=first)
self.assertIs(fused.kv_loc_plan, plan)
self.assertEqual(fused.kv_loc_cols, first)
self.assertTrue(
torch.equal(
fused.out_cache_loc,
plan.write_physical.view(bs, -1)[:, first].reshape(-1),
)
)
self.assertTrue(
torch.equal(
fused.out_cache_loc_virtual,
self.window.view(bs, -1)[:, first].reshape(-1),
)
)
self.assertTrue(fused.out_cache_loc_is_physical)
private = SimpleNamespace()
plan.bind(private, self.private_draft)
self.assertIs(private.out_cache_loc, self.window)
self.assertIsNone(private.out_cache_loc_virtual)
def test_an_npu_dcp_target_writes_rank_local_slots(self):
"""On NPU with DCP a target writes its rank's slots, while the
scheduler, `req_to_token` and the replicated draft pool keep
allocator-global ones: a plan's window stays global whichever runner
built it (a target prefill handing its plan to the draft, a draft
handing its window to the verify), the target binds it rank-local,
and the draft binds it global."""
parallel = SimpleNamespace(
dcp_enabled=True, dcp_size=2, dcp_rank=1, attn_dcp_size=1
)
with (
patch.object(kv_index_translator, "_is_npu", True),
patch.object(kv_index_translator, "get_parallel", return_value=parallel),
):
target, draft = (
KVIndexTranslator(
req_to_token=self.req_to_token,
token_to_kv_pool_allocator=self.allocator,
token_to_kv_pool=_FakeKVCache(64),
page_size=4,
device=_DEV,
is_draft_worker=is_draft_worker,
)
for is_draft_worker in (False, True)
)
global_ids = torch.arange(8, 16)
# Rank 1 of 2 owns every other 4-slot page, compacted.
local_ids = torch.tensor([-1, -1, -1, -1, 4, 5, 6, 7])
for source in (target, draft):
plan = source.plan(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=self.seq_lens.clone(),
write_virtual=global_ids,
)
target_batch, draft_batch = SimpleNamespace(), SimpleNamespace()
plan.bind(target_batch, target)
plan.bind(draft_batch, draft)
self.assertTrue(torch.equal(target_batch.out_cache_loc, local_ids))
self.assertTrue(torch.equal(draft_batch.out_cache_loc, global_ids))
self.assertIs(plan.write_virtual, global_ids)
def test_runner_slots_are_used_as_they_are(self):
for translator, writes_swa in (
(self.target, True),
(self.private_draft, False),
):
slots = torch.zeros(2, dtype=torch.int64)
batch = SimpleNamespace(
req_pool_indices=self.rpi,
seq_lens=self.seq_lens,
seq_lens_cpu=self.seq_lens.clone(),
out_cache_loc=slots,
)
translator.bind_runner_slots(batch)
# Kept by address: a captured graph writes this buffer on replay.
self.assertIs(batch.out_cache_loc, slots)
self.assertIsNone(batch.out_cache_loc_virtual)
self.assertTrue(batch.out_cache_loc_is_physical)
swa = translator.write_ids(batch, _SWA)
if writes_swa:
# The sink in the sliding-window sub-pool too.
self.assertTrue(torch.equal(swa, torch.zeros_like(slots)))
else:
self.assertIsNone(swa)
def test_a_replay_batch_writes_the_padded_slots(self):
plan = self._plan()
n = plan.write_physical.numel()
slots = torch.zeros(n + 2, dtype=torch.int64)
slots[:n].copy_(plan.write_physical)
batch = SimpleNamespace()
plan.bind_replay(batch, self.target, slots=slots)
self.assertIs(batch.kv_loc_plan, plan)
self.assertIs(batch.out_cache_loc, slots)
self.assertIsNone(batch.out_cache_loc_virtual)
swa = self.target.write_ids(batch, _SWA)
self.assertEqual(swa.shape[0], n + 2)
self.assertTrue(torch.equal(swa[:n], plan.write_ids(self.target, kind=_SWA)))
self.assertEqual(int(swa[n:].abs().sum()), 0)
if __name__ == "__main__":
unittest.main()
@@ -15,7 +15,7 @@
Under the token-major views a virtual id is in range and, by value, the same
kind of integer as a physical one, so a unified pool cannot tell a skipped
rebind from a translated loc by looking at it. `rebind_write_loc` marks the
bind from a translated loc by looking at it. A plan's `bind` marks the
batch's loc physical, producers carry the mark in `KVWriteLoc`, composites
forward it, and the unified write doors refuse a loc without it.
@@ -31,6 +31,7 @@ from types import SimpleNamespace
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
from sglang.srt.mem_cache.memory_pool import (
KVWriteLoc,
MHATokenToKVPool,
@@ -66,7 +67,11 @@ def _plain_translator():
def _translating_translator(v2p):
src = _plain_translator()
src.is_translating = True
src._translate_write_full = lambda t, out=None: v2p[t.to(torch.int64)]
src._spaces = {
IdSpaceKind.FULL: IdSpace(
key=(IdSpaceKind.FULL, "test"), write=lambda t: v2p[t.to(torch.int64)]
)
}
return src
@@ -111,7 +116,7 @@ class TestRebindMarksTheBatch(unittest.TestCase):
def test_non_translating_pool_marks_physical_without_rebinding(self):
loc = torch.tensor([3, 5], dtype=torch.int64)
fb = _batch(loc)
_plain_translator().rebind_write_loc(fb)
_plain_translator().bind_own_plan(fb)
self.assertIs(fb.out_cache_loc, loc) # physical by allocation: untouched
self.assertTrue(fb.out_cache_loc_is_physical)
@@ -119,13 +124,13 @@ class TestRebindMarksTheBatch(unittest.TestCase):
v2p = torch.tensor([7, 6, 5, 4], dtype=torch.int64)
loc = torch.tensor([1, 2], dtype=torch.int64)
fb = _batch(loc)
_translating_translator(v2p).rebind_write_loc(fb)
_translating_translator(v2p).bind_own_plan(fb)
self.assertTrue(torch.equal(fb.out_cache_loc, v2p[loc]))
self.assertTrue(fb.out_cache_loc_is_physical)
def test_no_loc_stays_unmarked(self):
fb = _batch(None)
_translating_translator(torch.arange(4)).rebind_write_loc(fb)
_translating_translator(torch.arange(4)).bind_own_plan(fb)
self.assertFalse(fb.out_cache_loc_is_physical)
@@ -30,6 +30,7 @@ from types import SimpleNamespace
import torch
from sglang.srt.environ import envs
from sglang.srt.mem_cache.kv_loc_plan import IdSpaceKind
from sglang.srt.mem_cache.layout.paged_view import paged_view
from sglang.srt.mem_cache.layout.token_major import ENTRY_ALIGN_BYTES, build_dense_views
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
@@ -550,10 +551,10 @@ class TestFactoryViews(unittest.TestCase):
self.assertEqual(pool.k_buffer[0].dtype, torch.uint8)
def test_rebind_emits_physical_full_and_build_derives_swa(self):
"""End-to-end over the real factory: rebind_write_loc rebinds
out_cache_loc to FULL-side physical ids (phase 1), and the per-batch
build derives the SWA write loc pointwise from those values (phase 2)
-- both checked against the v2p tables over the VIRTUAL ids."""
"""End-to-end over the real factory: a batch's own plan binds
out_cache_loc to FULL-side physical ids and derives the SWA write ids
from the virtual window -- both checked against the v2p tables over
the VIRTUAL ids."""
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.runtime_context import get_parallel
@@ -584,11 +585,11 @@ class TestFactoryViews(unittest.TestCase):
device="cpu",
)
self.assertTrue(source.is_translating)
source.rebind_write_loc(fb)
source.bind_own_plan(fb)
self.assertTrue(torch.equal(fb.out_cache_loc, expected_full))
self.assertTrue(
torch.equal(
source.sliding_window_write_loc_for(fb.out_cache_loc), expected_swa
source.write_ids(fb, IdSpaceKind.SLIDING_WINDOW), expected_swa
)
)
@@ -21,16 +21,15 @@ page ids:
kernel_page(virtual_page) = v2p[virtual_page]
Since the read-path translator, ONE builder computes that formula for every
family — `build_index_table` (the canonical) — and the backends only
differ in how they consume it:
- trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: rows filled straight
into their padded block tables (`KVIndexTranslator.fill_read_table`, prefix-only so
the backends' own -1 / stale tail sentinels survive);
- the flashinfer updaters: token ids reconstructed from the canonical by
ONE builder computes that formula for every family -- the iteration plan's
read table (`KVLocPlan.read_table`) -- and the backends only differ in how
they consume it:
- trtllm_mla / cutedsl_mla / tokenspeed_mla / flashmla: the plan's rows
copied into their padded block tables (`KVIndexTranslator.copy_page_table`);
- the flashinfer updaters: token ids reconstructed from the table by
`create_flashinfer_kv_indices_triton[ENTRY_PAGE_SIZE=ps]`;
- fa3's captured decode: `normal_decode_set_metadata` copies the canonical
rows' live prefixes (src_is_read_table=True).
- fa3's captured decode: the plan's rows copied into its captured page
table.
Covered here:
- the static `create_flashmla_kv_indices_triton` (no id-space knowledge left)
@@ -216,6 +215,44 @@ class TestBlockTable(unittest.TestCase):
torch.equal(gpu, cpu), f"ps={page_size}:\ngpu={gpu}\ncpu={cpu}"
)
def test_zero_tail_fills_a_fresh_table(self):
"""With ``zero_tail`` a fresh (garbage) table comes back whole in one
launch: each row's live prefix, the sink past it up to ``max_pages``,
and nothing past ``max_pages``. The CUDA kernel must agree with the
CPU path."""
from sglang.kernels.ops.kvcache.kv_read_table import build_kv_read_table
for page_size in (1, 32):
rt, rpi, sl, v2p = self._make_batch(page_size)
bs = rpi.shape[0]
max_pages = int((sl.max().item() + page_size - 1) // page_size) + 2
def fill(device):
# One guard column past the table catches a spill off the row.
out = torch.full(
(bs, max_pages + 1), 7, dtype=torch.int32, device=device
)
build_kv_read_table(
req_to_token=rt.to(device),
req_pool_indices=rpi.to(device),
seq_lens=sl.to(device=device, dtype=torch.int64),
v2p=v2p.to(device),
page_size=page_size,
max_pages=max_pages,
out=out,
zero_tail=True,
)
return out.cpu()
gpu, cpu = fill(_DEV), fill("cpu")
self.assertTrue(torch.equal(gpu, cpu), f"ps={page_size}")
self.assertTrue(bool((gpu[:, max_pages] == 7).all()), "spilled")
want = _reference(rt, rpi, sl, page_size, v2p=v2p).cpu().to(torch.int32)
for b in range(bs):
live = int((int(sl[b]) + page_size - 1) // page_size)
self.assertTrue(torch.equal(gpu[b, :live], want[b, :live]))
self.assertTrue(bool((gpu[b, live:max_pages] == 0).all()))
def test_static_kernel_matches_reference(self):
"""The stripped (id-space-free) flashmla kernel is byte-identical to the
plain token//ps reference -- guards the v2p-arg removal itself."""
@@ -303,10 +303,8 @@ class TestDefaultConfigurator(CustomTestCase):
self.assertEqual(mock_calculate_mla_kv_cache_dim.call_count, 2)
def test_fused_draft_entry_overrides_the_layer_ratio(self):
"""Unified mamba-MHA fusion prices the EXACT fused entry (host +
draft + alignment pad): the per-draft-layer ratio under-reserves the pad,
so a solve over it hands out more tokens than the fused pages hold.
The control pins that path-only drafts keep the ratio arm."""
"""A fused draft prices the exact fused entry (host + draft + pad);
the per-draft-layer ratio under-reserves the pad."""
mr = _make_model_runner(self, speculative_algorithm="EAGLE")
mr.spec_algorithm.is_eagle.return_value = True
mr.spec_algorithm.is_none.return_value = False
@@ -320,12 +318,7 @@ class TestDefaultConfigurator(CustomTestCase):
)
fused = create_memory_pool_configurator(mr)
mr.fused_entry_bytes.side_effect = None
mr.fused_entry_bytes.return_value = None
unfused = create_memory_pool_configurator(mr)
self.assertEqual(fused._cell_size, 77777)
base = _full_per_token(mr) * 32
self.assertEqual(unfused._cell_size, int(base * (1 + 1 / 32)))
class TestHybridSWAConfigurator(CustomTestCase):
@@ -1459,13 +1452,9 @@ class TestSWAPoolFloor(CustomTestCase):
class TestFusedDraftPricing(unittest.TestCase):
"""Fused draft KV: the boot solve must price the EXACT fused entry.
The fused full-side entry (host + draft + alignment pad) replaces BOTH the
per-token full term and the per-draft-layer approximation; charging both
would over-reserve, charging only the approximation would under-reserve
and OOM at pool construction (the factory allocates tokens x fused entry).
"""
"""A fused full-side entry (host + draft + pad) replaces both the
per-token full term and the per-draft-layer approximation in every solve:
charging both over-reserves, charging only the approximation under-reserves."""
def _make(self, fused_entry, draft_layers, **runner_kwargs):
mr = _make_model_runner(
@@ -1500,14 +1489,6 @@ class TestFusedDraftPricing(unittest.TestCase):
+ cfg._draft_cell_size
)
def test_unfused_keeps_the_draft_layer_approximation(self):
_, cfg = self._make(fused_entry=None, draft_layers=2)
self.assertEqual(cfg._draft_full_layers_num, 2)
self.assertEqual(
cfg._cell_size,
self._expected_cell(cfg, cfg._full_per_token * (4 + 2)),
)
def test_fused_entry_replaces_full_and_draft_terms(self):
fused_entry = 54_321
_, cfg = self._make(fused_entry=fused_entry, draft_layers=2)
@@ -1515,9 +1496,8 @@ class TestFusedDraftPricing(unittest.TestCase):
self.assertEqual(cfg._cell_size, self._expected_cell(cfg, fused_entry))
def _assert_unified_bytes_price_the_fused_entry(self, cfg, config, available):
"""The bytes handed to the unified pool must buy every full token at
the fused entry -- the factory divides them by it -- and must not leave
a full token's worth of the budget unspent."""
"""The unified pool's bytes buy every full token at the fused entry
(the factory divides by it) without leaving a full token unspent."""
fused_entry = cfg._fused_full_entry
swa_bytes = (
config.swa_max_total_num_tokens * cfg._swa_per_token * cfg._swa_layers_num
@@ -9,6 +9,7 @@ import torch
import sglang.srt.model_executor.model_runner_components.cuda_graph_setup as graph_setup
import sglang.srt.model_executor.runner.prefill_cuda_graph_runner as runner_module
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -31,6 +32,17 @@ from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=11, suite="base-a-test-cpu")
def _static_pool_translator():
"""A static pool's translator: its plans hand every batch its own ids."""
return KVIndexTranslator(
req_to_token=torch.zeros((1, 8), dtype=torch.int32),
token_to_kv_pool_allocator=None,
token_to_kv_pool=object(),
page_size=1,
device="cpu",
)
class _FakeAttentionBackend:
supports_full_cuda_graph_chunked_prefix = True
@@ -168,6 +180,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
prepare_dummy_forward_batch=lambda batch: prepared.append(batch) or batch,
attn_tp_sequence_sharded=lambda _: False,
attn_backend=attention_backend,
kv_index_translator=_static_pool_translator(),
)
runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner)
runner.model_runner = model_runner
@@ -321,9 +334,11 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
runner.max_context_size = None
runner._capture_chunked_prefix = False
runner.buffer_registry = _FakeBatchRegistry()
translator = _static_pool_translator()
runner.model_runner = SimpleNamespace(
attn_tp_sequence_sharded=lambda _: False,
prepare_dummy_forward_batch=lambda batch: batch,
kv_index_translator=translator,
)
runner.enable_cp_bcg_capture = False
runner._is_full_backend = False
@@ -357,6 +372,7 @@ class TestPrefillCudaGraphRunnerChunkedPrefix(CustomTestCase):
capture_hidden_mode=CaptureHiddenMode.NULL,
global_forward_mode=ForwardMode.EXTEND,
)
translator.bind_own_plan(forward_batch)
static_batch = runner.load_batch(forward_batch)
@@ -1,191 +0,0 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`resolve_spec_aux_hidden_state_config` runs before the TP group exists.
BUG REGRESSION. `ModelRunner.init_spec_aux_hidden_state()` runs BEFORE
`init_torch_distributed()`, so the attention-TP group does not exist while
this resolver runs. A `get_parallel().attn_tp_size` read inside it therefore
raises `AssertionError: attention tensor model parallel group is not
initialized` and kills the server at startup -- for EVERY EAGLE-family run
with a draft path, unified pool or not. It is invisible to CPU unit gates
(they never boot a runner) and was caught only by a GPU boot guard.
The resolver therefore records the draft's `ModelConfig` whole, and the
per-GPU geometry is derived at pool-build time (`draft_kv_profile`), where
the group exists -- the same point the target divides its own heads.
python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_boot_order.py -v
"""
import ast
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
_SRC = Path(_mod.__file__)
def _function_named(tree, name):
for node in ast.walk(tree):
if isinstance(node, ast.FunctionDef) and node.name == name:
return node
raise AssertionError(f"{name} not found in {_SRC}")
def _config(*, num_nextn_predict_layers, num_hidden_layers=1):
return SimpleNamespace(
num_nextn_predict_layers=num_nextn_predict_layers,
num_hidden_layers=num_hidden_layers,
num_attention_layers=num_hidden_layers,
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
)
def _spec_algorithm(*, eagle: bool, eagle3: bool = False):
return SimpleNamespace(
is_eagle=lambda: eagle,
is_standalone=lambda: False,
is_eagle3=lambda: eagle3,
)
class TestResolverTouchesNoParallelState(unittest.TestCase):
def test_no_parallel_state_in_the_eagle_resolver(self):
"""AST pin: the EAGLE resolver runs pre-distributed-init, so any
parallel-state read in it -- `get_parallel()` directly, or a per-GPU
head count that divides by it -- is a boot crash, not a wrong number."""
tree = ast.parse(_SRC.read_text())
fn = _function_named(tree, "_resolve_eagle_aux_hidden_state")
called = {
n.func.id
for n in ast.walk(fn)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
self.assertNotIn(
"get_parallel",
called,
"_resolve_eagle_aux_hidden_state reads parallel state, but it runs "
"before init_torch_distributed() -- the attention-TP group does not "
"exist yet and the server dies at startup.",
)
methods = {
n.func.attr
for n in ast.walk(fn)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
}
self.assertNotIn("get_num_kv_heads", methods)
imported = {
alias.name
for n in ast.walk(fn)
if isinstance(n, ast.ImportFrom)
for alias in n.names
}
self.assertNotIn("get_parallel", imported)
def _resolve_with(self, *, draft_path, target, draft):
"""Run the EAGLE resolver with the config factory stubbed: `draft`
answers the draft-mode read, `target` is the runner's own config."""
config = _mod.SpecAuxHiddenStateConfig()
spec = SimpleNamespace(
speculative_draft_model_path=draft_path,
speculative_draft_model_revision=None,
)
with (
patch.object(_mod, "get_spec", return_value=spec),
patch.object(
_mod.ModelConfig, "from_server_args", return_value=draft
) as factory,
):
_mod._resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=target,
spec_algorithm=_spec_algorithm(eagle=True),
is_draft_worker=False,
)
self.assertTrue(factory.call_args.kwargs["is_draft_model"])
return config
def test_path_less_nextn_fuses_without_resizing_the_private_pool(self):
"""Every `num_nextn_predict_layers = 1` assignment in model_config.py
is guarded by `if is_draft_model`, so a path-less MTP target answers
None for it. The fused placement must read the draft-mode config, or a
path-less NEXTN head never fuses; the private-pool budget must keep
reading the target, or every path-less MTP deployment's token budget
changes."""
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=1)
config = self._resolve_with(draft_path=None, target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 1)
self.assertIsNone(config.eagle_draft_num_layers)
def test_draft_path_sizes_both_from_the_draft_checkpoint(self):
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2)
config = self._resolve_with(draft_path="/draft", target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 2)
self.assertEqual(config.eagle_draft_num_layers, 2)
def test_the_draft_config_is_recorded_whole(self):
"""The aux config carries the draft `ModelConfig`, never a pre-divided
geometry: a per-GPU count recorded here would be computed without the
TP group (every unit fixture runs at tp=1, where that is invisible)."""
fields = _mod.SpecAuxHiddenStateConfig.__struct_fields__
self.assertIn("draft_model_config", fields)
self.assertNotIn("eagle_draft_total_kv_heads", fields)
self.assertNotIn("eagle_draft_kv_head_num", fields)
class TestResolverIsInertOffTheTargetEagleArm(unittest.TestCase):
"""BUG REGRESSION. The draft-config block is guarded by "EAGLE-family
target"; a rebase left the hybrid-SWA read outside that guard, so every
draft worker and every spec-off runner raised `UnboundLocalError` at
boot, and the EAGLE3 arm's try/except masked the same read into a silent
`None`. Off the target EAGLE arm the resolver must return untouched."""
def _resolve(self, *, spec_algorithm, is_draft_worker):
config = _mod.SpecAuxHiddenStateConfig()
_mod._resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=None,
spec_algorithm=spec_algorithm,
is_draft_worker=is_draft_worker,
)
return config
def test_draft_workers_and_spec_off_runners_resolve_nothing(self):
untouched = _mod.SpecAuxHiddenStateConfig()
for spec_algorithm, is_draft_worker in (
(_spec_algorithm(eagle=True), True),
(_spec_algorithm(eagle=True, eagle3=True), True),
(_spec_algorithm(eagle=False), False),
):
config = self._resolve(
spec_algorithm=spec_algorithm, is_draft_worker=is_draft_worker
)
self.assertEqual(config, untouched)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,90 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Which config the EAGLE aux-hidden-state resolver sizes the draft KV from.
python -m pytest test/registered/unit/model_executor/test_spec_aux_hidden_state_resolver.py -v
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state as _mod
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=3, stage="weekly", runner_config="cpu")
def _config(*, num_nextn_predict_layers, num_hidden_layers=1):
return SimpleNamespace(
num_nextn_predict_layers=num_nextn_predict_layers,
num_hidden_layers=num_hidden_layers,
num_attention_layers=num_hidden_layers,
is_hybrid_swa=False,
is_deepseek_v4_arch=False,
)
class TestEagleResolverDraftConfig(unittest.TestCase):
def _resolve_with(self, *, draft_path, target, draft):
"""Run the EAGLE resolver with the config factory stubbed: `draft`
answers the draft-mode read, `target` is the runner's own config."""
config = _mod.SpecAuxHiddenStateConfig()
spec = SimpleNamespace(
speculative_draft_model_path=draft_path,
speculative_draft_model_revision=None,
)
with (
patch.object(_mod, "get_spec", return_value=spec),
patch.object(
_mod.ModelConfig, "from_server_args", return_value=draft
) as factory,
):
_mod._resolve_eagle_aux_hidden_state(
config=config,
server_args=None,
model_config=target,
spec_algorithm=SimpleNamespace(
is_eagle=lambda: True,
is_standalone=lambda: False,
is_eagle3=lambda: False,
),
is_draft_worker=False,
)
self.assertTrue(factory.call_args.kwargs["is_draft_model"])
return config
def test_path_less_nextn_fuses_without_resizing_the_private_pool(self):
"""A path-less MTP head sizes its fused draft KV from the draft-mode
config (the only one that names its NEXTN layers), while the private
draft pool keeps reading the target config, so its token budget is
unchanged."""
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=1)
config = self._resolve_with(draft_path=None, target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 1)
self.assertIsNone(config.eagle_draft_num_layers)
def test_draft_path_sizes_both_from_the_draft_checkpoint(self):
target = _config(num_nextn_predict_layers=None)
draft = _config(num_nextn_predict_layers=None, num_hidden_layers=2)
config = self._resolve_with(draft_path="/draft", target=target, draft=draft)
self.assertIs(config.draft_model_config, draft)
self.assertEqual(config.draft_kv_num_layers, 2)
self.assertEqual(config.eagle_draft_num_layers, 2)
if __name__ == "__main__":
unittest.main()
@@ -13,11 +13,12 @@
# ==============================================================================
"""ForwardBatch construction wires the unified write-loc rebind.
`init_new` must call `kv_index_translator.rebind_write_loc`: a construction
path that skips it ships VIRTUAL write ids to the kernels, a silent
wrong-slot store. Also runs the REAL `_pad_inputs_to_size` against a live
translator, since pad lanes are zeros and zeros must derive to the slot-0
sink. Sliding-window semantics are pinned in test_kv_index_translator.py.
`init_new` must bind the batch to a plan (`translator.own_plan`,
`kv_loc_plan.bind`): a construction path that skips it ships VIRTUAL write
ids to the kernels, a silent wrong-slot store. Also runs the REAL
`_pad_inputs_to_size` against a live translator, since pad lanes are zeros
and must write the slot-0 sink in every id space. Sliding-window semantics
are pinned in test_kv_index_translator.py.
python -m pytest test/registered/unit/model_executor/test_unified_out_cache_loc_rebind.py -v
"""
@@ -32,6 +33,7 @@ from unittest.mock import create_autospec
import torch
from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator
from sglang.srt.mem_cache.kv_loc_plan import IdSpace, IdSpaceKind
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
@@ -70,22 +72,25 @@ def _armed_source(v2p, swa_map):
)
src.is_translating = True
src._translate_full = lambda t, out=None: v2p[t.to(torch.int64)]
# The WRITE loc has its own translate because under DCP it arrives widened;
# at dcp_size == 1 it is the read translate, so arm it with the same fake.
src._translate_write_full = src._translate_full
# Phase 2 derives from physical values through p2v + the swa v2p; arm the
# inverse of the fake v2p (ps=1, so the expected swa loc for virtual t is
# swa_map[t]).
p2v = torch.zeros(int(v2p.max()) + 1, dtype=torch.int64)
p2v[v2p] = torch.arange(v2p.numel(), dtype=torch.int64)
src._full_p2v_table = p2v
src._swa_v2p_table = swa_map
src._spaces = {
# The WRITE loc has its own translate because under DCP it arrives
# widened; at dcp_size == 1 it is the read translate.
IdSpaceKind.FULL: IdSpace(
key=(IdSpaceKind.FULL, "test"), write=src._translate_full
),
# The sliding-window ids derive from the virtual window (ps=1, so the
# swa id for virtual t is swa_map[t]).
IdSpaceKind.SLIDING_WINDOW: IdSpace(
key=(IdSpaceKind.SLIDING_WINDOW, "test"),
write=lambda t: swa_map[t.to(torch.int64)],
),
}
return src
def _call_names(func) -> list:
"""Dotted call targets appearing in `func`'s body, e.g.
'model_runner.kv_index_translator.rebind_write_loc'."""
'kv_loc_plan.bind'."""
tree = ast.parse(textwrap.dedent(inspect.getsource(func)))
names = []
for node in ast.walk(tree):
@@ -104,13 +109,15 @@ def _call_names(func) -> list:
class TestForwardBatchWiring(CustomTestCase):
"""Critical-path bookkeeping: the construction-time call sites."""
def test_init_new_calls_the_rebind(self):
self.assertIn(
"model_runner.kv_index_translator.rebind_write_loc",
_call_names(ForwardBatch.init_new.__func__),
"init_new must rebind the write loc through the source; a batch "
"built without it ships virtual ids to the kernels",
)
def test_init_new_binds_the_plan(self):
names = _call_names(ForwardBatch.init_new.__func__)
for call in ("translator.own_plan", "kv_loc_plan.bind"):
self.assertIn(
call,
names,
"init_new must take its write ids from a plan; a batch built "
"without one ships virtual ids to the kernels",
)
class TestPadComposesWithDerivation(CustomTestCase):
@@ -120,11 +127,11 @@ class TestPadComposesWithDerivation(CustomTestCase):
kv_index_translator=src,
)
def test_pad_lanes_derive_to_sink_and_slices_stay_pointwise(self):
"""The REAL `_pad_inputs_to_size` composes with phase 2: pad lanes are
zeros, zeros derive to the slot-0 sink, and any slice of the padded
tensor (the TBO-child shape) derives pointwise -- no handover call
exists for the pad to make."""
def test_pad_lanes_write_the_sink_in_both_spaces(self):
"""The REAL `_pad_inputs_to_size` pads the batch's write ids, and the
sliding-window ids its plan hands out follow them: pad lanes write the
slot-0 sink in both spaces, and both stay the same length as the
batch's tokens."""
n, padded = 3, 6
v2p = torch.arange(64, dtype=torch.int64) * 3
swa_map = torch.arange(64, dtype=torch.int64) * 5
@@ -133,36 +140,37 @@ class TestPadComposesWithDerivation(CustomTestCase):
fb = _make_fb(virt.clone())
fb.positions = torch.arange(n, dtype=torch.int64)
fb.lora_ids = [None] * fb.batch_size
src.rebind_write_loc(fb)
src.bind_own_plan(fb)
swa = IdSpaceKind.SLIDING_WINDOW
self.assertTrue(torch.equal(fb.out_cache_loc, v2p[virt]))
self.assertTrue(torch.equal(src.write_ids(fb, swa), swa_map[virt]))
fb._pad_inputs_to_size(self._fake_runner_for_pad(src), padded, fb.batch_size)
self.assertEqual(fb.out_cache_loc.shape[0], padded)
# Padded tail lanes go to slot 0 -- the reserved dummy-write sink.
self.assertTrue(bool((fb.out_cache_loc[n:] == 0).all()))
loc = src._swa_write_loc_unified(fb.out_cache_loc)
self.assertTrue(torch.equal(loc[:n], swa_map[virt]))
self.assertTrue(bool((loc[n:] == 0).all()))
self.assertEqual(loc.dtype, torch.int64)
# The TBO-child shape: a slice of the PADDED tensor derives pointwise.
sub = src._swa_write_loc_unified(fb.out_cache_loc[1:5])
self.assertTrue(torch.equal(sub, loc[1:5]))
for loc, want in (
(fb.out_cache_loc, v2p[virt]),
(src.write_ids(fb, swa), swa_map[virt]),
):
self.assertEqual(loc.shape[0], padded)
self.assertTrue(torch.equal(loc[:n], want))
self.assertTrue(bool((loc[n:] == 0).all()))
self.assertEqual(loc.dtype, torch.int64)
def test_empty_loc_rebinds_to_empty(self):
src = _armed_source(
torch.arange(8, dtype=torch.int64), torch.arange(8, dtype=torch.int64)
)
fb = _make_fb(torch.empty(0, dtype=torch.int64))
src.rebind_write_loc(fb)
src.bind_own_plan(fb)
self.assertEqual(fb.out_cache_loc.numel(), 0)
self.assertEqual(src._swa_write_loc_unified(fb.out_cache_loc).numel(), 0)
self.assertEqual(src.write_ids(fb, IdSpaceKind.SLIDING_WINDOW).numel(), 0)
class TestReadRailTranslatesAtProduction(CustomTestCase):
"""The model-door READ indices (req_to_token-derived, VIRTUAL under the
unified pool) are translated at their PRODUCTION site -- the cache then
holds the physical result and the pool door never translates."""
"""The model-door READ indices are physical from their PRODUCTION site
(a gather from the iteration's plan under the unified pool) -- the cache
then holds the physical result and the pool door never translates."""
def _fb_for_one_shot(self):
fb = _make_fb(torch.tensor([1, 2], dtype=torch.int64))
@@ -172,39 +180,84 @@ class TestReadRailTranslatesAtProduction(CustomTestCase):
fb.req_pool_indices = torch.tensor([0, 1], dtype=torch.int64)
return fb
def test_one_shot_indices_translated_once_and_cached(self):
def test_one_shot_indices_gathered_from_the_plan_once_and_cached(self):
from unittest.mock import patch
from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix
plans = []
def pack(plan, *, req_pool_indices, seq_lens, indptr, out, **_):
plans.append(plan)
out.fill_(7)
return True
fb = self._fb_for_one_shot()
fb.kv_loc_plan = object()
# autospec, not a bare namespace: setting a name the translator does
# not have raises, so renaming the method breaks this test loudly.
fake_translator = create_autospec(KVIndexTranslator, instance=True)
fake_translator.pack_read_stream = pack
fake_backend = SimpleNamespace(kv_index_translator=fake_translator)
with patch.object(mix, "get_attn_backend", return_value=fake_backend):
r1 = fb.fetch_mha_one_shot_kv_indices()
r2 = fb.fetch_mha_one_shot_kv_indices()
# One gather, from the iteration's plan; the cache holds its result.
self.assertEqual(plans, [fb.kv_loc_plan])
self.assertIs(r2, r1)
self.assertTrue(torch.equal(r1, torch.full((5,), 7, dtype=torch.int32)))
def _fb_for_chunks(self):
fb = self._fb_for_one_shot()
fb.kv_loc_plan = object()
fb.num_prefix_chunks = 2
fb.prefix_chunk_starts = [torch.tensor([0, 0]), torch.tensor([2, 1])]
fb.prefix_chunk_seq_lens = [torch.tensor([2, 1]), torch.tensor([1, 2])]
fb.prefix_chunk_cu_seq_lens = [
torch.tensor([0, 2, 3], dtype=torch.int32),
torch.tensor([0, 1, 3], dtype=torch.int32),
]
fb.prefix_chunk_num_tokens = [3, 3]
fb.prefix_chunk_starts_cpu = [[0, 0], [2, 1]]
fb.prefix_chunk_seq_lens_cpu = [[2, 1], [1, 2]]
return fb
def test_chunk_indices_gathered_from_the_plan(self):
"""On a translating pool each prefix chunk's ids come out of the plan's
table: one gather per chunk, at the chunk's start, no translate."""
from unittest.mock import patch
from sglang.srt.model_executor import forward_batch_deepseek_mha_mixin as mix
calls = []
sentinel = torch.arange(5, dtype=torch.int64) + 5000
def translate(t):
calls.append(t)
return sentinel
def pack(plan, *, req_pool_indices, seq_lens, indptr, out, kv_start_idx):
calls.append((plan, seq_lens, indptr, kv_start_idx))
out.fill_(9)
return True
fb = self._fb_for_one_shot()
fake_pool = SimpleNamespace(
req_to_token=torch.zeros((4, 16), dtype=torch.int32)
)
# autospec, not a bare namespace: setting a name the translator does
# not have raises, so renaming the method breaks this test loudly.
fb = self._fb_for_chunks()
fake_translator = create_autospec(KVIndexTranslator, instance=True)
fake_translator.translate_full_attn_ids = translate
fake_translator.reads_are_translated = True
fake_translator.pack_read_stream = pack
fake_backend = SimpleNamespace(kv_index_translator=fake_translator)
fake_pool = SimpleNamespace(req_to_token=torch.zeros((4, 8), dtype=torch.int32))
with (
patch.object(mix, "get_req_to_token_pool", return_value=fake_pool),
patch.object(mix, "get_attn_backend", return_value=fake_backend),
patch.object(mix, "create_flashinfer_kv_indices_triton"),
patch.object(mix, "get_req_to_token_pool", return_value=fake_pool),
):
r1 = fb.fetch_mha_one_shot_kv_indices()
r2 = fb.fetch_mha_one_shot_kv_indices()
fb.prepare_chunked_kv_indices(torch.device("cpu"))
self.assertIs(r1, sentinel) # production site translated
self.assertIs(r2, sentinel) # cache holds the TRANSLATED result
self.assertEqual(len(calls), 1) # translated exactly once
self.assertEqual(calls[0].dtype, torch.int32) # raw producer output
self.assertEqual(len(calls), 2)
for idx, (plan, seq_lens, indptr, kv_start_idx) in enumerate(calls):
self.assertIs(plan, fb.kv_loc_plan)
self.assertIs(seq_lens, fb.prefix_chunk_seq_lens[idx])
self.assertIs(indptr, fb.prefix_chunk_cu_seq_lens[idx])
self.assertIs(kv_start_idx, fb.prefix_chunk_starts[idx])
fake_translator.translate_dcp_read_ids.assert_not_called()
for chunk in fb.prefix_chunk_kv_indices:
self.assertTrue(bool((chunk == 9).all()))
def test_one_shot_indices_noop_on_unmigrated_backend(self):
from unittest.mock import patch
@@ -11,34 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""`--enable-unified-memory` speculative-decoding allow-list.
"""`--enable-unified-memory` speculative-decoding backend allow-lists.
Three audited arms (see `handle_unified_memory_pool`), each with its own
backend constraints:
* DSPARK: the target verifies on the audited verify set (`triton` /
`trtllm_mla` / `cutedsl_mla` / `tokenspeed_mla` / `flashmla` /
`flashinfer` / `fa3`, which includes the MLA family). Its draft fuses
into the target's pages when the fused-draft decision allows, else keeps
a private pool, so the gate leaves the draft backend unconstrained.
* DFLASH: the target verifies on `triton` / `fa3` / `flashinfer`; the MLA
verify family must not leak into this arm.
* EAGLE/EAGLE3: unified targets only (hybrid-SWA or mamba hybrids, either
full-pool kind) -- the draft's KV lives fused inside the full pool's
page envelope (`DenseDraftRegion`), and a draft that does not fuse is
refused at boot. The target's verify set follows the
host kind: the audited verify set on an MLA host, `triton` /
`flashinfer` / `fa3` on an MHA host, and an unresolved backend is
refused. The draft worker (its backend resolves separately: explicit
flag first, else it inherits the target's) runs on `triton` /
`flashinfer` / `fa3`.
Every arm drafts a linear chain (`--speculative-eagle-topk` in {None, 1}).
Everything else (NGRAM / STANDALONE / registered customs) stays refused.
NGRAM's refusal is load-bearing, not pending: like tree verify it relocates
accepted tokens one at a time inside the target pool, which the unified
pool's page-granular `move_kv_cache` cannot express. Pinned so no arm silently
widens to an unaudited algorithm, family, tree shape, or backend -- and so one
arm's addition never perturbs another.
A speculative backend that does not read through the KV-index translator
reads the unified pool with virtual ids and produces wrong tokens without
crashing, so each arm's admitted backend set is pinned exactly.
python -m pytest test/registered/unit/server_args/test_unified_memory_spec_gate.py -v
"""
@@ -55,14 +32,13 @@ from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
register_cpu_ci(est_time=4, suite="base-a-test-cpu")
class _PlainHFConfig:
"""Matches none of the hybrid-arch isinstance probes."""
"""Matches none of the hybrid-arch probes (`mambaish_config` screens on
the architecture name)."""
# `mambaish_config` screens on the architecture name, so a stub that
# declares none is what "not a hybrid arch" looks like to it.
architectures = ["LlamaForCausalLM"]
def get_text_config(self):
@@ -77,18 +53,10 @@ def _accepts(
backend: str | None = "triton",
draft_backend: str | None = None,
dcp_size: int = 1,
retraction_backup: str | None = None,
fields: dict | None = None,
attention_arch: AttentionArch = AttentionArch.MHA,
) -> bool:
"""Run just `handle_unified_memory_pool` against a minimal stand-in.
ServerArgs' real constructor pulls in a model config; this exercises the
single handler under test with the fields it reads (disaggregation off, so
only the speculative / cache / dcp / cuda-graph checks run). ServerArgs is
a msgspec Struct, so a field is restated with `force_setattr`, which the
Struct's own `__setattr__` guard allows and `object.__setattr__` does not.
"""
"""Run just `handle_unified_memory_pool` on a ServerArgs whose fields are
restated with `force_setattr` (ServerArgs is a msgspec Struct)."""
sa = ServerArgs(model_path="dummy")
for name, value in {
"enable_unified_memory": True,
@@ -98,13 +66,12 @@ def _accepts(
"speculative_draft_attention_backend": draft_backend,
"enable_hierarchical_cache": False,
"enable_lmcache": False,
"disaggregation_decode_retraction_backup": retraction_backup,
"disaggregation_decode_retraction_backup": None,
"dcp_size": dcp_size,
"cuda_graph_config": None,
"attention_backend": backend,
"prefill_attention_backend": None,
"decode_attention_backend": None,
**(fields or {}),
}.items():
msgspec.structs.force_setattr(sa, name, value)
sa._model_config = SimpleNamespace(
@@ -120,298 +87,103 @@ def _accepts(
return False
# The audited verify set (includes the MLA family).
_VERIFY_BACKENDS = (
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
)
_MHA_RAILS = ("triton", "flashinfer", "fa3")
class TestUnifiedMemorySpecGate(unittest.TestCase):
# Verify-audited backends for the DSPARK arm (includes the MLA family).
DSPARK_BACKENDS = (
"triton",
"trtllm_mla",
"cutedsl_mla",
"tokenspeed_mla",
"flashmla",
"flashinfer",
"fa3",
)
# The translated MHA rails: the EAGLE and DFLASH arms' target backends
# on an MHA host, and every fused draft's backend.
MHA_RAILS = ("triton", "flashinfer", "fa3")
# Algorithms with no audited unified-pool verify rails.
# "NEXTN" is deliberately absent: the CLI alias collapses it to
# "EAGLE" in handle_speculative_decoding BEFORE this gate runs
# (arg_groups/pipeline.py orders the hooks), so the raw string can
# never reach the gate -- a case on it would test an impossible
# input. The aliased spelling is covered by the EAGLE cases.
UNAUDITED_ALGORITHMS = ("NGRAM", "STANDALONE")
VERIFY_BACKENDS = _VERIFY_BACKENDS
MHA_RAILS = _MHA_RAILS
MLA_ONLY = tuple(b for b in _VERIFY_BACKENDS if b not in _MHA_RAILS)
UNAUDITED = ("fa4", "trtllm_mha")
def test_dspark_admitted_on_every_audited_backend(self):
"""Each entry is a claim that the backend's spec verify path
translates. Adding one here without the code is how a unified run
silently reads the pool with virtual ids."""
for backend in self.DSPARK_BACKENDS:
self.assertTrue(
_accepts("DSPARK", backend=backend),
f"DSPARK should pass on verify-audited backend {backend}",
)
def test_dspark_refused_on_unaudited_backends(self):
"""fa4 and trtllm_mha have no translated spec verify path."""
for backend in ("fa4", "trtllm_mha"):
self.assertFalse(_accepts("DSPARK", backend=backend))
def test_tree_drafting_refused(self):
"""Tree verify needs a per-token move inside the TARGET pool, which
the unified pool's page-granular `move_kv_cache` cannot express.
DSPARK demonstrates the rule here; the check itself is unconditional,
so it holds for every admitted arm."""
for topk in (2, 4, 8):
self.assertFalse(_accepts("DSPARK", topk=topk))
# The shape knob must not disturb spec-off.
self.assertTrue(_accepts(None, topk=None))
def test_dspark_draft_backend_is_not_constrained(self):
"""The gate does not constrain DSPARK's draft backend: a draft whose
backend is off the translated rails keeps a private pool (the
fused-draft decision declines fusion), where its translator is a
passthrough and any backend reads correctly. Model hooks declare one
on the operator's behalf -- Kimi-Linear / Kimi-K3 + DSPARK on SM100
declares `trtllm_mha` -- so refusing it would refuse a configuration
the operator never touched."""
for draft_backend in (None, *self.DSPARK_BACKENDS, "fa4", "trtllm_mha"):
self.assertTrue(
_accepts("DSPARK", draft_backend=draft_backend),
f"DSPARK draft backend {draft_backend} should pass",
)
def test_dspark_verify_backends(self):
for backend in self.VERIFY_BACKENDS:
self.assertTrue(_accepts("DSPARK", backend=backend), backend)
for backend in self.UNAUDITED:
self.assertFalse(_accepts("DSPARK", backend=backend), backend)
def test_dcp_keeps_flashinfer_off_the_verify_list(self):
"""Under --dcp-size > 1 the read ids stay DCP-widened for the consumer
to finish; flashinfer's spec verify gathers its CSR args with no DCP
read translation, so it is refused there while the MLA verify family,
which builds its own DCP block table, still passes."""
self.assertTrue(
_accepts(
"DSPARK",
backend="trtllm_mla",
dcp_size=2,
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
)
)
self.assertFalse(
_accepts(
"DSPARK",
backend="flashinfer",
dcp_size=2,
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
)
)
# Without DCP flashinfer stays admitted.
self.assertTrue(
_accepts(
"DSPARK",
backend="flashinfer",
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
)
)
"""Under DCP flashinfer's spec verify reads DCP-widened ids; the MLA
verify family builds its own DCP block table."""
mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA)
self.assertTrue(_accepts("DSPARK", backend="trtllm_mla", dcp_size=2, **mla))
self.assertFalse(_accepts("DSPARK", backend="flashinfer", dcp_size=2, **mla))
self.assertTrue(_accepts("DSPARK", backend="flashinfer", **mla))
def test_dflash_admitted_only_on_its_verify_backends(self):
"""DFLASH's target verifies on triton / fa3 / flashinfer. fa4 and
trtllm_mha have no translated spec verify path, and the MLA verify
family the DSPARK arm admits must not leak into this arm."""
def test_dflash_verify_backends(self):
"""DFLASH verifies on the MHA rails only; the MLA family DSPARK admits
must not leak into this arm."""
for backend in self.MHA_RAILS:
self.assertTrue(
_accepts("DFLASH", backend=backend),
f"DFLASH should pass on verify-audited backend {backend}",
)
for backend in ("fa4", "trtllm_mha") + tuple(
b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS
):
self.assertFalse(
_accepts("DFLASH", backend=backend),
f"DFLASH must be refused on backend {backend}",
)
self.assertTrue(_accepts("DFLASH", backend=backend), backend)
for backend in self.UNAUDITED + self.MLA_ONLY:
self.assertFalse(_accepts("DFLASH", backend=backend), backend)
def test_eagle_family_admitted_on_hybrid_swa(self):
"""EAGLE/EAGLE3 chain on a hybrid-SWA target with audited verify
backends is the fused-draft-KV configuration -- both spellings,
resolved or unset topk, every audited backend."""
def test_eagle_verify_backends_on_an_mha_host(self):
for algorithm in ("EAGLE", "EAGLE3"):
for topk in (None, 1):
for backend in self.MHA_RAILS:
self.assertTrue(
_accepts(algorithm, topk=topk, backend=backend),
f"{algorithm} topk={topk} backend={backend} should "
"pass on a hybrid-SWA target",
f"{algorithm} topk={topk} {backend}",
)
def test_eagle_refused_on_dense_targets(self):
"""No fused draft region outside the unified composites: a dense
(non-hybrid) target must be refused, not fail at boot."""
for algorithm in ("EAGLE", "EAGLE3"):
self.assertFalse(_accepts(algorithm, is_hybrid_swa=False))
def test_eagle_admitted_on_mamba_mha(self):
"""A mamba hybrid off the MLA backend provisions the fused draft
region in its MHA full sub-pool; refusing it here strands the whole
mamba x EAGLE matrix."""
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
for algorithm in ("EAGLE", "EAGLE3"):
self.assertTrue(_accepts(algorithm, is_hybrid_swa=False))
def test_eagle_on_mla_mamba_verifies_on_the_mla_family(self):
"""An MLA mamba hybrid fuses into MLA pages, so the target verifies on
the MLA backend family, while the fused draft is MHA-shaped and must run
on the translated MHA rails: an MLA-only backend is refused for the
draft whether named or inherited."""
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
for backend in self.DSPARK_BACKENDS:
self.assertTrue(
_accepts(
"EAGLE",
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
backend=backend,
draft_backend="triton",
),
f"EAGLE on an MLA host should pass on {backend}",
for backend in self.UNAUDITED + self.MLA_ONLY:
self.assertFalse(
_accepts(algorithm, backend=backend, draft_backend="triton"),
f"{algorithm} {backend}",
)
def test_eagle_draft_backend(self):
"""An unset draft backend inherits the target's; an explicit one must
be on the MHA rails."""
self.assertTrue(_accepts("EAGLE", draft_backend=None))
for draft_backend in self.MHA_RAILS:
self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend))
for draft_backend in self.UNAUDITED:
self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend))
def test_eagle_on_an_mla_host(self):
"""An MLA mamba hybrid verifies on the audited verify set, while its
fused draft is MHA-shaped: an MLA-only draft backend is refused whether
named or inherited."""
mla = dict(is_hybrid_swa=False, attention_arch=AttentionArch.MLA)
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
for backend in self.VERIFY_BACKENDS:
self.assertTrue(
_accepts("EAGLE", backend=backend, draft_backend="triton", **mla),
backend,
)
self.assertFalse(
_accepts("EAGLE", backend="fa4", draft_backend="triton", **mla)
)
for backend in ("trtllm_mla", "flashmla"):
for draft_backend in (None, backend):
self.assertFalse(
_accepts(
"EAGLE",
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
backend=backend,
draft_backend=draft_backend,
"EAGLE", backend=backend, draft_backend=draft_backend, **mla
),
f"MLA-only draft backend {draft_backend or backend}",
draft_backend or backend,
)
for backend in (None, "fa4"):
self.assertFalse(
_accepts(
"EAGLE",
is_hybrid_swa=False,
attention_arch=AttentionArch.MLA,
backend=backend,
)
)
def test_eagle_refused_tree_topk(self):
"""Tree verify is not audited for the unified pool; only a linear
chain passes."""
for topk in (2, 4, 8):
self.assertFalse(_accepts("EAGLE", topk=topk))
def test_eagle_refused_unaudited_backends(self):
"""Unaudited backends stay out, and the MLA verify set from the
DSPARK arm must not leak into the EAGLE arm."""
for backend in ("fa4", "trtllm_mha") + tuple(
b for b in self.DSPARK_BACKENDS if b not in self.MHA_RAILS
):
self.assertFalse(_accepts("EAGLE", backend=backend))
def test_eagle_refused_unset_backend(self):
"""A real boot resolves the default backend before this gate runs, so
an unset one here is unresolved: the gate refuses it rather than
trusting it."""
self.assertFalse(_accepts("EAGLE", backend=None))
def test_eagle_draft_backend_pinned(self):
"""The draft worker resolves its own backend: unset inherits the
target's (triton here), an explicit triton / flashinfer / fa3 passes,
and anything else refuses."""
self.assertTrue(_accepts("EAGLE", draft_backend=None))
for draft_backend in self.MHA_RAILS:
self.assertTrue(_accepts("EAGLE", draft_backend=draft_backend))
for draft_backend in ("fa4", "trtllm_mha"):
self.assertFalse(_accepts("EAGLE", draft_backend=draft_backend))
def test_hierarchical_cache_refused_with_an_eagle_draft(self):
"""Under HiCache an EAGLE draft keeps a private pool, and an MTP
head's KV would be packed into the target's page-envelope host pool,
which refuses per-layer draft loads. Without a draft HiCache is not
this gate's to refuse, and with DSPARK HiCache declines draft fusion,
so the draft keeps its private pool."""
hicache = {"enable_hierarchical_cache": True}
for algorithm in ("EAGLE", "EAGLE3"):
self.assertFalse(_accepts(algorithm, fields=hicache))
self.assertTrue(_accepts(None, fields=hicache))
self.assertTrue(_accepts("DSPARK", fields=hicache))
def test_pd_decode_refusals_raise_their_own_assertions(self):
"""The PD-decode branches screen the model with `mambaish_config` and
refuse with their own assertion, not an error from the screen."""
pd_decode = {
"disaggregation_mode": "decode",
"disaggregation_transfer_backend": "mooncake",
"pp_size": 1,
"enable_hisparse": False,
}
for extra in (
{
"disaggregation_decode_enable_offload_kvcache": True,
"hicache_storage_backend": "file",
},
{"disaggregation_decode_retraction_backup": "host_pool"},
):
with patch.object(kv_cache_hook, "mambaish_config", return_value=object()):
self.assertFalse(
_accepts(None, is_hybrid_swa=False, fields={**pd_decode, **extra})
)
def test_host_pool_retraction_refused_with_an_eagle_draft(self):
"""The host-pool retraction backup builds the draft's host pool off
its device pool, which an EAGLE draft, always fused on the unified
pool, does not have. Without a draft the backup is not this gate's to
refuse, and with DFLASH or DSPARK it declines draft fusion instead."""
for algorithm in ("EAGLE", "EAGLE3"):
with self.subTest(algorithm=algorithm):
self.assertTrue(_accepts(algorithm))
self.assertFalse(_accepts(algorithm, retraction_backup="host_pool"))
for algorithm in (None, "DFLASH", "DSPARK"):
with self.subTest(algorithm=algorithm):
self.assertTrue(_accepts(algorithm, retraction_backup="host_pool"))
def test_dspark_arm_unchanged(self):
"""The EAGLE addition must not perturb DSPARK: its MLA verify set
passes, its chain constraint holds, and unaudited backends refuse."""
for backend in self.DSPARK_BACKENDS:
self.assertTrue(
_accepts("DSPARK", backend=backend),
f"DSPARK should pass on verify-audited backend {backend}",
)
self.assertFalse(_accepts("DSPARK", backend="fa4"))
self.assertFalse(_accepts("DSPARK", topk=4))
def test_ngram_refused_because_it_relocates_target_kv_per_token(self):
"""BUG REGRESSION. NGRAM was admitted on the reasoning that
it carries no draft KV, so only the target-verify rails matter. But a
tree-shaped drafter finalizes a batch through
`move_accept_tokens_to_target_kvcache`, which relocates INDIVIDUAL
accepted tokens inside the target pool -- and the unified pool's
`move_kv_cache` is compaction-only (whole page envelopes). Every
unified NGRAM cell died there: a reshape past the end on the MHA/mamba
hosts, and the SWA composite's explicit refusal on gpt-oss. Admitting
it again without a per-token move primitive re-breaks every family."""
for backend in self.DSPARK_BACKENDS:
self.assertFalse(
_accepts("NGRAM", backend=backend),
f"NGRAM must stay refused (backend={backend})",
)
def test_spec_off_admitted(self):
"""The gate constrains only speculative configurations; spec-off must
keep booting regardless of family."""
for is_hybrid_swa in (True, False):
self.assertTrue(_accepts(None, is_hybrid_swa=is_hybrid_swa))
def test_unaudited_algorithms_refused(self):
"""Every other algorithm stays out until its verify id rails are
audited -- on every family."""
for algorithm in self.UNAUDITED_ALGORITHMS:
def test_unaudited_algorithms_and_tree_drafting_refused(self):
"""NGRAM and tree drafting relocate accepted tokens one at a time
inside the target pool, which the unified pool's page-granular move
cannot express."""
for algorithm in ("NGRAM", "STANDALONE"):
for is_hybrid_swa in (True, False):
self.assertFalse(_accepts(algorithm, is_hybrid_swa=is_hybrid_swa))
for algorithm in ("DSPARK", "EAGLE"):
for topk in (2, 4):
self.assertFalse(_accepts(algorithm, topk=topk), f"{algorithm} {topk}")
if __name__ == "__main__":
@@ -281,6 +281,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase):
hidden_states=object(), mm_input_embeds=None
),
next_token_ids=object(),
kv_loc_plan=None,
)
verify_input = object()
@@ -306,7 +307,7 @@ class TestDPSpecPrefillCoordinationWorker(CustomTestCase):
self.assertEqual(kwargs["grammar_barrier"], "grammar")
return result
def extend(current, *args):
def extend(current, *args, **kwargs):
record("draft_extend", current)
return object()
@@ -93,9 +93,7 @@ class DSparkTargetHiddenProjectionTest(CustomTestCase):
draft_model=draft_model,
draft_model_runner=SimpleNamespace(
token_to_kv_pool=pool,
kv_index_translator=SimpleNamespace(
translate_full_attn_ids=lambda ids: ids
),
kv_index_translator=SimpleNamespace(),
),
model_runner=SimpleNamespace(device=torch.device("cpu")),
device=torch.device("cpu"),
@@ -229,7 +229,9 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
max_context_len=1,
)
worker.target_worker = SimpleNamespace(
model_runner=SimpleNamespace(attn_backend=attn_backend)
model_runner=SimpleNamespace(
attn_backend=attn_backend, kv_index_translator=None
)
)
draft_input = SimpleNamespace(
bonus_tokens=torch.zeros((1,), dtype=torch.long, device=DEVICE),
@@ -253,7 +255,7 @@ class TestEagleWorkerV2BackendFallback(CustomTestCase):
),
patch(
"sglang.srt.speculative.eagle_worker_v2.prepare_for_draft",
return_value=(forward_batch, True),
return_value=(forward_batch, True, None),
),
):
worker.draft(batch)
@@ -1,194 +0,0 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A hand-built ForwardBatch view must carry the write rail.
BUG REGRESSION. The speculative draft-extend cuda-graph runners do not hand a
real ForwardBatch to `init_forward_metadata_out_graph`; they build a
`SimpleNamespace` view over their capture-stable buffers. On a translating
pool that hook reaches `fill_capture_write_loc`, which reads
`out_cache_loc_virtual` -- the pre-translate write loc -- and translates it
into the backend's capture buffer.
A view that omits the field fails two ways, and the quiet one is worse. A
`SimpleNamespace` raises AttributeError, which at least crashes the cell. But
`fill_capture_write_loc` also accepts the field being None, and answers it by
zeroing the buffer -- page-0 sink ids. So a view that "fixes" the crash by
passing None sends every draft-extend KV write to the sink, and speculative
decoding rejects the resulting drafts instead of failing: it surfaces as
accept length decaying toward 1.0, not as an error.
Three runners had the omission independently, so this guards the shape rather
than the three sites: any `SimpleNamespace` view that reaches
`init_forward_metadata_out_graph` must name `out_cache_loc_virtual`.
The field is only read on a unified (translating) pool, which is why the
default builds never caught it.
CPU-only, no torch.
python -m pytest test/registered/unit/spec/test_hand_built_view_write_rail.py -v
"""
import ast
import pathlib
import unittest
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
_SPEC_DIR = (
pathlib.Path(__file__).resolve().parents[4]
/ "python"
/ "sglang"
/ "srt"
/ "speculative"
)
_HOOK = "init_forward_metadata_out_graph"
_RAIL = "out_cache_loc_virtual"
def _views_missing_the_rail(path: pathlib.Path):
"""(function, variable, lineno) per hand-built view that drops the rail.
A view counts when it is a `SimpleNamespace(...)` bound to a name that is
later passed to the hook in the same function -- that is exactly the shape
the runners use, and it keeps the walk from flagging namespaces built for
unrelated purposes.
"""
tree = ast.parse(path.read_text())
offenders = []
for fn in ast.walk(tree):
if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
# Names handed to the hook anywhere in this function.
handed = set()
for node in ast.walk(fn):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == _HOOK
):
for arg in node.args:
if isinstance(arg, ast.Name):
handed.add(arg.id)
if not handed:
continue
for node in ast.walk(fn):
if not (isinstance(node, ast.Assign) and isinstance(node.value, ast.Call)):
continue
func = node.value.func
name = (
func.attr
if isinstance(func, ast.Attribute)
else getattr(func, "id", None)
)
if name != "SimpleNamespace":
continue
for target in node.targets:
if not (isinstance(target, ast.Name) and target.id in handed):
continue
# A literal None is the quiet failure: the hook zeroes the
# capture buffer, routing every write to the page-0 sink.
if not any(
kw.arg == _RAIL
and not (
isinstance(kw.value, ast.Constant) and kw.value.value is None
)
for kw in node.value.keywords
):
offenders.append((fn.name, target.id, node.lineno))
return sorted(offenders, key=lambda o: o[2])
class TestHandBuiltViewWriteRail(unittest.TestCase):
def test_every_hand_built_view_carries_the_rail(self):
offenders = []
for path in sorted(_SPEC_DIR.rglob("*.py")):
for fn, var, lineno in _views_missing_the_rail(path):
offenders.append(f"{path.name}:{lineno} {fn}() -> {var}")
self.assertEqual(
offenders,
[],
f"hand-built ForwardBatch view reaches {_HOOK} without {_RAIL}; on a "
"unified pool the draft-extend writes land on the page-0 sink: "
+ "; ".join(offenders),
)
def test_detector_catches_a_view_that_drops_the_rail(self):
"""The detector must catch the shape it guards, else it passes green
forever after a refactor of the walk above."""
src = (
"class R:\n"
" def replay(self, fb):\n"
" v = SimpleNamespace(\n"
" batch_size=8, out_cache_loc=self.buffers.out_cache_loc\n"
" )\n"
" self.backend.init_forward_metadata_out_graph(v)\n"
)
self.assertEqual(
_views_missing_the_rail(self._probe(src)), [("replay", "v", 3)]
)
def test_detector_catches_a_view_that_passes_none(self):
"""None is accepted by `fill_capture_write_loc` and becomes the page-0
sink, so naming the rail with None is the omission in disguise."""
src = (
"class R:\n"
" def replay(self, fb):\n"
" v = SimpleNamespace(batch_size=8, out_cache_loc_virtual=None)\n"
" self.backend.init_forward_metadata_out_graph(v)\n"
)
self.assertEqual(
_views_missing_the_rail(self._probe(src)), [("replay", "v", 3)]
)
def test_detector_accepts_a_view_that_carries_it(self):
"""...and must NOT fire once the rail is named, else it is unfixable."""
src = (
"class R:\n"
" def replay(self, fb):\n"
" v = SimpleNamespace(\n"
" batch_size=8,\n"
" out_cache_loc=self.buffers.out_cache_loc,\n"
" out_cache_loc_virtual=fb.out_cache_loc_virtual,\n"
" )\n"
" self.backend.init_forward_metadata_out_graph(v)\n"
)
self.assertEqual(_views_missing_the_rail(self._probe(src)), [])
def test_detector_ignores_a_namespace_that_never_reaches_the_hook(self):
"""A SimpleNamespace built for something else is not a view."""
src = (
"class R:\n"
" def replay(self, fb):\n"
" cfg = SimpleNamespace(batch_size=8)\n"
" return cfg\n"
)
self.assertEqual(_views_missing_the_rail(self._probe(src)), [])
def _probe(self, src):
# A temp dir, not the CWD: the checkout may be read-only.
import shutil
import tempfile
scratch = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, scratch, ignore_errors=True)
tmp = pathlib.Path(scratch) / (self.id().replace(".", "_") + ".py")
tmp.write_text(src)
return tmp
if __name__ == "__main__":
unittest.main()